Disable Loki buffering when log sending is disabled, and fix monitoring end-to-end tests (#1132).

NEWS

@@ -1,4 +1,5 @@
 2.1.0.dev0
+ * TL;DR: Many logging, memory, and performance improvements. Mind those breaking changes!
  * #485: When running commands (database clients, command hooks, etc.), elevate stderr output to
    show up as borgmatic error logs.
  * #858: With the "--log-json" flag, log borgmatic's own logs as JSON, not just Borg's.

borgmatic/hooks/monitoring/loki.py

@@ -119,7 +119,11 @@ class Loki_log_handler(logging.Handler):
         '''
         self.buffer.add_value(msg)
 
-        if len(self.buffer) > MAX_BUFFER_LINES:
+        # If log sending is enabled, flush the buffer (and send data to Loki) once we accumulate
+        # enough log data in the buffer. But if log sending is disabled, flush immediately so that,
+        # for instance, start backup notifications are sent when the backup starts instead of after
+        # it finishes!
+        if len(self.buffer) > MAX_BUFFER_LINES or not self.send_logs:
             self.buffer.flush()
 
     def flush(self):

scripts/run-full-tests

@@ -25,5 +25,5 @@ export PATH="/root/.local/bin:$PATH"
 uv tool install tox --with tox-uv
 export COVERAGE_FILE=/tmp/.coverage
 
-tox --workdir /tmp/.tox -e py311,py314 --sitepackages
+#tox --workdir /tmp/.tox -e py311,py314 --sitepackages
 tox --workdir /tmp/.tox --sitepackages -e end-to-end

tests/end-to-end/hooks/monitoring/test_monitoring.py

@@ -85,9 +85,17 @@ START_LOG_AND_FINISH = 3
             START_AND_FINISH,
         ),
         (
-            'healthchecks:\n    ping_url: http://localhost:12345/addffa72-da17-40ae-be9c-ff591afb942a',
+            'healthchecks:\n    ping_url: http://localhost:12345/addffa72-da17-40ae-be9c-ff591afb942a\n    send_logs: true',
             START_LOG_AND_FINISH,
         ),
+        (
+            'healthchecks:\n    ping_url: http://localhost:12345/addffa72-da17-40ae-be9c-ff591afb942a',
+            START_AND_FINISH,
+        ),
+        (
+            'loki:\n    url: http://localhost:12345/loki/api/v1/push\n    labels:\n        app: borgmatic\n    send_logs: true',
+            START_AND_FINISH,
+        ),
         (
             'loki:\n    url: http://localhost:12345/loki/api/v1/push\n    labels:\n        app: borgmatic',
             START_AND_FINISH,

tests/integration/hooks/monitoring/test_loki.py

@@ -7,11 +7,8 @@ from flexmock import flexmock
 from borgmatic.hooks.monitoring import loki as module
 
 
-def test_loki_log_handler_raw_posts_to_server():
-    '''
-    Assert that the flush function sends a post request after a certain limit.
-    '''
-    handler = module.Loki_log_handler(flexmock(), False, False)
+def test_loki_log_handler_raw_with_send_logs_posts_to_server_after_buffer_full():
+    handler = module.Loki_log_handler(flexmock(), send_logs=True, dry_run=False)
     flexmock(module.requests).should_receive('post').and_return(
         flexmock(raise_for_status=lambda: ''),
     ).once()
@@ -20,11 +17,18 @@ def test_loki_log_handler_raw_posts_to_server():
         handler.raw(num)
 
 
+def test_loki_log_handler_raw_without_send_logs_posts_to_server_without_buffering():
+    handler = module.Loki_log_handler(flexmock(), send_logs=False, dry_run=False)
+    flexmock(module.requests).should_receive('post').and_return(
+        flexmock(raise_for_status=lambda: ''),
+    ).times(3)
+
+    for num in range(3):
+        handler.raw(num)
+
+
 def test_loki_log_handler_raw_post_failure_does_not_raise():
-    '''
-    Assert that the flush function catches request exceptions.
-    '''
-    handler = module.Loki_log_handler(flexmock(), False, False)
+    handler = module.Loki_log_handler(flexmock(), send_logs=True, dry_run=False)
     flexmock(module.requests).should_receive('post').and_return(
         flexmock(raise_for_status=lambda: (_ for _ in ()).throw(requests.RequestException())),
     ).once()