Merge branch 'main' into fix-ntfy-tags-schema-array

NEWS

@@ -1,5 +1,7 @@
 2.1.2.dev0
  * #1250: Fix a regression in which the "--stats" flag hides statistics at default verbosity.
+ * #1258: Fix a "codec can't decode byte" error when running commands that output multi-byte unicode
+   characters.
 
 2.1.1
  * #1241: For the "recreate" action, actually pass the "--dry-run" flag through to Borg instead of

borgmatic/execute.py

@@ -237,10 +237,12 @@ def read_lines(buffer, process, line_separator='\n'):
     call to know when to read more lines. Otherwise, the generator will busywait if it's called in a
     tight loop.
     '''
-    data = ''
+    data = b''
+    encoded_separator = line_separator.encode()
+    separator_size = len(encoded_separator)
 
     while True:
-        chunk = os.read(buffer.fileno(), READ_CHUNK_SIZE).decode()
+        chunk = os.read(buffer.fileno(), READ_CHUNK_SIZE)
 
         if not chunk:  # EOF
             # The process is still running, so we keep running too.
@@ -255,19 +257,19 @@ def read_lines(buffer, process, line_separator='\n'):
         # Split the data into lines, holding back anything leftover that might
         # be a partial line.
         while True:
-            separator_position = data.find(line_separator)
+            separator_position = data.find(encoded_separator)
 
             if separator_position == -1:
                 break
 
-            lines.append(data[:separator_position].rstrip())
-            data = data[separator_position + 1 :]
+            lines.append(data[:separator_position].decode())
+            data = data[separator_position + separator_size :]
 
         yield tuple(lines)
 
     # Yield any leftover data from the end of the buffer.
     if data:
-        yield (data.rstrip(),)
+        yield (data.decode().rstrip(),)
 
 
 Buffer_reader = collections.namedtuple(

tests/integration/test_execute.py

@@ -26,6 +26,19 @@ def test_read_lines_yields_single_line_longer_than_chunk_size():
     )
 
 
+def test_read_lines_yields_single_line_with_multibyte_unicode_character_spanning_chunk_boundary():
+    # In case it's not clear, "ñ" is a multi-byte UTF-8 character. The "a" shifts it over one byte
+    # so it straddles the chunk boundary.
+    process = subprocess.Popen(['echo', 'aññññññññññññññññññññññññññññññ'], stdout=subprocess.PIPE)
+
+    assert tuple(flexmock(module, READ_CHUNK_SIZE=16).read_lines(process.stdout, process)) == (
+        (),
+        (),
+        (),
+        ('aññññññññññññññññññññññññññññññ',),
+    )
+
+
 def test_read_lines_yields_multiple_lines():
     process = subprocess.Popen(['echo', 'hi\nthere'], stdout=subprocess.PIPE)