diff --git a/NEWS b/NEWS index f821f5c3..35ae1c86 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,7 @@ 2.1.2.dev0 * #1250: Fix a regression in which the "--stats" flag hides statistics at default verbosity. + * #1258: Fix a "codec can't decode byte" error when running commands that output multi-byte unicode + characters. 2.1.1 * #1241: For the "recreate" action, actually pass the "--dry-run" flag through to Borg instead of diff --git a/borgmatic/execute.py b/borgmatic/execute.py index 798f5226..e358f0d3 100644 --- a/borgmatic/execute.py +++ b/borgmatic/execute.py @@ -237,10 +237,12 @@ def read_lines(buffer, process, line_separator='\n'): call to know when to read more lines. Otherwise, the generator will busywait if it's called in a tight loop. ''' - data = '' + data = b'' + encoded_separator = line_separator.encode() + separator_size = len(encoded_separator) while True: - chunk = os.read(buffer.fileno(), READ_CHUNK_SIZE).decode() + chunk = os.read(buffer.fileno(), READ_CHUNK_SIZE) if not chunk: # EOF # The process is still running, so we keep running too. @@ -255,19 +257,19 @@ def read_lines(buffer, process, line_separator='\n'): # Split the data into lines, holding back anything leftover that might # be a partial line. while True: - separator_position = data.find(line_separator) + separator_position = data.find(encoded_separator) if separator_position == -1: break - lines.append(data[:separator_position].rstrip()) - data = data[separator_position + 1 :] + lines.append(data[:separator_position].decode()) + data = data[separator_position + separator_size :] yield tuple(lines) # Yield any leftover data from the end of the buffer. if data: - yield (data.rstrip(),) + yield (data.decode().rstrip(),) Buffer_reader = collections.namedtuple( diff --git a/tests/integration/test_execute.py b/tests/integration/test_execute.py index 4617fe5d..d1de7f44 100644 --- a/tests/integration/test_execute.py +++ b/tests/integration/test_execute.py @@ -26,6 +26,19 @@ def test_read_lines_yields_single_line_longer_than_chunk_size(): ) +def test_read_lines_yields_single_line_with_multibyte_unicode_character_spanning_chunk_boundary(): + # In case it's not clear, "ñ" is a multi-byte UTF-8 character. The "a" shifts it over one byte + # so it straddles the chunk boundary. + process = subprocess.Popen(['echo', 'aññññññññññññññññññññññññññññññ'], stdout=subprocess.PIPE) + + assert tuple(flexmock(module, READ_CHUNK_SIZE=16).read_lines(process.stdout, process)) == ( + (), + (), + (), + ('aññññññññññññññññññññññññññññññ',), + ) + + def test_read_lines_yields_multiple_lines(): process = subprocess.Popen(['echo', 'hi\nthere'], stdout=subprocess.PIPE)