Bump version for release.

Merge pull request #42 from VTimofeenko/systemd_protecthome

.drone.yml

@@ -14,6 +14,9 @@ services:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
 
+clone:
+  skip_verify: true
+
 steps:
 - name: build
   image: alpine:3.9
@@ -36,6 +39,9 @@ services:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
 
+clone:
+  skip_verify: true
+
 steps:
 - name: build
   image: alpine:3.10
@@ -58,6 +64,9 @@ services:
       MYSQL_ROOT_PASSWORD: test
       MYSQL_DATABASE: test
 
+clone:
+  skip_verify: true
+
 steps:
 - name: build
   image: alpine:3.13
@@ -68,6 +77,9 @@ steps:
 kind: pipeline
 name: documentation
 
+clone:
+  skip_verify: true
+
 steps:
 - name: build
   #image: plugins/docker

NEWS

@@ -1,3 +1,19 @@
+1.5.20
+ * Re-release with correct version without dev0 tag.
+
+1.5.19
+ * #387: Fix error when configured source directories are not present on the filesystem at the time
+   of backup. Now, Borg will complain, but the backup will still continue.
+ * #455: Mention changing borgmatic path in cron documentation.
+ * Update sample systemd service file with more granular read-only filesystem settings.
+ * Move Gitea and GitHub hosting from a personal namespace to an organization for better
+   collaboration with related projects.
+ * 1k ★s on GitHub!
+
+1.5.18
+ * #389: Fix "message too long" error when logging to rsyslog.
+ * #440: Fix traceback that can occur when dumping a database.
+
 1.5.17
  * #437: Fix error when configuration file contains "umask" option.
  * Remove test dependency on vim and /dev/urandom.
@@ -564,7 +580,7 @@
  * #49: Support for Borg experimental --patterns-from and --patterns options for specifying mixed
    includes/excludes.
  * Moved issue tracker from Taiga to integrated Gitea tracker at
-   https://projects.torsion.org/witten/borgmatic/issues
+   https://projects.torsion.org/borgmatic-collective/borgmatic/issues
 
 1.1.12
  * #46: Declare dependency on pykwalify 1.6 or above, as older versions yield "Unknown key: version"

README.md

@@ -106,7 +106,7 @@ development or hosting.
 ### Issues
 
 You've got issues? Or an idea for a feature enhancement? We've got an [issue
-tracker](https://projects.torsion.org/witten/borgmatic/issues). In order to
+tracker](https://projects.torsion.org/borgmatic-collective/borgmatic/issues). In order to
 create a new issue or comment on an issue, you'll need to [login
 first](https://projects.torsion.org/user/login). Note that you can login with
 an existing GitHub account if you prefer.
@@ -129,15 +129,15 @@ Other questions or comments? Contact
 ### Contributing
 
 borgmatic [source code is
-available](https://projects.torsion.org/witten/borgmatic) and is also mirrored
-on [GitHub](https://github.com/witten/borgmatic) for convenience.
+available](https://projects.torsion.org/borgmatic-collective/borgmatic) and is also mirrored
+on [GitHub](https://github.com/borgmatic-collective/borgmatic) for convenience.
 
 borgmatic is licensed under the GNU General Public License version 3 or any
 later version.
 
 If you'd like to contribute to borgmatic development, please feel free to
-submit a [Pull Request](https://projects.torsion.org/witten/borgmatic/pulls)
-or open an [issue](https://projects.torsion.org/witten/borgmatic/issues) first
+submit a [Pull Request](https://projects.torsion.org/borgmatic-collective/borgmatic/pulls)
+or open an [issue](https://projects.torsion.org/borgmatic-collective/borgmatic/issues) first
 to discuss your idea. We also accept Pull Requests on GitHub, if that's more
 your thing. In general, contributions are very welcome. We don't bite! 
 
@@ -145,5 +145,5 @@ Also, please check out the [borgmatic development
 how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
 info on cloning source code, running tests, etc.
 
-<a href="https://build.torsion.org/witten/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/witten/borgmatic/status.svg?ref=refs/heads/master)</a>
+<a href="https://build.torsion.org/borgmatic-collective/borgmatic" alt="build status">![Build Status](https://build.torsion.org/api/badges/borgmatic-collective/borgmatic/status.svg?ref=refs/heads/master)</a>
 

borgmatic/borg/create.py

@@ -44,13 +44,18 @@ def _expand_home_directories(directories):
     return tuple(os.path.expanduser(directory) for directory in directories)
 
 
-def map_directories_to_devices(directories):  # pragma: no cover
+def map_directories_to_devices(directories):
     '''
     Given a sequence of directories, return a map from directory to an identifier for the device on
-    which that directory resides. This is handy for determining whether two different directories
-    are on the same filesystem (have the same device identifier).
+    which that directory resides or None if the path doesn't exist.
+
+    This is handy for determining whether two different directories are on the same filesystem (have
+    the same device identifier).
     '''
-    return {directory: os.stat(directory).st_dev for directory in directories}
+    return {
+        directory: os.stat(directory).st_dev if os.path.exists(directory) else None
+        for directory in directories
+    }
 
 
 def deduplicate_directories(directory_devices):
@@ -82,6 +87,7 @@ def deduplicate_directories(directory_devices):
             for parent in parents:
                 if (
                     pathlib.PurePath(other_directory) == parent
+                    and directory_devices[directory] is not None
                     and directory_devices[other_directory] == directory_devices[directory]
                 ):
                     if directory in deduplicated:

borgmatic/config/schema.yaml

@@ -135,12 +135,14 @@ properties:
                     type: string
                 description: |
                     Any paths matching these patterns are excluded from backups.
-                    Globs and tildes are expanded. Do not backslash spaces in
-                    path names. See the output of "borg help patterns" for more
-                    details.
+                    Globs and tildes are expanded. (Note however that a glob
+                    pattern must either start with a glob or be an absolute
+                    path.) Do not backslash spaces in path names. See the output
+                    of "borg help patterns" for more details.
                 example:
                     - '*.pyc'
                     - /home/*/.cache
+                    - '*/.vim*.tmp'
                     - /etc/ssl
                     - /home/user/path with spaces
             exclude_from:
@@ -633,7 +635,7 @@ properties:
                                 Password with which to connect to the database.
                                 Omitting a password will only work if PostgreSQL
                                 is configured to trust the configured username
-                                without a password, or you create a ~/.pgpass
+                                without a password or you create a ~/.pgpass
                                 file.
                             example: trustsome1
                         format:

borgmatic/execute.py

@@ -81,6 +81,7 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
                     for other_process in processes:
                         if (
                             other_process.poll() is None
+                            and other_process.stdout
                             and other_process.stdout not in output_buffers
                         ):
                             # Add the process's output to output_buffers to ensure it'll get read.
@@ -138,9 +139,12 @@ def log_outputs(processes, exclude_stdouts, output_log_level, borg_local_path):
         if not output_buffer:
             continue
 
-        remaining_output = output_buffer.read().rstrip().decode()
+        while True:  # pragma: no cover
+            remaining_output = output_buffer.readline().rstrip().decode()
+
+            if not remaining_output:
+                break
 
-        if remaining_output:  # pragma: no cover
             logger.log(output_log_level, remaining_output)
 
 

docs/_includes/components/suggestion-link.html

@@ -1,17 +1,5 @@
 <h2>Improve this documentation</h2>
 
 <p>Have an idea on how to make this documentation even better? Use our <a
-href="https://projects.torsion.org/witten/borgmatic/issues">issue tracker</a> to send your
+href="https://projects.torsion.org/borgmatic-collective/borgmatic/issues">issue tracker</a> to send your
 feedback!</p>
-
-<script>
-    document.getElementById('_page').value = window.location.href;
-    window.sk=window.sk||function(){(sk.q=sk.q||[]).push(arguments)};
-  
-    sk('form', 'init', {
-        id: '1d536680ab96',
-        element: '#suggestion-form'
-    });
-</script>
-
-<script defer src="https://js.statickit.com/statickit.js"></script>

docs/how-to/develop-on-borgmatic.md

@@ -10,13 +10,13 @@ eleventyNavigation:
 To get set up to hack on borgmatic, first clone master via HTTPS or SSH:
 
 ```bash
-git clone https://projects.torsion.org/witten/borgmatic.git
+git clone https://projects.torsion.org/borgmatic-collective/borgmatic.git
 ```
 
 Or:
 
 ```bash
-git clone ssh://git@projects.torsion.org:3022/witten/borgmatic.git
+git clone ssh://git@projects.torsion.org:3022/borgmatic-collective/borgmatic.git
 ```
 
 Then, install borgmatic
@@ -66,8 +66,6 @@ following:
 tox -e black
 ```
 
-Note that Black requires at minimum Python 3.6.
-
 And if you get a complaint from the
 [isort](https://github.com/timothycrosley/isort) Python import orderer, you
 can ask isort to order your imports for you:
@@ -118,7 +116,7 @@ See the Black, Flake8, and isort documentation for more information.
 
 Each pull request triggers a continuous integration build which runs the test
 suite. You can view these builds on
-[build.torsion.org](https://build.torsion.org/witten/borgmatic), and they're
+[build.torsion.org](https://build.torsion.org/borgmatic-collective/borgmatic), and they're
 also linked from the commits list on each pull request.
 
 ## Documentation development

docs/how-to/set-up-backups.md

@@ -28,7 +28,7 @@ sudo pip3 install --user --upgrade borgmatic
 This installs borgmatic and its commands at the `/root/.local/bin` path.
 
 Your pip binary may have a different name than "pip3". Make sure you're using
-Python 3, as borgmatic does not support Python 2.
+Python 3.6+, as borgmatic does not support Python 2.
 
 The next step is to ensure that borgmatic's commands available are on your
 system `PATH`, so that you can run borgmatic:
@@ -77,7 +77,7 @@ on a relatively dedicated system, then a global install can work out fine.
 Besides the approaches described above, there are several other options for
 installing borgmatic:
 
- * [Docker image with scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/)
+ * [Docker image with scheduled backups](https://hub.docker.com/r/b3vis/borgmatic/) (+ Docker Compose files)
  * [Docker base image](https://hub.docker.com/r/monachus/borgmatic/)
  * [Debian](https://tracker.debian.org/pkg/borgmatic)
  * [Ubuntu](https://launchpad.net/ubuntu/+source/borgmatic)
@@ -250,7 +250,7 @@ that, you can configure a separate job runner to invoke it periodically.
 ### cron
 
 If you're using cron, download the [sample cron
-file](https://projects.torsion.org/witten/borgmatic/src/master/sample/cron/borgmatic).
+file](https://projects.torsion.org/borgmatic-collective/borgmatic/src/master/sample/cron/borgmatic).
 Then, from the directory where you downloaded it:
 
 ```bash
@@ -258,7 +258,10 @@ sudo mv borgmatic /etc/cron.d/borgmatic
 sudo chmod +x /etc/cron.d/borgmatic
 ```
 
-You can modify the cron file if you'd like to run borgmatic more or less frequently.
+If borgmatic is installed at a different location than
+`/root/.local/bin/borgmatic`, edit the cron file with the correct path. You
+can also modify the cron file if you'd like to run borgmatic more or less
+frequently.
 
 ### systemd
 
@@ -271,9 +274,9 @@ you may already have borgmatic systemd service and timer files. If so, you may
 be able to skip some of the steps below.)
 
 First, download the [sample systemd service
-file](https://projects.torsion.org/witten/borgmatic/raw/branch/master/sample/systemd/borgmatic.service)
+file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/master/sample/systemd/borgmatic.service)
 and the [sample systemd timer
-file](https://projects.torsion.org/witten/borgmatic/raw/branch/master/sample/systemd/borgmatic.timer).
+file](https://projects.torsion.org/borgmatic-collective/borgmatic/raw/branch/master/sample/systemd/borgmatic.timer).
 
 Then, from the directory where you downloaded them:
 
@@ -294,7 +297,7 @@ borgmatic to run.
 If you run borgmatic in macOS with launchd, you may encounter permissions
 issues when reading files to backup. If that happens to you, you may be
 interested in an [unofficial work-around for Full Disk
-Access](https://projects.torsion.org/witten/borgmatic/issues/293).
+Access](https://projects.torsion.org/borgmatic-collective/borgmatic/issues/293).
 
 
 ## Colored output

sample/systemd/borgmatic.service

@@ -32,13 +32,16 @@ RestrictSUIDSGID=yes
 SystemCallArchitectures=native
 SystemCallFilter=@system-service
 SystemCallErrorNumber=EPERM
-# Restrict write access
-# Change to 'ProtectSystem=strict' and uncomment 'ProtectHome' to make the whole file
-# system read-only be default and uncomment 'ReadWritePaths' for the required write access.
-# Add local repositroy paths to the list of 'ReadWritePaths' like '-/mnt/my_backup_drive'.
+# To restrict write access further, change "ProtectSystem" to "strict" and uncomment
+# "ReadWritePaths", "ReadOnlyPaths", "ProtectHome", and "BindPaths". Then add any local repository
+# paths to the list of "ReadWritePaths" and local backup source paths to "ReadOnlyPaths". This
+# leaves most of the filesystem read-only to borgmatic.
 ProtectSystem=full
-# ProtectHome=read-only
-# ReadWritePaths=-/root/.config/borg -/root/.cache/borg -/root/.borgmatic
+# ReadWritePaths=-/mnt/my_backup_drive
+# ReadOnlyPaths=-/var/lib/my_backup_source
+# This will mount a tmpfs on top of /root and pass through needed paths
+# ProtectHome=tmpfs
+# BindPaths=-/root/.cache/borg -/root/.cache/borg -/root/.borgmatic
 
 CapabilityBoundingSet=CAP_DAC_READ_SEARCH CAP_NET_RAW
 

scripts/release

@@ -38,7 +38,7 @@ twine upload -r pypi dist/borgmatic-*-py3-none-any.whl dist/borgmatic-*-py3-none
 release_changelog="$(cat NEWS | sed '/^$/q' | grep -v '^\S')"
 escaped_release_changelog="$(echo "$release_changelog" | sed -z 's/\n/\\n/g' | sed -z 's/\"/\\"/g')"
 curl --silent --request POST \
-    "https://projects.torsion.org/api/v1/repos/witten/borgmatic/releases" \
+    "https://projects.torsion.org/api/v1/repos/borgmatic-collective/borgmatic/releases" \
     --header "Authorization: token $projects_token" \
     --header "Accept: application/json" \
     --header "Content-Type: application/json" \

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-VERSION = '1.5.17'
+VERSION = '1.5.20'
 
 
 setup(

tests/integration/test_execute.py

@@ -162,6 +162,37 @@ def test_log_outputs_vents_other_processes_when_one_exits():
     )
 
 
+def test_log_outputs_does_not_error_when_one_process_exits():
+    flexmock(module.logger).should_receive('log')
+    flexmock(module).should_receive('command_for_process').and_return('grep')
+
+    process = subprocess.Popen(
+        [
+            sys.executable,
+            '-c',
+            "import random, string; print(''.join(random.choice(string.ascii_letters) for _ in range(40000)))",
+        ],
+        stdout=None,  # Specifically test the case of a process without stdout captured.
+        stderr=None,
+    )
+    other_process = subprocess.Popen(
+        ['true'], stdin=process.stdout, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+    )
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        process, (process.stdout,)
+    ).and_return(process.stderr)
+    flexmock(module).should_receive('output_buffer_for_process').with_args(
+        other_process, (process.stdout,)
+    ).and_return(other_process.stdout)
+
+    module.log_outputs(
+        (process, other_process),
+        exclude_stdouts=(process.stdout,),
+        output_log_level=logging.INFO,
+        borg_local_path='borg',
+    )
+
+
 def test_log_outputs_truncates_long_error_output():
     flexmock(module).ERROR_OUTPUT_MAX_LINE_COUNT = 0
     flexmock(module.logger).should_receive('log')

tests/unit/borg/test_create.py

@@ -60,6 +60,30 @@ def test_expand_home_directories_considers_none_as_no_directories():
     assert paths == ()
 
 
+def test_map_directories_to_devices_gives_device_id_per_path():
+    flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
+    flexmock(module.os).should_receive('stat').with_args('/bar').and_return(flexmock(st_dev=66))
+
+    device_map = module.map_directories_to_devices(('/foo', '/bar'))
+
+    assert device_map == {
+        '/foo': 55,
+        '/bar': 66,
+    }
+
+
+def test_map_directories_to_devices_with_missing_path_does_not_error():
+    flexmock(module.os).should_receive('stat').with_args('/foo').and_return(flexmock(st_dev=55))
+    flexmock(module.os).should_receive('stat').with_args('/bar').and_raise(FileNotFoundError)
+
+    device_map = module.map_directories_to_devices(('/foo', '/bar'))
+
+    assert device_map == {
+        '/foo': 55,
+        '/bar': None,
+    }
+
+
 @pytest.mark.parametrize(
     'directories,expected_directories',
     (
@@ -72,6 +96,7 @@ def test_expand_home_directories_considers_none_as_no_directories():
         ({'/root': 1, '/root/foo/': 1}, ('/root',)),
         ({'/root': 1, '/root/foo': 2}, ('/root', '/root/foo')),
         ({'/root/foo': 1, '/root': 1}, ('/root',)),
+        ({'/root': None, '/root/foo': None}, ('/root', '/root/foo')),
         ({'/root': 1, '/etc': 1, '/root/foo/bar': 1}, ('/etc', '/root')),
         ({'/root': 1, '/root/foo': 1, '/root/foo/bar': 1}, ('/root',)),
         ({'/dup': 1, '/dup': 1}, ('/dup',)),