diff --git a/borgmatic/actions/config/generate.py b/borgmatic/actions/config/generate.py index 199046eb..1af8e688 100644 --- a/borgmatic/actions/config/generate.py +++ b/borgmatic/actions/config/generate.py @@ -19,26 +19,33 @@ def run_generate(generate_arguments, global_arguments): dry_run_label = ' (dry run; not actually writing anything)' if global_arguments.dry_run else '' logger.answer( - f'Generating a configuration file at: {generate_arguments.destination_filename}{dry_run_label}', + f'Generating configuration files within: {generate_arguments.destination_path}{dry_run_label}' + if generate_arguments.split + else f'Generating a configuration file at: {generate_arguments.destination_path}{dry_run_label}' ) borgmatic.config.generate.generate_sample_configuration( global_arguments.dry_run, generate_arguments.source_filename, - generate_arguments.destination_filename, + generate_arguments.destination_path, borgmatic.config.validate.schema_filename(), overwrite=generate_arguments.overwrite, + split=generate_arguments.split, ) if generate_arguments.source_filename: logger.answer( f''' -Merged in the contents of configuration file at: {generate_arguments.source_filename} -To review the changes made, run: - - diff --unified {generate_arguments.source_filename} {generate_arguments.destination_filename}''', +Merged in the contents of configuration file at: {generate_arguments.source_filename}''' ) + if not generate_arguments.split: + logger.answer( + '''To review the changes made, run: + + diff --unified {generate_arguments.source_filename} {generate_arguments.destination_path}''', + ) + logger.answer( ''' This includes all available configuration options with example values, the few diff --git a/borgmatic/commands/arguments.py b/borgmatic/commands/arguments.py index 9f77a7a8..9d5fe6dc 100644 --- a/borgmatic/commands/arguments.py +++ b/borgmatic/commands/arguments.py @@ -1213,9 +1213,9 @@ def make_parsers(schema, unparsed_arguments): # noqa: PLR0915 config_generate_group.add_argument( '-d', '--destination', - dest='destination_filename', + dest='destination_path', default=config_paths[0], - help=f'Destination configuration file, default: {unexpanded_config_paths[0]}', + help=f'Destination configuration file (or directory if using --split), default: {unexpanded_config_paths[0]}', ) config_generate_group.add_argument( '--overwrite', @@ -1223,6 +1223,11 @@ def make_parsers(schema, unparsed_arguments): # noqa: PLR0915 action='store_true', help='Whether to overwrite any existing destination file, defaults to false', ) + config_generate_group.add_argument( + '--split', + action='store_true', + help='Assuming the destination is a directory instead of a file, split the configuration into separate files within it, one per option, useful for documentation', + ) config_generate_group.add_argument( '-h', '--help', diff --git a/borgmatic/config/generate.py b/borgmatic/config/generate.py index fd287482..2cccdaa5 100644 --- a/borgmatic/config/generate.py +++ b/borgmatic/config/generate.py @@ -107,7 +107,7 @@ def comment_out_line(line): return '# '.join((indent_spaces, line[count_indent_spaces:])) -def comment_out_optional_configuration(rendered_config): +def transform_optional_configuration(rendered_config, comment_out=True): ''' Post-process a rendered configuration string to comment out optional key/values, as determined by a sentinel in the comment before each key. @@ -117,6 +117,9 @@ def comment_out_optional_configuration(rendered_config): Ideally ruamel.yaml would support commenting out keys during configuration generation, but it's not terribly easy to accomplish that way. + + If comment_out is False, then just strip the comment sentinel without actually commenting + anything out. ''' lines = [] optional = False @@ -129,6 +132,9 @@ def comment_out_optional_configuration(rendered_config): # Upon encountering an optional configuration option, comment out lines until the next blank # line. if line.strip().startswith(f'# {COMMENTED_OUT_SENTINEL}'): + if comment_out is False: + continue + optional = True indent_characters_at_sentinel = indent_characters continue @@ -313,16 +319,18 @@ def merge_source_configuration_into_destination(destination_config, source_confi def generate_sample_configuration( dry_run, source_filename, - destination_filename, + destination_path, schema_filename, overwrite=False, + split=False, ): ''' - Given an optional source configuration filename, and a required destination configuration - filename, the path to a schema filename in a YAML rendition of the JSON Schema format, and - whether to overwrite a destination file, write out a sample configuration file based on that - schema. If a source filename is provided, merge the parsed contents of that configuration into - the generated configuration. + Given an optional source configuration filename, a required destination configuration path, the + path to a schema filename in a YAML rendition of the JSON Schema format, whether to overwrite a + destination file, and whether to split the configuration into multiple files (one per option) in + the assumed destination directory, write out sample configuration file(s) based on that schema. + If a source filename is provided, merge the parsed contents of that configuration into the + generated configuration. ''' schema = ruamel.yaml.YAML(typ='safe').load(open(schema_filename, encoding='utf-8')) source_config = None @@ -345,8 +353,31 @@ def generate_sample_configuration( if dry_run: return + if split: + if os.path.exists(destination_path) and not os.path.isdir(destination_path): + raise ValueError('With the --split flag, the destination path must be a directory') + + os.makedirs(destination_path, exist_ok=True) + + for option_name, option_config in destination_config.items(): + write_configuration( + os.path.join(destination_path, f'{option_name}.yaml'), + transform_optional_configuration( + render_configuration({option_name: option_config}), + comment_out=False, + ), + overwrite=overwrite, + ) + + return + + if os.path.exists(destination_path) and not os.path.isfile(destination_path): + raise ValueError('Without the --split flag, the destination path must be a file') + write_configuration( - destination_filename, - comment_out_optional_configuration(render_configuration(destination_config)), + destination_path, + transform_optional_configuration( + render_configuration(destination_config), comment_out=True + ), overwrite=overwrite, ) diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index 4c2ee89c..abd0c388 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -2139,17 +2139,17 @@ properties: type: string description: | The message body to publish. - example: Your backups have failed. + example: Your backups have started. priority: type: string description: | The priority to set. - example: urgent + example: min tags: type: string description: | Tags to attach to the message. - example: incoming_envelope + example: borgmatic finish: type: object additionalProperties: false @@ -2163,17 +2163,17 @@ properties: type: string description: | The message body to publish. - example: Your backups have failed. + example: Your backups have finished. priority: type: string description: | The priority to set. - example: urgent + example: min tags: type: string description: | Tags to attach to the message. - example: incoming_envelope + example: borgmatic,+1 fail: type: object additionalProperties: false @@ -2192,12 +2192,12 @@ properties: type: string description: | The priority to set. - example: urgent + example: max tags: type: string description: | Tags to attach to the message. - example: incoming_envelope + example: borgmatic,-1,skull states: type: array items: diff --git a/docs/Dockerfile b/docs/Dockerfile index 0b2d3eb4..e2d6f2e3 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -2,7 +2,7 @@ FROM docker.io/alpine:3.20.1 AS borgmatic COPY . /app RUN apk add --no-cache py3-pip py3-ruamel.yaml py3-ruamel.yaml.clib -RUN pip install --break-system-packages --no-cache /app && borgmatic config generate && chmod +r /etc/borgmatic/config.yaml +RUN pip install --break-system-packages --no-cache /app && borgmatic config generate && borgmatic config generate --destination /etc/borgmatic --split && chmod +r /etc/borgmatic/*.yaml RUN borgmatic --help > /command-line.txt \ && for action in repo-create transfer create prune compact check delete extract config "config bootstrap" "config generate" "config validate" export-tar mount umount repo-delete restore repo-list list repo-info info break-lock "key export" "key import" "key change-passphrase" recreate borg; do \ echo -e "\n--------------------------------------------------------------------------------\n" >> /command-line.txt \ @@ -23,7 +23,7 @@ RUN npm install @11ty/eleventy \ markdown-it \ markdown-it-anchor \ markdown-it-replace-link -COPY --from=borgmatic /etc/borgmatic/config.yaml /source/docs/_includes/borgmatic/config.yaml +COPY --from=borgmatic /etc/borgmatic/* /source/docs/_includes/borgmatic/ COPY --from=borgmatic /command-line.txt /source/docs/_includes/borgmatic/command-line.txt COPY --from=borgmatic /contributors.html /source/docs/_includes/borgmatic/contributors.html COPY . /source diff --git a/docs/_includes/header.njk b/docs/_includes/header.njk index 33ce9cec..60e281e4 100644 --- a/docs/_includes/header.njk +++ b/docs/_includes/header.njk @@ -2,7 +2,8 @@ {% if page.url != '/' %}

borgmatic

{% endif %}

{{ title | safe }}

{% if page.url == '/' %}

It's your data. Keep it that way.

{% endif %} diff --git a/docs/docker-compose.yaml b/docs/docker-compose.yaml index cbf2eda7..8c280a60 100644 --- a/docs/docker-compose.yaml +++ b/docs/docker-compose.yaml @@ -23,14 +23,13 @@ services: labels: - "traefik.enable=true" - "traefik.http.routers.borgmatic-docs.rule=PathPrefix(`/borgmatic`)" -# - "traefik.http.routers.borgmatic-docs.middlewares=borgmatic-trailing-slash-redirectregex,borgmatic-docs-redirectregex,borgmatic-stripprefix" - - "traefik.http.routers.borgmatic-docs.middlewares=borgmatic-trailing-slash-redirectregex,borgmatic-stripprefix" + - "traefik.http.routers.borgmatic-docs.middlewares=borgmatic-trailing-slash-redirectregex,borgmatic-docs-redirectregex,borgmatic-stripprefix" - "traefik.http.middlewares.borgmatic-trailing-slash-redirectregex.redirectregex.regex=^(.*)/borgmatic$$" - "traefik.http.middlewares.borgmatic-trailing-slash-redirectregex.redirectregex.replacement=$${1}/borgmatic/" - "traefik.http.middlewares.borgmatic-trailing-slash-redirectregex.redirectregex.permanent=true" -# - "traefik.http.middlewares.borgmatic-docs-redirectregex.redirectregex.regex=^(.*)/borgmatic/docs/(.*)$$" -# - "traefik.http.middlewares.borgmatic-docs-redirectregex.redirectregex.replacement=$${1}/borgmatic/$${2}" -# - "traefik.http.middlewares.borgmatic-docs-redirectregex.redirectregex.permanent=true" + - "traefik.http.middlewares.borgmatic-docs-redirectregex.redirectregex.regex=^(.*)/borgmatic/docs/(.*)$$" + - "traefik.http.middlewares.borgmatic-docs-redirectregex.redirectregex.replacement=$${1}/borgmatic/$${2}" + - "traefik.http.middlewares.borgmatic-docs-redirectregex.redirectregex.permanent=true" - "traefik.http.middlewares.borgmatic-stripprefix.stripprefix.prefixes=/borgmatic" - "traefik.http.routers.borgmatic-docs.entrypoints=web" build: diff --git a/docs/how-to/backup-your-databases.md b/docs/how-to/backup-your-databases.md index 0caf775f..13564b7d 100644 --- a/docs/how-to/backup-your-databases.md +++ b/docs/how-to/backup-your-databases.md @@ -27,33 +27,6 @@ mysql_databases: these and other database options in the `hooks:` section of your configuration. -New in version 1.5.22 You can -also dump MongoDB databases. For example: - -```yaml -mongodb_databases: - - name: messages -``` - -New in version 1.7.9 -Additionally, you can dump SQLite databases. For example: - -```yaml -sqlite_databases: - - name: mydb - path: /var/lib/sqlite3/mydb.sqlite -``` - -New in version 1.8.2 If you're -using MariaDB, use the MariaDB database hook instead of `mysql_databases:` as -the MariaDB hook calls native MariaDB commands instead of the deprecated MySQL -ones. For instance: - -```yaml -mariadb_databases: - - name: comments -``` - As part of each backup, borgmatic streams a database dump for each configured database directly to Borg, so it's included in the backup without consuming additional disk space. (The exceptions are the PostgreSQL/MongoDB `directory` @@ -107,47 +80,17 @@ sqlite_databases: path: /var/lib/sqlite3/mydb.sqlite ``` -See your [borgmatic configuration -file](https://torsion.org/borgmatic/reference/configuration/) for -additional customization of the options passed to database commands (when -listing databases, restoring databases, etc.). +See the [data sources +documentation](https://torsion.org/borgmatic/reference/configuration/data-sources/) +for details on additional options, including customizing the flags passed to +database commands when listing databases, restoring databases, etc. + -### Runtime directory - -New in version 1.9.0 To support -streaming database dumps to Borg, borgmatic uses a runtime directory for -temporary file storage, probing the following locations (in order) to find it: - - 1. The `user_runtime_directory` borgmatic configuration option. - 2. The `XDG_RUNTIME_DIR` environment variable, usually `/run/user/$UID` - (where `$UID` is the current user's ID), automatically set by PAM on Linux - for a user with a session. - 3. New in version 1.9.2The - `RUNTIME_DIRECTORY` environment variable, set by systemd if - `RuntimeDirectory=borgmatic` is added to borgmatic's systemd service file. - 4. New in version 1.9.1The - `TMPDIR` environment variable, set on macOS for a user with a session, - among other operating systems. - 5. New in version 1.9.1The - `TEMP` environment variable, set on various systems. - 6. New in version 1.9.2 - Hard-coded `/tmp`. Prior to - version 1.9.2This was instead hard-coded to `/run/user/$UID`. - -You can see the runtime directory path that borgmatic selects by running with -`--verbosity 2` and looking for "Using runtime directory" in the output. - -Regardless of the runtime directory selected, borgmatic stores its files -within a `borgmatic` subdirectory of the runtime directory. Additionally, in -the case of `TMPDIR`, `TEMP`, and the hard-coded `/tmp`, borgmatic creates a -randomly named subdirectory in an effort to reduce path collisions in shared -system temporary directories. - -Prior to version 1.9.0 -borgmatic created temporary streaming database dumps within the `~/.borgmatic` -directory by default. At that time, the path was configurable by the -`borgmatic_source_directory` configuration option (now deprecated). +To support streaming database dumps to Borg, borgmatic uses a runtime directory +for temporary file storage. See the [runtime directory +documentation](https://torsion.org/borgmatic/reference/configuration/runtime-directory/) +for details. ### All databases @@ -412,8 +355,6 @@ most up-to-date files and therefore the latest timestamp, run a command like: borgmatic restore --archive host-2023-01-02T04:06:07.080910 ``` -(No borgmatic `restore` action? Upgrade borgmatic!) - Or you can simplify this to: ```bash diff --git a/docs/how-to/extract-a-backup.md b/docs/how-to/extract-a-backup.md index dbd02db9..38cea493 100644 --- a/docs/how-to/extract-a-backup.md +++ b/docs/how-to/extract-a-backup.md @@ -30,8 +30,6 @@ and therefore the latest timestamp, run a command like: borgmatic extract --archive host-2023-01-02T04:06:07.080910 ``` -(No borgmatic `extract` action? Upgrade borgmatic!) - Or simplify this to: ```bash diff --git a/docs/reference/configuration/credentials/container.md b/docs/reference/configuration/credentials/container.md index 516bbb3d..83fa2f62 100644 --- a/docs/reference/configuration/credentials/container.md +++ b/docs/reference/configuration/credentials/container.md @@ -1,5 +1,5 @@ --- -title: Container secerts +title: Container secrets eleventyNavigation: key: • Container parent: 🔒 Credentials diff --git a/docs/reference/configuration/data-sources/btrfs.md b/docs/reference/configuration/data-sources/btrfs.md new file mode 100644 index 00000000..c3c2b1a3 --- /dev/null +++ b/docs/reference/configuration/data-sources/btrfs.md @@ -0,0 +1,10 @@ +--- +title: Btrfs +eleventyNavigation: + key: • Btrfs + parent: 🗄️ Data sources +--- + +```yaml +{% include borgmatic/btrfs.yaml %} +``` diff --git a/docs/reference/configuration/data-sources/index.md b/docs/reference/configuration/data-sources/index.md new file mode 100644 index 00000000..c7720975 --- /dev/null +++ b/docs/reference/configuration/data-sources/index.md @@ -0,0 +1,27 @@ +--- +title: Data sources +eleventyNavigation: + key: 🗄️ Data sources + parent: ⚙️ Configuration +--- +Data sources are built-in borgmatic integrations that, instead of backing up +plain filesystem data, can pull data directly from database servers and +filesystem snapshots. + +In the case of supported database systems, borgmatic dumps your configured +databases, streaming them directly to Borg when creating a backup. Here are the +supported databases and how to configure their borgmatic integrations: + + * [MariaDB](https://torsion.org/borgmatic/reference/configuration/data-sources/mariadb/) + * [MongoDB](https://torsion.org/borgmatic/reference/configuration/data-sources/mongodb/) + * [MySQL](https://torsion.org/borgmatic/reference/configuration/data-sources/mysql/) + * [PostgreSQL](https://torsion.org/borgmatic/reference/configuration/data-sources/postgresql/) + * [SQLite](https://torsion.org/borgmatic/reference/configuration/data-sources/sqlite/) + +For supported filesystems, borgmatic takes on-demand snapshots of configured +source directories and feeds them to Borg. Here are the supported filesystems / +volume managers and how to configure their borgmatic integrations: + + * [Btrfs](https://torsion.org/borgmatic/reference/configuration/data-sources/btrfs/) + * [LVM](https://torsion.org/borgmatic/reference/configuration/data-sources/lvm/) + * [ZFS](https://torsion.org/borgmatic/reference/configuration/data-sources/zfs/) diff --git a/docs/reference/configuration/data-sources/lvm.md b/docs/reference/configuration/data-sources/lvm.md new file mode 100644 index 00000000..3818d9e1 --- /dev/null +++ b/docs/reference/configuration/data-sources/lvm.md @@ -0,0 +1,10 @@ +--- +title: LVM +eleventyNavigation: + key: • LVM + parent: 🗄️ Data sources +--- + +```yaml +{% include borgmatic/lvm.yaml %} +``` diff --git a/docs/reference/configuration/data-sources/mariadb.md b/docs/reference/configuration/data-sources/mariadb.md new file mode 100644 index 00000000..9298e3ee --- /dev/null +++ b/docs/reference/configuration/data-sources/mariadb.md @@ -0,0 +1,23 @@ +--- +title: MariaDB +eleventyNavigation: + key: • MariaDB + parent: 🗄️ Data sources +--- + +New in version 1.8.2 To backup +MariaDB with borgmatic, use the `mariadb_databases:` hook instead of +`mysql_databases:` as the MariaDB hook calls native MariaDB commands instead of +the deprecated MySQL ones. For instance: + +```yaml +mariadb_databases: + - name: comments +``` + + +### Full configuration + +```yaml +{% include borgmatic/mariadb_databases.yaml %} +``` diff --git a/docs/reference/configuration/data-sources/mongodb.md b/docs/reference/configuration/data-sources/mongodb.md new file mode 100644 index 00000000..61a2394e --- /dev/null +++ b/docs/reference/configuration/data-sources/mongodb.md @@ -0,0 +1,21 @@ +--- +title: MongoDB +eleventyNavigation: + key: • MongoDB + parent: 🗄️ Data sources +--- + +New in version 1.5.22 To backup +MongoDB with borgmatic, use the `mongodb_databases:` hook. For example: + +```yaml +mongodb_databases: + - name: messages +``` + + +### Full configuration + +```yaml +{% include borgmatic/mongodb_databases.yaml %} +``` diff --git a/docs/reference/configuration/data-sources/mysql.md b/docs/reference/configuration/data-sources/mysql.md new file mode 100644 index 00000000..6b096f13 --- /dev/null +++ b/docs/reference/configuration/data-sources/mysql.md @@ -0,0 +1,21 @@ +--- +title: MySQL +eleventyNavigation: + key: • MySQL + parent: 🗄️ Data sources +--- + +New in version 1.4.9 To backup +MySQL with borgmatic, use the `mysql_databases:` hook. For instance: + +```yaml +mysql_databases: + - name: posts +``` + + +## Full configuration + +```yaml +{% include borgmatic/mysql_databases.yaml %} +``` diff --git a/docs/reference/configuration/data-sources/postgresql.md b/docs/reference/configuration/data-sources/postgresql.md new file mode 100644 index 00000000..90d08f18 --- /dev/null +++ b/docs/reference/configuration/data-sources/postgresql.md @@ -0,0 +1,21 @@ +--- +title: PostgreSQL +eleventyNavigation: + key: • PostgreSQL + parent: 🗄️ Data sources +--- + +New in version 1.4.0 To backup +PostgreSQL with borgmatic, use the `postgresql_databases:` hook. For instance: + +```yaml +postgresql_databases: + - name: users +``` + + +## Full configuration + +```yaml +{% include borgmatic/postgresql_databases.yaml %} +``` diff --git a/docs/reference/configuration/data-sources/sqlite.md b/docs/reference/configuration/data-sources/sqlite.md new file mode 100644 index 00000000..3255c177 --- /dev/null +++ b/docs/reference/configuration/data-sources/sqlite.md @@ -0,0 +1,22 @@ +--- +title: SQLite +eleventyNavigation: + key: • SQLite + parent: 🗄️ Data sources +--- +New in version 1.7.9 To backup +SQLite with borgmatic, use the `sqlite_databases:` hook. For example: + + +```yaml +sqlite_databases: + - name: mydb + path: /var/lib/sqlite3/mydb.sqlite +``` + + +## Full configuration + +```yaml +{% include borgmatic/sqlite_databases.yaml %} +``` diff --git a/docs/reference/configuration/data-sources/zfs.md b/docs/reference/configuration/data-sources/zfs.md new file mode 100644 index 00000000..33a88709 --- /dev/null +++ b/docs/reference/configuration/data-sources/zfs.md @@ -0,0 +1,10 @@ +--- +title: ZFS +eleventyNavigation: + key: • ZFS + parent: 🗄️ Data sources +--- + +```yaml +{% include borgmatic/zfs.yaml %} +``` diff --git a/docs/reference/configuration/monitoring/apprise.md b/docs/reference/configuration/monitoring/apprise.md index dee0b6ed..ce82987b 100644 --- a/docs/reference/configuration/monitoring/apprise.md +++ b/docs/reference/configuration/monitoring/apprise.md @@ -104,3 +104,10 @@ This may be necessary for some services that reject large requests. See the [configuration reference](https://torsion.org/borgmatic/reference/configuration/) for details. + + +### Full configuration + +```yaml +{% include borgmatic/apprise.yaml %} +``` diff --git a/docs/reference/configuration/monitoring/healthchecks.md b/docs/reference/configuration/monitoring/healthchecks.md index 68fd63e5..4119c143 100644 --- a/docs/reference/configuration/monitoring/healthchecks.md +++ b/docs/reference/configuration/monitoring/healthchecks.md @@ -48,3 +48,10 @@ defaults for these flags in your borgmatic configuration via the You can configure Healthchecks to notify you by a [variety of mechanisms](https://healthchecks.io/#welcome-integrations) when backups fail or it doesn't hear from borgmatic for a certain period of time. + + +### Full configuration + +```yaml +{% include borgmatic/healthchecks.yaml %} +``` diff --git a/docs/reference/configuration/monitoring/ntfy.md b/docs/reference/configuration/monitoring/ntfy.md index b114b5b4..8c4f4c5e 100644 --- a/docs/reference/configuration/monitoring/ntfy.md +++ b/docs/reference/configuration/monitoring/ntfy.md @@ -17,9 +17,7 @@ to issues. The `states` list can override this. Each state can have its own custom messages, priorities and tags or, if none are provided, will use the default. -An example configuration is shown here with all the available options, -including [priorities](https://ntfy.sh/docs/publish/#message-priority) and -[tags](https://ntfy.sh/docs/publish/#tags-emojis): +Here's a basic configuration that notifies on failure: ```yaml ntfy: @@ -28,24 +26,12 @@ ntfy: username: myuser password: secret - start: - title: A borgmatic backup started - message: Watch this space... - tags: borgmatic - priority: min - finish: - title: A borgmatic backup completed successfully - message: Nice! - tags: borgmatic,+1 - priority: min fail: title: A borgmatic backup failed message: You should probably fix it tags: borgmatic,-1,skull priority: max states: - - start - - finish - fail ``` @@ -62,3 +48,14 @@ ntfy: server: https://ntfy.my-domain.com access_token: tk_AgQdq7mVBoFD37zQVN29RhuMzNIz2 ```` + + +### Full configuration + +Here's an example configuration with all the available options, +including [priorities](https://ntfy.sh/docs/publish/#message-priority) and +[tags](https://ntfy.sh/docs/publish/#tags-emojis): + +```yaml +{% include borgmatic/ntfy.yaml %} +``` diff --git a/docs/reference/configuration/monitoring/pagerduty.md b/docs/reference/configuration/monitoring/pagerduty.md index 01579120..db187ce3 100644 --- a/docs/reference/configuration/monitoring/pagerduty.md +++ b/docs/reference/configuration/monitoring/pagerduty.md @@ -65,5 +65,3 @@ pagerduty: integration_key: a177cad45bd374409f78906a810a3074 send_logs: false ``` - - diff --git a/docs/reference/configuration/monitoring/pushover.md b/docs/reference/configuration/monitoring/pushover.md index bcb1b2c9..a83d5d48 100644 --- a/docs/reference/configuration/monitoring/pushover.md +++ b/docs/reference/configuration/monitoring/pushover.md @@ -52,3 +52,11 @@ pushover: expire: 600 # Used only for priority 2. Default is 600 seconds. retry: 30 # Used only for priority 2. Default is 30 seconds. device: "pixel8" +``` + + +### Full configuration + +```yaml +{% include borgmatic/pushover.yaml %} +``` diff --git a/docs/reference/configuration/monitoring/sentry.md b/docs/reference/configuration/monitoring/sentry.md index ca9e5d42..4c3fcc18 100644 --- a/docs/reference/configuration/monitoring/sentry.md +++ b/docs/reference/configuration/monitoring/sentry.md @@ -17,7 +17,7 @@ displayed environment variable into borgmatic's Sentry `data_source_name_url` configuration option. For example: -``` +```yaml sentry: data_source_name_url: https://5f80ec@o294220.ingest.us.sentry.io/203069 monitor_slug: mymonitor @@ -32,7 +32,7 @@ finishes, or fails, but only when any of the `create`, `prune`, `compact`, or behavior with the `states` configuration option. For instance, to only ping Sentry on failure: -``` +```yaml sentry: data_source_name_url: https://5f80ec@o294220.ingest.us.sentry.io/203069 monitor_slug: mymonitor diff --git a/docs/reference/configuration/monitoring/uptime-kuma.md b/docs/reference/configuration/monitoring/uptime-kuma.md index e40f3872..e927ced2 100644 --- a/docs/reference/configuration/monitoring/uptime-kuma.md +++ b/docs/reference/configuration/monitoring/uptime-kuma.md @@ -57,3 +57,10 @@ Heartbeat Retry = 360 # = 10 minutes # is sent each time. Resend Notification every X times = 1 ``` + + +### Full configuration + +```yaml +{% include borgmatic/uptime_kuma.yaml %} +``` diff --git a/docs/reference/configuration/monitoring/zabbix.md b/docs/reference/configuration/monitoring/zabbix.md index 46606298..5e990d44 100644 --- a/docs/reference/configuration/monitoring/zabbix.md +++ b/docs/reference/configuration/monitoring/zabbix.md @@ -16,7 +16,7 @@ states will trigger the hook. The value defined in the configuration of each state is used to populate the data of the configured Zabbix item. If none are provided, it defaults to a lower-case string of the state. -An example configuration is shown here with all the available options. +Here's an example configuration: ```yaml zabbix: @@ -24,21 +24,13 @@ zabbix: username: myuser password: secret - api_key: b2ecba64d8beb47fc161ae48b164cfd7104a79e8e48e6074ef5b141d8a0aeeca host: "borg-server" key: borg.status - itemid: 55105 - start: - value: "STARTED" - finish: - value: "OK" fail: value: "ERROR" states: - - start - - finish - fail ``` @@ -69,3 +61,10 @@ is used. Keep in mind that `host` refers to the "Host name" on the Zabbix server and not the "Visual name". + + +### Full configuration + +```yaml +{% include borgmatic/zabbix.yaml %} +``` diff --git a/docs/reference/configuration/runtime-directory.md b/docs/reference/configuration/runtime-directory.md new file mode 100644 index 00000000..548c281d --- /dev/null +++ b/docs/reference/configuration/runtime-directory.md @@ -0,0 +1,41 @@ +--- +title: Runtime directory +eleventyNavigation: + key: 📁 Runtime directory + parent: ⚙️ Configuration +--- +New in version 1.9.0 borgmatic +uses a runtime directory for temporary file storage, such as for streaming +database dumps to Borg, creating filesystem snapshots, saving bootstrap +metadata, and so on. To determine the path for this runtime directory, borgmatic +probes the following values: + + 1. The `user_runtime_directory` borgmatic configuration option. + 2. The `XDG_RUNTIME_DIR` environment variable, usually `/run/user/$UID` + (where `$UID` is the current user's ID), automatically set by PAM on Linux + for a user with a session. + 3. New in version 1.9.2The + `RUNTIME_DIRECTORY` environment variable, set by systemd if + `RuntimeDirectory=borgmatic` is added to borgmatic's systemd service file. + 4. New in version 1.9.1The + `TMPDIR` environment variable, set on macOS for a user with a session, + among other operating systems. + 5. New in version 1.9.1The + `TEMP` environment variable, set on various systems. + 6. New in version 1.9.2 + Hard-coded `/tmp`. Prior to + version 1.9.2This was instead hard-coded to `/run/user/$UID`. + +You can see the runtime directory path that borgmatic selects by running with +`--verbosity 2` and looking for `Using runtime directory` in the output. + +Regardless of the runtime directory selected, borgmatic stores its files +within a `borgmatic` subdirectory of the runtime directory. Additionally, in +the case of `TMPDIR`, `TEMP`, and the hard-coded `/tmp`, borgmatic creates a +randomly named subdirectory in an effort to reduce path collisions in shared +system temporary directories. + +Prior to version 1.9.0 +borgmatic created temporary streaming database dumps within the `~/.borgmatic` +directory by default. At that time, the path was configurable by the +`borgmatic_source_directory` configuration option (now deprecated). diff --git a/tests/integration/config/test_generate.py b/tests/integration/config/test_generate.py index 25448449..26e75573 100644 --- a/tests/integration/config/test_generate.py +++ b/tests/integration/config/test_generate.py @@ -195,7 +195,7 @@ def test_comment_out_line_comments_twice_indented_option(): assert module.comment_out_line(line) == ' # - item' -def test_comment_out_optional_configuration_comments_optional_config_only(): +def test_transform_optional_configuration_comments_optional_config_only(): # The "# COMMENT_OUT" comment is a sentinel used to express that the following key is optional. # It's stripped out of the final output. flexmock(module).comment_out_line = lambda line: '# ' + line @@ -236,7 +236,54 @@ repositories: # other: thing ''' - assert module.comment_out_optional_configuration(config.strip()) == expected_config.strip() + assert module.transform_optional_configuration(config.strip()) == expected_config.strip() + + +def test_transform_optional_configuration_with_comment_out_false_leaves_in_optional_config(): + # The "# COMMENT_OUT" comment is a sentinel used to express that the following key is optional. + # It's stripped out of the final output. + flexmock(module).comment_out_line = lambda line: '# ' + line + config = ''' +# COMMENT_OUT +foo: + # COMMENT_OUT + bar: + - baz + - quux + +repositories: + - path: foo + # COMMENT_OUT + label: bar + - path: baz + label: quux + +# This comment should be kept. +# COMMENT_OUT +other: thing + ''' + + # flake8: noqa + expected_config = ''' +foo: + bar: + - baz + - quux + +repositories: + - path: foo + label: bar + - path: baz + label: quux + +# This comment should be kept. +other: thing + ''' + + assert ( + module.transform_optional_configuration(config.strip(), comment_out=False) + == expected_config.strip() + ) def test_render_configuration_converts_configuration_to_yaml_string(): @@ -377,13 +424,32 @@ def test_generate_sample_configuration_does_not_raise(): ) flexmock(module).should_receive('schema_to_sample_configuration') flexmock(module).should_receive('merge_source_configuration_into_destination') + flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module).should_receive('render_configuration') - flexmock(module).should_receive('comment_out_optional_configuration') + flexmock(module).should_receive('transform_optional_configuration') flexmock(module).should_receive('write_configuration') module.generate_sample_configuration(False, None, 'dest.yaml', 'schema.yaml') +def test_generate_sample_configuration_with_destination_directory_error(): + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('schema.yaml', encoding='utf-8').and_return('') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return( + flexmock(load=lambda filename: {}) + ) + flexmock(module).should_receive('schema_to_sample_configuration') + flexmock(module).should_receive('merge_source_configuration_into_destination') + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.os.path).should_receive('isfile').and_return(False) + flexmock(module).should_receive('render_configuration').never() + flexmock(module).should_receive('transform_optional_configuration').never() + flexmock(module).should_receive('write_configuration').never() + + with pytest.raises(ValueError): + module.generate_sample_configuration(False, None, 'dest.yaml', 'schema.yaml') + + def test_generate_sample_configuration_with_source_filename_omits_empty_bootstrap_field(): builtins = flexmock(sys.modules['builtins']) builtins.should_receive('open').with_args('schema.yaml', encoding='utf-8').and_return('') @@ -398,8 +464,9 @@ def test_generate_sample_configuration_with_source_filename_omits_empty_bootstra object, {'foo': 'bar'} ).once() flexmock(module).should_receive('merge_source_configuration_into_destination') + flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module).should_receive('render_configuration') - flexmock(module).should_receive('comment_out_optional_configuration') + flexmock(module).should_receive('transform_optional_configuration') flexmock(module).should_receive('write_configuration') module.generate_sample_configuration(False, 'source.yaml', 'dest.yaml', 'schema.yaml') @@ -418,8 +485,9 @@ def test_generate_sample_configuration_with_source_filename_keeps_non_empty_boot object, source_config ).once() flexmock(module).should_receive('merge_source_configuration_into_destination') + flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module).should_receive('render_configuration') - flexmock(module).should_receive('comment_out_optional_configuration') + flexmock(module).should_receive('transform_optional_configuration') flexmock(module).should_receive('write_configuration') module.generate_sample_configuration(False, 'source.yaml', 'dest.yaml', 'schema.yaml') @@ -433,8 +501,58 @@ def test_generate_sample_configuration_with_dry_run_does_not_write_file(): ) flexmock(module).should_receive('schema_to_sample_configuration') flexmock(module).should_receive('merge_source_configuration_into_destination') + flexmock(module.os.path).should_receive('exists').and_return(False) flexmock(module).should_receive('render_configuration') - flexmock(module).should_receive('comment_out_optional_configuration') + flexmock(module).should_receive('transform_optional_configuration') flexmock(module).should_receive('write_configuration').never() module.generate_sample_configuration(True, None, 'dest.yaml', 'schema.yaml') + + +def test_generate_sample_configuration_with_split_writes_each_option_to_file(): + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('schema.yaml', encoding='utf-8').and_return('') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return( + flexmock(load=lambda filename: {}) + ) + flexmock(module).should_receive('schema_to_sample_configuration') + flexmock(module).should_receive('merge_source_configuration_into_destination').and_return( + {'foo': 1, 'bar': 2} + ) + flexmock(module.os.path).should_receive('exists').and_return(False) + flexmock(module).should_receive('render_configuration') + flexmock(module).should_receive('transform_optional_configuration') + flexmock(module.os).should_receive('makedirs') + flexmock(module).should_receive('write_configuration').with_args( + 'dest/foo.yaml', + None, + overwrite=False, + ).once() + flexmock(module).should_receive('write_configuration').with_args( + 'dest/bar.yaml', + None, + overwrite=False, + ).once() + + module.generate_sample_configuration(False, None, 'dest', 'schema.yaml', split=True) + + +def test_generate_sample_configuration_with_split_and_file_destination_errors(): + builtins = flexmock(sys.modules['builtins']) + builtins.should_receive('open').with_args('schema.yaml', encoding='utf-8').and_return('') + flexmock(module.ruamel.yaml).should_receive('YAML').and_return( + flexmock(load=lambda filename: {}) + ) + flexmock(module).should_receive('schema_to_sample_configuration') + flexmock(module).should_receive('merge_source_configuration_into_destination').and_return( + {'foo': 1, 'bar': 2} + ) + flexmock(module.os.path).should_receive('exists').and_return(True) + flexmock(module.os.path).should_receive('isdir').and_return(False) + flexmock(module).should_receive('render_configuration').never() + flexmock(module).should_receive('transform_optional_configuration').never() + flexmock(module.os).should_receive('makedirs').never() + flexmock(module).should_receive('write_configuration').never() + + with pytest.raises(ValueError): + module.generate_sample_configuration(False, None, 'dest', 'schema.yaml', split=True) diff --git a/tests/unit/actions/config/test_generate.py b/tests/unit/actions/config/test_generate.py index d25a58e9..e1821dd5 100644 --- a/tests/unit/actions/config/test_generate.py +++ b/tests/unit/actions/config/test_generate.py @@ -6,8 +6,9 @@ from borgmatic.actions.config import generate as module def test_run_generate_does_not_raise(): generate_arguments = flexmock( source_filename=None, - destination_filename='destination.yaml', + destination_path='destination.yaml', overwrite=False, + split=False, ) global_arguments = flexmock(dry_run=False) flexmock(module.borgmatic.config.generate).should_receive('generate_sample_configuration') @@ -18,8 +19,9 @@ def test_run_generate_does_not_raise(): def test_run_generate_with_dry_run_does_not_raise(): generate_arguments = flexmock( source_filename=None, - destination_filename='destination.yaml', + destination_path='destination.yaml', overwrite=False, + split=False, ) global_arguments = flexmock(dry_run=True) flexmock(module.borgmatic.config.generate).should_receive('generate_sample_configuration') @@ -30,8 +32,22 @@ def test_run_generate_with_dry_run_does_not_raise(): def test_run_generate_with_source_filename_does_not_raise(): generate_arguments = flexmock( source_filename='source.yaml', - destination_filename='destination.yaml', + destination_path='destination.yaml', overwrite=False, + split=False, + ) + global_arguments = flexmock(dry_run=False) + flexmock(module.borgmatic.config.generate).should_receive('generate_sample_configuration') + + module.run_generate(generate_arguments, global_arguments) + + +def test_run_generate_with_split_does_not_raise(): + generate_arguments = flexmock( + source_filename=None, + destination_path='destination.yaml', + overwrite=False, + split=True, ) global_arguments = flexmock(dry_run=False) flexmock(module.borgmatic.config.generate).should_receive('generate_sample_configuration')