mirror of
https://github.com/borgmatic-collective/borgmatic.git
synced 2026-07-22 10:13:00 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4af31db99c | ||
|
|
7cd9e6b361 | ||
|
|
4ee057a909 | ||
|
|
3e7e7879bc | ||
|
|
d29cfaf227 | ||
|
|
23a8d665d6 | ||
|
|
2a026a4842 | ||
|
|
16c8f81a5a | ||
|
|
1670ba2aeb | ||
|
|
5accda1a65 | ||
|
|
ec8e52944c | ||
|
|
30ca942f76 | ||
|
|
9748a0298a | ||
|
|
dc91f81a39 | ||
|
|
64ceebb2c8 | ||
|
|
7439d7cb8f | ||
|
|
f832a361ed | ||
|
|
3a4bdfb3c5 | ||
|
|
ed955c2a73 | ||
|
|
174aa3d522 | ||
|
|
5a74b3d081 | ||
|
|
a876ce2058 | ||
|
|
e82c1bb195 | ||
|
|
16061f4f6d | ||
|
|
59e6e61786 | ||
|
|
b7d0f006d6 | ||
|
|
6869a5d5f2 | ||
|
|
9da6d0f90a | ||
|
|
ac65e65302 | ||
|
|
8820a1eeab | ||
|
|
3abe025a80 | ||
|
|
b979e0356d | ||
|
|
ea0ed8345a | ||
|
|
4b8a6d1a34 | ||
|
|
60201541e0 | ||
|
|
bfd79a5500 | ||
|
|
43e995a4f3 | ||
|
|
94a2d68198 | ||
|
|
c9ba0762f0 | ||
|
|
13031a4ce4 | ||
|
|
9a2e853ea7 | ||
|
|
b1f42eb233 | ||
|
|
7e7c3c273e | ||
|
|
704db9c9d7 | ||
|
|
61d86c55b4 | ||
|
|
ecedccadee | ||
|
|
006825838f | ||
|
|
1102bf3787 | ||
|
|
220c76b3c0 | ||
|
|
9aea0b1d90 | ||
|
|
b6ab326bb4 | ||
|
|
c4170bb126 | ||
|
|
9935d0ef8d | ||
|
|
ec11e8817d | ||
|
|
73222e7b5b | ||
|
|
fd98f4fdd1 | ||
|
|
16e743d8d9 | ||
|
|
e93fe4cc15 | ||
|
|
da0698f6e5 | ||
|
|
5075c1e539 | ||
|
|
4b4851e78a | ||
|
|
8e91af6820 | ||
|
|
9be1698762 | ||
|
|
9310b42d9f | ||
|
|
d1d757a5d1 | ||
|
|
3f4a87517b | ||
|
|
a2dcd4747f | ||
|
|
0396a89f72 | ||
|
|
f170ef19df | ||
|
|
9729809f6f | ||
|
|
dfb3c0830e | ||
|
|
b22a068a60 | ||
|
|
1286ccce46 | ||
|
|
e339ce2fd1 | ||
|
|
aaf5812a3f | ||
|
|
47a6691886 | ||
|
|
596d59ef60 | ||
|
|
9839b3dada | ||
|
|
411685280c | ||
|
|
cce679248f | ||
|
|
68c9516424 | ||
|
|
a508cabe3f | ||
|
|
0e1659bd73 | ||
|
|
ad61ad356e | ||
|
|
5c7d03910b | ||
|
|
468af1de0b | ||
|
|
524d8263a7 | ||
|
|
1d2aea0951 | ||
|
|
961ff7c724 | ||
|
|
3e80056956 | ||
|
|
d8f558ce0d | ||
|
|
9da75fdc58 | ||
|
|
ac9c8bb644 | ||
|
|
d085fc2398 | ||
|
|
40d2d521a7 | ||
|
|
0b34ef0e1a | ||
|
|
3f34d0848e | ||
|
|
54289e3ee0 | ||
|
|
6eea2d5323 | ||
|
|
0ca5333fd4 | ||
|
|
462e1392da | ||
|
|
71e2762aa7 | ||
|
|
edfa708fa3 | ||
|
|
aee16e32e2 | ||
|
|
f3ae04225d | ||
|
|
3f70cf0b29 | ||
|
|
731e8d7c37 | ||
|
|
3bcf592d53 | ||
|
|
9bb5791e9f | ||
|
|
8a60fb6398 | ||
|
|
9db2bb2b54 | ||
|
|
87bfd6e97f | ||
|
|
52f9442377 | ||
|
|
af841e0c89 | ||
|
|
ed8320c1bb | ||
|
|
26b3a03721 | ||
|
|
52234c47e6 | ||
|
|
b74b6aa18d | ||
|
|
0380ecd8fb | ||
|
|
32b7d1a0f7 | ||
|
|
da873c09f8 | ||
|
|
d612d398e7 | ||
|
|
1301bec702 | ||
|
|
c7fc68a49a | ||
|
|
c66e29906e | ||
|
|
475389a094 | ||
|
|
9f59bf2827 | ||
|
|
f4e9569297 | ||
|
|
f43c2f7130 | ||
|
|
889b599d55 | ||
|
|
53791d4dc9 | ||
|
|
ee58adb4eb | ||
|
|
d2903640a8 | ||
|
|
ec25a40ddc | ||
|
|
23e451e641 | ||
|
|
43d8c25234 | ||
|
|
e693e42b9e | ||
|
|
1abbfe8ee6 | ||
|
|
8a65b43ae3 | ||
|
|
dbbce4167f | ||
|
|
68c2a4d231 | ||
|
|
12e92acd15 | ||
|
|
1ada61a1d3 | ||
|
|
75dcd41050 | ||
|
|
a6878829a2 | ||
|
|
7a44be38b7 | ||
|
|
92add238d9 | ||
|
|
24e1e38615 | ||
|
|
cb8b298648 | ||
|
|
eaf3d959e6 | ||
|
|
c2449ad811 | ||
|
|
89703334f3 | ||
|
|
ee32cc19e9 | ||
|
|
ceb487a6b9 | ||
|
|
f90d7a99c6 | ||
|
|
012c02d962 | ||
|
|
607ff20971 | ||
|
|
bf2a1c6b25 | ||
|
|
7dbdd103d1 | ||
|
|
06e2b4f2c7 | ||
|
|
6cfa31e2c2 | ||
|
|
3473f034ae | ||
|
|
7dcb4e3c3a | ||
|
|
42c006cf1d | ||
|
|
ea2a253134 | ||
|
|
7ca42a8f4f | ||
|
|
4f25581a12 | ||
|
|
394f33f28a | ||
|
|
f5ff2f9ae3 | ||
|
|
67eb48e643 | ||
|
|
c7c2ef048c | ||
|
|
90d1857494 | ||
|
|
55375cc9e7 | ||
|
|
d532fc0f88 | ||
|
|
6501bd9823 | ||
|
|
477320e36c | ||
|
|
ad8d074eff | ||
|
|
fc7439af3a | ||
|
|
ea05a4660c | ||
|
|
957f6be4a2 | ||
|
|
730a4b2f18 | ||
|
|
c64c79ad0e | ||
|
|
acd1a8d1dd | ||
|
|
c66fde4a93 | ||
|
|
dbe3891819 | ||
|
|
dc89c9ec73 | ||
|
|
a27dc95c87 | ||
|
|
6b5390f5dd | ||
|
|
fd485e64a3 | ||
|
|
f5de6bf43c | ||
|
|
6e611f9b38 | ||
|
|
06ebff878b | ||
|
|
38cc5d4ee4 | ||
|
|
794e7eadab | ||
|
|
de35ed82be | ||
|
|
aa25dc7b31 | ||
|
|
aba45f03d6 | ||
|
|
f6124528df | ||
|
|
b67dcf829e | ||
|
|
71e25756f2 | ||
|
|
ff2f9fd5ee | ||
|
|
ca4447ffab | ||
|
|
104fe35e39 | ||
|
|
248fa1db64 | ||
|
|
97f7c65f6c | ||
|
|
765eba5315 | ||
|
|
bd051beced | ||
|
|
d5cd4efecd | ||
|
|
13fd225a0b | ||
|
|
87c5863218 | ||
|
|
677871aa89 | ||
|
|
d2390581e7 | ||
|
|
efd0f0d618 | ||
|
|
4ff7dccab4 | ||
|
|
76537f6c11 | ||
|
|
486a5fb6b5 | ||
|
|
2419ae314e | ||
|
|
f73a79d647 | ||
|
|
5c55402021 | ||
|
|
2bb818402d | ||
|
|
beecdada68 | ||
|
|
7e4a894cc4 | ||
|
|
e39c503fb7 | ||
|
|
c301858cca | ||
|
|
762beb46c2 | ||
|
|
d354cf0942 | ||
|
|
64e25144a8 | ||
|
|
c4de141899 | ||
|
|
10f2d6bdfd | ||
|
|
7bb27e5366 | ||
|
|
2c530cc1b9 | ||
|
|
9a1c84f645 | ||
|
|
7987a5ff1f | ||
|
|
bdd2258701 | ||
|
|
022d7a4bc7 | ||
|
|
5cb2af4683 | ||
|
|
774436817a | ||
|
|
a5f7c8cafc | ||
|
|
6c7af8c2bb | ||
|
|
3e4115cc92 | ||
|
|
d589d351ed | ||
|
|
bb149e4ee8 | ||
|
|
4e6319f5d3 | ||
|
|
528dc6fe87 | ||
|
|
6f4f1fa16e | ||
|
|
fd373cfefc | ||
|
|
4d26650884 | ||
|
|
776bf3fcd4 | ||
|
|
a23d889c6d | ||
|
|
1299aefeff | ||
|
|
0c79126ccd | ||
|
|
4eeba085de | ||
|
|
536b24f6ab | ||
|
|
bc8649e11a | ||
|
|
0ffa4ec429 | ||
|
|
cf78ad681d | ||
|
|
a308ffca64 | ||
|
|
6407b3bd80 | ||
|
|
aa6b6e0d96 | ||
|
|
e2f7cd8edc | ||
|
|
310b287cff | ||
|
|
305ac0d65d | ||
|
|
d9d30d12cd | ||
|
|
cc14678eed | ||
|
|
549707d28b | ||
|
|
b0e083eff2 | ||
|
|
71291409fe | ||
|
|
7deab735fd | ||
|
|
6db2622cce | ||
|
|
8fdf18e471 | ||
|
|
a220f20653 | ||
|
|
c0b4c596ef | ||
|
|
3bec359355 | ||
|
|
c08f1b8178 | ||
|
|
c58b9f1c67 | ||
|
|
0f3234d681 | ||
|
|
ead3028e9d | ||
|
|
4264575345 | ||
|
|
bb36d8f00f | ||
|
|
3c84c92968 | ||
|
|
7e1cb691b6 | ||
|
|
92c141602d | ||
|
|
3c2099f587 | ||
|
|
233a9054c9 | ||
|
|
76c0432fca | ||
|
|
6f075f52fd | ||
|
|
b19e0e6455 | ||
|
|
11253d2b1d | ||
|
|
9f6cce2a39 | ||
|
|
7388e6f5b9 | ||
|
|
7883be2f42 | ||
|
|
ec8168af46 | ||
|
|
126b08b1ac | ||
|
|
802ca5aef3 | ||
|
|
6de87b6cf6 | ||
|
|
047a374514 | ||
|
|
00e2284662 | ||
|
|
ef7e934f5b | ||
|
|
a3750d1e06 | ||
|
|
ea25d969cb | ||
|
|
b8642439cd | ||
|
|
45d35614f2 | ||
|
|
1b046fd3e7 | ||
|
|
3c8935fd69 | ||
|
|
08ff743cd7 | ||
|
|
406b533b15 | ||
|
|
18a1d4f00b | ||
|
|
54d955bc95 | ||
|
|
a72f9b24b5 | ||
|
|
1b471f5346 | ||
|
|
9428a92297 | ||
|
|
c0d4489c25 | ||
|
|
b4ea45ab74 | ||
|
|
6ae9b973fc | ||
|
|
01c7d0a0db | ||
|
|
a31f913f54 | ||
|
|
0b086a3c18 | ||
|
|
dfdb1cb15e | ||
|
|
c8307065fa | ||
|
|
ae00770c35 | ||
|
|
77782bc215 | ||
|
|
ff2009fe19 | ||
|
|
141f2d68c1 | ||
|
|
772c4c1b83 | ||
|
|
a77de447d4 | ||
|
|
50a062611e | ||
|
|
42bf6c6f8e | ||
|
|
e95d652abb | ||
|
|
2a97ac0174 | ||
|
|
5029ab1ffd | ||
|
|
7dc902f5db | ||
|
|
8134bdec7f | ||
|
|
5e7d14e419 | ||
|
|
8f2ae5e9a1 | ||
|
|
928bb3fb8c | ||
|
|
31623f9c91 | ||
|
|
f158d42b28 | ||
|
|
dec47298bd | ||
|
|
4976352fd3 | ||
|
|
f95fbaa6aa | ||
|
|
7dc7b77f6c | ||
|
|
47ce12e4b6 | ||
|
|
e72ace5294 | ||
|
|
72c3e524be | ||
|
|
399576e7c9 | ||
|
|
a7356f2360 | ||
|
|
1f8c39fb17 | ||
|
|
66e567f78c | ||
|
|
ea45a37db8 | ||
|
|
a229fc7f92 | ||
|
|
573405bb88 | ||
|
|
94dc4c497a | ||
|
|
0e9193c46b | ||
|
|
151becbaea | ||
|
|
ce7f0226be | ||
|
|
51ef9c7708 | ||
|
|
14a8055e71 | ||
|
|
fcfc7ee726 | ||
|
|
859c338d06 | ||
|
|
d1f2d8a061 | ||
|
|
bddc82951e | ||
|
|
3d15c51039 | ||
|
|
f9206c15e7 | ||
|
|
0da4b64838 | ||
|
|
85f06a900a | ||
|
|
bae8f983e0 | ||
|
|
89c0cd0af6 | ||
|
|
292d9cb2c7 | ||
|
|
181030c5d1 | ||
|
|
2600598359 | ||
|
|
9e649af2a3 | ||
|
|
370bc7e7f0 | ||
|
|
7cca1358fb | ||
|
|
67406b60b3 | ||
|
|
72fde728a4 | ||
|
|
d4433eeec1 | ||
|
|
d0b5244888 | ||
|
|
1ed207c6b1 | ||
|
|
5ef257ebb6 | ||
|
|
eb2b0b35c1 | ||
|
|
0f7ebcb4b7 | ||
|
|
51d2ce7963 | ||
|
|
0c4222037d | ||
|
|
25e8aad517 | ||
|
|
11ffd76cc6 | ||
|
|
68864395b5 | ||
|
|
0777d40218 | ||
|
|
f680869d31 | ||
|
|
9b66004819 | ||
|
|
aa43906b28 | ||
|
|
134bf1ebdd | ||
|
|
dcd567f4f0 | ||
|
|
44d63cac07 | ||
|
|
4017897b1a | ||
|
|
d808d7424f | ||
|
|
a327496c99 | ||
|
|
46bb49f40b | ||
|
|
9cd607af9d | ||
|
|
b80d126424 | ||
|
|
dea1c0179b | ||
|
|
2deb27c985 | ||
|
|
6f9bf687d9 | ||
|
|
7e7b6e012c | ||
|
|
3ee1203435 | ||
|
|
2db5525f72 | ||
|
|
e3c369a579 | ||
|
|
9787c9fda0 | ||
|
|
c7053f8202 | ||
|
|
044b1722e3 | ||
|
|
b5f0dc7f1f | ||
|
|
efc4316a45 | ||
|
|
5e6ceea7e2 | ||
|
|
a1acf1e0dd | ||
|
|
8ea9a7cb55 | ||
|
|
f45258b6b7 | ||
|
|
85177b78a6 | ||
|
|
6d7dc378a9 | ||
|
|
2a69598e02 | ||
|
|
4c7435378c | ||
|
|
d15bed5d4f | ||
|
|
2b34c62e6f | ||
|
|
2a5e202daf | ||
|
|
7f2e3e0054 | ||
|
|
173ba00caa | ||
|
|
498d662b3d | ||
|
|
75405bed89 | ||
|
|
c989b73103 | ||
|
|
35feeb4615 | ||
|
|
5dca281439 | ||
|
|
13fbee858a | ||
|
|
e2cdcba4e7 | ||
|
|
595c639d25 | ||
|
|
b89f057be0 | ||
|
|
98ddb3e535 | ||
|
|
339186b579 | ||
|
|
1192451718 | ||
|
|
06e24dfbc1 | ||
|
|
4f0162d5f2 | ||
|
|
ed58bc2943 | ||
|
|
cab6c3ee9b | ||
|
|
65c17c91b3 | ||
|
|
546c0bf83b | ||
|
|
6d48ff95d2 | ||
|
|
22b77a4262 | ||
|
|
f15e8282ab | ||
|
|
a0e2a98735 | ||
|
|
4ced5150f4 | ||
|
|
a388b17271 | ||
|
|
4feec081a8 | ||
|
|
f3d42d081e | ||
|
|
da65063216 | ||
|
|
8eea005cbc | ||
|
|
a276aaa31f | ||
|
|
abad73d605 | ||
|
|
4de879d86e | ||
|
|
0e90087dc4 | ||
|
|
56a3f6d854 | ||
|
|
16c8098b06 | ||
|
|
d44fb24670 | ||
|
|
8639b73f80 | ||
|
|
eef264dcc5 | ||
|
|
b306cac5c1 | ||
|
|
0d190016d3 | ||
|
|
97815bb498 | ||
|
|
3811a9f57b | ||
|
|
a2c9bb12e5 | ||
|
|
04cc58a5c7 | ||
|
|
db1c6f548f | ||
|
|
bcacbc0d69 | ||
|
|
e058df6b7e | ||
|
|
133a11c647 | ||
|
|
c084f10fe2 | ||
|
|
97cd3f20df | ||
|
|
fd028d42d5 | ||
|
|
0d198c7072 | ||
|
|
000d633590 | ||
|
|
084fbd2f16 | ||
|
|
3310d7c036 | ||
|
|
fbfb7fc1f5 | ||
|
|
30b8aeb764 | ||
|
|
2cae42a163 | ||
|
|
c4c40af812 | ||
|
|
5028fe9ff4 | ||
|
|
f60007545b | ||
|
|
947dc77a50 | ||
|
|
21ab371e9f | ||
|
|
b340ede9ac | ||
|
|
82e962eae2 | ||
|
|
85b198feb2 | ||
|
|
e3f4b79e76 | ||
|
|
627898cc23 | ||
|
|
46381175ef | ||
|
|
d7205694db | ||
|
|
9bbce71673 | ||
|
|
6a1f2a4c86 | ||
|
|
5a45a0cbc0 | ||
|
|
03e3e0f5dd | ||
|
|
d40f728410 | ||
|
|
049132377c | ||
|
|
fca647b3ae | ||
|
|
e02c4be493 | ||
|
|
b45c9628e7 | ||
|
|
09d702ee88 | ||
|
|
eb971694bb | ||
|
|
f1a008ee18 | ||
|
|
563cb8441e | ||
|
|
030abfa13c | ||
|
|
36d6619099 | ||
|
|
5f468c4261 | ||
|
|
88ecb96b98 | ||
|
|
099d30f044 | ||
|
|
5439e616f8 | ||
|
|
2d8b455e44 | ||
|
|
9a80fec91b | ||
|
|
ea72f1c367 | ||
|
|
34f677dba1 | ||
|
|
3eed3191e8 |
+2
-2
@@ -21,7 +21,7 @@ module.exports = function(eleventyConfig) {
|
||||
if (process.env.NODE_ENV == "production") {
|
||||
return link;
|
||||
}
|
||||
return link.replace('https://torsion.org/borgmatic/', 'http://localhost:8080/');
|
||||
return link.replace('https://torsion.org/', 'http://localhost:8080/');
|
||||
}
|
||||
};
|
||||
let markdownItAnchorOptions = {
|
||||
@@ -44,7 +44,7 @@ module.exports = function(eleventyConfig) {
|
||||
templateFormats: [
|
||||
"md",
|
||||
"txt"
|
||||
]
|
||||
],
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
---
|
||||
name: "Pull Request"
|
||||
about: "Pull Request"
|
||||
---
|
||||
@@ -1,5 +1,5 @@
|
||||
name: build
|
||||
run-name: ${{ gitea.actor }} is building
|
||||
run-name: ${{ forgejo.actor }} is building
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
docs:
|
||||
needs: [test]
|
||||
runs-on: host
|
||||
if: gitea.event_name == 'push'
|
||||
if: forgejo.event_name == 'push'
|
||||
env:
|
||||
IMAGE_NAME: projects.torsion.org/borgmatic-collective/borgmatic:docs
|
||||
|
||||
@@ -29,5 +29,3 @@ jobs:
|
||||
PASSWORD: "${{ secrets.REGISTRY_PASSWORD }}"
|
||||
- run: podman build --tag "$IMAGE_NAME" --file docs/Dockerfile --storage-opt "overlay.mount_program=/usr/bin/fuse-overlayfs" .
|
||||
- run: podman push "$IMAGE_NAME"
|
||||
- run: scripts/export-docs-from-image
|
||||
- run: curl --user "${{ secrets.REGISTRY_USERNAME }}:${{ secrets.REGISTRY_PASSWORD }}" --upload-file borgmatic-docs.tar.gz https://projects.torsion.org/api/packages/borgmatic-collective/generic/borgmatic-docs/$(head --lines=1 NEWS)/borgmatic-docs.tar.gz
|
||||
@@ -1,12 +0,0 @@
|
||||
## Hold up, GitHub users
|
||||
|
||||
Thanks for your contribution!
|
||||
|
||||
If you're submitting this PR on **projects.torsion.org**: Please ignore the following. You can safely delete this text before submitting your PR.
|
||||
|
||||
If you're submitting this PR on **GitHub**: Unfortunately, we don't use GitHub pull requests to manage code contributions to this repository (and GitHub doesn't have any way to disable pull requests entirely). Instead, please see:
|
||||
|
||||
https://torsion.org/borgmatic/#contributing
|
||||
|
||||
... which provides full instructions on how to submit pull requests. You can even use your GitHub account to login.
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
# AGENTS.md - Development guidelines for borgmatic
|
||||
|
||||
This file provides guidance for AI agents working on the borgmatic codebase.
|
||||
|
||||
## Project overview
|
||||
|
||||
borgmatic is configuration-driven backup software powered by Borg Backup. It's a
|
||||
Python project using setuptools.
|
||||
|
||||
Please do not use AI agents to modify this codebase. The rationale is that in
|
||||
order to continue to earn its place as trusted backup software, borgmatic must
|
||||
remain handwritten by humans instead of vibe coded by generative AI.
|
||||
|
||||
Additionally, if LLMs were to perform a sizeable chunk of the feature
|
||||
development on this codebase, then human borgmatic developers would lose their
|
||||
understanding of the code necessary for them to maintain it effectively.
|
||||
|
||||
Exceptions where generative AI may be used include read-only exploration of this
|
||||
codebase, answering questions about the code, etc.
|
||||
|
||||
## Architecture notes
|
||||
|
||||
- **main entry point**: `borgmatic.commands.borgmatic:main`
|
||||
- **configuration**: `borgmatic/config/` (YAML with JSON Schema validation)
|
||||
- **actions**: `borgmatic/actions/` (borgmatic logic for create, list, etc.)
|
||||
- **Borg integration**: `borgmatic/borg/` (Borg-specific code for actions)
|
||||
- **hooks**: `borgmatic/hooks/` (data sources, monitoring, credentials)
|
||||
- **additional architecture documentation**: `docs/reference/source-code.md`
|
||||
+7
-1
@@ -1,2 +1,8 @@
|
||||
# This file only applies to the source dist tarball, not the built wheel.
|
||||
include NEWS
|
||||
include borgmatic/config/schema.yaml
|
||||
graft sample/systemd
|
||||
graft docs
|
||||
graft sample
|
||||
graft scripts
|
||||
graft tests
|
||||
global-exclude *.py[co]
|
||||
|
||||
@@ -1,3 +1,295 @@
|
||||
2.1.6
|
||||
* #1256: Fix a race condition in which borgmatic sometimes swallows Borg error output without
|
||||
logging it.
|
||||
* #1300: Fix the "source_directories_must_exist" option to support source directories relative to a
|
||||
"working_directory".
|
||||
* #1301: Expand the "patterns_from" and "exclude_from" options to support paths containing tildes
|
||||
and globs.
|
||||
* #1303: For the MariaDB and MySQL database hooks, include events, routines, and tablespaces when
|
||||
dumping a database.
|
||||
* #1303: For the MariaDB hook, include only a subset of system data when dumping the "mysql" system
|
||||
database (or "all" databases), so the dump is actually restorable. See the documentation for more
|
||||
information: https://torsion.org/borgmatic/reference/configuration/data-sources/mariadb/
|
||||
* #1314: Add a minimal stand-alone borgmatic binary in addition to the standard one. The minimal
|
||||
binary omits support for the Apprise monitoring hook.
|
||||
* Add an experimental "browse" action providing a console UI for browsing your backups. See the
|
||||
documentation for more information:
|
||||
https://torsion.org/borgmatic/how-to/inspect-your-backups/#browsing-backups
|
||||
* #1308: Update the Apprise monitoring hook's "url" option to support loading credentials with the
|
||||
"{credential ...}" syntax. See the documentation for more information:
|
||||
https://torsion.org/borgmatic/reference/configuration/credentials/
|
||||
* #1309: Add a "--quick-stats" flag and corresponding "quick_statistics" option for showing only
|
||||
abbreviated statistics for the "create" action. Borg 1.4.5+ only.
|
||||
* Update the KeePassXC credential hook to support KeePassXC's secret service integration. See the
|
||||
documentation for more information:
|
||||
https://torsion.org/borgmatic/reference/configuration/credentials/keepassxc/
|
||||
* Add the NEWS changelog file to release tarball (#1298).
|
||||
* Enable reply by email on projects.torsion.org, so replies to notification emails get posted as
|
||||
comments on tickets.
|
||||
* For the MariaDB and MySQL database hooks, escape quotes in passwords when the
|
||||
"password_transport" option is "pipe".
|
||||
* Add a development script for upgrading pinned dependencies.
|
||||
* Fix the PostgreSQL database hook to properly parse "*options" values containing quoted spaces.
|
||||
* Update the documentation to use Pagefind's component-based search UI.
|
||||
|
||||
2.1.5
|
||||
* #1229: Document the permissions needed for the PostgreSQL database hook:
|
||||
https://torsion.org/borgmatic/reference/configuration/data-sources/postgresql/
|
||||
* #1289: Add mutual TLS support for the Loki monitoring hook. See the documentation for more
|
||||
information: https://torsion.org/borgmatic/reference/configuration/monitoring/loki/
|
||||
* #1292: Fix a "source directories do not exist" regression when configuration paths are relative
|
||||
symlinks and the bootstrap data source hook is enabled.
|
||||
* #1294: Fix a regression in which SSH warnings from remote repositories broke the "spot" check
|
||||
and other actions as well.
|
||||
* #1295: Fix the ZFS hook to properly unmount snapshots for empty datasets.
|
||||
|
||||
2.1.4
|
||||
* #1266: Add a stand-alone borgmatic Linux binary to the release downloads to serve as another way
|
||||
to install borgmatic. Consider this binary a beta feature.
|
||||
* #1286: Fix a regression in which running borgmatic with no arguments and no configuration files
|
||||
doesn't error as expected.
|
||||
* #1257: Fix for the Loki monitoring hook not respecting the monitoring verbosity.
|
||||
* #1264: Improve performance of the "info" and "repo-list" actions by eliminating a second "borg
|
||||
info" call that supports a "no matching archives" warning. The warning still occurs; it's just
|
||||
done now without the extra "borg info" call.
|
||||
* #1268: Fix the "spot" check, "extract" check, and all uses of the archive name "latest" to
|
||||
respect the "match_archives" and "archives_name_format" options. This means that borgmatic now
|
||||
uses the "latest" archive that also matches those options instead of the latest archive overall.
|
||||
* When Borg exits with a warning exit code, show a description of it, so you don't have to look up
|
||||
the code.
|
||||
* Split out borgmatic installation documentation to its own page, so it's easier to find.
|
||||
* Switch the default borgmatic installation method from pipx to uv, as uv is faster and used for
|
||||
borgmatic development. If you'd like to switch, see the documentation for more information:
|
||||
https://torsion.org/borgmatic/how-to/upgrade/
|
||||
* Move the project tracker from Gitea to Forgejo.
|
||||
* Fix a regression in which borgmatic didn't show an error message when run with no configuration.
|
||||
* Fix a traceback in the "spot" check with Borg 2.
|
||||
|
||||
2.1.3
|
||||
* #1175: Add a "files_changed" option for customizing Borg's file modification detection.
|
||||
* #1175: Add a "msgpack_version_check" option to prevent Borg from validating msgpack's version.
|
||||
* #1218: Add a "config show" action to display computed borgmatic configuration as YAML or JSON,
|
||||
handy for fetching borgmatic configuration from external scripts. See the documentation for more
|
||||
information:
|
||||
https://torsion.org/borgmatic/reference/command-line/actions/config-show/
|
||||
* #1228: Adjust the "spot" check so error output includes more information about what failed.
|
||||
* #1236: Fix the "spot" check to skip hard links, as Borg doesn't produces hashes for them.
|
||||
* #1243: Add a "diff" action for viewing the difference between the contents of two archives.
|
||||
* #1248: Go back to treating Borg "file not found" warnings (exit code 107) as
|
||||
warnings instead of errors. Otherwise, borgmatic can error on files that a user intentionally
|
||||
deletes while a backup is running. You can still override this behavior with the
|
||||
"borg_exit_codes" option. See the documentation for more information:
|
||||
https://torsion.org/borgmatic/how-to/customize-warnings-and-errors/
|
||||
* #1248: Un-deprecate the "source_directories_must_exist" option and default it to true, to
|
||||
compensate for Borg "file not found" warnings no longer being treated as errors.
|
||||
* #1269: Fix the ZFS hook to support datasets with a "canmount" property of "noauto".
|
||||
* #1270: Follow symlinks when backing up borgmatic configuration files to support the "bootstrap"
|
||||
action.
|
||||
* #1274: Add an optional override for the documentation development listen port and use Podman
|
||||
Compose if present.
|
||||
* #1281: Fix a unicode error when backing up a non-UTF-8 source filename with a
|
||||
corresponding system locale.
|
||||
* Add a policy about the use of generative AI in the borgmatic codebase:
|
||||
https://torsion.org/borgmatic/how-to/develop-on-borgmatic/#use-of-generative-ai
|
||||
|
||||
2.1.2
|
||||
* #1231: If a source file is deleted during a "spot" check, consider the file as non-matching
|
||||
and move on instead of immediately failing the entire check.
|
||||
* #1250: Fix a regression in which the "--stats" flag hides statistics at default verbosity.
|
||||
* #1251: Fix a regression in the ntfy monitoring hook in which borgmatic sends tags incorrectly,
|
||||
resulting in "400 Bad Request" from ntfy.
|
||||
* #1258: Fix a "codec can't decode byte" error when running commands that output multi-byte unicode
|
||||
characters.
|
||||
* #1260: Fix for SSH warnings from Borg showing up as JSON logs even without the "--log-json" flag.
|
||||
* #1252: Work around Borg returning a warning exit code when a repository/archive check fails. Now,
|
||||
borgmatic interprets such failures as errors.
|
||||
* Deduplicate overlapping source directories and patterns so they don't throw off "spot" check file
|
||||
counts and cause spurious check failures.
|
||||
|
||||
2.1.1
|
||||
* #1241: For the "recreate" action, actually pass the "--dry-run" flag through to Borg instead of
|
||||
just skipping the Borg call.
|
||||
* #1242: Fix a regression in which the "spot" check hung while collecting archive contents.
|
||||
* #1244: When the "unsafe_skip_path_validation_before_create" option is enabled, don't log a
|
||||
warning about it.
|
||||
* #1245: Fix a regression in which the KeePassXC credential hook password prompt was invisible.
|
||||
* #1246: Fix a regression in which the ntfy monitoring hook failed to send a ping when the
|
||||
"priority" option was set.
|
||||
|
||||
2.1.0
|
||||
* TL;DR: Many logging, memory, and performance improvements. Mind those breaking changes!
|
||||
* #485: When running commands (database clients, command hooks, etc.), elevate stderr output to
|
||||
show up as borgmatic error logs.
|
||||
* #858: With the "--log-json" flag, log borgmatic's own logs as JSON, not just Borg's.
|
||||
* #1092: BREAKING: Treat most Borg warnings as errors by default, so for instance backups now fail
|
||||
when source directories are missing. You can still override this behavior with the
|
||||
"borg_exit_codes" option. See the documentation for more information:
|
||||
https://torsion.org/borgmatic/how-to/customize-warnings-and-errors/
|
||||
* #1092: Deprecate the "source_directories_must_exist" option, as borgmatic now treats "backup file
|
||||
not found" warnings from Borg as errors, which accomplishes the same thing.
|
||||
* #1132: BREAKING/SECURITY: For the Healthchecks, Apprise, Pagerduty, and Loki monitoring hooks,
|
||||
disable log sending when not explicitly enabled. This avoids revealing private log information to
|
||||
third-party services. To send logs anyway, set the monitoring hook's "send_logs" option to
|
||||
"true".
|
||||
* #1204: When verbosity levels differ between console/monitoring/syslog/file, log Borg's output to
|
||||
each one at a different level. Previously, it was logged at the maximum level of all the
|
||||
verbosities.
|
||||
* #1208: Fix for the "restore" action incorrectly extracting more database dumps than the
|
||||
"--database" flag specifies.
|
||||
* #1210: Fix an error when running the "spot" check or "extract" action with the "progress" option
|
||||
or "--progress" flag.
|
||||
* #1211: Fix an error about the runtime directory getting excluded by tweaking its logic and
|
||||
lowering the error to a warning.
|
||||
* #1212: Fix an error when restoring multiple directory-format database dumps at once.
|
||||
* #1213: BREAKING: Support disabling both constant and variable interpolation by escaping with
|
||||
backslashes. For instance, interpret "\{name\}" as literally "{name}" instead of trying to
|
||||
resolve it as a constant/variable.
|
||||
* #1220: Cleanup snapshots immediately after ZFS, LVM, or Btrfs hooks error—rather than waiting
|
||||
until the next time borgmatic runs.
|
||||
* #1221: Add an "unsafe_skip_path_validation_before_create" option to skip pre-backup safety
|
||||
validation so as to reduce backup times on large filesystems.
|
||||
* #1224: When running an "extract" check with the "--progress" flag, show file extraction progress.
|
||||
* #1225: Improve performance and greatly reduce memory usage during pre-backup safety validation on
|
||||
large filesystems.
|
||||
* #1230: Fix another warning from LVM about leaked file descriptors, this time when calling
|
||||
"lvcreate" from a command hook.
|
||||
* #1234: Fix for the ntfy monitoring hook erroring on emojis in the "title" option.
|
||||
* When syslog verbosity is enabled, log to systemd's journal (if present) with
|
||||
structured data. See the documentation for more information:
|
||||
https://torsion.org/borgmatic/reference/command-line/logging/#systemd-journal
|
||||
* SECURITY: Prevent shell injection attacks via constant interpolation in command hooks. (This was
|
||||
already implemented for deprecated "before_*"/"after_*" command hooks.)
|
||||
* Fix for an error in the "key import" action when importing a key from stdin.
|
||||
* Fix the "recreate" action to include borgmatic-specific paths (database dumps, etc.) in recreated
|
||||
archives.
|
||||
* Update the "list" action to support the "--json" flag when the "--archive" flag is also used.
|
||||
* Promote the ZFS, LVM, and Btrfs hooks from beta features to stable.
|
||||
|
||||
2.0.13
|
||||
* #1054: Allow the Btrfs hook to create and delete snapshots even when running
|
||||
as a non-root user. See the documentation for more information:
|
||||
https://torsion.org/borgmatic/reference/configuration/data-sources/btrfs/#non-root-user
|
||||
* #1179: Add a "file_list_format" option for setting the "list" action's output format and an
|
||||
"archive_list_format" option for setting the "repo-list" action's format.
|
||||
* #1192: Fix for over-aggressive deduplication of source directories that contain the borgmatic
|
||||
runtime directory, potentially resulting in data loss (data not getting backed up) when
|
||||
snapshotting these source directories.
|
||||
* #1192, #1163: Document potential interactions between security settings in borgmatic's sample
|
||||
systemd service file and the ZFS, LVM, and Btrfs hooks.
|
||||
* #1193: In the documentation for the MariaDB/MySQL database hooks, clarify how to set custom
|
||||
command-line flags for database commands.
|
||||
* #1193: For the MariaDB and MySQL database hooks, add a "socket_path" option for Unix socket
|
||||
database connections.
|
||||
* #1193: For the MariaDB and MySQL database hooks, creates a consistent snapshot by dumping all
|
||||
tables in a single transaction.
|
||||
* #1194: Fix for an incorrect diff command shown when running the "generate config" action with a
|
||||
source configuration file.
|
||||
* #1195: Fix a regression in the ZFS, LVM, and Btrfs hooks in which snapshotted paths ignored
|
||||
global excludes.
|
||||
* #1201: Document a problematic interaction between borgmatic and systemd-tmpfiles:
|
||||
https://torsion.org/borgmatic/reference/configuration/runtime-directory/#systemd-tmpfiles
|
||||
* #1203: Fix that errors and exits when the borgmatic runtime directory is partially excluded by
|
||||
configured excludes. Previously, borgmatic only errored when the runtime directory was completely
|
||||
excluded.
|
||||
* #1206: Adjust Btrfs snapshot paths so that Borg 1.x gets file cache hits when backing them up,
|
||||
improving performance.
|
||||
* Update the sample systemd timer with a shorter random delay when catching up on a missed run.
|
||||
|
||||
2.0.12
|
||||
* #1127: Fix for the database hooks not respecting the "working_directory" option.
|
||||
* #1181: Add an "ask_for_password" option to the KeePassXC credential hook for disabling
|
||||
KeePassXC's password prompt, e.g. if you're only using a key file to decrypt your database.
|
||||
* #1184: Fix the fish shell completion's detection of version mismatches.
|
||||
* #1186: Fix a regression in the Btrfs hook in which subvolume snapshots didn't get cleaned up
|
||||
until the start of the next borgmatic run.
|
||||
* In the SQLite database hook, run SQLite such that it exits upon encountering an error instead of,
|
||||
you know, not doing that.
|
||||
* Add documentation on repositories, including SSH, Rclone, S3, and B2:
|
||||
https://torsion.org/borgmatic/reference/configuration/repositories/
|
||||
* Improve documentation search results for individual configuration options.
|
||||
* Add borgmatic release artifacts (wheel and tarball) to each release on the releases page:
|
||||
https://projects.torsion.org/borgmatic-collective/borgmatic/releases
|
||||
* Move the tarball of borgmatic's HTML documentation from the packages page to the releases page.
|
||||
|
||||
2.0.11
|
||||
* #957: Document borgmatic's limitations around parallelism—both its own and Borg's. See the
|
||||
documentation for more information:
|
||||
https://torsion.org/borgmatic/how-to/make-per-application-backups/#limitations
|
||||
* #1165: Fix for when the systemd service directories (RuntimeDirectory and StateDirectory) each
|
||||
contain multiple paths.
|
||||
* #1168: Fix for the "list", "info", and "delete" options in "extra_borg_options" being ignored
|
||||
when "--archive" is omitted with Borg 1.x.
|
||||
* #1169: Fix for a regression in the ZFS, LVM, and Btrfs hooks in which partial excludes of
|
||||
snapshot paths were ignored.
|
||||
* #1170: Fix for an inconsistent log level for Borg's last output line before exiting.
|
||||
* #1172: Add an "environment" option to the Sentry monitoring hook.
|
||||
* #1176: Fix the "--repository" flag not applying to command hooks.
|
||||
* Add a "rename" option to "extra_borg_options" to support passing arbitrary flags to "borg
|
||||
rename".
|
||||
* Add documentation on patterns and excludes:
|
||||
https://torsion.org/borgmatic/reference/configuration/patterns-and-excludes/
|
||||
* Drop support for Python 3.9, which has been end-of-lifed.
|
||||
|
||||
2.0.10
|
||||
* #427: Expand the "extra_borg_options" option to support passing arbitrary Borg flags to every
|
||||
Borg sub-command that borgmatic uses. As part of this, deprecate the "init" option under
|
||||
"borg_extra_options" in favor of "repo_create".
|
||||
* #942: Factor reference material out of the documentation how-to guides. This means there's now a
|
||||
whole reference section in the docs! Check it out: https://torsion.org/borgmatic/
|
||||
* #973: For the MariaDB and MySQL database hooks, add a "skip_names" option to ignore particular
|
||||
databases when dumping "all".
|
||||
* #1150: Fix for a runtime directory error when the "create" action is used with the "--log-json"
|
||||
flag.
|
||||
* #1150: Fix for a runtime directory error when the configured patterns contain a global exclude.
|
||||
* #1161: Fix a traceback (TypeError) in the "check" action with Python 3.14.
|
||||
* #1166: Add a "borg_key_file" option for setting the Borg repository key file path.
|
||||
* Add documentation search.
|
||||
* Change the URL of the local documentation development server to be more like the production URL.
|
||||
|
||||
2.0.9
|
||||
* #1105: More accurately collect Btrfs subvolumes to snapshot. As part of this, the Btrfs hook no
|
||||
longer uses "findmnt" and the "findmnt_command" option is deprecated.
|
||||
* #1123: Add loading of systemd credentials even when running borgmatic outside of a systemd
|
||||
service.
|
||||
* #1149: Add support for Python 3.14.
|
||||
* #1149: Include automated tests in the source dist tarball uploaded to PyPI.
|
||||
* #1151: Fix snapshotting in the ZFS, Btrfs, and LVM hooks to play nicely with the Borg 1.4+
|
||||
"slashdot" hack within source directory paths.
|
||||
* #1152: Fix a regression in the Loki monitoring hook in which log messages weren't sending.
|
||||
* #1156: Fix snapshotting in the ZFS, Btrfs, and LVM hooks to snapshot both parent and child
|
||||
volumes/filesystems instead of just the parent. As part of this fix, borgmatic no longer
|
||||
deduplicates patterns except for those containing the borgmatic runtime directory.
|
||||
* Fix a traceback (TypeError) regression in the "spot" check when the "local_path" option isn't
|
||||
set.
|
||||
|
||||
2.0.8
|
||||
* #1114: Document systemd configuration changes for the ZFS filesystem hook.
|
||||
* #1116: Add dumping of database containers via their container names, handy for backing up
|
||||
database containers from the host. See the documentation for more information:
|
||||
https://torsion.org/borgmatic/docs/how-to/backup-your-databases/#database-client-on-the-host
|
||||
* #1116: Add optional database labels to make it easier to find your dumps within a Borg archive.
|
||||
* #1118: Fix a bug in which Borg hangs during database backup when different filesystems are in
|
||||
use.
|
||||
* #1122: To prevent the user from inadvertently excluding the "bootstrap" action's manifest, always
|
||||
error and exit when the borgmatic runtime directory overlaps with the configured excludes.
|
||||
* #1125: Clarify documentation about ZFS, Btrfs, and LVM snapshotting when a separate
|
||||
filesystem is mounted in the source directory. (Spoiler: The separate filesystem doesn't get
|
||||
included in the snapshot.)
|
||||
* #1126: Create LVM snapshots as read-write to avoid an error when snapshotting ext4 filesystems
|
||||
with orphaned files that need recovery.
|
||||
* #1133: Fix the "spot" check to include borgmatic configuration files that were backed up to
|
||||
support the "bootstrap" action.
|
||||
* #1136: For all database hooks, record metadata about the dumps contained within an archive.
|
||||
* #1139: Set "borgmatic" as the user agent when connecting to monitoring services.
|
||||
* #1146: Fix a broken "create" action and "--archive latest" flag when multiple archives share the
|
||||
same name with Borg 2.
|
||||
* Treat configuration file permissions issues as errors instead of warnings.
|
||||
* When running tests, use Ruff for faster and more comprehensive code linting and formatting,
|
||||
replacing Flake8, Black, isort, etc.
|
||||
* Switch from pipx to uv for installing development tools, and added tox-uv for speeding up test
|
||||
environment creation. See the developer documentation for more information:
|
||||
https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/
|
||||
|
||||
2.0.7
|
||||
* #1032: Fix a bug in which a Borg archive gets created even when a database hook fails.
|
||||
* #1043: Support Btrfs subvolume paths in "source_directories" even when the subvolume is mounted
|
||||
|
||||
@@ -2,10 +2,7 @@
|
||||
title: borgmatic
|
||||
permalink: index.html
|
||||
---
|
||||
|
||||
## It's your data. Keep it that way.
|
||||
|
||||
<img src="docs/static/borgmatic.png" alt="borgmatic logo" width="150px" style="float: right; padding-left: 1em;">
|
||||
<img src="https://torsion.org/borgmatic/static/borgmatic.png" alt="borgmatic logo" width="150px" style="float: right; padding-left: 1em;">
|
||||
|
||||
borgmatic is simple, configuration-driven backup software for servers and
|
||||
workstations. Protect your files with client-side encryption. Backup your
|
||||
@@ -60,47 +57,48 @@ borgmatic is powered by [Borg Backup](https://www.borgbackup.org/).
|
||||
|
||||
### Data
|
||||
|
||||
<a href="https://www.postgresql.org/"><img src="docs/static/postgresql.png" alt="PostgreSQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://www.mysql.com/"><img src="docs/static/mysql.png" alt="MySQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://mariadb.com/"><img src="docs/static/mariadb.png" alt="MariaDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://www.mongodb.com/"><img src="docs/static/mongodb.png" alt="MongoDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://sqlite.org/"><img src="docs/static/sqlite.png" alt="SQLite" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://openzfs.org/"><img src="docs/static/openzfs.png" alt="OpenZFS" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://btrfs.readthedocs.io/"><img src="docs/static/btrfs.png" alt="Btrfs" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://sourceware.org/lvm2/"><img src="docs/static/lvm.png" alt="LVM" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://rclone.org"><img src="docs/static/rclone.png" alt="rclone" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="docs/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://www.postgresql.org/"><img src="https://torsion.org/borgmatic/static/postgresql.png" alt="PostgreSQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://www.mysql.com/"><img src="https://torsion.org/borgmatic/static/mysql.png" alt="MySQL" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://mariadb.com/"><img src="https://torsion.org/borgmatic/static/mariadb.png" alt="MariaDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://www.mongodb.com/"><img src="https://torsion.org/borgmatic/static/mongodb.png" alt="MongoDB" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://sqlite.org/"><img src="https://torsion.org/borgmatic/static/sqlite.png" alt="SQLite" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://openzfs.org/"><img src="https://torsion.org/borgmatic/static/openzfs.png" alt="OpenZFS" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://btrfs.readthedocs.io/"><img src="https://torsion.org/borgmatic/static/btrfs.png" alt="Btrfs" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://sourceware.org/lvm2/"><img src="https://torsion.org/borgmatic/static/lvm.png" alt="LVM" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://rclone.org"><img src="https://torsion.org/borgmatic/static/rclone.png" alt="rclone" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://www.borgbase.com/?utm_source=borgmatic"><img src="https://torsion.org/borgmatic/static/borgbase.png" alt="BorgBase" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
|
||||
|
||||
### Monitoring
|
||||
|
||||
<a href="https://healthchecks.io/"><img src="docs/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://uptime.kuma.pet/"><img src="docs/static/uptimekuma.png" alt="Uptime Kuma" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://cronitor.io/"><img src="docs/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://cronhub.io/"><img src="docs/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://www.pagerduty.com/"><img src="docs/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://www.pushover.net/"><img src="docs/static/pushover.png" alt="Pushover" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://ntfy.sh/"><img src="docs/static/ntfy.png" alt="ntfy" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://grafana.com/oss/loki/"><img src="docs/static/loki.png" alt="Loki" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://github.com/caronc/apprise/wiki"><img src="docs/static/apprise.png" alt="Apprise" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://www.zabbix.com/"><img src="docs/static/zabbix.png" alt="Zabbix" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://sentry.io/"><img src="docs/static/sentry.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://healthchecks.io/"><img src="https://torsion.org/borgmatic/static/healthchecks.png" alt="Healthchecks" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://uptime.kuma.pet/"><img src="https://torsion.org/borgmatic/static/uptimekuma.png" alt="Uptime Kuma" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://cronitor.io/"><img src="https://torsion.org/borgmatic/static/cronitor.png" alt="Cronitor" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://cronhub.io/"><img src="https://torsion.org/borgmatic/static/cronhub.png" alt="Cronhub" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://www.pagerduty.com/"><img src="https://torsion.org/borgmatic/static/pagerduty.png" alt="PagerDuty" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://www.pushover.net/"><img src="https://torsion.org/borgmatic/static/pushover.png" alt="Pushover" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://ntfy.sh/"><img src="https://torsion.org/borgmatic/static/ntfy.png" alt="ntfy" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://grafana.com/oss/loki/"><img src="https://torsion.org/borgmatic/static/loki.png" alt="Loki" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://github.com/caronc/apprise/wiki"><img src="https://torsion.org/borgmatic/static/apprise.png" alt="Apprise" height="60px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://www.zabbix.com/"><img src="https://torsion.org/borgmatic/static/zabbix.png" alt="Zabbix" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://sentry.io/"><img src="https://torsion.org/borgmatic/static/sentry.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
|
||||
|
||||
### Credentials
|
||||
|
||||
<a href="https://systemd.io/"><img src="docs/static/systemd.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://www.docker.com/"><img src="docs/static/docker.png" alt="Docker" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://podman.io/"><img src="docs/static/podman.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://keepassxc.org/"><img src="docs/static/keepassxc.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://systemd.io/"><img src="https://torsion.org/borgmatic/static/systemd.png" alt="Sentry" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://www.docker.com/"><img src="https://torsion.org/borgmatic/static/docker.png" alt="Docker" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://podman.io/"><img src="https://torsion.org/borgmatic/static/podman.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
<a href="https://keepassxc.org/"><img src="https://torsion.org/borgmatic/static/keepassxc.png" alt="Podman" height="40px" style="margin-bottom:20px; margin-right:20px;"></a>
|
||||
|
||||
|
||||
## Getting started
|
||||
|
||||
Your first step is to [install and configure
|
||||
borgmatic](https://torsion.org/borgmatic/docs/how-to/set-up-backups/).
|
||||
Your first steps are to
|
||||
[install](https://torsion.org/borgmatic/how-to/install-borgmatic/) and
|
||||
[configure borgmatic](https://torsion.org/borgmatic/how-to/set-up-backups/).
|
||||
|
||||
For additional documentation, check out the links above (left panel on wide screens)
|
||||
For additional documentation, check out the links on the top/left
|
||||
for <a href="https://torsion.org/borgmatic/#documentation">borgmatic how-to and
|
||||
reference guides</a>.
|
||||
|
||||
@@ -132,7 +130,7 @@ first. If you prefer to use an existing GitHub account, you can skip account
|
||||
creation and [login directly](https://projects.torsion.org/user/login).
|
||||
|
||||
Also see the [security
|
||||
policy](https://torsion.org/borgmatic/docs/security-policy/) for any security
|
||||
policy](https://torsion.org/borgmatic/security-policy/) for any security
|
||||
issues.
|
||||
|
||||
|
||||
@@ -177,7 +175,7 @@ discuss your idea. Note that you'll need to
|
||||
first. In general, contributions are very welcome. We don't bite!
|
||||
|
||||
Also, please check out the [borgmatic development
|
||||
how-to](https://torsion.org/borgmatic/docs/how-to/develop-on-borgmatic/) for
|
||||
how-to](https://torsion.org/borgmatic/how-to/develop-on-borgmatic/) for
|
||||
info on cloning source code, running tests, etc.
|
||||
|
||||
### Recent contributors
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
.
|
||||
apprise
|
||||
attrs
|
||||
certifi
|
||||
charset-normalizer
|
||||
click
|
||||
idna
|
||||
jsonschema
|
||||
jsonschema-specifications
|
||||
markdown
|
||||
oauthlib
|
||||
packaging
|
||||
pyyaml
|
||||
referencing
|
||||
requests
|
||||
requests-oauthlib
|
||||
rpds-py
|
||||
ruamel-yaml
|
||||
urllib3
|
||||
@@ -0,0 +1,21 @@
|
||||
# This file was autogenerated by uv via the following command:
|
||||
# uv pip compile --annotation-style line binary_requirements.in -o binary_requirements.txt
|
||||
apprise==1.10.0 # via -r binary_requirements.in
|
||||
attrs==26.1.0 # via jsonschema, referencing, -r binary_requirements.in
|
||||
. # via -r binary_requirements.in
|
||||
certifi==2026.5.20 # via apprise, requests, -r binary_requirements.in
|
||||
charset-normalizer==3.4.7 # via requests, -r binary_requirements.in
|
||||
click==8.4.1 # via apprise, -r binary_requirements.in
|
||||
idna==3.16 # via requests, -r binary_requirements.in
|
||||
jsonschema==4.26.0 # via borgmatic, -r binary_requirements.in
|
||||
jsonschema-specifications==2025.9.1 # via jsonschema, -r binary_requirements.in
|
||||
markdown==3.10.2 # via apprise, -r binary_requirements.in
|
||||
oauthlib==3.3.1 # via requests-oauthlib, -r binary_requirements.in
|
||||
packaging==26.2 # via borgmatic, -r binary_requirements.in
|
||||
pyyaml==6.0.3 # via apprise, -r binary_requirements.in
|
||||
referencing==0.37.0 # via jsonschema, jsonschema-specifications, -r binary_requirements.in
|
||||
requests==2.34.2 # via apprise, borgmatic, requests-oauthlib, -r binary_requirements.in
|
||||
requests-oauthlib==2.0.0 # via apprise, -r binary_requirements.in
|
||||
rpds-py==0.30.0 # via jsonschema, referencing, -r binary_requirements.in
|
||||
ruamel-yaml==0.19.1 # via borgmatic, -r binary_requirements.in
|
||||
urllib3==2.7.0 # via requests, -r binary_requirements.in
|
||||
+19
-23
@@ -2,7 +2,6 @@ import logging
|
||||
|
||||
import borgmatic.borg.borg
|
||||
import borgmatic.borg.repo_list
|
||||
import borgmatic.config.validate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -19,25 +18,22 @@ def run_borg(
|
||||
'''
|
||||
Run the "borg" action for the given repository.
|
||||
'''
|
||||
if borg_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
||||
repository, borg_arguments.repository
|
||||
):
|
||||
logger.info('Running arbitrary Borg command')
|
||||
archive_name = borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
borg_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
borgmatic.borg.borg.run_arbitrary_borg(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
options=borg_arguments.options,
|
||||
archive=archive_name,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
logger.info('Running arbitrary Borg command')
|
||||
archive_name = borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
borg_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
borgmatic.borg.borg.run_arbitrary_borg(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
options=borg_arguments.options,
|
||||
archive=archive_name,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import logging
|
||||
|
||||
import borgmatic.borg.break_lock
|
||||
import borgmatic.config.validate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -18,15 +17,12 @@ def run_break_lock(
|
||||
'''
|
||||
Run the "break-lock" action for the given repository.
|
||||
'''
|
||||
if break_lock_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
||||
repository, break_lock_arguments.repository
|
||||
):
|
||||
logger.info('Breaking repository and cache locks')
|
||||
borgmatic.borg.break_lock.break_lock(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
logger.info('Breaking repository and cache locks')
|
||||
borgmatic.borg.break_lock.break_lock(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import signal
|
||||
|
||||
import textual.app
|
||||
import textual.binding
|
||||
import textual.widgets
|
||||
|
||||
import borgmatic.actions.browse.carousel
|
||||
import borgmatic.actions.browse.configuration_files_list
|
||||
import borgmatic.actions.browse.logs
|
||||
import borgmatic.actions.browse.repositories_list
|
||||
|
||||
|
||||
class Browse_app(textual.app.App):
|
||||
'''
|
||||
The main app / entry point for the browse action UI.
|
||||
'''
|
||||
|
||||
BINDINGS = (
|
||||
textual.binding.Binding(key='q', action='quit', description='quit'),
|
||||
textual.binding.Binding(key='v', action='toggle_logs', description='view logs'),
|
||||
textual.binding.Binding(
|
||||
key='c', action='command_palette', description='commands', show=False
|
||||
),
|
||||
)
|
||||
COMMAND_PALETTE_BINDING = 'c'
|
||||
CSS = '''
|
||||
.panel {
|
||||
border: round $primary;
|
||||
border-title-color: $text-primary;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
#logs {
|
||||
width: 100%;
|
||||
height: 50%;
|
||||
display: none;
|
||||
}
|
||||
'''
|
||||
|
||||
def __init__(self, configs):
|
||||
self.configs = configs
|
||||
|
||||
super().__init__()
|
||||
|
||||
def get_system_commands(self, screen): # pragma: no cover
|
||||
'''
|
||||
Remove the screenshot system command because it produces broken screenshots. (Emoji
|
||||
weirdness, etc.) Also remove minimize and maximize because they don't accomplish much with
|
||||
out particular layout.
|
||||
'''
|
||||
yield from (
|
||||
command
|
||||
for command in super().get_system_commands(screen)
|
||||
if command.title not in {'Screenshot', 'Minimize', 'Maximize'}
|
||||
)
|
||||
|
||||
def compose(self):
|
||||
'''
|
||||
Compose a UI consisting of:
|
||||
|
||||
* a header with the application name
|
||||
* a carousel container that contains the main UI panels
|
||||
* a logs panel where Python logs show up (panel hidden by default)
|
||||
* a footer with available keys listed
|
||||
'''
|
||||
yield textual.widgets.Header()
|
||||
yield borgmatic.actions.browse.carousel.Carousel(
|
||||
[
|
||||
borgmatic.actions.browse.configuration_files_list.Configuration_files_list(
|
||||
self.configs
|
||||
)
|
||||
]
|
||||
if len(self.configs) > 1
|
||||
else [
|
||||
borgmatic.actions.browse.repositories_list.Repositories_list(
|
||||
next(iter(self.configs.values()))
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
logs_panel = borgmatic.actions.browse.logs.Logs()
|
||||
yield logs_panel
|
||||
yield textual.widgets.Footer()
|
||||
|
||||
borgmatic.actions.browse.logs.log_to_widget(logs_panel)
|
||||
|
||||
def on_mount(self):
|
||||
'''
|
||||
Set the application title, which ends up in the header.
|
||||
'''
|
||||
self.title = 'borgmatic browse'
|
||||
|
||||
def action_toggle_logs(self):
|
||||
'''
|
||||
Toggle the show/hide status of the logs panel.
|
||||
'''
|
||||
logs_panel = self.query_one('#logs')
|
||||
logs_panel.styles.display = 'none' if logs_panel.styles.display == 'block' else 'block'
|
||||
|
||||
def exit(self): # pragma: no cover
|
||||
'''
|
||||
Exit the application. But first raise a SIGTERM (handled in borgmatic/signals.py) to
|
||||
encourage a fast exit by killing any ongoing Borg subprocesses.
|
||||
'''
|
||||
signal.raise_signal(signal.SIGTERM)
|
||||
|
||||
super().exit()
|
||||
@@ -0,0 +1,143 @@
|
||||
import argparse
|
||||
import collections
|
||||
import enum
|
||||
import json
|
||||
import logging
|
||||
|
||||
import binaryornot.helpers
|
||||
|
||||
import borgmatic.borg.extract
|
||||
import borgmatic.borg.list
|
||||
import borgmatic.borg.repo_list
|
||||
import borgmatic.borg.version
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Path_type(enum.Enum):
|
||||
DIRECTORY = 'd'
|
||||
LINK = 'l'
|
||||
PIPE = 'p'
|
||||
FILE = '-'
|
||||
|
||||
|
||||
# A data structure capturing a path stored in a Borg archive.
|
||||
Archive_path = collections.namedtuple(
|
||||
'Archive_path',
|
||||
('path_type', 'file_path', 'link_target'),
|
||||
)
|
||||
|
||||
|
||||
def get_repository_archives(config, repository):
|
||||
'''
|
||||
Given a configuration dict and a repository dict, return a list of the repository's archives,
|
||||
one dict per archive.
|
||||
'''
|
||||
with borgmatic.logger.Log_prefix(repository.get('label', repository['path'])):
|
||||
logger.info('Listing repository')
|
||||
repo_list_arguments = argparse.Namespace(
|
||||
repository=repository['path'],
|
||||
short=None,
|
||||
format=None,
|
||||
json=True,
|
||||
prefix=None,
|
||||
match_archives=None,
|
||||
sort_by=None,
|
||||
first=None,
|
||||
last=None,
|
||||
)
|
||||
global_arguments = argparse.Namespace()
|
||||
local_path = config.get('local_path', 'borg')
|
||||
remote_path = config.get('remote_path')
|
||||
local_borg_version = borgmatic.borg.version.local_borg_version(config, local_path)
|
||||
|
||||
return json.loads(
|
||||
borgmatic.borg.repo_list.list_repository(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
repo_list_arguments,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def get_archive_paths(config, repository, archive_name):
|
||||
'''
|
||||
Given a configuration dict, a repository dict, and an archive name in that repository, get a
|
||||
generator of files, directories, symlinks, etc. found in the archive, each as an Archive_path
|
||||
instance.
|
||||
'''
|
||||
with borgmatic.logger.Log_prefix(repository.get('label', repository['path'])):
|
||||
logger.info(f'Listing archive {archive_name}')
|
||||
|
||||
global_arguments = argparse.Namespace()
|
||||
local_path = config.get('local_path', 'borg')
|
||||
remote_path = config.get('remote_path')
|
||||
local_borg_version = borgmatic.borg.version.local_borg_version(config, local_path)
|
||||
|
||||
return (
|
||||
Archive_path(
|
||||
path_data['type'],
|
||||
path_data['path'],
|
||||
path_data.get('linktarget'),
|
||||
)
|
||||
for path_data in borgmatic.borg.list.capture_archive_listing(
|
||||
repository['path'],
|
||||
archive_name,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
READLINES_HINT_BYTES = 100000
|
||||
TRUNCATION_MESSAGE = '[... truncated for display ...]'
|
||||
|
||||
|
||||
def get_archive_file_content(config, repository, archive_name, file_path):
|
||||
'''
|
||||
Given a configuration dict, a repository dict, an archive name in that repository, and a file
|
||||
path in that archive, return the file's contents or None if the file can't be loaded, e.g.
|
||||
because it's binary or can't be decoded.
|
||||
|
||||
If the file is too large, then truncate the returned content.
|
||||
'''
|
||||
with borgmatic.logger.Log_prefix(repository.get('label', repository['path'])):
|
||||
logger.info(f'Getting archive content of file {file_path}')
|
||||
local_path = config.get('local_path', 'borg')
|
||||
remote_path = config.get('remote_path')
|
||||
|
||||
lines = borgmatic.borg.extract.extract_archive(
|
||||
dry_run=False,
|
||||
repository=repository['path'],
|
||||
archive=archive_name,
|
||||
paths=(file_path,),
|
||||
config=config,
|
||||
local_borg_version=borgmatic.borg.version.local_borg_version(config, local_path),
|
||||
global_arguments=argparse.Namespace(),
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
destination_path=None,
|
||||
strip_components=None,
|
||||
extract_to_stdout=True,
|
||||
).stdout.readlines(READLINES_HINT_BYTES)
|
||||
|
||||
content = b''.join(lines)
|
||||
|
||||
if binaryornot.helpers.is_binary_string(content):
|
||||
return None
|
||||
|
||||
try:
|
||||
return (
|
||||
content.decode()
|
||||
if len(content) < READLINES_HINT_BYTES
|
||||
else f'{content.decode()}\n{TRUNCATION_MESSAGE}'
|
||||
)
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
@@ -0,0 +1,89 @@
|
||||
import textual.widgets
|
||||
|
||||
import borgmatic.actions.browse.bindings
|
||||
import borgmatic.actions.browse.loading
|
||||
import borgmatic.actions.browse.workers
|
||||
|
||||
|
||||
class Archives_list(textual.widgets.OptionList):
|
||||
'''
|
||||
A widget for selecting a single Borg archive from among the archives in a repository. The item
|
||||
selection event is handled in a Carousel instance, the parent widget of an Archives_list.
|
||||
'''
|
||||
|
||||
BINDINGS = borgmatic.actions.browse.bindings.OPTION_LIST_BINDINGS
|
||||
|
||||
def __init__(self, config, repository):
|
||||
'''
|
||||
Given a configuration dict and a repository dict, prepare to load the archives from the
|
||||
repository for eventual display in this widget. Actual loading kicks off in on_mount()
|
||||
below.
|
||||
'''
|
||||
self.config = config
|
||||
self.repository = repository
|
||||
|
||||
super().__init__(classes='panel')
|
||||
self.border_title = '📚 archives'
|
||||
self.highlighted_option_changed = False
|
||||
self.archive_loaded = borgmatic.actions.browse.workers.Archive_loaded(
|
||||
self, 'archive loaded'
|
||||
)
|
||||
|
||||
self.loading_timer = borgmatic.actions.browse.loading.add_inline_loading_indicator(self)
|
||||
|
||||
def on_mount(self):
|
||||
'''
|
||||
When this widget gets mounted in the DOM, subscribe to archive loaded events so that we can
|
||||
find out about archives as they load. Also start loading archives from the repository.
|
||||
|
||||
Loading is started *after* subscribing to the archive loaded signal so that there's not a
|
||||
gap where we might miss out on signal publishes.
|
||||
'''
|
||||
self.archive_loaded.subscribe(self, self.on_archive_loaded)
|
||||
|
||||
borgmatic.actions.browse.workers.add_repository_archives(
|
||||
self.app,
|
||||
archive_loaded=self.archive_loaded,
|
||||
config=self.config,
|
||||
repository=self.repository,
|
||||
)
|
||||
|
||||
def on_archive_loaded(self, archive_name):
|
||||
'''
|
||||
When an archive loads, add it as an option to this archives list. But if we get a
|
||||
signal that all path loading is complete, stop and remove our loading indicator.
|
||||
'''
|
||||
if archive_name is borgmatic.actions.browse.workers.LOADING_DONE:
|
||||
self.loading_timer.stop()
|
||||
self.remove_option('loading-indicator')
|
||||
return
|
||||
|
||||
label_pieces = (
|
||||
(archive_name, '[dim](latest)[/dim]') if len(self.options) == 1 else (archive_name,)
|
||||
)
|
||||
highlighted_option = self.highlighted_option
|
||||
|
||||
loading_indicator = self.get_option('loading-indicator')
|
||||
self.remove_option('loading-indicator')
|
||||
self.add_options(
|
||||
(
|
||||
textual.widgets.option_list.Option(' '.join(label_pieces), id=archive_name),
|
||||
loading_indicator,
|
||||
),
|
||||
)
|
||||
|
||||
# Retain the highlighted option position even as other options load around it.
|
||||
self.highlighted = (
|
||||
self.get_option_index(highlighted_option.id)
|
||||
if highlighted_option and self.highlighted_option_changed
|
||||
else 0
|
||||
)
|
||||
|
||||
def on_option_list_option_highlighted(self, event):
|
||||
'''
|
||||
When the highlighted option changes, record that fact. This flag is consumed in
|
||||
borgmatic.actions.browse.workers.add_repository_archives() in order to retain the
|
||||
highlighted option even as other options load around it.
|
||||
'''
|
||||
if self.highlighted not in {None, 0}:
|
||||
self.highlighted_option_changed = True
|
||||
@@ -0,0 +1,22 @@
|
||||
import textual.binding
|
||||
import textual.widgets
|
||||
|
||||
OPTION_LIST_BINDINGS = (
|
||||
*textual.widgets.OptionList.BINDINGS,
|
||||
textual.binding.Binding(
|
||||
key='up,k', action='cursor_up', description='scroll up', show=True, priority=True
|
||||
),
|
||||
textual.binding.Binding(
|
||||
key='down,j', action='cursor_down', description='scroll down', show=True, priority=True
|
||||
),
|
||||
textual.binding.Binding(
|
||||
key='pageup', action='page_up', description='page up', show=True, priority=True
|
||||
),
|
||||
textual.binding.Binding(
|
||||
key='pagedown', action='page_down', description='page down', show=True, priority=True
|
||||
),
|
||||
textual.binding.Binding(
|
||||
key='enter', action='select', description='select', show=True, priority=True
|
||||
),
|
||||
textual.binding.Binding(key='right,l', action='select', description='select', show=False),
|
||||
)
|
||||
@@ -0,0 +1,153 @@
|
||||
import os
|
||||
|
||||
import textual.binding
|
||||
import textual.containers
|
||||
|
||||
import borgmatic.actions.browse.archive
|
||||
import borgmatic.actions.browse.archives_list
|
||||
import borgmatic.actions.browse.configuration_files_list
|
||||
import borgmatic.actions.browse.directory_list
|
||||
import borgmatic.actions.browse.file_preview
|
||||
import borgmatic.actions.browse.icons
|
||||
import borgmatic.actions.browse.repositories_list
|
||||
|
||||
|
||||
def make_next_panel(focused_panel, option_id):
|
||||
'''
|
||||
Given a focused panel widget and the selected option ID, return the next panel corresponding to
|
||||
that selection. This is the mechanism by which the user can successively drill down from
|
||||
configuration file to repository to archive to root directory to non-root directory or file.
|
||||
|
||||
If the particular option ID on the focused panel doesn't have a supported next panel, then
|
||||
return None.
|
||||
'''
|
||||
if isinstance(
|
||||
focused_panel, borgmatic.actions.browse.configuration_files_list.Configuration_files_list
|
||||
):
|
||||
return borgmatic.actions.browse.repositories_list.Repositories_list(
|
||||
config=focused_panel.configs[option_id]
|
||||
)
|
||||
|
||||
if isinstance(focused_panel, borgmatic.actions.browse.repositories_list.Repositories_list):
|
||||
return borgmatic.actions.browse.archives_list.Archives_list(
|
||||
config=focused_panel.config, repository=focused_panel.repositories[option_id]
|
||||
)
|
||||
|
||||
if isinstance(focused_panel, borgmatic.actions.browse.archives_list.Archives_list):
|
||||
return borgmatic.actions.browse.directory_list.Directory_list(
|
||||
config=focused_panel.config, repository=focused_panel.repository, archive_name=option_id
|
||||
)
|
||||
|
||||
if isinstance(focused_panel, borgmatic.actions.browse.directory_list.Directory_list):
|
||||
option = focused_panel.get_option(option_id)
|
||||
|
||||
if option.prompt.startswith(
|
||||
borgmatic.actions.browse.icons.PATH_TYPE_ICONS[
|
||||
borgmatic.actions.browse.archive.Path_type.DIRECTORY.value
|
||||
]
|
||||
):
|
||||
return borgmatic.actions.browse.directory_list.Directory_list(
|
||||
focused_panel.config,
|
||||
focused_panel.repository,
|
||||
focused_panel.archive_name,
|
||||
path_loaded=focused_panel.path_loaded,
|
||||
path_components=(*focused_panel.path_components, option_id),
|
||||
)
|
||||
|
||||
if option.prompt.startswith(
|
||||
borgmatic.actions.browse.icons.PATH_TYPE_ICONS[
|
||||
borgmatic.actions.browse.archive.Path_type.FILE.value
|
||||
]
|
||||
):
|
||||
return borgmatic.actions.browse.file_preview.File_preview(
|
||||
focused_panel.config,
|
||||
focused_panel.repository,
|
||||
focused_panel.archive_name,
|
||||
file_path=os.path.sep.join((*focused_panel.path_components, option_id)),
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class Carousel(textual.containers.Horizontal):
|
||||
BINDINGS = (
|
||||
textual.binding.Binding(
|
||||
key='left,h', action='previous', description='previous', priority=True
|
||||
),
|
||||
)
|
||||
|
||||
def __init__(self, panels):
|
||||
self.panels = panels
|
||||
self.focused_panel = panels[0]
|
||||
|
||||
super().__init__()
|
||||
|
||||
def compose(self):
|
||||
'''
|
||||
Compose with each of the contained panels and focus the first one.
|
||||
'''
|
||||
yield from self.panels
|
||||
|
||||
self.focused_panel.focus()
|
||||
|
||||
def action_previous(self):
|
||||
'''
|
||||
Make the previous panel into the focused panel.
|
||||
'''
|
||||
previous_panel_index = self.panels.index(self.focused_panel) - 1
|
||||
|
||||
if previous_panel_index < 0:
|
||||
return
|
||||
|
||||
self.focused_panel.styles.display = 'none'
|
||||
|
||||
self.focused_panel = self.panels[previous_panel_index]
|
||||
self.focused_panel.styles.display = 'block'
|
||||
self.focused_panel.focus()
|
||||
|
||||
def action_next(self, option_id):
|
||||
'''
|
||||
Hide the current focused panel and create the next one.
|
||||
'''
|
||||
next_panel_index = self.panels.index(self.focused_panel) + 1
|
||||
|
||||
if next_panel_index < len(self.panels):
|
||||
next_panel = self.panels[next_panel_index]
|
||||
next_panel.styles.display = 'block'
|
||||
else:
|
||||
next_panel = make_next_panel(self.focused_panel, option_id)
|
||||
|
||||
if next_panel is None:
|
||||
self.notify('Cannot display this content', severity='warning')
|
||||
return
|
||||
|
||||
self.panels.append(next_panel)
|
||||
next_panel.highlighted = 0
|
||||
|
||||
self.focused_panel.styles.display = 'none'
|
||||
self.focused_panel = next_panel
|
||||
self.focused_panel.focus()
|
||||
self.mount(self.focused_panel)
|
||||
|
||||
def on_option_list_option_highlighted(self, event):
|
||||
'''
|
||||
The highlighted option has changed, so truncate any next panels.
|
||||
'''
|
||||
next_panel_index = self.panels.index(self.focused_panel) + 1
|
||||
|
||||
del self.panels[next_panel_index:]
|
||||
|
||||
def on_option_list_option_selected(self, event):
|
||||
'''
|
||||
An option has been selected, so advance to the next panel—unless the option selected is
|
||||
"..", in which case go to the previous panel.
|
||||
'''
|
||||
if (
|
||||
event.option_list != self.focused_panel or event.option_id == 'loading-indicator'
|
||||
): # pragma: no cover
|
||||
return
|
||||
|
||||
if event.option_id == '..':
|
||||
self.action_previous()
|
||||
else:
|
||||
self.action_next(event.option_id)
|
||||
@@ -0,0 +1,33 @@
|
||||
import os
|
||||
|
||||
import textual.widgets
|
||||
|
||||
import borgmatic.actions.browse.bindings
|
||||
|
||||
|
||||
class Configuration_files_list(textual.widgets.OptionList):
|
||||
'''
|
||||
A widget for selecting a single borgmatic configuration file from among available configuration
|
||||
files. The item selection event is handled in a Carousel instance, the parent widget of an
|
||||
Configuration_files_list.
|
||||
'''
|
||||
|
||||
BINDINGS = borgmatic.actions.browse.bindings.OPTION_LIST_BINDINGS
|
||||
|
||||
def __init__(self, configs):
|
||||
'''
|
||||
Given a dict mapping from configuration path to corresponding configuration dict, add each
|
||||
configuration path as an option to this widget.
|
||||
'''
|
||||
self.configs = configs
|
||||
home_directory = os.path.expanduser('~')
|
||||
|
||||
super().__init__(
|
||||
*(
|
||||
textual.widgets.option_list.Option(unexpanded_path, id=config_path)
|
||||
for config_path in configs
|
||||
for unexpanded_path in (config_path.replace(home_directory, '~'),)
|
||||
),
|
||||
classes='panel',
|
||||
)
|
||||
self.border_title = '📄 configuration files'
|
||||
@@ -0,0 +1,211 @@
|
||||
import os
|
||||
|
||||
import textual.widgets
|
||||
|
||||
import borgmatic.actions.browse.bindings
|
||||
import borgmatic.actions.browse.icons
|
||||
import borgmatic.actions.browse.loading
|
||||
import borgmatic.actions.browse.workers
|
||||
|
||||
|
||||
def get_relative_archive_path_components(archive_path, current_directory_path_components):
|
||||
'''
|
||||
Given an Archive_path instance and a tuple of path components for the currently browsed
|
||||
directory, get the path components as a tuple for the archive path relative to that directory.
|
||||
|
||||
For instance, given an archive path with a path of 'foo/bar/baz/quux.txt' and current
|
||||
directory path components of ('foo', 'bar'), return ('baz', 'quux.txt').
|
||||
|
||||
If the archive path is not actually relative to the current directory, return None.
|
||||
'''
|
||||
archive_path_components = tuple(archive_path.file_path.split(os.path.sep))
|
||||
|
||||
if not current_directory_path_components:
|
||||
return archive_path_components
|
||||
|
||||
# If the loaded path doesn't match this directory list's own path, then we don't care about
|
||||
# it for purposes of displaying this particular directory.
|
||||
if (
|
||||
tuple(archive_path_components[: len(current_directory_path_components)])
|
||||
!= current_directory_path_components
|
||||
):
|
||||
return None
|
||||
|
||||
# Strip off the portion of the archive path that matches the directory list's own path.
|
||||
return archive_path_components[len(current_directory_path_components) :]
|
||||
|
||||
|
||||
def make_directory_list_option(archive_path, relative_path_components):
|
||||
'''
|
||||
Given an Archive_path instance and a tuple of relative path components for it, make a
|
||||
textual.widgets.option_list.Option for the path. Use an the icon based on whether this looks
|
||||
like a terminal filename or a directory.
|
||||
'''
|
||||
pieces = (
|
||||
borgmatic.actions.browse.icons.PATH_TYPE_ICONS.get(
|
||||
archive_path.path_type if len(relative_path_components) == 1 else 'd', '❓'
|
||||
),
|
||||
relative_path_components[0],
|
||||
) + (('→', archive_path.link_target) if archive_path.link_target else ())
|
||||
|
||||
return textual.widgets.option_list.Option(
|
||||
prompt=' '.join(pieces), id=relative_path_components[0]
|
||||
)
|
||||
|
||||
|
||||
def add_archive_paths(
|
||||
directory_list,
|
||||
config,
|
||||
repository,
|
||||
archive_name,
|
||||
archive_paths,
|
||||
):
|
||||
'''
|
||||
Given a DirectoryList instance, a configuration dict, a repository dict, an archive name, and a
|
||||
sequence of ArchivePath instances, add the paths to the directory list as options, sorting and
|
||||
deduplicating the resulting directory list's options.
|
||||
|
||||
After all of this reshuffling, make sure the original highlighted option remains highlighted.
|
||||
'''
|
||||
highlighted_option = directory_list.highlighted_option
|
||||
original_options_count = len(directory_list.options)
|
||||
|
||||
sorted_options = sorted(
|
||||
(
|
||||
*directory_list.options,
|
||||
*(
|
||||
make_directory_list_option(archive_path, relative_path_components)
|
||||
for archive_path in archive_paths
|
||||
for relative_path_components in (
|
||||
get_relative_archive_path_components(
|
||||
archive_path,
|
||||
directory_list.path_components,
|
||||
),
|
||||
)
|
||||
if relative_path_components
|
||||
if relative_path_components[0] not in directory_list._id_to_option
|
||||
),
|
||||
),
|
||||
# The loading indicator "option" always goes to the bottom.
|
||||
key=lambda option: ((option.id == 'loading-indicator'), option.prompt),
|
||||
)
|
||||
|
||||
# If there aren't actually any options to add (due to deduplication), bail.
|
||||
if len(sorted_options) == original_options_count:
|
||||
return
|
||||
|
||||
# Retain the highlighted option position even as other options load around it.
|
||||
directory_list.set_options(sorted_options)
|
||||
directory_list.highlighted = (
|
||||
directory_list.get_option_index(highlighted_option.id)
|
||||
if highlighted_option and directory_list.highlighted_option_changed
|
||||
else 0
|
||||
)
|
||||
|
||||
|
||||
class Directory_list(textual.widgets.OptionList):
|
||||
'''
|
||||
A widget for selecting a path from among the contents of a particular directory in a Borg
|
||||
archive. The item selection event is handled in a Carousel instance, the parent widget of a
|
||||
Directory_list.
|
||||
'''
|
||||
|
||||
BINDINGS = borgmatic.actions.browse.bindings.OPTION_LIST_BINDINGS
|
||||
|
||||
def __init__(self, config, repository, archive_name, path_loaded=None, path_components=None):
|
||||
'''
|
||||
Given a configuration dict, a repository dict, an archive name, an optional
|
||||
Archive_path_loaded instance for signalling new paths as they load, and an optional tuple of
|
||||
path components indicating this directory's position in the backed up filesystem, prepare to
|
||||
load paths from the archive for eventual display in this widget. Actual loading kicks off in
|
||||
on_mount() below.
|
||||
'''
|
||||
self.config = config
|
||||
self.repository = repository
|
||||
self.archive_name = archive_name
|
||||
self.path_components = path_components or ()
|
||||
self.highlighted_option_changed = False
|
||||
|
||||
super().__init__(classes='panel')
|
||||
|
||||
self.border_title = ' '.join(
|
||||
(
|
||||
'📁',
|
||||
os.path.sep.join(self.path_components)
|
||||
if self.path_components
|
||||
else f'{archive_name}',
|
||||
)
|
||||
)
|
||||
|
||||
if self.path_components:
|
||||
self.add_option(
|
||||
textual.widgets.option_list.Option(
|
||||
'📁 ..',
|
||||
id='..',
|
||||
),
|
||||
)
|
||||
|
||||
self.path_loaded = path_loaded or borgmatic.actions.browse.workers.Archive_path_loaded(
|
||||
self, 'archive path loaded'
|
||||
)
|
||||
|
||||
if not self.path_loaded.complete:
|
||||
self.timer = borgmatic.actions.browse.loading.add_inline_loading_indicator(self)
|
||||
|
||||
def on_mount(self):
|
||||
'''
|
||||
When this widget gets mounted in the DOM, subscribe to path loaded events so that we can
|
||||
find out about relevant archive paths as they load. And if this is a root directory list,
|
||||
start loading paths from the archive. If this is a non-root directory list, add any already
|
||||
loaded archive paths to this widget as options.
|
||||
|
||||
Loading is started *after* subscribing to path loaded signals so that there's not a gap
|
||||
where we might miss out on any paths.
|
||||
'''
|
||||
self.path_loaded.subscribe(self, self.on_archive_path_loaded)
|
||||
|
||||
if self.path_components:
|
||||
add_archive_paths(
|
||||
directory_list=self,
|
||||
config=self.config,
|
||||
repository=self.repository,
|
||||
archive_name=self.archive_name,
|
||||
archive_paths=borgmatic.actions.browse.workers.get_paths(
|
||||
self.path_loaded.path_hierarchy, self.path_components
|
||||
),
|
||||
)
|
||||
else:
|
||||
borgmatic.actions.browse.workers.load_archive_paths(
|
||||
self.app,
|
||||
path_loaded=self.path_loaded,
|
||||
config=self.config,
|
||||
repository=self.repository,
|
||||
archive_name=self.archive_name,
|
||||
)
|
||||
|
||||
def on_archive_path_loaded(self, data):
|
||||
'''
|
||||
When an archive path loads, add it as an option to this directory list. But if we get a
|
||||
signal that all path loading is complete, stop and remove our loading indicator.
|
||||
'''
|
||||
if data is borgmatic.actions.browse.workers.LOADING_DONE:
|
||||
self.timer.stop()
|
||||
self.remove_option('loading-indicator')
|
||||
return
|
||||
|
||||
add_archive_paths(
|
||||
directory_list=self,
|
||||
config=self.config,
|
||||
repository=self.repository,
|
||||
archive_name=self.archive_name,
|
||||
archive_paths=(data,),
|
||||
)
|
||||
|
||||
def on_option_list_option_highlighted(self, event):
|
||||
'''
|
||||
When the highlighted option changes, record that fact. This flag is consumed in
|
||||
add_archive_paths() in order to retain the highlighted option even as other options load
|
||||
around it.
|
||||
'''
|
||||
if self.highlighted not in {None, 0}:
|
||||
self.highlighted_option_changed = True
|
||||
@@ -0,0 +1,87 @@
|
||||
import logging
|
||||
|
||||
import rich.syntax
|
||||
import textual.binding
|
||||
import textual.widgets
|
||||
|
||||
import borgmatic.actions.browse.loading
|
||||
import borgmatic.actions.browse.workers
|
||||
|
||||
logger = logging.getLogger('__name__')
|
||||
|
||||
|
||||
class File_preview(textual.widgets.RichLog):
|
||||
'''
|
||||
A widget for extracting and previewing the contents of a file stored in a Borg archive.
|
||||
'''
|
||||
|
||||
BINDINGS = (
|
||||
*textual.widgets.RichLog.BINDINGS,
|
||||
textual.binding.Binding(
|
||||
key='up,k', action='scroll_up', description='scroll up', show=True, priority=True
|
||||
),
|
||||
textual.binding.Binding(
|
||||
key='down,j', action='scroll_down', description='scroll down', show=True, priority=True
|
||||
),
|
||||
textual.binding.Binding(
|
||||
key='pageup', action='page_up', description='page up', show=True, priority=True
|
||||
),
|
||||
textual.binding.Binding(
|
||||
key='pagedown', action='page_down', description='page down', show=True, priority=True
|
||||
),
|
||||
)
|
||||
|
||||
def __init__(self, config, repository, archive_name, file_path):
|
||||
'''
|
||||
Given a configuration dict, a repository dict, an archive name, and the path of a file in
|
||||
the archive, prepare to load the file's contents for eventual display in this widget. Actual
|
||||
loading kicks off in on_mount() below.
|
||||
'''
|
||||
self.config = config
|
||||
self.repository = repository
|
||||
self.archive_name = archive_name
|
||||
self.file_path = file_path
|
||||
|
||||
super().__init__(classes='panel')
|
||||
self.border_title = f'📄 {self.file_path} preview'
|
||||
self.auto_scroll = False
|
||||
self.file_preview_loaded = borgmatic.actions.browse.workers.File_preview_loaded(
|
||||
self, 'file preview loaded'
|
||||
)
|
||||
|
||||
self.loading_timer = borgmatic.actions.browse.loading.add_inline_loading_indicator(self)
|
||||
|
||||
def on_mount(self):
|
||||
'''
|
||||
When this widget gets mounted in the DOM, subscribe to archive loaded events so that we can
|
||||
find out about archives as they load. Also start loading file contents from the archive.
|
||||
|
||||
Loading is started *after* subscribing to the file preview loaded signal so that there's not
|
||||
a gap where we might miss out on signal publishes.
|
||||
'''
|
||||
self.file_preview_loaded.subscribe(self, self.on_file_preview_loaded)
|
||||
|
||||
borgmatic.actions.browse.workers.load_file_preview(
|
||||
self.app,
|
||||
file_preview_loaded=self.file_preview_loaded,
|
||||
config=self.config,
|
||||
repository=self.repository,
|
||||
archive_name=self.archive_name,
|
||||
file_path=self.file_path,
|
||||
)
|
||||
|
||||
def on_file_preview_loaded(self, file_contents):
|
||||
'''
|
||||
When a file loads, write its contents (syntax highlighted) to this file preview widget.
|
||||
'''
|
||||
self.loading_timer.stop()
|
||||
self.clear()
|
||||
|
||||
if file_contents is None:
|
||||
self.write('Cannot display a preview for this file')
|
||||
else:
|
||||
# Only pass the file path and not its contents to guess_lexer(). Passing the contents is
|
||||
# more accurate, but also much slower.
|
||||
self.write(
|
||||
rich.syntax.Syntax(file_contents, rich.syntax.Syntax.guess_lexer(self.file_path))
|
||||
)
|
||||
@@ -0,0 +1,8 @@
|
||||
import borgmatic.actions.browse.archive
|
||||
|
||||
PATH_TYPE_ICONS = {
|
||||
borgmatic.actions.browse.archive.Path_type.DIRECTORY.value: '📁',
|
||||
borgmatic.actions.browse.archive.Path_type.LINK.value: '🔗',
|
||||
borgmatic.actions.browse.archive.Path_type.PIPE.value: '🚰',
|
||||
borgmatic.actions.browse.archive.Path_type.FILE.value: '📄',
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import contextlib
|
||||
import functools
|
||||
import logging
|
||||
|
||||
import textual.widgets
|
||||
import textual.widgets.option_list
|
||||
|
||||
LOADING_DOT_INTERVAL_SECONDS = 0.3
|
||||
|
||||
|
||||
logger = logging.getLogger('__name__')
|
||||
|
||||
|
||||
def update_inline_loading_indicator(widget):
|
||||
'''
|
||||
Given a textual.widgets.OptionList or a textual.widgets.RichLog instance, animate the existing
|
||||
loading indicator inside it.
|
||||
'''
|
||||
if isinstance(widget, textual.widgets.OptionList):
|
||||
with contextlib.suppress(textual.widgets.option_list.OptionDoesNotExist):
|
||||
widget.replace_option_prompt(
|
||||
'loading-indicator',
|
||||
(str(widget.get_option('loading-indicator').prompt) + '.').replace('....', ''),
|
||||
)
|
||||
elif isinstance(widget, textual.widgets.RichLog):
|
||||
with contextlib.suppress(IndexError):
|
||||
loading_message = str(widget.lines[0].text)
|
||||
widget.clear()
|
||||
widget.write((loading_message + '.').replace('....', ''))
|
||||
else:
|
||||
raise ValueError(f'Unsupported widget type: {type(widget)}')
|
||||
|
||||
|
||||
LOADING_MESSAGE = '⏳ loading...'
|
||||
|
||||
|
||||
def add_inline_loading_indicator(widget):
|
||||
'''
|
||||
Given a textual.widgets.OptionList or a textual.widgets.RichLog instance, add a loading
|
||||
indicator to it.
|
||||
'''
|
||||
if isinstance(widget, textual.widgets.OptionList):
|
||||
loading_option = textual.widgets.option_list.Option(LOADING_MESSAGE, id='loading-indicator')
|
||||
widget.add_option(loading_option)
|
||||
widget.highlighted = None
|
||||
elif isinstance(widget, textual.widgets.RichLog):
|
||||
widget.write(LOADING_MESSAGE)
|
||||
else:
|
||||
raise ValueError(f'Unsupported widget type: {type(widget)}')
|
||||
|
||||
return widget.set_interval(
|
||||
LOADING_DOT_INTERVAL_SECONDS,
|
||||
functools.partial(update_inline_loading_indicator, widget),
|
||||
)
|
||||
@@ -0,0 +1,102 @@
|
||||
import contextlib
|
||||
import logging
|
||||
|
||||
import textual._context
|
||||
import textual.widgets
|
||||
import textual.worker
|
||||
|
||||
import borgmatic.logger
|
||||
|
||||
|
||||
class Rich_color_formatter(logging.Formatter):
|
||||
'''
|
||||
A Python logging formatter that formats log records with Rich-compatible color markup according
|
||||
to their levels.
|
||||
'''
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.prefix = None
|
||||
super().__init__(
|
||||
'{prefix}{message}',
|
||||
*args,
|
||||
style='{',
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def format(self, record):
|
||||
'''
|
||||
Given a log record, format it with Rich-compatibe color markup corresponding to its log
|
||||
level.
|
||||
'''
|
||||
borgmatic.logger.add_custom_log_levels()
|
||||
|
||||
color = {
|
||||
logging.CRITICAL: 'bright_red',
|
||||
logging.ERROR: 'bright_red',
|
||||
logging.WARNING: 'bright_yellow',
|
||||
logging.ANSWER: 'bright_magenta',
|
||||
logging.INFO: 'bright_green',
|
||||
logging.DEBUG: 'bright_cyan',
|
||||
}.get(record.levelno)
|
||||
record.prefix = f'{self.prefix}: ' if self.prefix else ''
|
||||
|
||||
return f'[{color}]{super().format(record)}[/{color}]'
|
||||
|
||||
|
||||
class Browse_log_handler(logging.Handler):
|
||||
'''
|
||||
A Python log handler that writes any log records to a logging widget.
|
||||
'''
|
||||
|
||||
def __init__(self, logs_widget):
|
||||
'''
|
||||
Given a logs widget, save it for use below.
|
||||
'''
|
||||
self.logs_widget = logs_widget
|
||||
|
||||
super().__init__()
|
||||
|
||||
def emit(self, record):
|
||||
'''
|
||||
Given a log record, format it and log it to the logs widgets. This works whether or not the
|
||||
logging is happening in the main thread.
|
||||
'''
|
||||
message = self.format(record)
|
||||
|
||||
try:
|
||||
textual.worker.get_current_worker()
|
||||
self.logs_widget.app.call_from_thread(self.logs_widget.write, message)
|
||||
except (RuntimeError, textual.worker.NoActiveWorker):
|
||||
with contextlib.suppress(textual._context.NoActiveAppError):
|
||||
self.logs_widget.write(message)
|
||||
|
||||
|
||||
def log_to_widget(logs_widget):
|
||||
'''
|
||||
Given a Textual RichLog logs widget, add a log handler and formatter that logs to it. Also
|
||||
remove the default borgmatic console log handler so it doesn't try to log all over our UI.
|
||||
'''
|
||||
handler = Browse_log_handler(logs_widget)
|
||||
handler.setFormatter(Rich_color_formatter())
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(min(handler.level for handler in logger.handlers))
|
||||
logger.addHandler(handler)
|
||||
|
||||
with contextlib.suppress(StopIteration):
|
||||
console_handler = next(
|
||||
handler
|
||||
for handler in logging.getLogger().handlers
|
||||
if isinstance(handler, borgmatic.logger.Multi_stream_handler)
|
||||
)
|
||||
logger.removeHandler(console_handler)
|
||||
|
||||
|
||||
class Logs(textual.widgets.RichLog):
|
||||
'''
|
||||
A widget for viewing borgmatic logs in realtime. The log level is determined by borgmatic's
|
||||
current verbosity level.
|
||||
'''
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(markup=True, id='logs', classes='panel')
|
||||
self.border_title = '🪵 logs'
|
||||
@@ -0,0 +1,30 @@
|
||||
import textual.widgets
|
||||
|
||||
import borgmatic.actions.browse.bindings
|
||||
|
||||
|
||||
class Repositories_list(textual.widgets.OptionList):
|
||||
'''
|
||||
A widget for selecting a single Borg repository from among the repositories in a borgmatic
|
||||
configuration file. The item selection event is handled in a Carousel instance, the parent
|
||||
widget of a Repositories_list.
|
||||
'''
|
||||
|
||||
BINDINGS = borgmatic.actions.browse.bindings.OPTION_LIST_BINDINGS
|
||||
|
||||
def __init__(self, config):
|
||||
'''
|
||||
Given a configuration dict, populate the repositories in this widget.
|
||||
'''
|
||||
self.config = config
|
||||
self.repositories = config['repositories']
|
||||
|
||||
super().__init__(
|
||||
*(
|
||||
textual.widgets.option_list.Option(label, id=index)
|
||||
for index, repository in enumerate(self.repositories)
|
||||
for label in (repository.get('label', repository.get('path')),)
|
||||
),
|
||||
classes='panel',
|
||||
)
|
||||
self.border_title = '📦 repositories'
|
||||
@@ -0,0 +1,29 @@
|
||||
import logging
|
||||
|
||||
|
||||
def run_browse(
|
||||
diff_arguments,
|
||||
global_arguments,
|
||||
configs,
|
||||
):
|
||||
'''
|
||||
Run the "browse" action for the given borgmatic configurations. This launches a console UI.
|
||||
|
||||
Raise ValueError if the Textual library (a prerequisite for this action) can't be imported.
|
||||
'''
|
||||
if not configs:
|
||||
return
|
||||
|
||||
logging.getLogger('asyncio').setLevel(logging.WARNING)
|
||||
|
||||
try:
|
||||
import textual # noqa: F401, PLC0415
|
||||
except ImportError: # pragma: no cover
|
||||
raise ValueError(
|
||||
'Unable to import the Textual library for the browse action; try installing "borgmatic[browse]"'
|
||||
)
|
||||
|
||||
import borgmatic.actions.browse.app # noqa: PLC0415
|
||||
|
||||
app = borgmatic.actions.browse.app.Browse_app(configs)
|
||||
app.run()
|
||||
@@ -0,0 +1,192 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
import textual
|
||||
import textual.signal
|
||||
|
||||
import borgmatic.actions.browse.archive
|
||||
|
||||
logger = logging.getLogger('__name__')
|
||||
|
||||
|
||||
LOADING_DONE = object()
|
||||
|
||||
|
||||
class Archive_loaded(textual.signal.Signal):
|
||||
'''
|
||||
A signal that publishes when each subsequent archive is loaded from a repository, intended for
|
||||
consumption in widgets that display archives as they are loaded. This signal also publishes
|
||||
when loading is complete.
|
||||
|
||||
Each subscribed callback call includes the archive as an archive name string. Given the lack of
|
||||
other identifying information (configuration file, repository), there should be a separate
|
||||
Archive_loaded instance per repository.
|
||||
'''
|
||||
|
||||
|
||||
@textual.work(thread=True)
|
||||
def add_repository_archives(browse_app, archive_loaded, config, repository):
|
||||
'''
|
||||
Given a running Browse_app instance, an Archive_loaded instance, a configuration dict, and a
|
||||
repository dict, load a list of the archives from the repository and add them as options in the
|
||||
archives list. Reverse the order so the most recent archive is first.
|
||||
|
||||
This function runs in a separate thread from the main UI. When loading is complete, publish a
|
||||
loading done signal.
|
||||
'''
|
||||
archives_data = borgmatic.actions.browse.archive.get_repository_archives(config, repository)
|
||||
|
||||
# Reverse the archives, so the common case of accessing the latest archive is easy because it's
|
||||
# at the top.
|
||||
for archive in reversed(archives_data['archives']):
|
||||
archive_loaded.publish(archive['archive'])
|
||||
|
||||
archive_loaded.publish(LOADING_DONE)
|
||||
|
||||
|
||||
def record_path(archive_path, hierarchy, path_components):
|
||||
'''
|
||||
Given an Archive_path instance, a dict capturing a filesystem hierarchy of paths, and a tuple of
|
||||
path components for the archive path, set the archive path into the hierarchy data structure.
|
||||
|
||||
For instance, if given an archive path and path components representing "foo/bar/baz.txt",
|
||||
produce a hierarchy that looks like:
|
||||
|
||||
{'foo': {'bar': {'baz.txt': Archive_path('-', 'foo/bar/baz.txt', '')}}}
|
||||
|
||||
Note that the hierarchy is modified in place, so any existing paths there are retained.
|
||||
'''
|
||||
if len(path_components) == 1:
|
||||
hierarchy[path_components[0]] = {} if archive_path.path_type == 'd' else archive_path
|
||||
return
|
||||
|
||||
record_path(archive_path, hierarchy.setdefault(path_components[0], {}), path_components[1:])
|
||||
|
||||
|
||||
def get_paths(hierarchy, path_components, full_path_components=None):
|
||||
'''
|
||||
Given a dict capturing a filesystem hierarchy of paths (or a subset thereof), a tuple of path
|
||||
components for a directory path relative to the hierarchy root, and an optional tuple of
|
||||
*absolute* path components for the same path (if different), return a generator of the
|
||||
contained file and directory Archive_path instances from the hierarchy.
|
||||
|
||||
For instance, given the following hierarchy:
|
||||
|
||||
{'foo': {'bar': {'baz.txt': Archive_path('-', 'foo/bar/baz.txt', ''), 'quux': {}}}}
|
||||
|
||||
... and path components of ('foo', 'bar'), return a generator with the following:
|
||||
|
||||
* Archive_path('-', 'foo/bar/baz.txt', '')
|
||||
* Archive_path('d', 'foo/bar/quux', '')
|
||||
|
||||
The given absolute path components are use to construct directory paths like that last archive
|
||||
path.
|
||||
'''
|
||||
if full_path_components is None:
|
||||
full_path_components = path_components
|
||||
|
||||
if len(path_components) == 1:
|
||||
try:
|
||||
return (
|
||||
archive_path
|
||||
if isinstance(archive_path, borgmatic.actions.browse.archive.Archive_path)
|
||||
else borgmatic.actions.browse.archive.Archive_path(
|
||||
'd', os.path.join(*full_path_components, component), ''
|
||||
)
|
||||
for component, archive_path in hierarchy[path_components[0]].items()
|
||||
)
|
||||
except KeyError:
|
||||
raise ValueError(f'Unknown file or directory: {path_components[0]}')
|
||||
|
||||
try:
|
||||
return get_paths(hierarchy[path_components[0]], path_components[1:], full_path_components)
|
||||
except KeyError:
|
||||
raise ValueError(f'Unknown directory: {path_components[0]}')
|
||||
|
||||
|
||||
class Archive_path_loaded(textual.signal.Signal):
|
||||
'''
|
||||
A signal that publishes when each subsequent path is loaded from an archive, intended for
|
||||
consumption in widgets that display paths as they are loaded. This signal also tracks the
|
||||
complete filesystem hierarchy seen thus far, so new widgets that get created after loading has
|
||||
started can "catch up" with existing known paths. Lastly, this signal publishes and tracks when
|
||||
loading is complete.
|
||||
|
||||
Each subscrided callback call includes the loaded path as an Archive_path instance. There is
|
||||
intended to be a separate Archive_path_loaded instance per archive, but that instance should be
|
||||
shared among several different widgets for the same archive for performance reasons.
|
||||
'''
|
||||
|
||||
def __init__(self, owner, name):
|
||||
self.path_hierarchy = {}
|
||||
self.complete = False
|
||||
|
||||
super().__init__(owner, name)
|
||||
|
||||
def publish(self, archive_path):
|
||||
'''
|
||||
Publish the given archive path to subscribers and record its path locally. But if the
|
||||
archive path is actually LOADING_DONE, then record loading as complete.
|
||||
'''
|
||||
super().publish(archive_path)
|
||||
|
||||
if archive_path is LOADING_DONE:
|
||||
self.complete = True
|
||||
else:
|
||||
record_path(
|
||||
archive_path, self.path_hierarchy, archive_path.file_path.split(os.path.sep)
|
||||
)
|
||||
|
||||
|
||||
@textual.work(thread=True)
|
||||
def load_archive_paths(browse_app, path_loaded, config, repository, archive_name):
|
||||
'''
|
||||
Given a running Browse_app instance, an Archive_path_loaded instance, a configuration dict, a
|
||||
repository dict, and an archive name, load the paths in this archive and publish each one via
|
||||
the Archive_path_loaded signal, so interested widgets can subscribe. Also send a "loading done"
|
||||
signal when loading completes.
|
||||
|
||||
This function runs in a separate thread from the main UI. When loading is complete, publish a
|
||||
loading done signal.
|
||||
'''
|
||||
for archive_path in borgmatic.actions.browse.archive.get_archive_paths(
|
||||
config, repository, archive_name
|
||||
):
|
||||
path_loaded.publish(archive_path)
|
||||
|
||||
path_loaded.publish(LOADING_DONE)
|
||||
|
||||
|
||||
class File_preview_loaded(textual.signal.Signal):
|
||||
'''
|
||||
A signal that publishes when file contents are loaded from an archive, intended for consumption
|
||||
in widgets that display loaded files. This signal also publishes when loading is complete.
|
||||
|
||||
Each published callback includes a the file's contents as a string. Given the lack of other
|
||||
identifying information (configuration file, repository, archive), there should be a separate
|
||||
Archive_loaded instance per previewed file.
|
||||
'''
|
||||
|
||||
|
||||
@textual.work(thread=True)
|
||||
def load_file_preview(
|
||||
browse_app,
|
||||
file_preview_loaded,
|
||||
config,
|
||||
repository,
|
||||
archive_name,
|
||||
file_path,
|
||||
):
|
||||
'''
|
||||
Given a running Browse_app instance, a File_preview_loaded instance, a configuration dict, a
|
||||
repository dict, an archive name, and the path of a file in that archive, load the contents of
|
||||
the file and write it into the given file preview widget.
|
||||
|
||||
This function runs in a separate thread from the main UI. When loading is complete, publish a
|
||||
loading done signal.
|
||||
'''
|
||||
file_contents = borgmatic.actions.browse.archive.get_archive_file_content(
|
||||
config, repository, archive_name, file_path
|
||||
)
|
||||
|
||||
file_preview_loaded.publish(file_contents)
|
||||
@@ -1,7 +1,6 @@
|
||||
import logging
|
||||
|
||||
import borgmatic.borg.change_passphrase
|
||||
import borgmatic.config.validate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -18,19 +17,13 @@ def run_change_passphrase(
|
||||
'''
|
||||
Run the "key change-passphrase" action for the given repository.
|
||||
'''
|
||||
if (
|
||||
change_passphrase_arguments.repository is None
|
||||
or borgmatic.config.validate.repositories_match(
|
||||
repository, change_passphrase_arguments.repository
|
||||
)
|
||||
):
|
||||
logger.info('Changing repository passphrase')
|
||||
borgmatic.borg.change_passphrase.change_passphrase(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
change_passphrase_arguments,
|
||||
global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
logger.info('Changing repository passphrase')
|
||||
borgmatic.borg.change_passphrase.change_passphrase(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
change_passphrase_arguments,
|
||||
global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
|
||||
+188
-127
@@ -1,4 +1,5 @@
|
||||
import calendar
|
||||
import contextlib
|
||||
import datetime
|
||||
import hashlib
|
||||
import itertools
|
||||
@@ -8,20 +9,20 @@ import pathlib
|
||||
import random
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import textwrap
|
||||
|
||||
import borgmatic.actions.config.bootstrap
|
||||
import borgmatic.actions.pattern
|
||||
import borgmatic.borg.check
|
||||
import borgmatic.borg.create
|
||||
import borgmatic.borg.environment
|
||||
import borgmatic.borg.extract
|
||||
import borgmatic.borg.list
|
||||
import borgmatic.borg.pattern
|
||||
import borgmatic.borg.repo_list
|
||||
import borgmatic.borg.state
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.config.validate
|
||||
import borgmatic.execute
|
||||
import borgmatic.hooks.command
|
||||
|
||||
DEFAULT_CHECKS = (
|
||||
{'name': 'repository', 'frequency': '1 month'},
|
||||
@@ -55,12 +56,14 @@ def parse_checks(config, only_checks=None):
|
||||
|
||||
if 'disabled' in checks:
|
||||
logger.warning(
|
||||
'The "disabled" value for the "checks" option is deprecated and will be removed from a future release; use "skip_actions" instead'
|
||||
'The "disabled" value for the "checks" option is deprecated and will be removed from a future release; use "skip_actions" instead',
|
||||
)
|
||||
|
||||
if len(checks) > 1:
|
||||
logger.warning(
|
||||
'Multiple checks are configured, but one of them is "disabled"; not running any checks'
|
||||
'Multiple checks are configured, but one of them is "disabled"; not running any checks',
|
||||
)
|
||||
|
||||
return ()
|
||||
|
||||
return checks
|
||||
@@ -164,6 +167,7 @@ def filter_checks_on_frequency(
|
||||
**dict.fromkeys(day for day in days if day != 'weekday'),
|
||||
**dict.fromkeys(WEEKDAY_DAYS),
|
||||
}
|
||||
|
||||
if 'weekend' in days:
|
||||
days = {
|
||||
**dict.fromkeys(day for day in days if day != 'weekend'),
|
||||
@@ -172,7 +176,7 @@ def filter_checks_on_frequency(
|
||||
|
||||
if calendar.day_name[datetime_now().weekday()] not in days:
|
||||
logger.info(
|
||||
f"Skipping {check} check due to day of the week; check only runs on {'/'.join(day.title() for day in days)} (use --force to check anyway)"
|
||||
f"Skipping {check} check due to day of the week; check only runs on {'/'.join(day.title() for day in days)} (use --force to check anyway)",
|
||||
)
|
||||
filtered_checks.remove(check)
|
||||
continue
|
||||
@@ -190,7 +194,7 @@ def filter_checks_on_frequency(
|
||||
if datetime_now() < check_time + frequency_delta:
|
||||
remaining = check_time + frequency_delta - datetime_now()
|
||||
logger.info(
|
||||
f'Skipping {check} check due to configured frequency; {remaining} until next check (use --force to check anyway)'
|
||||
f'Skipping {check} check due to configured frequency; {remaining} until next check (use --force to check anyway)',
|
||||
)
|
||||
filtered_checks.remove(check)
|
||||
|
||||
@@ -216,7 +220,7 @@ def make_check_time_path(config, borg_repository_id, check_type, archives_check_
|
||||
'''
|
||||
borgmatic_state_directory = borgmatic.config.paths.get_borgmatic_state_directory(config)
|
||||
|
||||
if check_type in ('archives', 'data'):
|
||||
if check_type in {'archives', 'data'}:
|
||||
return os.path.join(
|
||||
borgmatic_state_directory,
|
||||
'checks',
|
||||
@@ -240,7 +244,7 @@ def write_check_time(path): # pragma: no cover
|
||||
logger.debug(f'Writing check time at {path}')
|
||||
|
||||
os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
|
||||
pathlib.Path(path, mode=0o600).touch()
|
||||
pathlib.Path(path).touch(mode=0o600)
|
||||
|
||||
|
||||
def read_check_time(path):
|
||||
@@ -251,7 +255,7 @@ def read_check_time(path):
|
||||
logger.debug(f'Reading check time from {path}')
|
||||
|
||||
try:
|
||||
return datetime.datetime.fromtimestamp(os.stat(path).st_mtime)
|
||||
return datetime.datetime.fromtimestamp(os.stat(path).st_mtime) # noqa: DTZ006
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
||||
@@ -282,7 +286,7 @@ def probe_for_check_time(config, borg_repository_id, check, archives_check_id):
|
||||
(
|
||||
make_check_time_path(config, borg_repository_id, check, archives_check_id),
|
||||
make_check_time_path(config, borg_repository_id, check),
|
||||
)
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -314,16 +318,17 @@ def upgrade_check_times(config, borg_repository_id):
|
||||
{borgmatic_state_directory}/checks/1234567890/archives/all
|
||||
'''
|
||||
borgmatic_source_checks_path = os.path.join(
|
||||
borgmatic.config.paths.get_borgmatic_source_directory(config), 'checks'
|
||||
borgmatic.config.paths.get_borgmatic_source_directory(config),
|
||||
'checks',
|
||||
)
|
||||
borgmatic_state_path = borgmatic.config.paths.get_borgmatic_state_directory(config)
|
||||
borgmatic_state_checks_path = os.path.join(borgmatic_state_path, 'checks')
|
||||
|
||||
if os.path.exists(borgmatic_source_checks_path) and not os.path.exists(
|
||||
borgmatic_state_checks_path
|
||||
borgmatic_state_checks_path,
|
||||
):
|
||||
logger.debug(
|
||||
f'Upgrading archives check times directory from {borgmatic_source_checks_path} to {borgmatic_state_checks_path}'
|
||||
f'Upgrading archives check times directory from {borgmatic_source_checks_path} to {borgmatic_state_checks_path}',
|
||||
)
|
||||
os.makedirs(borgmatic_state_path, mode=0o700, exist_ok=True)
|
||||
shutil.move(borgmatic_source_checks_path, borgmatic_state_checks_path)
|
||||
@@ -338,10 +343,8 @@ def upgrade_check_times(config, borg_repository_id):
|
||||
|
||||
logger.debug(f'Upgrading archives check time file from {old_path} to {new_path}')
|
||||
|
||||
try:
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
shutil.move(old_path, temporary_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
os.mkdir(old_path)
|
||||
shutil.move(temporary_path, new_path)
|
||||
@@ -355,45 +358,52 @@ def collect_spot_check_source_paths(
|
||||
local_path,
|
||||
remote_path,
|
||||
borgmatic_runtime_directory,
|
||||
bootstrap_config_paths,
|
||||
):
|
||||
'''
|
||||
Given a repository configuration dict, a configuration dict, the local Borg version, global
|
||||
arguments as an argparse.Namespace instance, the local Borg path, and the remote Borg path,
|
||||
collect the source paths that Borg would use in an actual create (but only include files).
|
||||
arguments as an argparse.Namespace instance, the local Borg path, the remote Borg path, and the
|
||||
bootstrap configuration paths as read from an archive's manifest, collect the source paths that
|
||||
Borg would use in an actual create (but only include files). As part of this, include the
|
||||
bootstrap configuration paths, so that any configuration files included in the archive to
|
||||
support bootstrapping are also spot checked.
|
||||
'''
|
||||
stream_processes = any(
|
||||
borgmatic.hooks.dispatch.call_hooks(
|
||||
'use_streaming',
|
||||
config,
|
||||
borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
|
||||
).values()
|
||||
).values(),
|
||||
)
|
||||
working_directory = borgmatic.config.paths.get_working_directory(config)
|
||||
|
||||
(create_flags, create_positional_arguments, pattern_file) = (
|
||||
borgmatic.borg.create.make_base_create_command(
|
||||
dry_run=True,
|
||||
repository_path=repository['path'],
|
||||
# Omit "progress" because it interferes with "list_details".
|
||||
config=dict(
|
||||
{option: value for option, value in config.items() if option != 'progress'},
|
||||
list_details=True,
|
||||
(create_flags, create_positional_arguments, _) = borgmatic.borg.create.make_base_create_command(
|
||||
dry_run=True,
|
||||
repository_path=repository['path'],
|
||||
# Omit "progress" because it interferes with "list_details".
|
||||
config=dict(config, progress=False, list_details=True),
|
||||
patterns=borgmatic.actions.pattern.process_patterns(
|
||||
borgmatic.actions.pattern.collect_patterns(config, working_directory)
|
||||
+ tuple(
|
||||
borgmatic.borg.pattern.Pattern(
|
||||
config_path,
|
||||
source=borgmatic.borg.pattern.Pattern_source.INTERNAL,
|
||||
)
|
||||
for config_path in bootstrap_config_paths
|
||||
),
|
||||
patterns=borgmatic.actions.pattern.process_patterns(
|
||||
borgmatic.actions.pattern.collect_patterns(config),
|
||||
working_directory,
|
||||
),
|
||||
local_borg_version=local_borg_version,
|
||||
global_arguments=global_arguments,
|
||||
borgmatic_runtime_directory=borgmatic_runtime_directory,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
stream_processes=stream_processes,
|
||||
)
|
||||
config,
|
||||
working_directory,
|
||||
),
|
||||
local_borg_version=local_borg_version,
|
||||
global_arguments=global_arguments,
|
||||
borgmatic_runtime_directory=borgmatic_runtime_directory,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
stream_processes=stream_processes,
|
||||
)
|
||||
working_directory = borgmatic.config.paths.get_working_directory(config)
|
||||
|
||||
paths_output = borgmatic.execute.execute_command_and_capture_output(
|
||||
path_lines = borgmatic.execute.execute_command_and_capture_output(
|
||||
create_flags + create_positional_arguments,
|
||||
capture_stderr=True,
|
||||
environment=borgmatic.borg.environment.make_environment(config),
|
||||
@@ -402,14 +412,20 @@ def collect_spot_check_source_paths(
|
||||
borg_exit_codes=config.get('borg_exit_codes'),
|
||||
)
|
||||
|
||||
paths = tuple(
|
||||
paths = (
|
||||
path_line.split(' ', 1)[1]
|
||||
for path_line in paths_output.splitlines()
|
||||
if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
|
||||
for path_line in path_lines
|
||||
if path_line and path_line.startswith(('- ', '+ '))
|
||||
)
|
||||
|
||||
return tuple(
|
||||
path for path in paths if os.path.isfile(os.path.join(working_directory or '', path))
|
||||
# Use dict.fromkeys() to deduplicate file paths, which are present in Borg's dry run output
|
||||
# when there are overlapping source patterns. For instance, if both "/foo" and
|
||||
# "/foo/file.txt" are in configured patterns, then "/foo/file.txt" will show up in Borg's
|
||||
# dry run output twice.
|
||||
dict.fromkeys(
|
||||
path for path in paths if os.path.isfile(os.path.join(working_directory or '', path))
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -439,24 +455,22 @@ def collect_spot_check_archive_paths(
|
||||
borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(config)
|
||||
|
||||
return tuple(
|
||||
path
|
||||
for line in borgmatic.borg.list.capture_archive_listing(
|
||||
entry['path']
|
||||
for entry in borgmatic.borg.list.capture_archive_listing(
|
||||
repository['path'],
|
||||
archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
path_format='{type} {path}{NUL}', # noqa: FS003
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
for (file_type, path) in (line.split(' ', 1),)
|
||||
if file_type not in (BORG_DIRECTORY_FILE_TYPE, BORG_PIPE_FILE_TYPE)
|
||||
if pathlib.Path('borgmatic') not in pathlib.Path(path).parents
|
||||
if entry['type'] not in {BORG_DIRECTORY_FILE_TYPE, BORG_PIPE_FILE_TYPE}
|
||||
if pathlib.Path('borgmatic') not in pathlib.Path(entry['path']).parents
|
||||
if pathlib.Path(borgmatic_source_directory.lstrip(os.path.sep))
|
||||
not in pathlib.Path(path).parents
|
||||
not in pathlib.Path(entry['path']).parents
|
||||
if pathlib.Path(borgmatic_runtime_directory.lstrip(os.path.sep))
|
||||
not in pathlib.Path(path).parents
|
||||
not in pathlib.Path(entry['path']).parents
|
||||
)
|
||||
|
||||
|
||||
@@ -484,7 +498,8 @@ def compare_spot_check_hashes(
|
||||
# source directories.
|
||||
spot_check_config = next(check for check in config['checks'] if check['name'] == 'spot')
|
||||
sample_count = max(
|
||||
int(len(source_paths) * (min(spot_check_config['data_sample_percentage'], 100) / 100)), 1
|
||||
int(len(source_paths) * (min(spot_check_config['data_sample_percentage'], 100) / 100)),
|
||||
1,
|
||||
)
|
||||
source_sample_paths = tuple(random.SystemRandom().sample(source_paths, sample_count))
|
||||
working_directory = borgmatic.config.paths.get_working_directory(config)
|
||||
@@ -496,7 +511,7 @@ def compare_spot_check_hashes(
|
||||
if not os.path.islink(full_source_path)
|
||||
}
|
||||
logger.debug(
|
||||
f'Sampling {sample_count} source paths (~{spot_check_config["data_sample_percentage"]}%) for spot check'
|
||||
f'Sampling {sample_count} source paths (~{spot_check_config["data_sample_percentage"]}%) for spot check',
|
||||
)
|
||||
|
||||
source_sample_paths_iterator = iter(source_sample_paths)
|
||||
@@ -508,54 +523,102 @@ def compare_spot_check_hashes(
|
||||
while True:
|
||||
# Hash each file in the sample paths (if it exists).
|
||||
source_sample_paths_subset = tuple(
|
||||
itertools.islice(source_sample_paths_iterator, SAMPLE_PATHS_SUBSET_COUNT)
|
||||
itertools.islice(source_sample_paths_iterator, SAMPLE_PATHS_SUBSET_COUNT),
|
||||
)
|
||||
|
||||
if not source_sample_paths_subset:
|
||||
break
|
||||
|
||||
hash_output = borgmatic.execute.execute_command_and_capture_output(
|
||||
tuple(
|
||||
shlex.quote(part)
|
||||
for part in shlex.split(spot_check_config.get('xxh64sum_command', 'xxh64sum'))
|
||||
)
|
||||
+ tuple(
|
||||
path for path in source_sample_paths_subset if path in hashable_source_sample_path
|
||||
),
|
||||
working_directory=working_directory,
|
||||
hash_paths = tuple(
|
||||
path for path in source_sample_paths_subset if path in hashable_source_sample_path
|
||||
)
|
||||
|
||||
source_hashes.update(
|
||||
**dict(
|
||||
(reversed(line.split(' ', 1)) for line in hash_output.splitlines()),
|
||||
# Represent non-existent files as having empty hashes so the comparison below still
|
||||
# works. Same thing for filesystem links, since Borg produces empty archive hashes
|
||||
# for them.
|
||||
**{
|
||||
path: ''
|
||||
for path in source_sample_paths_subset
|
||||
if path not in hashable_source_sample_path
|
||||
},
|
||||
try:
|
||||
hash_lines = borgmatic.execute.execute_command_and_capture_output(
|
||||
tuple(
|
||||
shlex.quote(part)
|
||||
for part in shlex.split(spot_check_config.get('xxh64sum_command', 'xxh64sum'))
|
||||
)
|
||||
+ hash_paths,
|
||||
working_directory=working_directory,
|
||||
)
|
||||
)
|
||||
source_hashes.update(
|
||||
**dict(
|
||||
zip(
|
||||
# xxh64sum rewrites/escapes the paths that it returns alongside its hashes, for
|
||||
# instance if they contain special characters. When that happens, they don't
|
||||
# match the original source paths and therefore hash lookups fail. So when
|
||||
# building this lookup dict, use the original unaltered paths we provided as
|
||||
# input to xxh64sum.
|
||||
hash_paths,
|
||||
(
|
||||
# For some reason, xxh64sum prefixes the hash with a backslash if the path
|
||||
# contains a newline. Work around that.
|
||||
line.split(' ', 1)[0].lstrip('\\')
|
||||
for line in hash_lines
|
||||
),
|
||||
),
|
||||
# Represent non-existent files as having empty hashes so the comparison below still
|
||||
# works. Same thing for filesystem links, since Borg produces empty archive hashes
|
||||
# for them.
|
||||
**{
|
||||
path: ''
|
||||
for path in source_sample_paths_subset
|
||||
if path not in hashable_source_sample_path
|
||||
},
|
||||
),
|
||||
)
|
||||
except subprocess.CalledProcessError:
|
||||
# This can happen if a file we planned to hash gets deleted right before we try to hash
|
||||
# it. Falling back to individual file hashing allows us to find and mark just the
|
||||
# file(s) with problems instead of failing the whole batch.
|
||||
logger.warning(
|
||||
'Bulk source path hashing failed for this batch; falling back to individual file hashing'
|
||||
)
|
||||
|
||||
for hash_path in hash_paths:
|
||||
try:
|
||||
hash_lines = borgmatic.execute.execute_command_and_capture_output(
|
||||
(
|
||||
*(
|
||||
shlex.quote(part)
|
||||
for part in shlex.split(
|
||||
spot_check_config.get('xxh64sum_command', 'xxh64sum')
|
||||
)
|
||||
),
|
||||
hash_path,
|
||||
),
|
||||
working_directory=working_directory,
|
||||
)
|
||||
source_hashes[hash_path] = next(hash_lines).split(' ', 1)[0].lstrip('\\')
|
||||
except (subprocess.CalledProcessError, StopIteration): # noqa: PERF203
|
||||
logger.warning(
|
||||
f'Source path hashing failed for {hash_path}; treating as missing'
|
||||
)
|
||||
source_hashes[hash_path] = ''
|
||||
|
||||
# Get the hash for each file in the archive.
|
||||
archive_hashes.update(
|
||||
**dict(
|
||||
reversed(line.split(' ', 1))
|
||||
for line in borgmatic.borg.list.capture_archive_listing(
|
||||
repository['path'],
|
||||
archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
list_paths=source_sample_paths_subset,
|
||||
path_format='{xxh64} {path}{NUL}', # noqa: FS003
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
if line
|
||||
)
|
||||
)
|
||||
for entry in borgmatic.borg.list.capture_archive_listing(
|
||||
repository['path'],
|
||||
archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
list_paths=source_sample_paths_subset,
|
||||
path_format='{xxh64}{path}{linktarget}{target}',
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
):
|
||||
if not entry:
|
||||
continue
|
||||
|
||||
# Borg can't get hashes of stored hard links. So if this is a hard link path (and not
|
||||
# deemed as the "original" by Borg), then skip hashing of it.
|
||||
if entry.get('linktarget') or entry.get('target'):
|
||||
source_hashes.pop(os.path.join('/', entry['path']), None)
|
||||
continue
|
||||
|
||||
archive_hashes[entry['path']] = entry['xxh64']
|
||||
|
||||
# Compare the source hashes with the archive hashes to see how many match.
|
||||
failing_paths = []
|
||||
@@ -592,8 +655,6 @@ def spot_check(
|
||||
disk to those stored in the latest archive. If any differences are beyond configured tolerances,
|
||||
then the check fails.
|
||||
'''
|
||||
logger.debug('Running spot check')
|
||||
|
||||
try:
|
||||
spot_check_config = next(
|
||||
check for check in config.get('checks', ()) if check.get('name') == 'spot'
|
||||
@@ -603,20 +664,9 @@ def spot_check(
|
||||
|
||||
if spot_check_config['data_tolerance_percentage'] > spot_check_config['data_sample_percentage']:
|
||||
raise ValueError(
|
||||
'The data_tolerance_percentage must be less than or equal to the data_sample_percentage'
|
||||
'The data_tolerance_percentage must be less than or equal to the data_sample_percentage',
|
||||
)
|
||||
|
||||
source_paths = collect_spot_check_source_paths(
|
||||
repository,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
borgmatic_runtime_directory,
|
||||
)
|
||||
logger.debug(f'{len(source_paths)} total source paths for spot check')
|
||||
|
||||
archive = borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
'latest',
|
||||
@@ -628,6 +678,25 @@ def spot_check(
|
||||
)
|
||||
logger.debug(f'Using archive {archive} for spot check')
|
||||
|
||||
source_paths = collect_spot_check_source_paths(
|
||||
repository,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
borgmatic_runtime_directory,
|
||||
bootstrap_config_paths=borgmatic.actions.config.bootstrap.load_config_paths_from_archive(
|
||||
repository['path'],
|
||||
archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
borgmatic_runtime_directory,
|
||||
),
|
||||
)
|
||||
logger.debug(f'{len(source_paths)} total source paths for spot check')
|
||||
|
||||
archive_paths = collect_spot_check_archive_paths(
|
||||
repository,
|
||||
archive,
|
||||
@@ -648,7 +717,7 @@ def spot_check(
|
||||
)
|
||||
logger.debug(f'Paths in latest archive but not source paths: {truncated_archive_paths}')
|
||||
raise ValueError(
|
||||
'Spot check failed: There are no source paths to compare against the archive'
|
||||
'Spot check failed; there are no source paths to compare against the archive',
|
||||
)
|
||||
|
||||
# Calculate the percentage delta between the source paths count and the archive paths count, and
|
||||
@@ -656,25 +725,19 @@ def spot_check(
|
||||
count_delta_percentage = abs(len(source_paths) - len(archive_paths)) / len(source_paths) * 100
|
||||
|
||||
if count_delta_percentage > spot_check_config['count_tolerance_percentage']:
|
||||
rootless_source_paths = set(path.lstrip(os.path.sep) for path in source_paths)
|
||||
rootless_source_paths = {path.lstrip(os.path.sep) for path in source_paths}
|
||||
truncated_exclusive_source_paths = textwrap.shorten(
|
||||
', '.join(rootless_source_paths - set(archive_paths)) or 'none',
|
||||
width=MAX_SPOT_CHECK_PATHS_LENGTH,
|
||||
placeholder=' ...',
|
||||
)
|
||||
logger.debug(
|
||||
f'Paths in source paths but not latest archive: {truncated_exclusive_source_paths}'
|
||||
)
|
||||
truncated_exclusive_archive_paths = textwrap.shorten(
|
||||
', '.join(set(archive_paths) - rootless_source_paths) or 'none',
|
||||
width=MAX_SPOT_CHECK_PATHS_LENGTH,
|
||||
placeholder=' ...',
|
||||
)
|
||||
logger.debug(
|
||||
f'Paths in latest archive but not source paths: {truncated_exclusive_archive_paths}'
|
||||
)
|
||||
raise ValueError(
|
||||
f'Spot check failed: {count_delta_percentage:.2f}% file count delta between source paths and latest archive (tolerance is {spot_check_config["count_tolerance_percentage"]}%)'
|
||||
f'Spot check failed\n{count_delta_percentage:.2f}% file count delta between source paths ({len(source_paths)} total) and latest archive ({len(archive_paths)} total); tolerance is {spot_check_config["count_tolerance_percentage"]}%\nOnly in source paths: {truncated_exclusive_source_paths}\nOnly in latest archive: {truncated_exclusive_archive_paths}',
|
||||
)
|
||||
|
||||
failing_paths = compare_spot_check_hashes(
|
||||
@@ -699,15 +762,12 @@ def spot_check(
|
||||
width=MAX_SPOT_CHECK_PATHS_LENGTH,
|
||||
placeholder=' ...',
|
||||
)
|
||||
logger.debug(
|
||||
f'Source paths with data not matching the latest archive: {truncated_failing_paths}'
|
||||
)
|
||||
raise ValueError(
|
||||
f'Spot check failed: {failing_percentage:.2f}% of source paths with data not matching the latest archive (tolerance is {data_tolerance_percentage}%)'
|
||||
f'Spot check failed\n{failing_percentage:.2f}% of source paths ({len(failing_paths)} out of {len(source_paths)} checked) with data not matching the latest archive; tolerance is {data_tolerance_percentage}%\nSource paths with non-matching data: {truncated_failing_paths}',
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f'Spot check passed with a {count_delta_percentage:.2f}% file count delta and a {failing_percentage:.2f}% file data delta'
|
||||
f'Spot check passed with a {count_delta_percentage:.2f}% file count delta and a {failing_percentage:.2f}% file data delta',
|
||||
)
|
||||
|
||||
|
||||
@@ -726,11 +786,6 @@ def run_check(
|
||||
|
||||
Raise ValueError if the Borg repository ID cannot be determined.
|
||||
'''
|
||||
if check_arguments.repository and not borgmatic.config.validate.repositories_match(
|
||||
repository, check_arguments.repository
|
||||
):
|
||||
return
|
||||
|
||||
logger.info('Running consistency checks')
|
||||
|
||||
repository_id = borgmatic.borg.check.get_repository_id(
|
||||
@@ -744,7 +799,10 @@ def run_check(
|
||||
upgrade_check_times(config, repository_id)
|
||||
configured_checks = parse_checks(config, check_arguments.only_checks)
|
||||
archive_filter_flags = borgmatic.borg.check.make_archive_filter_flags(
|
||||
local_borg_version, config, configured_checks, check_arguments
|
||||
local_borg_version,
|
||||
config,
|
||||
configured_checks,
|
||||
check_arguments,
|
||||
)
|
||||
archives_check_id = make_archives_check_id(archive_filter_flags)
|
||||
checks = filter_checks_on_frequency(
|
||||
@@ -772,6 +830,7 @@ def run_check(
|
||||
write_check_time(make_check_time_path(config, repository_id, check, archives_check_id))
|
||||
|
||||
if 'extract' in checks:
|
||||
logger.info('Running extract check')
|
||||
borgmatic.borg.extract.extract_last_archive_dry_run(
|
||||
config,
|
||||
local_borg_version,
|
||||
@@ -784,6 +843,7 @@ def run_check(
|
||||
write_check_time(make_check_time_path(config, repository_id, 'extract'))
|
||||
|
||||
if 'spot' in checks:
|
||||
logger.info('Running spot check')
|
||||
with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
|
||||
spot_check(
|
||||
repository,
|
||||
@@ -794,4 +854,5 @@ def run_check(
|
||||
remote_path,
|
||||
borgmatic_runtime_directory,
|
||||
)
|
||||
|
||||
write_check_time(make_check_time_path(config, repository_id, 'spot'))
|
||||
|
||||
@@ -2,8 +2,6 @@ import logging
|
||||
|
||||
import borgmatic.borg.compact
|
||||
import borgmatic.borg.feature
|
||||
import borgmatic.config.validate
|
||||
import borgmatic.hooks.command
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -22,11 +20,6 @@ def run_compact(
|
||||
'''
|
||||
Run the "compact" action for the given repository.
|
||||
'''
|
||||
if compact_arguments.repository and not borgmatic.config.validate.repositories_match(
|
||||
repository, compact_arguments.repository
|
||||
):
|
||||
return
|
||||
|
||||
if borgmatic.borg.feature.available(borgmatic.borg.feature.Feature.COMPACT, local_borg_version):
|
||||
logger.info(f'Compacting segments{dry_run_label}')
|
||||
borgmatic.borg.compact.compact_segments(
|
||||
|
||||
@@ -5,8 +5,6 @@ import os
|
||||
import borgmatic.borg.extract
|
||||
import borgmatic.borg.repo_list
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.config.validate
|
||||
import borgmatic.hooks.command
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -16,78 +14,81 @@ def make_bootstrap_config(bootstrap_arguments):
|
||||
Given the bootstrap arguments as an argparse.Namespace, return a corresponding config dict.
|
||||
'''
|
||||
return {
|
||||
'ssh_command': bootstrap_arguments.ssh_command,
|
||||
'borgmatic_source_directory': bootstrap_arguments.borgmatic_source_directory,
|
||||
'local_path': bootstrap_arguments.local_path,
|
||||
'remote_path': bootstrap_arguments.remote_path,
|
||||
# In case the repo has been moved or is accessed from a different path at the point of
|
||||
# bootstrapping.
|
||||
'relocated_repo_access_is_ok': True,
|
||||
'ssh_command': bootstrap_arguments.ssh_command,
|
||||
'user_runtime_directory': bootstrap_arguments.user_runtime_directory,
|
||||
}
|
||||
|
||||
|
||||
def get_config_paths(archive_name, bootstrap_arguments, global_arguments, local_borg_version):
|
||||
def load_config_paths_from_archive(
|
||||
repository_path,
|
||||
archive_name,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
borgmatic_runtime_directory,
|
||||
):
|
||||
'''
|
||||
Given an archive name, the bootstrap arguments as an argparse.Namespace (containing the
|
||||
repository and archive name, Borg local path, Borg remote path, borgmatic runtime directory,
|
||||
borgmatic source directory, destination directory, and whether to strip components), the global
|
||||
arguments as an argparse.Namespace (containing the dry run flag and the local borg version),
|
||||
return the config paths from the manifest.json file in the borgmatic source directory or runtime
|
||||
directory after extracting it from the repository archive.
|
||||
Given a repository path, an archive name, a configuration dict, the local Borg version, the
|
||||
global arguments as an argparse.Namespace, and the borgmatic runtime directory, return the
|
||||
config paths from the manifest.json file in the borgmatic source directory or runtime directory
|
||||
within the repository archive.
|
||||
|
||||
Raise ValueError if the manifest JSON is missing, can't be decoded, or doesn't contain the
|
||||
expected configuration path data.
|
||||
'''
|
||||
borgmatic_source_directory = borgmatic.config.paths.get_borgmatic_source_directory(
|
||||
{'borgmatic_source_directory': bootstrap_arguments.borgmatic_source_directory}
|
||||
)
|
||||
config = make_bootstrap_config(bootstrap_arguments)
|
||||
|
||||
# Probe for the manifest file in multiple locations, as the default location has moved to the
|
||||
# borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But we
|
||||
# still want to support reading the manifest from previously created archives as well.
|
||||
with borgmatic.config.paths.Runtime_directory(
|
||||
{'user_runtime_directory': bootstrap_arguments.user_runtime_directory},
|
||||
) as borgmatic_runtime_directory:
|
||||
for base_directory in (
|
||||
'borgmatic',
|
||||
borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
|
||||
borgmatic_source_directory,
|
||||
):
|
||||
borgmatic_manifest_path = 'sh:' + os.path.join(
|
||||
base_directory, 'bootstrap', 'manifest.json'
|
||||
)
|
||||
for base_directory in (
|
||||
'borgmatic',
|
||||
borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
|
||||
borgmatic.config.paths.get_borgmatic_source_directory(config),
|
||||
):
|
||||
borgmatic_manifest_path = 'sh:' + os.path.join(
|
||||
base_directory,
|
||||
'bootstrap',
|
||||
'manifest.json',
|
||||
)
|
||||
|
||||
extract_process = borgmatic.borg.extract.extract_archive(
|
||||
global_arguments.dry_run,
|
||||
bootstrap_arguments.repository,
|
||||
archive_name,
|
||||
[borgmatic_manifest_path],
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path=bootstrap_arguments.local_path,
|
||||
remote_path=bootstrap_arguments.remote_path,
|
||||
extract_to_stdout=True,
|
||||
)
|
||||
manifest_json = extract_process.stdout.read()
|
||||
extract_process = borgmatic.borg.extract.extract_archive(
|
||||
global_arguments.dry_run,
|
||||
repository_path,
|
||||
archive_name,
|
||||
[borgmatic_manifest_path],
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path=config.get('local_path', 'borg'),
|
||||
remote_path=config.get('remote_path'),
|
||||
extract_to_stdout=True,
|
||||
)
|
||||
manifest_json = extract_process.stdout.read()
|
||||
|
||||
if manifest_json:
|
||||
break
|
||||
else:
|
||||
raise ValueError(
|
||||
'Cannot read configuration paths from archive due to missing bootstrap manifest'
|
||||
)
|
||||
if manifest_json:
|
||||
break
|
||||
else:
|
||||
raise ValueError(
|
||||
'Cannot read configuration paths from archive due to missing archive or bootstrap manifest',
|
||||
)
|
||||
|
||||
try:
|
||||
manifest_data = json.loads(manifest_json)
|
||||
except json.JSONDecodeError as error:
|
||||
raise ValueError(
|
||||
f'Cannot read configuration paths from archive due to invalid bootstrap manifest JSON: {error}'
|
||||
f'Cannot read configuration paths from archive due to invalid bootstrap manifest JSON: {error}',
|
||||
)
|
||||
|
||||
try:
|
||||
return manifest_data['config_paths']
|
||||
except KeyError:
|
||||
raise ValueError(
|
||||
'Cannot read configuration paths from archive due to invalid bootstrap manifest'
|
||||
'Cannot read configuration paths from archive due to invalid bootstrap manifest',
|
||||
)
|
||||
|
||||
|
||||
@@ -108,19 +109,24 @@ def run_bootstrap(bootstrap_arguments, global_arguments, local_borg_version):
|
||||
local_path=bootstrap_arguments.local_path,
|
||||
remote_path=bootstrap_arguments.remote_path,
|
||||
)
|
||||
manifest_config_paths = get_config_paths(
|
||||
archive_name, bootstrap_arguments, global_arguments, local_borg_version
|
||||
)
|
||||
|
||||
logger.info(f"Bootstrapping config paths: {', '.join(manifest_config_paths)}")
|
||||
with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
|
||||
manifest_config_paths = load_config_paths_from_archive(
|
||||
bootstrap_arguments.repository,
|
||||
archive_name,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
borgmatic_runtime_directory,
|
||||
)
|
||||
|
||||
logger.info(f"Bootstrapping configuration paths: {', '.join(manifest_config_paths)}")
|
||||
|
||||
borgmatic.borg.extract.extract_archive(
|
||||
global_arguments.dry_run,
|
||||
bootstrap_arguments.repository,
|
||||
archive_name,
|
||||
[config_path.lstrip(os.path.sep) for config_path in manifest_config_paths],
|
||||
# Only add progress here and not the extract_archive() call above, because progress
|
||||
# conflicts with extract_to_stdout.
|
||||
dict(config, progress=bootstrap_arguments.progress or False),
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
|
||||
@@ -19,30 +19,37 @@ def run_generate(generate_arguments, global_arguments):
|
||||
dry_run_label = ' (dry run; not actually writing anything)' if global_arguments.dry_run else ''
|
||||
|
||||
logger.answer(
|
||||
f'Generating a configuration file at: {generate_arguments.destination_filename}{dry_run_label}'
|
||||
f'Generating configuration files within: {generate_arguments.destination_path}{dry_run_label}'
|
||||
if generate_arguments.split
|
||||
else f'Generating a configuration file at: {generate_arguments.destination_path}{dry_run_label}'
|
||||
)
|
||||
|
||||
borgmatic.config.generate.generate_sample_configuration(
|
||||
global_arguments.dry_run,
|
||||
generate_arguments.source_filename,
|
||||
generate_arguments.destination_filename,
|
||||
generate_arguments.destination_path,
|
||||
borgmatic.config.validate.schema_filename(),
|
||||
overwrite=generate_arguments.overwrite,
|
||||
split=generate_arguments.split,
|
||||
)
|
||||
|
||||
if generate_arguments.source_filename:
|
||||
logger.answer(
|
||||
f'''
|
||||
Merged in the contents of configuration file at: {generate_arguments.source_filename}
|
||||
To review the changes made, run:
|
||||
|
||||
diff --unified {generate_arguments.source_filename} {generate_arguments.destination_filename}'''
|
||||
Merged the contents of the source configuration file: {generate_arguments.source_filename}'''
|
||||
)
|
||||
|
||||
if not generate_arguments.split:
|
||||
logger.answer(
|
||||
f'''To review the changes made, run:
|
||||
|
||||
diff --unified {generate_arguments.source_filename} {generate_arguments.destination_path}''',
|
||||
)
|
||||
|
||||
logger.answer(
|
||||
'''
|
||||
This includes all available configuration options with example values, the few
|
||||
required options as indicated. Please edit the file to suit your needs.
|
||||
|
||||
If you ever need help: https://torsion.org/borgmatic/#issues'''
|
||||
If you ever need help: https://torsion.org/borgmatic/#issues''',
|
||||
)
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
|
||||
import borgmatic.config.generate
|
||||
import borgmatic.logger
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_show(show_arguments, configs):
|
||||
'''
|
||||
Given the show arguments as an argparse.Namespace instance and a dict of configuration filename
|
||||
to corresponding parsed configuration, run the "show" action. That consists of rendering and
|
||||
logging the computed configuration as YAML, separating the configuration for each file with
|
||||
"---".
|
||||
|
||||
If show_arguments.option is set, limit the results to the value of that single option. If
|
||||
show_arguments.json is True, render the results as JSON with one array element per configuration
|
||||
file.
|
||||
'''
|
||||
borgmatic.logger.add_custom_log_levels()
|
||||
|
||||
if show_arguments.json:
|
||||
sys.stdout.write(
|
||||
json.dumps(
|
||||
[
|
||||
config.get(show_arguments.option) if show_arguments.option else config
|
||||
for config in configs.values()
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
for config in configs.values():
|
||||
if len(configs) > 1:
|
||||
logger.answer('---')
|
||||
|
||||
logger.answer(
|
||||
borgmatic.config.generate.render_configuration(
|
||||
config.get(show_arguments.option) if show_arguments.option else config
|
||||
).rstrip()
|
||||
)
|
||||
@@ -18,7 +18,7 @@ def run_validate(validate_arguments, configs):
|
||||
borgmatic.logger.add_custom_log_levels()
|
||||
|
||||
if validate_arguments.show:
|
||||
for config_path, config in configs.items():
|
||||
for config in configs.values():
|
||||
if len(configs) > 1:
|
||||
logger.answer('---')
|
||||
|
||||
|
||||
+79
-73
@@ -1,11 +1,12 @@
|
||||
import logging
|
||||
|
||||
import borgmatic.actions.dump
|
||||
import borgmatic.actions.json
|
||||
import borgmatic.borg.create
|
||||
import borgmatic.borg.feature
|
||||
import borgmatic.borg.rename
|
||||
import borgmatic.borg.repo_list
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.config.validate
|
||||
import borgmatic.hooks.dispatch
|
||||
from borgmatic.actions import pattern
|
||||
|
||||
@@ -29,108 +30,107 @@ def run_create(
|
||||
|
||||
If create_arguments.json is True, yield the JSON output from creating the archive.
|
||||
'''
|
||||
if create_arguments.repository and not borgmatic.config.validate.repositories_match(
|
||||
repository, create_arguments.repository
|
||||
):
|
||||
return
|
||||
|
||||
if config.get('list_details') and config.get('progress'):
|
||||
raise ValueError(
|
||||
'With the create action, only one of --list/--files/list_details and --progress/progress can be used.'
|
||||
'With the create action, only one of --list/--files/list_details and --progress/progress can be used.',
|
||||
)
|
||||
|
||||
if config.get('list_details') and create_arguments.json:
|
||||
raise ValueError(
|
||||
'With the create action, only one of --list/--files/list_details and --json can be used.'
|
||||
'With the create action, only one of --list/--files/list_details and --json can be used.',
|
||||
)
|
||||
|
||||
logger.info(f'Creating archive{dry_run_label}')
|
||||
working_directory = borgmatic.config.paths.get_working_directory(config)
|
||||
|
||||
with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
|
||||
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
|
||||
'remove_data_source_dumps',
|
||||
config,
|
||||
borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
|
||||
borgmatic_runtime_directory,
|
||||
global_arguments.dry_run,
|
||||
)
|
||||
patterns = pattern.process_patterns(pattern.collect_patterns(config), working_directory)
|
||||
active_dumps = borgmatic.hooks.dispatch.call_hooks(
|
||||
'dump_data_sources',
|
||||
config,
|
||||
borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
|
||||
config_paths,
|
||||
borgmatic_runtime_directory,
|
||||
patterns,
|
||||
global_arguments.dry_run,
|
||||
)
|
||||
|
||||
# Process the patterns again in case any data source hooks updated them. Without this step,
|
||||
# we could end up with duplicate paths that cause Borg to hang when it tries to read from
|
||||
# the same named pipe twice.
|
||||
patterns = pattern.process_patterns(
|
||||
patterns, working_directory, skip_expand_paths=config_paths
|
||||
)
|
||||
stream_processes = [process for processes in active_dumps.values() for process in processes]
|
||||
|
||||
# If we have stream processes, we first create an archive with .checkpoint suffix. This is
|
||||
# to make sure we only create a real archive if all the streaming processes completed
|
||||
# successfully (create_archive will fail if a streaming process fails, but the archive might
|
||||
# have already been created at this point).
|
||||
use_checkpoint = bool(stream_processes)
|
||||
|
||||
json_output = borgmatic.borg.create.create_archive(
|
||||
global_arguments.dry_run,
|
||||
repository['path'],
|
||||
pattern.collect_patterns(config, working_directory),
|
||||
config,
|
||||
patterns,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
working_directory,
|
||||
borgmatic_runtime_directory,
|
||||
archive_suffix='.checkpoint' if use_checkpoint else '',
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
json=create_arguments.json,
|
||||
comment=create_arguments.comment,
|
||||
stream_processes=stream_processes,
|
||||
)
|
||||
|
||||
if use_checkpoint:
|
||||
rename_checkpoint_archive(
|
||||
repository['path'],
|
||||
global_arguments,
|
||||
original_patterns = list(patterns)
|
||||
|
||||
# Use the original patterns so as to disregard any modifications made by any data source
|
||||
# hooks, e.g. via dump_data_sources() below.
|
||||
with borgmatic.actions.dump.Dump_cleanup(
|
||||
config, borgmatic_runtime_directory, original_patterns, global_arguments.dry_run
|
||||
):
|
||||
active_dumps = borgmatic.hooks.dispatch.call_hooks(
|
||||
'dump_data_sources',
|
||||
config,
|
||||
borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
|
||||
config_paths,
|
||||
borgmatic_runtime_directory,
|
||||
patterns,
|
||||
global_arguments.dry_run,
|
||||
)
|
||||
|
||||
# Process the patterns again in case any data source hooks updated them. Without this step,
|
||||
# we could end up with duplicate paths that cause Borg to hang when it tries to read from
|
||||
# the same named pipe twice.
|
||||
patterns = pattern.process_patterns(
|
||||
patterns,
|
||||
config,
|
||||
working_directory,
|
||||
borgmatic_runtime_directory,
|
||||
skip_expand_paths=config_paths,
|
||||
)
|
||||
stream_processes = [
|
||||
process for processes in active_dumps.values() for process in processes
|
||||
]
|
||||
|
||||
# If we have stream processes, we first create an archive with .checkpoint suffix. This is
|
||||
# to make sure we only create a real archive if all the streaming processes completed
|
||||
# successfully (create_archive will fail if a streaming process fails, but the archive might
|
||||
# have already been created at this point).
|
||||
use_checkpoint = bool(stream_processes)
|
||||
|
||||
json_output = borgmatic.borg.create.create_archive(
|
||||
global_arguments.dry_run,
|
||||
repository['path'],
|
||||
config,
|
||||
patterns,
|
||||
local_borg_version,
|
||||
local_path,
|
||||
remote_path,
|
||||
global_arguments,
|
||||
borgmatic_runtime_directory,
|
||||
archive_suffix='.checkpoint' if use_checkpoint else '',
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
json=create_arguments.json,
|
||||
comment=create_arguments.comment,
|
||||
stream_processes=stream_processes,
|
||||
)
|
||||
|
||||
if json_output:
|
||||
output = borgmatic.actions.json.parse_json(json_output, repository.get('label'))
|
||||
if use_checkpoint:
|
||||
# Patch archive name and ID
|
||||
renamed_archive = borgmatic.borg.repo_list.get_latest_archive(
|
||||
rename_checkpoint_archive(
|
||||
repository['path'],
|
||||
global_arguments,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
|
||||
output['archive']['name'] = renamed_archive['name']
|
||||
output['archive']['id'] = renamed_archive['id']
|
||||
if json_output:
|
||||
output = borgmatic.actions.json.parse_json(json_output, repository.get('label'))
|
||||
if use_checkpoint:
|
||||
# Patch archive name and ID
|
||||
renamed_archive = borgmatic.borg.repo_list.get_latest_archive(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
|
||||
yield output
|
||||
output['archive']['name'] = renamed_archive['name']
|
||||
output['archive']['id'] = renamed_archive['id']
|
||||
|
||||
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
|
||||
'remove_data_source_dumps',
|
||||
config,
|
||||
borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
|
||||
borgmatic_runtime_directory,
|
||||
global_arguments.dry_run,
|
||||
)
|
||||
yield output
|
||||
|
||||
|
||||
def rename_checkpoint_archive(
|
||||
@@ -174,7 +174,13 @@ def rename_checkpoint_archive(
|
||||
|
||||
borgmatic.borg.rename.rename_archive(
|
||||
repository_path,
|
||||
archive_name,
|
||||
(
|
||||
archive['id']
|
||||
if borgmatic.borg.feature.available(
|
||||
borgmatic.borg.feature.Feature.ARCHIVE_SERIES, local_borg_version
|
||||
)
|
||||
else archive['name']
|
||||
),
|
||||
new_archive_name,
|
||||
global_arguments.dry_run,
|
||||
config,
|
||||
|
||||
+18
-22
@@ -2,7 +2,6 @@ import logging
|
||||
|
||||
import borgmatic.actions.arguments
|
||||
import borgmatic.borg.delete
|
||||
import borgmatic.borg.repo_delete
|
||||
import borgmatic.borg.repo_list
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -20,31 +19,28 @@ def run_delete(
|
||||
'''
|
||||
Run the "delete" action for the given repository and archive(s).
|
||||
'''
|
||||
if delete_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
||||
repository, delete_arguments.repository
|
||||
):
|
||||
logger.answer('Deleting archives')
|
||||
logger.answer('Deleting archives')
|
||||
|
||||
archive_name = (
|
||||
borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
delete_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
if delete_arguments.archive
|
||||
else None
|
||||
)
|
||||
|
||||
borgmatic.borg.delete.delete_archives(
|
||||
repository,
|
||||
archive_name = (
|
||||
borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
delete_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
borgmatic.actions.arguments.update_arguments(delete_arguments, archive=archive_name),
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
if delete_arguments.archive
|
||||
else None
|
||||
)
|
||||
|
||||
borgmatic.borg.delete.delete_archives(
|
||||
repository,
|
||||
config,
|
||||
local_borg_version,
|
||||
borgmatic.actions.arguments.update_arguments(delete_arguments, archive=archive_name),
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import logging
|
||||
|
||||
import borgmatic.actions.pattern
|
||||
import borgmatic.borg.diff
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def run_diff(
|
||||
repository,
|
||||
config,
|
||||
local_borg_version,
|
||||
diff_arguments,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
):
|
||||
'''
|
||||
Run the "diff" action for the given repository.
|
||||
'''
|
||||
|
||||
# Only process patterns if only_patterns flag is set
|
||||
if diff_arguments.only_patterns:
|
||||
working_directory = borgmatic.config.paths.get_working_directory(config)
|
||||
processed_patterns = borgmatic.actions.pattern.process_patterns(
|
||||
(*borgmatic.actions.pattern.collect_patterns(config, working_directory),),
|
||||
config,
|
||||
working_directory,
|
||||
)
|
||||
else:
|
||||
processed_patterns = None
|
||||
|
||||
archive = borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
diff_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
second_archive = borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
diff_arguments.second_archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
|
||||
borgmatic.borg.diff.diff(
|
||||
repository['path'],
|
||||
archive,
|
||||
second_archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
diff_arguments,
|
||||
global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
patterns=processed_patterns,
|
||||
)
|
||||
@@ -0,0 +1,53 @@
|
||||
import borgmatic.hooks.dispatch
|
||||
|
||||
|
||||
class Dump_cleanup:
|
||||
'''
|
||||
A Python context manager for removing data source dumps before and after wrapped code. Besides
|
||||
doing this for general filesystem cleanliness reasons, leaving old dumps lying around can
|
||||
prevent future dumping from working (in the case of filesystem snapshots) or cause Borg hangs
|
||||
(in the case of database dump named pipes).
|
||||
|
||||
Example use as a context manager:
|
||||
|
||||
with borgmatic.actions.dump.Dump_cleanup(
|
||||
config, borgmatic_runtime_directory, patterns, dry_run,
|
||||
):
|
||||
do_something_like_perform_a_dump_or_restore()
|
||||
'''
|
||||
|
||||
def __init__(self, config, borgmatic_runtime_directory, patterns, dry_run):
|
||||
'''
|
||||
Given a configuration dict, the borgmatic runtime directory, the configured patterns, and
|
||||
whether this is a dry-run, store these values for use below.
|
||||
'''
|
||||
self.config = config
|
||||
self.borgmatic_runtime_directory = borgmatic_runtime_directory
|
||||
self.patterns = patterns
|
||||
self.dry_run = dry_run
|
||||
|
||||
def __enter__(self):
|
||||
'''
|
||||
Remove all data source dumps that exist prior to the wrapped code running.
|
||||
'''
|
||||
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
|
||||
'remove_data_source_dumps',
|
||||
self.config,
|
||||
borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
|
||||
self.borgmatic_runtime_directory,
|
||||
self.patterns,
|
||||
self.dry_run,
|
||||
)
|
||||
|
||||
def __exit__(self, exception_type, exception, traceback):
|
||||
'''
|
||||
Remove all data source dumps, including any created by the wrapped code.
|
||||
'''
|
||||
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
|
||||
'remove_data_source_dumps',
|
||||
self.config,
|
||||
borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
|
||||
self.borgmatic_runtime_directory,
|
||||
self.patterns,
|
||||
self.dry_run,
|
||||
)
|
||||
@@ -1,7 +1,6 @@
|
||||
import logging
|
||||
|
||||
import borgmatic.borg.export_key
|
||||
import borgmatic.config.validate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -18,16 +17,13 @@ def run_export_key(
|
||||
'''
|
||||
Run the "key export" action for the given repository.
|
||||
'''
|
||||
if export_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
||||
repository, export_arguments.repository
|
||||
):
|
||||
logger.info('Exporting repository key')
|
||||
borgmatic.borg.export_key.export_key(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
export_arguments,
|
||||
global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
logger.info('Exporting repository key')
|
||||
borgmatic.borg.export_key.export_key(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
export_arguments,
|
||||
global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
|
||||
@@ -2,7 +2,6 @@ import logging
|
||||
|
||||
import borgmatic.borg.export_tar
|
||||
import borgmatic.borg.repo_list
|
||||
import borgmatic.config.validate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -19,29 +18,26 @@ def run_export_tar(
|
||||
'''
|
||||
Run the "export-tar" action for the given repository.
|
||||
'''
|
||||
if export_tar_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
||||
repository, export_tar_arguments.repository
|
||||
):
|
||||
logger.info(f'Exporting archive {export_tar_arguments.archive} as tar file')
|
||||
borgmatic.borg.export_tar.export_tar_archive(
|
||||
global_arguments.dry_run,
|
||||
logger.info(f'Exporting archive {export_tar_arguments.archive} as tar file')
|
||||
borgmatic.borg.export_tar.export_tar_archive(
|
||||
global_arguments.dry_run,
|
||||
repository['path'],
|
||||
borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
export_tar_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
),
|
||||
export_tar_arguments.paths,
|
||||
export_tar_arguments.destination,
|
||||
export_tar_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
tar_filter=export_tar_arguments.tar_filter,
|
||||
strip_components=export_tar_arguments.strip_components,
|
||||
)
|
||||
local_path,
|
||||
remote_path,
|
||||
),
|
||||
export_tar_arguments.paths,
|
||||
export_tar_arguments.destination,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
tar_filter=export_tar_arguments.tar_filter,
|
||||
strip_components=export_tar_arguments.strip_components,
|
||||
)
|
||||
|
||||
@@ -2,8 +2,6 @@ import logging
|
||||
|
||||
import borgmatic.borg.extract
|
||||
import borgmatic.borg.repo_list
|
||||
import borgmatic.config.validate
|
||||
import borgmatic.hooks.command
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -21,28 +19,25 @@ def run_extract(
|
||||
'''
|
||||
Run the "extract" action for the given repository.
|
||||
'''
|
||||
if extract_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
||||
repository, extract_arguments.repository
|
||||
):
|
||||
logger.info(f'Extracting archive {extract_arguments.archive}')
|
||||
borgmatic.borg.extract.extract_archive(
|
||||
global_arguments.dry_run,
|
||||
logger.info(f'Extracting archive {extract_arguments.archive}')
|
||||
borgmatic.borg.extract.extract_archive(
|
||||
global_arguments.dry_run,
|
||||
repository['path'],
|
||||
borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
extract_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
),
|
||||
extract_arguments.paths,
|
||||
extract_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
destination_path=extract_arguments.destination,
|
||||
strip_components=extract_arguments.strip_components,
|
||||
)
|
||||
local_path,
|
||||
remote_path,
|
||||
),
|
||||
extract_arguments.paths,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
destination_path=extract_arguments.destination,
|
||||
strip_components=extract_arguments.strip_components,
|
||||
)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import logging
|
||||
|
||||
import borgmatic.borg.import_key
|
||||
import borgmatic.config.validate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -18,16 +17,13 @@ def run_import_key(
|
||||
'''
|
||||
Run the "key import" action for the given repository.
|
||||
'''
|
||||
if import_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
||||
repository, import_arguments.repository
|
||||
):
|
||||
logger.info('Importing repository key')
|
||||
borgmatic.borg.import_key.import_key(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
import_arguments,
|
||||
global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
logger.info('Importing repository key')
|
||||
borgmatic.borg.import_key.import_key(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
import_arguments,
|
||||
global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
|
||||
+23
-26
@@ -4,7 +4,6 @@ import borgmatic.actions.arguments
|
||||
import borgmatic.actions.json
|
||||
import borgmatic.borg.info
|
||||
import borgmatic.borg.repo_list
|
||||
import borgmatic.config.validate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -23,28 +22,26 @@ def run_info(
|
||||
|
||||
If info_arguments.json is True, yield the JSON output from the info for the archive.
|
||||
'''
|
||||
if info_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
||||
repository, info_arguments.repository
|
||||
):
|
||||
if not info_arguments.json:
|
||||
logger.answer('Displaying archive summary information')
|
||||
archive_name = borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
info_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
json_output = borgmatic.borg.info.display_archives_info(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
borgmatic.actions.arguments.update_arguments(info_arguments, archive=archive_name),
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
if json_output:
|
||||
yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
|
||||
if not info_arguments.json:
|
||||
logger.answer('Displaying archive summary information')
|
||||
|
||||
archive_name = borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
info_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
json_output = borgmatic.borg.info.display_archives_info(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
borgmatic.actions.arguments.update_arguments(info_arguments, archive=archive_name),
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
if json_output:
|
||||
yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
|
||||
|
||||
+25
-29
@@ -3,7 +3,6 @@ import logging
|
||||
import borgmatic.actions.arguments
|
||||
import borgmatic.actions.json
|
||||
import borgmatic.borg.list
|
||||
import borgmatic.config.validate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -22,32 +21,29 @@ def run_list(
|
||||
|
||||
If list_arguments.json is True, yield the JSON output from listing the archive.
|
||||
'''
|
||||
if list_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
||||
repository, list_arguments.repository
|
||||
):
|
||||
if not list_arguments.json:
|
||||
if list_arguments.find_paths: # pragma: no cover
|
||||
logger.answer('Searching archives')
|
||||
elif not list_arguments.archive: # pragma: no cover
|
||||
logger.answer('Listing archives')
|
||||
if not list_arguments.json:
|
||||
if list_arguments.find_paths: # pragma: no cover
|
||||
logger.answer('Searching archives')
|
||||
elif not list_arguments.archive: # pragma: no cover
|
||||
logger.answer('Listing archives')
|
||||
|
||||
archive_name = borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
list_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
json_output = borgmatic.borg.list.list_archive(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
borgmatic.actions.arguments.update_arguments(list_arguments, archive=archive_name),
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
if json_output:
|
||||
yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
|
||||
archive_name = borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
list_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
json_output = borgmatic.borg.list.list_archive(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
borgmatic.actions.arguments.update_arguments(list_arguments, archive=archive_name),
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
if json_output:
|
||||
yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
|
||||
|
||||
+18
-22
@@ -2,7 +2,6 @@ import logging
|
||||
|
||||
import borgmatic.borg.mount
|
||||
import borgmatic.borg.repo_list
|
||||
import borgmatic.config.validate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -19,29 +18,26 @@ def run_mount(
|
||||
'''
|
||||
Run the "mount" action for the given repository.
|
||||
'''
|
||||
if mount_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
||||
repository, mount_arguments.repository
|
||||
):
|
||||
if mount_arguments.archive:
|
||||
logger.info(f'Mounting archive {mount_arguments.archive}')
|
||||
else: # pragma: nocover
|
||||
logger.info('Mounting repository')
|
||||
if mount_arguments.archive:
|
||||
logger.info(f'Mounting archive {mount_arguments.archive}')
|
||||
else: # pragma: nocover
|
||||
logger.info('Mounting repository')
|
||||
|
||||
borgmatic.borg.mount.mount_archive(
|
||||
borgmatic.borg.mount.mount_archive(
|
||||
repository['path'],
|
||||
borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
mount_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
),
|
||||
mount_arguments,
|
||||
mount_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
local_path,
|
||||
remote_path,
|
||||
),
|
||||
mount_arguments,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
|
||||
@@ -34,10 +34,11 @@ def parse_pattern(pattern_line, default_style=borgmatic.borg.pattern.Pattern_sty
|
||||
)
|
||||
|
||||
|
||||
def collect_patterns(config):
|
||||
def collect_patterns(config, working_directory):
|
||||
'''
|
||||
Given a configuration dict, produce a single sequence of patterns comprised of the configured
|
||||
source directories, patterns, excludes, pattern files, and exclude files.
|
||||
Given a configuration dict and the working directory, produce a single sequence of patterns
|
||||
comprised of the configured source directories, patterns, excludes, pattern files, and exclude
|
||||
files.
|
||||
|
||||
The idea is that Borg has all these different ways of specifying includes, excludes, source
|
||||
directories, etc., but we'd like to collapse them all down to one common format (patterns) for
|
||||
@@ -47,7 +48,8 @@ def collect_patterns(config):
|
||||
return (
|
||||
tuple(
|
||||
borgmatic.borg.pattern.Pattern(
|
||||
source_directory, source=borgmatic.borg.pattern.Pattern_source.CONFIG
|
||||
source_directory,
|
||||
source=borgmatic.borg.pattern.Pattern_source.CONFIG,
|
||||
)
|
||||
for source_directory in config.get('source_directories', ())
|
||||
)
|
||||
@@ -67,7 +69,8 @@ def collect_patterns(config):
|
||||
+ tuple(
|
||||
parse_pattern(pattern_line.strip())
|
||||
for filename in config.get('patterns_from', ())
|
||||
for pattern_line in open(filename).readlines()
|
||||
for expanded_path in expand_directory(filename, working_directory)
|
||||
for pattern_line in open(expanded_path, encoding='utf-8')
|
||||
if not pattern_line.lstrip().startswith('#')
|
||||
if pattern_line.strip()
|
||||
)
|
||||
@@ -77,7 +80,8 @@ def collect_patterns(config):
|
||||
borgmatic.borg.pattern.Pattern_style.FNMATCH,
|
||||
)
|
||||
for filename in config.get('exclude_from', ())
|
||||
for exclude_line in open(filename).readlines()
|
||||
for expanded_path in expand_directory(filename, working_directory)
|
||||
for exclude_line in open(expanded_path, encoding='utf-8')
|
||||
if not exclude_line.lstrip().startswith('#')
|
||||
if exclude_line.strip()
|
||||
)
|
||||
@@ -112,9 +116,8 @@ def expand_directory(directory, working_directory):
|
||||
glob_path
|
||||
# If these are equal, that means we didn't add any working directory prefix above.
|
||||
if normalized_directory == expanded_directory
|
||||
# Remove the working directory prefix that we added above in order to make glob() work.
|
||||
# We can't use os.path.relpath() here because it collapses any use of Borg's slashdot
|
||||
# hack.
|
||||
# Remove the working directory prefix added above in order to make glob() work. We
|
||||
# can't use os.path.relpath() here because it collapses any use of Borg's slashdot hack.
|
||||
else glob_path.removeprefix(working_directory_prefix)
|
||||
)
|
||||
for glob_path in glob_paths
|
||||
@@ -161,7 +164,7 @@ def expand_patterns(patterns, working_directory=None, skip_paths=None):
|
||||
)
|
||||
)
|
||||
for pattern in patterns
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -180,8 +183,10 @@ def get_existent_path_or_parent(path):
|
||||
try:
|
||||
return next(
|
||||
candidate_path
|
||||
for candidate_path in (path,)
|
||||
+ tuple(str(parent) for parent in pathlib.PurePath(path).parents)
|
||||
for candidate_path in (
|
||||
path,
|
||||
*tuple(str(parent) for parent in pathlib.PurePath(path).parents),
|
||||
)
|
||||
if os.path.exists(candidate_path)
|
||||
)
|
||||
except StopIteration:
|
||||
@@ -219,49 +224,75 @@ def device_map_patterns(patterns, working_directory=None):
|
||||
for pattern in patterns
|
||||
for existent_path in (
|
||||
get_existent_path_or_parent(
|
||||
os.path.join(working_directory or '', pattern.path.lstrip('^'))
|
||||
os.path.join(working_directory or '', pattern.path.lstrip('^')),
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def deduplicate_patterns(patterns):
|
||||
def deduplicate_runtime_directory_patterns(patterns, config, borgmatic_runtime_directory=None):
|
||||
'''
|
||||
Given a sequence of borgmatic.borg.pattern.Pattern instances, return them with all duplicate
|
||||
root child patterns removed. For instance, if two root patterns are given with paths "/foo" and
|
||||
"/foo/bar", return just the one with "/foo". Non-root patterns are passed through without
|
||||
Given a sequence of borgmatic.borg.pattern.Pattern instances, the borgmatic runtime directory,
|
||||
and a configuration dict, return them without any duplicate root child patterns that contain the
|
||||
runtime directory. For instance, if two root patterns are given with paths "/foo" and
|
||||
"/foo/bar", and the runtime directory is "/foo/bar", return just the "/foo" pattern. Non-root
|
||||
patterns and patterns not containing the runtime directory are passed through without
|
||||
modification.
|
||||
|
||||
The one exception to deduplication is two paths are on different filesystems (devices). In that
|
||||
case, they won't get deduplicated, in case they both need to be passed to Borg (e.g. the
|
||||
one_file_system option is true).
|
||||
One exception to deduplication is if two paths are on different filesystems (devices) and
|
||||
"one_file_system" is True in the given configuration. In that case, the paths won't get
|
||||
deduplicated, because Borg won't cross filesystem boundaries when "one_file_system" is True.
|
||||
|
||||
The idea is that if Borg is given a root parent pattern, then it doesn't also need to be given
|
||||
child patterns, because it will naturally spider the contents of the parent pattern's path. And
|
||||
there are cases where Borg coming across the same file twice will result in duplicate reads and
|
||||
even hangs, e.g. when a database hook is using a named pipe for streaming database dumps to
|
||||
Borg.
|
||||
The idea is that if Borg is given a root parent pattern containing the borgmatic runtime
|
||||
directory, then Borg doesn't also need to be given child patterns, because it will naturally
|
||||
spider the contents of the parent pattern's path. Additionally, there are cases where Borg
|
||||
coming across the same file twice will result in duplicate reads and even hangs, e.g. when a
|
||||
database hook in the borgmatic runtime directory is using a named pipe for streaming database
|
||||
dumps to Borg.
|
||||
|
||||
This deduplication is limited to the borgmatic runtime directory (where borgmatic's named pipes
|
||||
exist), because there are other legitimate use cases for parent and child patterns to both exist
|
||||
in patterns. For instance, with some snapshotted filesystems, snapshots don't traverse from a
|
||||
parent filesystem to a child and therefore both need to remain in patterns.
|
||||
|
||||
And for the case of named pipes outside of the borgmatic runtime directory, there is code
|
||||
elsewhere (in the "create" action) that auto-excludes special files to prevent Borg hangs.
|
||||
'''
|
||||
if borgmatic_runtime_directory is None:
|
||||
return patterns
|
||||
|
||||
deduplicated = {} # Use just the keys as an ordered set.
|
||||
runtime_directory_parents = set(pathlib.PurePath(borgmatic_runtime_directory).parents).union(
|
||||
{pathlib.PurePath(borgmatic_runtime_directory)}
|
||||
)
|
||||
|
||||
for pattern in patterns:
|
||||
if pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
|
||||
deduplicated[pattern] = True
|
||||
continue
|
||||
|
||||
parents = pathlib.PurePath(pattern.path).parents
|
||||
pattern_parents = pathlib.PurePath(pattern.path).parents
|
||||
|
||||
# If another directory in the given list is a parent of current directory (even n levels up)
|
||||
# and both are on the same filesystem, then the current directory is a duplicate.
|
||||
# If:
|
||||
#
|
||||
# 1. another pattern is a parent of the current pattern (even n levels up),
|
||||
# 2. both patterns are parents of the runtime directory (even n levels up),
|
||||
# 3. and both patterns are on the same filesystem (or one_file_system is not set)
|
||||
#
|
||||
# ... then consider the current pattern as a duplicate.
|
||||
for other_pattern in patterns:
|
||||
if other_pattern.type != borgmatic.borg.pattern.Pattern_type.ROOT:
|
||||
continue
|
||||
|
||||
if any(
|
||||
pathlib.PurePath(other_pattern.path) == parent
|
||||
and pattern.device is not None
|
||||
and other_pattern.device == pattern.device
|
||||
for parent in parents
|
||||
device_matches = pattern.device is not None and (
|
||||
other_pattern.device == pattern.device or config.get('one_file_system') is not True
|
||||
)
|
||||
|
||||
if (
|
||||
pathlib.PurePath(other_pattern.path) in pattern_parents
|
||||
and pathlib.PurePosixPath(other_pattern.path) in runtime_directory_parents
|
||||
and pathlib.PurePosixPath(pattern.path) in runtime_directory_parents
|
||||
and device_matches
|
||||
):
|
||||
break
|
||||
else:
|
||||
@@ -270,23 +301,30 @@ def deduplicate_patterns(patterns):
|
||||
return tuple(deduplicated.keys())
|
||||
|
||||
|
||||
def process_patterns(patterns, working_directory, skip_expand_paths=None):
|
||||
def process_patterns(
|
||||
patterns, config, working_directory, borgmatic_runtime_directory=None, skip_expand_paths=None
|
||||
):
|
||||
'''
|
||||
Given a sequence of Borg patterns and a configured working directory, expand and deduplicate any
|
||||
"root" patterns, returning the resulting root and non-root patterns as a list.
|
||||
Given a sequence of Borg patterns, a configuration dict, a configured working directory, the
|
||||
borgmatic runtime directory, and a sequence of paths to skip path expansion for, expand and
|
||||
deduplicate any "root" patterns, returning the resulting root and non-root patterns as a list.
|
||||
|
||||
If any paths are given to skip, don't expand them.
|
||||
If the borgmatic runtime directory is None, then don't deduplicate patterns. Deduplication is
|
||||
really only necessary for the "create" action when the runtime directory might contain named
|
||||
pipes for database dumps.
|
||||
'''
|
||||
skip_paths = set(skip_expand_paths or ())
|
||||
|
||||
return list(
|
||||
deduplicate_patterns(
|
||||
deduplicate_runtime_directory_patterns(
|
||||
device_map_patterns(
|
||||
expand_patterns(
|
||||
patterns,
|
||||
working_directory=working_directory,
|
||||
skip_paths=skip_paths,
|
||||
)
|
||||
)
|
||||
)
|
||||
),
|
||||
),
|
||||
config,
|
||||
borgmatic_runtime_directory,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import logging
|
||||
|
||||
import borgmatic.borg.prune
|
||||
import borgmatic.config.validate
|
||||
import borgmatic.hooks.command
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -21,11 +19,6 @@ def run_prune(
|
||||
'''
|
||||
Run the "prune" action for the given repository.
|
||||
'''
|
||||
if prune_arguments.repository and not borgmatic.config.validate.repositories_match(
|
||||
repository, prune_arguments.repository
|
||||
):
|
||||
return
|
||||
|
||||
logger.info(f'Pruning archives{dry_run_label}')
|
||||
borgmatic.borg.prune.prune_archives(
|
||||
global_arguments.dry_run,
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
import logging
|
||||
import subprocess
|
||||
|
||||
import borgmatic.borg.info
|
||||
import borgmatic.actions.pattern
|
||||
import borgmatic.borg.pattern
|
||||
import borgmatic.borg.recreate
|
||||
import borgmatic.borg.repo_list
|
||||
import borgmatic.config.validate
|
||||
from borgmatic.actions.pattern import collect_patterns, process_patterns
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -19,66 +18,77 @@ def run_recreate(
|
||||
local_borg_version,
|
||||
recreate_arguments,
|
||||
global_arguments,
|
||||
dry_run_label,
|
||||
local_path,
|
||||
remote_path,
|
||||
):
|
||||
'''
|
||||
Run the "recreate" action for the given repository.
|
||||
'''
|
||||
if recreate_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
||||
repository, recreate_arguments.repository
|
||||
):
|
||||
if recreate_arguments.archive:
|
||||
logger.answer(f'Recreating archive {recreate_arguments.archive}')
|
||||
else:
|
||||
logger.answer('Recreating repository')
|
||||
if recreate_arguments.archive:
|
||||
logger.answer(f'Recreating archive {recreate_arguments.archive}{dry_run_label}')
|
||||
else:
|
||||
logger.answer(f'Recreating repository{dry_run_label}')
|
||||
|
||||
# Collect and process patterns.
|
||||
processed_patterns = process_patterns(
|
||||
collect_patterns(config), borgmatic.config.paths.get_working_directory(config)
|
||||
working_directory = borgmatic.config.paths.get_working_directory(config)
|
||||
|
||||
# Collect and process patterns.
|
||||
processed_patterns = borgmatic.actions.pattern.process_patterns(
|
||||
(
|
||||
*borgmatic.actions.pattern.collect_patterns(config, working_directory),
|
||||
# Also add borgmatic-specific paths, so they don't get excluded from the recreated
|
||||
# archive. Note that this doesn't currently work for archives created with Borg 1.2 or
|
||||
# below.
|
||||
borgmatic.borg.pattern.Pattern(
|
||||
'/borgmatic', source=borgmatic.borg.pattern.Pattern_source.INTERNAL
|
||||
),
|
||||
),
|
||||
config,
|
||||
working_directory,
|
||||
)
|
||||
|
||||
archive = borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
recreate_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
|
||||
if archive and archive.endswith('.recreate'):
|
||||
if recreate_arguments.archive == 'latest':
|
||||
raise ValueError(
|
||||
f'The latest archive "{archive}" is leftover from a prior recreate. Delete it first or select a different archive.',
|
||||
)
|
||||
|
||||
raise ValueError(
|
||||
f'The archive "{recreate_arguments.archive}" is leftover from a prior recreate. Select a different archive.',
|
||||
)
|
||||
|
||||
archive = borgmatic.borg.repo_list.resolve_archive_name(
|
||||
try:
|
||||
borgmatic.borg.recreate.recreate_archive(
|
||||
repository['path'],
|
||||
recreate_arguments.archive,
|
||||
archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
recreate_arguments,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
patterns=processed_patterns,
|
||||
)
|
||||
|
||||
if archive and archive.endswith('.recreate'):
|
||||
if recreate_arguments.archive == 'latest':
|
||||
except subprocess.CalledProcessError as error:
|
||||
if error.returncode == BORG_EXIT_CODE_ARCHIVE_ALREADY_EXISTS:
|
||||
if recreate_arguments.target:
|
||||
raise ValueError(
|
||||
f'The latest archive "{archive}" is leftover from a prior recreate. Delete it first or select a different archive.'
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f'The archive "{recreate_arguments.archive}" is leftover from a prior recreate. Select a different archive.'
|
||||
f'The archive "{recreate_arguments.target}" already exists. Delete it first or set a different target archive name.',
|
||||
)
|
||||
|
||||
try:
|
||||
borgmatic.borg.recreate.recreate_archive(
|
||||
repository['path'],
|
||||
archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
recreate_arguments,
|
||||
global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
patterns=processed_patterns,
|
||||
)
|
||||
except subprocess.CalledProcessError as error:
|
||||
if error.returncode == BORG_EXIT_CODE_ARCHIVE_ALREADY_EXISTS:
|
||||
if recreate_arguments.target:
|
||||
raise ValueError(
|
||||
f'The archive "{recreate_arguments.target}" already exists. Delete it first or set a different target archive name.'
|
||||
)
|
||||
elif archive:
|
||||
raise ValueError(
|
||||
f'The archive "{archive}.recreate" is leftover from a prior recreate. Delete it first or select a different archive.'
|
||||
)
|
||||
if archive:
|
||||
raise ValueError(
|
||||
f'The archive "{archive}.recreate" is leftover from a prior recreate. Delete it first or select a different archive.',
|
||||
)
|
||||
|
||||
raise
|
||||
raise
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import logging
|
||||
|
||||
import borgmatic.borg.repo_create
|
||||
import borgmatic.config.validate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -18,18 +17,13 @@ def run_repo_create(
|
||||
'''
|
||||
Run the "repo-create" action for the given repository.
|
||||
'''
|
||||
if repo_create_arguments.repository and not borgmatic.config.validate.repositories_match(
|
||||
repository, repo_create_arguments.repository
|
||||
):
|
||||
return
|
||||
|
||||
logger.info('Creating repository')
|
||||
|
||||
encryption_mode = repo_create_arguments.encryption_mode or repository.get('encryption')
|
||||
|
||||
if not encryption_mode:
|
||||
raise ValueError(
|
||||
'With the repo-create action, either the --encryption flag or the repository encryption option is required.'
|
||||
'With the repo-create action, either the --encryption flag or the repository encryption option is required.',
|
||||
)
|
||||
|
||||
borgmatic.borg.repo_create.create_repository(
|
||||
|
||||
@@ -17,19 +17,16 @@ def run_repo_delete(
|
||||
'''
|
||||
Run the "repo-delete" action for the given repository.
|
||||
'''
|
||||
if repo_delete_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
||||
repository, repo_delete_arguments.repository
|
||||
):
|
||||
logger.answer(
|
||||
'Deleting repository' + (' cache' if repo_delete_arguments.cache_only else '')
|
||||
)
|
||||
logger.answer(
|
||||
'Deleting repository' + (' cache' if repo_delete_arguments.cache_only else ''),
|
||||
)
|
||||
|
||||
borgmatic.borg.repo_delete.delete_repository(
|
||||
repository,
|
||||
config,
|
||||
local_borg_version,
|
||||
repo_delete_arguments,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
borgmatic.borg.repo_delete.delete_repository(
|
||||
repository,
|
||||
config,
|
||||
local_borg_version,
|
||||
repo_delete_arguments,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
|
||||
@@ -2,7 +2,6 @@ import logging
|
||||
|
||||
import borgmatic.actions.json
|
||||
import borgmatic.borg.repo_info
|
||||
import borgmatic.config.validate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -21,20 +20,17 @@ def run_repo_info(
|
||||
|
||||
If repo_info_arguments.json is True, yield the JSON output from the info for the repository.
|
||||
'''
|
||||
if repo_info_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
||||
repository, repo_info_arguments.repository
|
||||
):
|
||||
if not repo_info_arguments.json:
|
||||
logger.answer('Displaying repository summary information')
|
||||
if not repo_info_arguments.json:
|
||||
logger.answer('Displaying repository summary information')
|
||||
|
||||
json_output = borgmatic.borg.repo_info.display_repository_info(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
repo_info_arguments=repo_info_arguments,
|
||||
global_arguments=global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
if json_output:
|
||||
yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
|
||||
json_output = borgmatic.borg.repo_info.display_repository_info(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
repo_info_arguments=repo_info_arguments,
|
||||
global_arguments=global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
if json_output:
|
||||
yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
|
||||
|
||||
@@ -2,7 +2,6 @@ import logging
|
||||
|
||||
import borgmatic.actions.json
|
||||
import borgmatic.borg.repo_list
|
||||
import borgmatic.config.validate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -21,20 +20,17 @@ def run_repo_list(
|
||||
|
||||
If repo_list_arguments.json is True, yield the JSON output from listing the repository.
|
||||
'''
|
||||
if repo_list_arguments.repository is None or borgmatic.config.validate.repositories_match(
|
||||
repository, repo_list_arguments.repository
|
||||
):
|
||||
if not repo_list_arguments.json:
|
||||
logger.answer('Listing repository')
|
||||
if not repo_list_arguments.json:
|
||||
logger.answer('Listing repository')
|
||||
|
||||
json_output = borgmatic.borg.repo_list.list_repository(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
repo_list_arguments=repo_list_arguments,
|
||||
global_arguments=global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
if json_output:
|
||||
yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
|
||||
json_output = borgmatic.borg.repo_list.list_repository(
|
||||
repository['path'],
|
||||
config,
|
||||
local_borg_version,
|
||||
repo_list_arguments=repo_list_arguments,
|
||||
global_arguments=global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
)
|
||||
if json_output:
|
||||
yield borgmatic.actions.json.parse_json(json_output, repository.get('label'))
|
||||
|
||||
+230
-137
@@ -1,16 +1,17 @@
|
||||
import collections
|
||||
import locale
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
import borgmatic.actions.dump
|
||||
import borgmatic.actions.pattern
|
||||
import borgmatic.borg.extract
|
||||
import borgmatic.borg.list
|
||||
import borgmatic.borg.mount
|
||||
import borgmatic.borg.repo_list
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.config.validate
|
||||
import borgmatic.hooks.data_source.dump
|
||||
import borgmatic.hooks.dispatch
|
||||
|
||||
@@ -22,8 +23,8 @@ UNSPECIFIED = object()
|
||||
|
||||
Dump = collections.namedtuple(
|
||||
'Dump',
|
||||
('hook_name', 'data_source_name', 'hostname', 'port'),
|
||||
defaults=('localhost', None),
|
||||
('hook_name', 'data_source_name', 'hostname', 'port', 'label', 'container'),
|
||||
defaults=(None, None, None, None),
|
||||
)
|
||||
|
||||
|
||||
@@ -33,17 +34,24 @@ def dumps_match(first, second, default_port=None):
|
||||
indicates that the field should match any value. If a default port is given, then consider any
|
||||
dump having that port to match with a dump having a None port.
|
||||
'''
|
||||
for field_name in first._fields:
|
||||
# label kinda counts as an unique id, if they match ignore host/container/port
|
||||
if first.label not in {None, UNSPECIFIED} and first.label == second.label:
|
||||
field_list = ('hook_name', 'data_source_name')
|
||||
else:
|
||||
field_list = Dump._fields
|
||||
|
||||
for field_name in field_list:
|
||||
first_value = getattr(first, field_name)
|
||||
second_value = getattr(second, field_name)
|
||||
|
||||
if default_port is not None and field_name == 'port':
|
||||
if first_value == default_port and second_value is None:
|
||||
continue
|
||||
|
||||
if second_value == default_port and first_value is None:
|
||||
continue
|
||||
|
||||
if first_value == UNSPECIFIED or second_value == UNSPECIFIED:
|
||||
if first_value == UNSPECIFIED or second_value == UNSPECIFIED: # noqa: PLR1714
|
||||
continue
|
||||
|
||||
if first_value != second_value:
|
||||
@@ -56,16 +64,19 @@ def render_dump_metadata(dump):
|
||||
'''
|
||||
Given a Dump instance, make a display string describing it for use in log messages.
|
||||
'''
|
||||
label = dump.label or UNSPECIFIED
|
||||
name = 'unspecified' if dump.data_source_name is UNSPECIFIED else dump.data_source_name
|
||||
hostname = dump.hostname or UNSPECIFIED
|
||||
host = dump.container or dump.hostname or UNSPECIFIED
|
||||
port = None if dump.port is UNSPECIFIED else dump.port
|
||||
|
||||
if port:
|
||||
metadata = f'{name}@:{port}' if hostname is UNSPECIFIED else f'{name}@{hostname}:{port}'
|
||||
if label is not UNSPECIFIED:
|
||||
metadata = f'{name}@{label}'
|
||||
elif port:
|
||||
metadata = f'{name}@:{port}' if host is UNSPECIFIED else f'{name}@{host}:{port}'
|
||||
else:
|
||||
metadata = f'{name}' if hostname is UNSPECIFIED else f'{name}@{hostname}'
|
||||
metadata = f'{name}' if host is UNSPECIFIED else f'{name}@{host}'
|
||||
|
||||
if dump.hook_name not in (None, UNSPECIFIED):
|
||||
if dump.hook_name not in {None, UNSPECIFIED}:
|
||||
return f'{metadata} ({dump.hook_name})'
|
||||
|
||||
return metadata
|
||||
@@ -98,8 +109,10 @@ def get_configured_data_source(config, restore_dump):
|
||||
Dump(
|
||||
hook_name,
|
||||
hook_data_source.get('name'),
|
||||
hook_data_source.get('hostname', 'localhost'),
|
||||
hook_data_source.get('hostname'),
|
||||
hook_data_source.get('port'),
|
||||
hook_data_source.get('label') or UNSPECIFIED,
|
||||
hook_data_source.get('container'),
|
||||
),
|
||||
restore_dump,
|
||||
default_port,
|
||||
@@ -111,20 +124,21 @@ def get_configured_data_source(config, restore_dump):
|
||||
|
||||
if len(matching_dumps) > 1:
|
||||
raise ValueError(
|
||||
f'Cannot restore data source {render_dump_metadata(restore_dump)} because there are multiple matching data sources configured'
|
||||
f'Cannot restore data source {render_dump_metadata(restore_dump)} because there are multiple matching data sources configured',
|
||||
)
|
||||
|
||||
return matching_dumps[0]
|
||||
|
||||
|
||||
def strip_path_prefix_from_extracted_dump_destination(
|
||||
destination_path, borgmatic_runtime_directory
|
||||
extract_path,
|
||||
borgmatic_runtime_directory,
|
||||
):
|
||||
'''
|
||||
Directory-format dump files get extracted into a temporary directory containing a path prefix
|
||||
that depends how the files were stored in the archive. So, given the destination path where the
|
||||
dump was extracted and the borgmatic runtime directory, move the dump files such that the
|
||||
restore doesn't have to deal with that varying path prefix.
|
||||
that depends how the files were stored in the archive. So, given the path where the dump was
|
||||
extracted and the borgmatic runtime directory, move the dump files such that the restore doesn't
|
||||
have to deal with that varying path prefix.
|
||||
|
||||
For instance, if the dump was extracted to:
|
||||
|
||||
@@ -138,15 +152,16 @@ def strip_path_prefix_from_extracted_dump_destination(
|
||||
|
||||
/run/user/0/borgmatic/postgresql_databases/test/...
|
||||
'''
|
||||
for subdirectory_path, _, _ in os.walk(destination_path):
|
||||
for subdirectory_path, _, _ in os.walk(extract_path):
|
||||
databases_directory = os.path.basename(subdirectory_path)
|
||||
|
||||
if not databases_directory.endswith('_databases'):
|
||||
continue
|
||||
|
||||
shutil.move(
|
||||
subdirectory_path, os.path.join(borgmatic_runtime_directory, databases_directory)
|
||||
)
|
||||
destination_path = os.path.join(borgmatic_runtime_directory, databases_directory)
|
||||
shutil.rmtree(destination_path, ignore_errors=True)
|
||||
shutil.move(subdirectory_path, destination_path)
|
||||
|
||||
break
|
||||
|
||||
|
||||
@@ -169,7 +184,14 @@ def restore_single_dump(
|
||||
that data source from the archive.
|
||||
'''
|
||||
dump_metadata = render_dump_metadata(
|
||||
Dump(hook_name, data_source['name'], data_source.get('hostname'), data_source.get('port'))
|
||||
Dump(
|
||||
hook_name,
|
||||
data_source['name'],
|
||||
data_source.get('hostname'),
|
||||
data_source.get('port'),
|
||||
data_source.get('label') or UNSPECIFIED,
|
||||
data_source.get('container'),
|
||||
),
|
||||
)
|
||||
|
||||
logger.info(f'Restoring data source {dump_metadata}')
|
||||
@@ -180,6 +202,10 @@ def restore_single_dump(
|
||||
borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
|
||||
borgmatic_runtime_directory,
|
||||
data_source['name'],
|
||||
data_source.get('hostname'),
|
||||
data_source.get('port'),
|
||||
data_source.get('container'),
|
||||
data_source.get('label'),
|
||||
)[hook_name.split('_databases', 1)[0]]
|
||||
|
||||
destination_path = (
|
||||
@@ -197,8 +223,8 @@ def restore_single_dump(
|
||||
archive=archive_name,
|
||||
paths=[
|
||||
borgmatic.hooks.data_source.dump.convert_glob_patterns_to_borg_pattern(
|
||||
dump_patterns
|
||||
)
|
||||
dump_patterns,
|
||||
),
|
||||
],
|
||||
config=config,
|
||||
local_borg_version=local_borg_version,
|
||||
@@ -213,7 +239,8 @@ def restore_single_dump(
|
||||
|
||||
if destination_path and not global_arguments.dry_run:
|
||||
strip_path_prefix_from_extracted_dump_destination(
|
||||
destination_path, borgmatic_runtime_directory
|
||||
destination_path,
|
||||
borgmatic_runtime_directory,
|
||||
)
|
||||
finally:
|
||||
if destination_path and not global_arguments.dry_run:
|
||||
@@ -246,16 +273,12 @@ def collect_dumps_from_archive(
|
||||
Given a local or remote repository path, a resolved archive name, a configuration dict, the
|
||||
local Borg version, global arguments an argparse.Namespace, local and remote Borg paths, and the
|
||||
borgmatic runtime directory, query the archive for the names of data sources dumps it contains
|
||||
and return them as a set of Dump instances.
|
||||
and return them as a tuple of Dump instances.
|
||||
'''
|
||||
borgmatic_source_directory = str(
|
||||
pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config))
|
||||
)
|
||||
dumps_from_archive = {} # Use a dict as an ordered set.
|
||||
|
||||
# Probe for the data source dumps in multiple locations, as the default location has moved to
|
||||
# the borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But
|
||||
# we still want to support reading dumps from previously created archives as well.
|
||||
dump_paths = borgmatic.borg.list.capture_archive_listing(
|
||||
# There is (at most) one dump metadata file per data source hook. Load each.
|
||||
for dumps_metadata_entry in borgmatic.borg.list.capture_archive_listing(
|
||||
repository,
|
||||
archive,
|
||||
config,
|
||||
@@ -264,7 +287,69 @@ def collect_dumps_from_archive(
|
||||
list_paths=[
|
||||
'sh:'
|
||||
+ borgmatic.hooks.data_source.dump.make_data_source_dump_path(
|
||||
base_directory, '*_databases/*/*'
|
||||
base_directory,
|
||||
'*_databases/dumps.json',
|
||||
)
|
||||
# Probe for dump metadata files in multiple locations, as the default location is
|
||||
# "/borgmatic/*_databases/dumps.json" with Borg 1.4+, but instead begins with the
|
||||
# borgmatic runtime directory for older versions of Borg.
|
||||
for base_directory in (
|
||||
'borgmatic',
|
||||
borgmatic.config.paths.make_runtime_directory_glob(borgmatic_runtime_directory),
|
||||
)
|
||||
],
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
):
|
||||
if not dumps_metadata_entry.get('path'):
|
||||
continue
|
||||
|
||||
for dump in borgmatic.hooks.data_source.dump.parse_data_source_dumps_metadata(
|
||||
borgmatic.borg.extract.extract_archive(
|
||||
global_arguments.dry_run,
|
||||
repository,
|
||||
archive,
|
||||
[dumps_metadata_entry['path']],
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
extract_to_stdout=True,
|
||||
)
|
||||
.stdout.read()
|
||||
.decode(locale.getpreferredencoding()),
|
||||
dumps_metadata_entry['path'],
|
||||
):
|
||||
dumps_from_archive[dump] = None
|
||||
|
||||
# If we've successfully loaded any dumps metadata, we're done.
|
||||
if dumps_from_archive:
|
||||
logger.debug('Collecting database dumps from archive data source dumps metadata files')
|
||||
|
||||
return tuple(dumps_from_archive.keys())
|
||||
|
||||
# No dumps metadata files were found, so for backwards compatibility, fall back to parsing the
|
||||
# paths of dumps found in the archive to get their respective dump metadata.
|
||||
logger.debug('Collecting database dumps from archive data source dump paths (fallback)')
|
||||
borgmatic_source_directory = str(
|
||||
pathlib.Path(borgmatic.config.paths.get_borgmatic_source_directory(config)),
|
||||
)
|
||||
|
||||
# Probe for the data source dumps in multiple locations, as the default location has moved to
|
||||
# the borgmatic runtime directory (which gets stored as just "/borgmatic" with Borg 1.4+). But
|
||||
# we still want to support reading dumps from previously created archives as well.
|
||||
dump_entries = borgmatic.borg.list.capture_archive_listing(
|
||||
repository,
|
||||
archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
list_paths=[
|
||||
'sh:'
|
||||
+ borgmatic.hooks.data_source.dump.make_data_source_dump_path(
|
||||
base_directory,
|
||||
'*_databases/*/*',
|
||||
)
|
||||
for base_directory in (
|
||||
'borgmatic',
|
||||
@@ -276,10 +361,9 @@ def collect_dumps_from_archive(
|
||||
remote_path=remote_path,
|
||||
)
|
||||
|
||||
# Parse the paths of dumps found in the archive to get their respective dump metadata.
|
||||
dumps_from_archive = set()
|
||||
for dump_entry in dump_entries:
|
||||
dump_path = dump_entry.get('path')
|
||||
|
||||
for dump_path in dump_paths:
|
||||
if not dump_path:
|
||||
continue
|
||||
|
||||
@@ -291,7 +375,8 @@ def collect_dumps_from_archive(
|
||||
):
|
||||
try:
|
||||
(hook_name, host_and_port, data_source_name) = dump_path.split(
|
||||
base_directory + os.path.sep, 1
|
||||
base_directory + os.path.sep,
|
||||
1,
|
||||
)[1].split(os.path.sep)[0:3]
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
@@ -308,29 +393,34 @@ def collect_dumps_from_archive(
|
||||
except (ValueError, TypeError):
|
||||
port = None
|
||||
|
||||
dumps_from_archive.add(Dump(hook_name, data_source_name, hostname, port))
|
||||
dumps_from_archive[
|
||||
Dump(
|
||||
hook_name, data_source_name, None if hostname == 'localhost' else hostname, port
|
||||
)
|
||||
] = None
|
||||
|
||||
# We've successfully parsed the dump path, so need to probe any further.
|
||||
break
|
||||
else:
|
||||
logger.warning(
|
||||
f'Ignoring invalid data source dump path "{dump_path}" in archive {archive}'
|
||||
f'Ignoring invalid data source dump path "{dump_path}" in archive {archive}',
|
||||
)
|
||||
|
||||
return dumps_from_archive
|
||||
return tuple(dumps_from_archive.keys())
|
||||
|
||||
|
||||
def get_dumps_to_restore(restore_arguments, dumps_from_archive):
|
||||
'''
|
||||
Given restore arguments as an argparse.Namespace instance indicating which dumps to restore and
|
||||
a set of Dump instances representing the dumps found in an archive, return a set of specific
|
||||
a tuple of Dump instances representing the dumps found in an archive, return a tuple of specific
|
||||
Dump instances from the archive to restore. As part of this, replace any Dump having a data
|
||||
source name of "all" with multiple named Dump instances as appropriate.
|
||||
|
||||
Raise ValueError if any of the requested data source names cannot be found in the archive or if
|
||||
there are multiple archive dump matches for a given requested dump.
|
||||
'''
|
||||
requested_dumps = (
|
||||
requested_dumps = tuple(
|
||||
# Use a dict comprehension as an ordered set.
|
||||
{
|
||||
Dump(
|
||||
hook_name=(
|
||||
@@ -345,28 +435,35 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
|
||||
data_source_name=name,
|
||||
hostname=restore_arguments.original_hostname or UNSPECIFIED,
|
||||
port=restore_arguments.original_port,
|
||||
)
|
||||
label=restore_arguments.original_label or UNSPECIFIED,
|
||||
container=restore_arguments.original_container or UNSPECIFIED,
|
||||
): None
|
||||
for name in restore_arguments.data_sources or (UNSPECIFIED,)
|
||||
}
|
||||
}.keys()
|
||||
if restore_arguments.hook
|
||||
or restore_arguments.data_sources
|
||||
or restore_arguments.original_hostname
|
||||
or restore_arguments.original_port
|
||||
else {
|
||||
or restore_arguments.original_label
|
||||
or restore_arguments.original_container
|
||||
else (
|
||||
Dump(
|
||||
hook_name=UNSPECIFIED,
|
||||
data_source_name='all',
|
||||
hostname=UNSPECIFIED,
|
||||
port=UNSPECIFIED,
|
||||
)
|
||||
}
|
||||
label=UNSPECIFIED,
|
||||
container=UNSPECIFIED,
|
||||
),
|
||||
)
|
||||
)
|
||||
missing_dumps = set()
|
||||
dumps_to_restore = set()
|
||||
dumps_to_restore = {} # Use a dict as an ordered set.
|
||||
|
||||
# If there's a requested "all" dump, add every dump from the archive to the dumps to restore.
|
||||
if any(dump for dump in requested_dumps if dump.data_source_name == 'all'):
|
||||
dumps_to_restore.update(dumps_from_archive)
|
||||
for dump in dumps_from_archive:
|
||||
dumps_to_restore[dump] = None
|
||||
|
||||
# If any archive dump matches a requested dump, add the archive dump to the dumps to restore.
|
||||
for requested_dump in requested_dumps:
|
||||
@@ -382,10 +479,10 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
|
||||
if len(matching_dumps) == 0:
|
||||
missing_dumps.add(requested_dump)
|
||||
elif len(matching_dumps) == 1:
|
||||
dumps_to_restore.add(matching_dumps[0])
|
||||
dumps_to_restore[matching_dumps[0]] = None
|
||||
else:
|
||||
raise ValueError(
|
||||
f'Cannot restore data source {render_dump_metadata(requested_dump)} because there are multiple matching dumps in the archive. Try adding flags to disambiguate.'
|
||||
f'Cannot restore data source {render_dump_metadata(requested_dump)} because there are multiple matching dumps in the archive. Try adding flags to disambiguate.',
|
||||
)
|
||||
|
||||
if missing_dumps:
|
||||
@@ -394,30 +491,31 @@ def get_dumps_to_restore(restore_arguments, dumps_from_archive):
|
||||
)
|
||||
|
||||
raise ValueError(
|
||||
f"Cannot restore data source dump{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from archive"
|
||||
f"Cannot restore data source dump{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from archive",
|
||||
)
|
||||
|
||||
return dumps_to_restore
|
||||
return tuple(dumps_to_restore.keys())
|
||||
|
||||
|
||||
def ensure_requested_dumps_restored(dumps_to_restore, dumps_actually_restored):
|
||||
'''
|
||||
Given a set of requested dumps to restore and a set of dumps actually restored, raise ValueError
|
||||
if any requested dumps to restore weren't restored, indicating that they were missing from the
|
||||
configuration.
|
||||
Given a tuple of requested dumps to restore and a set of dumps actually restored, raise
|
||||
ValueError if any requested dumps to restore weren't restored, indicating that they were missing
|
||||
from the configuration.
|
||||
'''
|
||||
if not dumps_actually_restored:
|
||||
if not dumps_to_restore:
|
||||
raise ValueError('No data source dumps were found to restore')
|
||||
|
||||
missing_dumps = sorted(
|
||||
dumps_to_restore - dumps_actually_restored, key=lambda dump: dump.data_source_name
|
||||
set(dumps_to_restore) - dumps_actually_restored,
|
||||
key=lambda dump: dump.data_source_name,
|
||||
)
|
||||
|
||||
if missing_dumps:
|
||||
rendered_dumps = ', '.join(f'{render_dump_metadata(dump)}' for dump in missing_dumps)
|
||||
|
||||
raise ValueError(
|
||||
f"Cannot restore data source{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from borgmatic's configuration"
|
||||
f"Cannot restore data source{'s' if len(missing_dumps) > 1 else ''} {rendered_dumps} missing from borgmatic's configuration",
|
||||
)
|
||||
|
||||
|
||||
@@ -431,101 +529,96 @@ def run_restore(
|
||||
remote_path,
|
||||
):
|
||||
'''
|
||||
Run the "restore" action for the given repository, but only if the repository matches the
|
||||
requested repository in restore arguments.
|
||||
Run the "restore" action for the given repository.
|
||||
|
||||
Raise ValueError if a configured data source could not be found to restore or there's no
|
||||
matching dump in the archive.
|
||||
'''
|
||||
if restore_arguments.repository and not borgmatic.config.validate.repositories_match(
|
||||
repository, restore_arguments.repository
|
||||
):
|
||||
return
|
||||
|
||||
logger.info(f'Restoring data sources from archive {restore_arguments.archive}')
|
||||
working_directory = borgmatic.config.paths.get_working_directory(config)
|
||||
|
||||
with borgmatic.config.paths.Runtime_directory(config) as borgmatic_runtime_directory:
|
||||
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
|
||||
'remove_data_source_dumps',
|
||||
patterns = borgmatic.actions.pattern.process_patterns(
|
||||
borgmatic.actions.pattern.collect_patterns(config, working_directory),
|
||||
config,
|
||||
borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
|
||||
borgmatic_runtime_directory,
|
||||
global_arguments.dry_run,
|
||||
working_directory,
|
||||
)
|
||||
|
||||
archive_name = borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
restore_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
dumps_from_archive = collect_dumps_from_archive(
|
||||
repository['path'],
|
||||
archive_name,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
borgmatic_runtime_directory,
|
||||
)
|
||||
dumps_to_restore = get_dumps_to_restore(restore_arguments, dumps_from_archive)
|
||||
|
||||
dumps_actually_restored = set()
|
||||
connection_params = {
|
||||
'hostname': restore_arguments.hostname,
|
||||
'port': restore_arguments.port,
|
||||
'username': restore_arguments.username,
|
||||
'password': restore_arguments.password,
|
||||
'restore_path': restore_arguments.restore_path,
|
||||
}
|
||||
|
||||
# Restore each dump.
|
||||
for restore_dump in dumps_to_restore:
|
||||
found_data_source = get_configured_data_source(
|
||||
config,
|
||||
restore_dump,
|
||||
)
|
||||
|
||||
# For a dump that wasn't found via an exact match in the configuration, try to fallback
|
||||
# to an "all" data source.
|
||||
if not found_data_source:
|
||||
found_data_source = get_configured_data_source(
|
||||
config,
|
||||
Dump(restore_dump.hook_name, 'all', restore_dump.hostname, restore_dump.port),
|
||||
)
|
||||
|
||||
if not found_data_source:
|
||||
continue
|
||||
|
||||
found_data_source = dict(found_data_source)
|
||||
found_data_source['name'] = restore_dump.data_source_name
|
||||
|
||||
dumps_actually_restored.add(restore_dump)
|
||||
|
||||
restore_single_dump(
|
||||
repository,
|
||||
with borgmatic.actions.dump.Dump_cleanup(
|
||||
config, borgmatic_runtime_directory, patterns, global_arguments.dry_run
|
||||
):
|
||||
archive_name = borgmatic.borg.repo_list.resolve_archive_name(
|
||||
repository['path'],
|
||||
restore_arguments.archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
dumps_from_archive = collect_dumps_from_archive(
|
||||
repository['path'],
|
||||
archive_name,
|
||||
restore_dump.hook_name,
|
||||
dict(found_data_source, **{'schemas': restore_arguments.schemas}),
|
||||
connection_params,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
borgmatic_runtime_directory,
|
||||
)
|
||||
dumps_to_restore = get_dumps_to_restore(restore_arguments, dumps_from_archive)
|
||||
|
||||
borgmatic.hooks.dispatch.call_hooks_even_if_unconfigured(
|
||||
'remove_data_source_dumps',
|
||||
config,
|
||||
borgmatic.hooks.dispatch.Hook_type.DATA_SOURCE,
|
||||
borgmatic_runtime_directory,
|
||||
global_arguments.dry_run,
|
||||
)
|
||||
dumps_actually_restored = set()
|
||||
connection_params = {
|
||||
'container': restore_arguments.container,
|
||||
'hostname': restore_arguments.hostname,
|
||||
'port': restore_arguments.port,
|
||||
'username': restore_arguments.username,
|
||||
'password': restore_arguments.password,
|
||||
'restore_path': restore_arguments.restore_path,
|
||||
}
|
||||
|
||||
# Restore each dump.
|
||||
for restore_dump in dumps_to_restore:
|
||||
found_data_source = get_configured_data_source(
|
||||
config,
|
||||
restore_dump,
|
||||
)
|
||||
# For a dump that wasn't found via an exact match in the configuration, try to fallback
|
||||
# to an "all" data source.
|
||||
if not found_data_source:
|
||||
found_data_source = get_configured_data_source(
|
||||
config,
|
||||
Dump(
|
||||
restore_dump.hook_name,
|
||||
'all',
|
||||
restore_dump.hostname,
|
||||
restore_dump.port,
|
||||
restore_dump.label,
|
||||
restore_dump.container,
|
||||
),
|
||||
)
|
||||
|
||||
if not found_data_source:
|
||||
continue
|
||||
|
||||
found_data_source = dict(found_data_source)
|
||||
found_data_source['name'] = restore_dump.data_source_name
|
||||
|
||||
dumps_actually_restored.add(restore_dump)
|
||||
|
||||
restore_single_dump(
|
||||
repository,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
archive_name,
|
||||
restore_dump.hook_name,
|
||||
dict(found_data_source, schemas=restore_arguments.schemas),
|
||||
connection_params,
|
||||
borgmatic_runtime_directory,
|
||||
)
|
||||
|
||||
ensure_requested_dumps_restored(dumps_to_restore, dumps_actually_restored)
|
||||
|
||||
@@ -19,7 +19,7 @@ def run_transfer(
|
||||
'''
|
||||
if transfer_arguments.archive and config.get('match_archives'):
|
||||
raise ValueError(
|
||||
'With the transfer action, only one of --archive and --match-archives/match_archives can be used.'
|
||||
'With the transfer action, only one of --archive and --match-archives/match_archives can be used.',
|
||||
)
|
||||
|
||||
logger.info('Transferring archives to repository')
|
||||
|
||||
@@ -39,9 +39,9 @@ def run_arbitrary_borg(
|
||||
borg_command = tuple(options[:command_options_start_index])
|
||||
command_options = tuple(options[command_options_start_index:])
|
||||
|
||||
if borg_command and borg_command[0] in borgmatic.commands.arguments.ACTION_ALIASES.keys():
|
||||
if borg_command and borg_command[0] in borgmatic.commands.arguments.ACTION_ALIASES:
|
||||
logger.warning(
|
||||
f"Borg's {borg_command[0]} subcommand is supported natively by borgmatic. Try this instead: borgmatic {borg_command[0]}"
|
||||
f"Borg's {borg_command[0]} subcommand is supported natively by borgmatic. Try this instead: borgmatic {borg_command[0]}",
|
||||
)
|
||||
except IndexError:
|
||||
borg_command = ()
|
||||
@@ -57,16 +57,14 @@ def run_arbitrary_borg(
|
||||
+ command_options
|
||||
)
|
||||
|
||||
return execute_command(
|
||||
return execute_command( # noqa: S604
|
||||
tuple(shlex.quote(part) for part in full_command),
|
||||
output_file=DO_NOT_CAPTURE,
|
||||
shell=True, # noqa: S604
|
||||
shell=True,
|
||||
environment=dict(
|
||||
(environment.make_environment(config) or {}),
|
||||
**{
|
||||
'BORG_REPO': repository_path,
|
||||
'ARCHIVE': archive if archive else '',
|
||||
},
|
||||
BORG_REPO=repository_path,
|
||||
ARCHIVE=archive if archive else '',
|
||||
),
|
||||
working_directory=borgmatic.config.paths.get_working_directory(config),
|
||||
borg_local_path=local_path,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
import shlex
|
||||
|
||||
import borgmatic.config.paths
|
||||
from borgmatic.borg import environment, flags
|
||||
@@ -22,15 +23,17 @@ def break_lock(
|
||||
'''
|
||||
umask = config.get('umask', None)
|
||||
lock_wait = config.get('lock_wait', None)
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get('break_lock', '')
|
||||
|
||||
full_command = (
|
||||
(local_path, 'break-lock')
|
||||
+ (('--remote-path', remote_path) if remote_path else ())
|
||||
+ (('--umask', str(umask)) if umask else ())
|
||||
+ (('--log-json',) if config.get('log_json') else ())
|
||||
+ ('--log-json',)
|
||||
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
||||
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
|
||||
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ flags.make_repository_flags(repository_path, local_borg_version)
|
||||
)
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
import shlex
|
||||
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.execute
|
||||
@@ -25,15 +26,16 @@ def change_passphrase(
|
||||
borgmatic.logger.add_custom_log_levels()
|
||||
umask = config.get('umask', None)
|
||||
lock_wait = config.get('lock_wait', None)
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get('key_change_passphrase', '')
|
||||
|
||||
full_command = (
|
||||
(local_path, 'key', 'change-passphrase')
|
||||
+ (('--remote-path', remote_path) if remote_path else ())
|
||||
+ (('--umask', str(umask)) if umask else ())
|
||||
+ (('--log-json',) if config.get('log_json') else ())
|
||||
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
||||
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
|
||||
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ flags.make_repository_flags(
|
||||
repository_path,
|
||||
local_borg_version,
|
||||
@@ -49,7 +51,7 @@ def change_passphrase(
|
||||
config_without_passphrase = {
|
||||
option_name: value
|
||||
for (option_name, value) in config.items()
|
||||
if option_name not in ('encryption_passphrase', 'encryption_passcommand')
|
||||
if option_name not in {'encryption_passphrase', 'encryption_passcommand'}
|
||||
}
|
||||
|
||||
borgmatic.execute.execute_command(
|
||||
@@ -63,5 +65,5 @@ def change_passphrase(
|
||||
)
|
||||
|
||||
logger.answer(
|
||||
f"{repository_path}: Don't forget to update your encryption_passphrase option (if needed)"
|
||||
"Don't forget to update your encryption_passphrase option (if needed)",
|
||||
)
|
||||
|
||||
+23
-7
@@ -42,11 +42,12 @@ def make_archive_filter_flags(local_borg_version, config, checks, check_argument
|
||||
|
||||
if check_last:
|
||||
logger.warning(
|
||||
'Ignoring check_last option, as "archives" or "data" are not in consistency checks'
|
||||
'Ignoring check_last option, as "archives" or "data" are not in consistency checks',
|
||||
)
|
||||
|
||||
if prefix:
|
||||
logger.warning(
|
||||
'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks'
|
||||
'Ignoring consistency prefix option, as "archives" or "data" are not in consistency checks',
|
||||
)
|
||||
|
||||
return ()
|
||||
@@ -76,13 +77,18 @@ def make_check_name_flags(checks, archive_filter_flags):
|
||||
return common_flags
|
||||
|
||||
return (
|
||||
tuple(f'--{check}-only' for check in checks if check in ('repository', 'archives'))
|
||||
tuple(f'--{check}-only' for check in checks if check in {'repository', 'archives'})
|
||||
+ common_flags
|
||||
)
|
||||
|
||||
|
||||
def get_repository_id(
|
||||
repository_path, config, local_borg_version, global_arguments, local_path, remote_path
|
||||
repository_path,
|
||||
config,
|
||||
local_borg_version,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
):
|
||||
'''
|
||||
Given a local or remote repository path, a configuration dict, the local Borg version, global
|
||||
@@ -100,7 +106,7 @@ def get_repository_id(
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
),
|
||||
)['repository']['id']
|
||||
except (json.JSONDecodeError, KeyError):
|
||||
raise ValueError(f'Cannot determine Borg repository ID for {repository_path}')
|
||||
@@ -127,8 +133,10 @@ def check_archives(
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get('check', '')
|
||||
|
||||
verbosity_flags = ()
|
||||
|
||||
if logger.isEnabledFor(logging.INFO):
|
||||
verbosity_flags = ('--info',)
|
||||
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
verbosity_flags = ('--debug', '--show-rc')
|
||||
|
||||
@@ -141,8 +149,10 @@ def check_archives(
|
||||
|
||||
max_duration = check_arguments.max_duration or repository_check_config.get('max_duration')
|
||||
|
||||
# If not configured, elevate Borg's exit code 1 (an ostensible warning) to error, because Borg
|
||||
# returns exit code 1 for repository check errors!
|
||||
borg_exit_codes = [*config.get('borg_exit_codes', []), *[{'code': 1, 'treat_as': 'error'}]]
|
||||
umask = config.get('umask')
|
||||
borg_exit_codes = config.get('borg_exit_codes')
|
||||
working_directory = borgmatic.config.paths.get_working_directory(config)
|
||||
|
||||
if 'data' in checks:
|
||||
@@ -168,7 +178,13 @@ def check_archives(
|
||||
+ make_check_name_flags(checks_subset, archive_filter_flags)
|
||||
+ (('--remote-path', remote_path) if remote_path else ())
|
||||
+ (('--umask', str(umask)) if umask else ())
|
||||
+ (('--log-json',) if config.get('log_json') else ())
|
||||
+ (
|
||||
('--log-json',)
|
||||
if (
|
||||
config.get('log_json') or not (check_arguments.repair or config.get('progress'))
|
||||
)
|
||||
else ()
|
||||
)
|
||||
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
||||
+ verbosity_flags
|
||||
+ (('--progress',) if config.get('progress') else ())
|
||||
|
||||
@@ -31,7 +31,7 @@ def compact_segments(
|
||||
(local_path, 'compact')
|
||||
+ (('--remote-path', remote_path) if remote_path else ())
|
||||
+ (('--umask', str(umask)) if umask else ())
|
||||
+ (('--log-json',) if config.get('log_json') else ())
|
||||
+ (('--log-json',) if (config.get('log_json') or not config.get('progress')) else ())
|
||||
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
||||
+ (('--progress',) if config.get('progress') else ())
|
||||
+ (('--cleanup-commits',) if cleanup_commits else ())
|
||||
@@ -48,7 +48,7 @@ def compact_segments(
|
||||
)
|
||||
|
||||
if dry_run and not feature.available(feature.Feature.DRY_RUN_COMPACT, local_borg_version):
|
||||
logging.info('Skipping compact (dry run)')
|
||||
logger.info('Skipping compact (dry run)')
|
||||
return
|
||||
|
||||
execute_command(
|
||||
|
||||
+181
-107
@@ -45,35 +45,46 @@ def any_parent_directories(path, candidate_parents):
|
||||
return False
|
||||
|
||||
|
||||
def collect_special_file_paths(
|
||||
def validate_planned_backup_paths(
|
||||
dry_run,
|
||||
create_command,
|
||||
config,
|
||||
patterns,
|
||||
local_path,
|
||||
working_directory,
|
||||
borgmatic_runtime_directory,
|
||||
find_special_files=False,
|
||||
):
|
||||
'''
|
||||
Given a dry-run flag, a Borg create command as a tuple, a configuration dict, a local Borg path,
|
||||
a working directory, and the borgmatic runtime directory, collect the paths for any special
|
||||
files (character devices, block devices, and named pipes / FIFOs) that Borg would encounter
|
||||
during a create. These are all paths that could cause Borg to hang if its --read-special flag is
|
||||
used.
|
||||
a working directory, and the borgmatic runtime directory, perform a "borg create --dry-run" to
|
||||
determine whether Borg's planned paths to include in a backup look good. Specifically, if the
|
||||
given runtime directory exists, validate that it will be included in a backup and hasn't been
|
||||
excluded.
|
||||
|
||||
Skip looking for special files in the given borgmatic runtime directory, as borgmatic creates
|
||||
its own special files there for database dumps and we don't want those omitted.
|
||||
If find special files is True, then return the subset of planned backup paths that are special
|
||||
files. Otherwise, return an empty tuple.
|
||||
|
||||
Additionally, if the borgmatic runtime directory is not contained somewhere in the files Borg
|
||||
plans to backup, that means the user must have excluded the runtime directory (e.g. via
|
||||
"exclude_patterns" or similar). Therefore, raise, because this means Borg won't be able to
|
||||
consume any database dumps and therefore borgmatic will hang when it tries to do so.
|
||||
Raise ValueError if the runtime directory has been excluded via "exclude_patterns" or similar,
|
||||
because any features that rely on the runtime directory getting backed up will break. For
|
||||
instance, without the runtime directory, Borg can't consume any database dumps and borgmatic may
|
||||
hang waiting for them to be consumed.
|
||||
'''
|
||||
# Omit "--exclude-nodump" from the Borg dry run command, because that flag causes Borg to open
|
||||
# files including any named pipe we've created. And omit "--filter" because that can break the
|
||||
# paths output parsing below such that path lines no longer start with the expected "- ".
|
||||
paths_output = execute_command_and_capture_output(
|
||||
flags.omit_flag_and_value(flags.omit_flag(create_command, '--exclude-nodump'), '--filter')
|
||||
+ ('--dry-run', '--list'),
|
||||
path_lines = execute_command_and_capture_output(
|
||||
(
|
||||
*flags.omit_flag_and_value(
|
||||
flags.omit_flag(
|
||||
flags.omit_flag(create_command, '--exclude-nodump'),
|
||||
'--log-json',
|
||||
),
|
||||
'--filter',
|
||||
),
|
||||
'--dry-run',
|
||||
'--list',
|
||||
),
|
||||
capture_stderr=True,
|
||||
working_directory=working_directory,
|
||||
environment=environment.make_environment(config),
|
||||
@@ -83,38 +94,68 @@ def collect_special_file_paths(
|
||||
|
||||
# These are all the individual files that Borg is planning to backup as determined by the Borg
|
||||
# create dry run above.
|
||||
paths = tuple(
|
||||
paths = (
|
||||
path_line.split(' ', 1)[1]
|
||||
for path_line in paths_output.split('\n')
|
||||
if path_line and path_line.startswith('- ') or path_line.startswith('+ ')
|
||||
for path_line in path_lines
|
||||
if path_line and path_line.startswith(('- ', '+ '))
|
||||
)
|
||||
|
||||
# These are the subset of those files that contain the borgmatic runtime directory.
|
||||
paths_containing_runtime_directory = {}
|
||||
|
||||
if os.path.exists(borgmatic_runtime_directory):
|
||||
paths_containing_runtime_directory = {
|
||||
path for path in paths if any_parent_directories(path, (borgmatic_runtime_directory,))
|
||||
}
|
||||
|
||||
# If no paths to backup contain the runtime directory, it must've been excluded.
|
||||
if not paths_containing_runtime_directory and not dry_run:
|
||||
raise ValueError(
|
||||
f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes or patterns with excludes. Please ensure the runtime directory is not excluded.'
|
||||
)
|
||||
|
||||
return tuple(
|
||||
path
|
||||
for path in paths
|
||||
if special_file(path, working_directory)
|
||||
if path not in paths_containing_runtime_directory
|
||||
include_pattern_paths = {
|
||||
pattern.path
|
||||
for pattern in patterns
|
||||
if pattern.type == borgmatic.borg.pattern.Pattern_type.INCLUDE
|
||||
}
|
||||
runtime_directory_root_patterns = tuple(
|
||||
pattern
|
||||
for pattern in patterns
|
||||
if any_parent_directories(pattern.path, (borgmatic_runtime_directory,))
|
||||
if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
|
||||
# Skip root patterns that have corresponding include patterns, because those will "punch
|
||||
# through" any subsequent excludes.
|
||||
if pattern.path not in include_pattern_paths
|
||||
)
|
||||
|
||||
special_paths = []
|
||||
validate_runtime_directory = bool(
|
||||
not dry_run
|
||||
and os.path.exists(borgmatic_runtime_directory)
|
||||
and runtime_directory_root_patterns
|
||||
)
|
||||
runtime_directory_in_path = False
|
||||
|
||||
# Do everything in this one loop because we only want to consume the paths generator once.
|
||||
for path in paths:
|
||||
# If all root patterns in the runtime directory are missing from the paths Borg is planning
|
||||
# to backup, then they must've gotten excluded, e.g. by user-configured excludes. Warn
|
||||
# accordingly (below).
|
||||
if validate_runtime_directory and any(
|
||||
any_parent_directories(path, (pattern.path,))
|
||||
for pattern in runtime_directory_root_patterns
|
||||
):
|
||||
runtime_directory_in_path = True
|
||||
|
||||
# Return the subset of output paths that are special files but *not* contained within the
|
||||
# borgmatic runtime directory. The intent is to skip runtime paths that borgmatic uses for its
|
||||
# own bookkeeping, instead focusing on user-configured paths.
|
||||
if (
|
||||
find_special_files
|
||||
and not any_parent_directories(path, (borgmatic_runtime_directory,))
|
||||
and special_file(path, working_directory)
|
||||
):
|
||||
special_paths.append(path)
|
||||
|
||||
if validate_runtime_directory and not runtime_directory_in_path:
|
||||
logger.warning(
|
||||
f'The runtime directory {os.path.normpath(borgmatic_runtime_directory)} overlaps with the configured excludes (or the snapshotted source directories are empty). Please ensure the runtime directory is not excluded.'
|
||||
)
|
||||
|
||||
return tuple(special_paths)
|
||||
|
||||
|
||||
MAX_SPECIAL_FILE_PATHS_LENGTH = 1000
|
||||
|
||||
|
||||
def make_base_create_command(
|
||||
def make_base_create_command( # noqa: PLR0912
|
||||
dry_run,
|
||||
repository_path,
|
||||
config,
|
||||
@@ -138,11 +179,14 @@ def make_base_create_command(
|
||||
return a tuple of (base Borg create command flags, Borg create command positional arguments,
|
||||
open pattern file handle).
|
||||
'''
|
||||
if config.get('source_directories_must_exist', False):
|
||||
borgmatic.borg.pattern.check_all_root_patterns_exist(patterns)
|
||||
if config.get('source_directories_must_exist', True):
|
||||
borgmatic.borg.pattern.check_all_root_patterns_exist(
|
||||
patterns, borgmatic.config.paths.get_working_directory(config)
|
||||
)
|
||||
|
||||
patterns_file = borgmatic.borg.pattern.write_patterns_file(
|
||||
patterns, borgmatic_runtime_directory
|
||||
patterns,
|
||||
borgmatic_runtime_directory,
|
||||
)
|
||||
checkpoint_interval = config.get('checkpoint_interval', None)
|
||||
checkpoint_volume = config.get('checkpoint_volume', None)
|
||||
@@ -153,6 +197,7 @@ def make_base_create_command(
|
||||
umask = config.get('umask', None)
|
||||
lock_wait = config.get('lock_wait', None)
|
||||
list_filter_flags = flags.make_list_filter_flags(local_borg_version, dry_run)
|
||||
files_changed = config.get('files_changed')
|
||||
files_cache = config.get('files_cache')
|
||||
archive_name_format = (
|
||||
config.get('archive_name_format', flags.get_default_archive_name_format(local_borg_version))
|
||||
@@ -203,11 +248,12 @@ def make_base_create_command(
|
||||
+ (('--nobirthtime',) if config.get('birthtime') is False else ())
|
||||
+ (('--read-special',) if config.get('read_special') or stream_processes else ())
|
||||
+ noflags_flags
|
||||
+ (('--files-changed', files_changed) if files_changed else ())
|
||||
+ (('--files-cache', files_cache) if files_cache else ())
|
||||
+ (('--remote-path', remote_path) if remote_path else ())
|
||||
+ (('--umask', str(umask)) if umask else ())
|
||||
+ (('--log-json',) if config.get('log_json') else ())
|
||||
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
||||
+ (('--log-json',) if (config.get('log_json') or not config.get('progress')) else ())
|
||||
+ (
|
||||
('--list', '--filter', list_filter_flags)
|
||||
if config.get('list_details') and not json and not config.get('progress')
|
||||
@@ -218,52 +264,64 @@ def make_base_create_command(
|
||||
)
|
||||
|
||||
create_positional_arguments = flags.make_repository_archive_flags(
|
||||
repository_path, archive_name_format, local_borg_version
|
||||
repository_path,
|
||||
archive_name_format,
|
||||
local_borg_version,
|
||||
)
|
||||
working_directory = borgmatic.config.paths.get_working_directory(config)
|
||||
|
||||
if config.get('unsafe_skip_path_validation_before_create'):
|
||||
logger.debug(
|
||||
'Skipping pre-backup path validation due to "unsafe_skip_path_validation_before_create" option.'
|
||||
)
|
||||
|
||||
return (create_flags, create_positional_arguments, patterns_file)
|
||||
|
||||
logger.debug('Checking file paths Borg plans to include')
|
||||
|
||||
special_file_paths = validate_planned_backup_paths(
|
||||
dry_run,
|
||||
create_flags + create_positional_arguments,
|
||||
config,
|
||||
patterns,
|
||||
local_path,
|
||||
working_directory,
|
||||
borgmatic_runtime_directory=borgmatic_runtime_directory,
|
||||
find_special_files=bool(stream_processes),
|
||||
)
|
||||
|
||||
if stream_processes and config.get('read_special') is False:
|
||||
logger.warning(
|
||||
'Ignoring configured "read_special" value of false, as true is needed for database hooks.',
|
||||
)
|
||||
|
||||
# If database hooks are enabled (as indicated by streaming processes), exclude files that might
|
||||
# cause Borg to hang. But skip this if the user has explicitly set the "read_special" to True.
|
||||
if stream_processes and not config.get('read_special'):
|
||||
if special_file_paths:
|
||||
truncated_special_file_paths = textwrap.shorten(
|
||||
', '.join(special_file_paths),
|
||||
width=MAX_SPECIAL_FILE_PATHS_LENGTH,
|
||||
placeholder=' ...',
|
||||
)
|
||||
logger.warning(
|
||||
'Ignoring configured "read_special" value of false, as true is needed for database hooks.'
|
||||
f'Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}',
|
||||
)
|
||||
working_directory = borgmatic.config.paths.get_working_directory(config)
|
||||
|
||||
logger.debug('Collecting special file paths')
|
||||
special_file_paths = collect_special_file_paths(
|
||||
dry_run,
|
||||
create_flags + create_positional_arguments,
|
||||
config,
|
||||
local_path,
|
||||
working_directory,
|
||||
borgmatic_runtime_directory=borgmatic_runtime_directory,
|
||||
patterns_file = borgmatic.borg.pattern.write_patterns_file(
|
||||
tuple(
|
||||
borgmatic.borg.pattern.Pattern(
|
||||
special_file_path,
|
||||
borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
|
||||
borgmatic.borg.pattern.Pattern_style.FNMATCH,
|
||||
source=borgmatic.borg.pattern.Pattern_source.INTERNAL,
|
||||
)
|
||||
for special_file_path in special_file_paths
|
||||
),
|
||||
borgmatic_runtime_directory,
|
||||
patterns_file=patterns_file,
|
||||
)
|
||||
|
||||
if special_file_paths:
|
||||
truncated_special_file_paths = textwrap.shorten(
|
||||
', '.join(special_file_paths),
|
||||
width=MAX_SPECIAL_FILE_PATHS_LENGTH,
|
||||
placeholder=' ...',
|
||||
)
|
||||
logger.warning(
|
||||
f'Excluding special files to prevent Borg from hanging: {truncated_special_file_paths}'
|
||||
)
|
||||
patterns_file = borgmatic.borg.pattern.write_patterns_file(
|
||||
tuple(
|
||||
borgmatic.borg.pattern.Pattern(
|
||||
special_file_path,
|
||||
borgmatic.borg.pattern.Pattern_type.NO_RECURSE,
|
||||
borgmatic.borg.pattern.Pattern_style.FNMATCH,
|
||||
source=borgmatic.borg.pattern.Pattern_source.INTERNAL,
|
||||
)
|
||||
for special_file_path in special_file_paths
|
||||
),
|
||||
borgmatic_runtime_directory,
|
||||
patterns_file=patterns_file,
|
||||
)
|
||||
|
||||
if '--patterns-from' not in create_flags:
|
||||
create_flags += ('--patterns-from', patterns_file.name)
|
||||
if '--patterns-from' not in create_flags:
|
||||
create_flags += ('--patterns-from', patterns_file.name)
|
||||
|
||||
return (create_flags, create_positional_arguments, patterns_file)
|
||||
|
||||
@@ -298,7 +356,7 @@ def create_archive(
|
||||
|
||||
working_directory = borgmatic.config.paths.get_working_directory(config)
|
||||
|
||||
(create_flags, create_positional_arguments, patterns_file) = make_base_create_command(
|
||||
(create_flags, create_positional_arguments, _) = make_base_create_command(
|
||||
dry_run,
|
||||
repository_path,
|
||||
config,
|
||||
@@ -316,7 +374,9 @@ def create_archive(
|
||||
|
||||
if json:
|
||||
output_log_level = None
|
||||
elif config.get('list_details') or (config.get('statistics') and not dry_run):
|
||||
elif config.get('list_details') or (
|
||||
(config.get('statistics') or config.get('quick_statistics')) and not dry_run
|
||||
):
|
||||
output_log_level = logging.ANSWER
|
||||
else:
|
||||
output_log_level = logging.INFO
|
||||
@@ -328,6 +388,11 @@ def create_archive(
|
||||
create_flags += (
|
||||
(('--info',) if logger.getEffectiveLevel() == logging.INFO and not json else ())
|
||||
+ (('--stats',) if config.get('statistics') and not json and not dry_run else ())
|
||||
+ (
|
||||
('--quick-stats',)
|
||||
if config.get('quick_statistics') and not json and not dry_run
|
||||
else ()
|
||||
)
|
||||
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) and not json else ())
|
||||
+ (('--progress',) if config.get('progress') else ())
|
||||
+ (('--json',) if json else ())
|
||||
@@ -335,31 +400,40 @@ def create_archive(
|
||||
borg_exit_codes = config.get('borg_exit_codes')
|
||||
|
||||
if stream_processes:
|
||||
return execute_command_with_processes(
|
||||
create_flags + create_positional_arguments,
|
||||
stream_processes,
|
||||
output_log_level,
|
||||
output_file,
|
||||
working_directory=working_directory,
|
||||
environment=environment.make_environment(config),
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
output = '\n'.join(
|
||||
execute_command_with_processes(
|
||||
create_flags + create_positional_arguments,
|
||||
stream_processes,
|
||||
output_log_level,
|
||||
output_file,
|
||||
working_directory=working_directory,
|
||||
environment=environment.make_environment(config),
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
)
|
||||
elif output_log_level is None:
|
||||
return execute_command_and_capture_output(
|
||||
create_flags + create_positional_arguments,
|
||||
working_directory=working_directory,
|
||||
environment=environment.make_environment(config),
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
else:
|
||||
execute_command(
|
||||
create_flags + create_positional_arguments,
|
||||
output_log_level,
|
||||
output_file,
|
||||
working_directory=working_directory,
|
||||
environment=environment.make_environment(config),
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
return output if json else None
|
||||
|
||||
if output_log_level is None:
|
||||
output = '\n'.join(
|
||||
execute_command_and_capture_output(
|
||||
create_flags + create_positional_arguments,
|
||||
working_directory=working_directory,
|
||||
environment=environment.make_environment(config),
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
)
|
||||
return output if json else None
|
||||
|
||||
execute_command(
|
||||
create_flags + create_positional_arguments,
|
||||
output_log_level,
|
||||
output_file,
|
||||
working_directory=working_directory,
|
||||
environment=environment.make_environment(config),
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import argparse
|
||||
import logging
|
||||
import shlex
|
||||
|
||||
import borgmatic.borg.environment
|
||||
import borgmatic.borg.feature
|
||||
@@ -11,6 +12,9 @@ import borgmatic.execute
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
FORCE_HARDER_FLAG_COUNT = 2
|
||||
|
||||
|
||||
def make_delete_command(
|
||||
repository,
|
||||
config,
|
||||
@@ -25,6 +29,8 @@ def make_delete_command(
|
||||
arguments to the delete action as an argparse.Namespace, and global arguments, return a command
|
||||
as a tuple to delete archives from the repository.
|
||||
'''
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get('delete', '')
|
||||
|
||||
return (
|
||||
(local_path, 'delete')
|
||||
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
|
||||
@@ -32,11 +38,14 @@ def make_delete_command(
|
||||
+ borgmatic.borg.flags.make_flags('dry-run', global_arguments.dry_run)
|
||||
+ borgmatic.borg.flags.make_flags('remote-path', remote_path)
|
||||
+ borgmatic.borg.flags.make_flags('umask', config.get('umask'))
|
||||
+ borgmatic.borg.flags.make_flags('log-json', config.get('log_json'))
|
||||
+ ('--log-json',)
|
||||
+ borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
|
||||
+ borgmatic.borg.flags.make_flags('list', config.get('list_details'))
|
||||
+ (
|
||||
(('--force',) + (('--force',) if delete_arguments.force >= 2 else ()))
|
||||
(
|
||||
('--force',)
|
||||
+ (('--force',) if delete_arguments.force >= FORCE_HARDER_FLAG_COUNT else ())
|
||||
)
|
||||
if delete_arguments.force
|
||||
else ()
|
||||
)
|
||||
@@ -60,6 +69,7 @@ def make_delete_command(
|
||||
'repository',
|
||||
),
|
||||
)
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version)
|
||||
)
|
||||
|
||||
@@ -98,10 +108,11 @@ def delete_archives(
|
||||
for argument_name in ARCHIVE_RELATED_ARGUMENT_NAMES
|
||||
):
|
||||
if borgmatic.borg.feature.available(
|
||||
borgmatic.borg.feature.Feature.REPO_DELETE, local_borg_version
|
||||
borgmatic.borg.feature.Feature.REPO_DELETE,
|
||||
local_borg_version,
|
||||
):
|
||||
logger.warning(
|
||||
'Deleting an entire repository with the delete action is deprecated when using Borg 2.x+. Use the repo-delete action instead.'
|
||||
'Deleting an entire repository with the delete action is deprecated when using Borg 2.x+. Use the repo-delete action instead.',
|
||||
)
|
||||
|
||||
repo_delete_arguments = argparse.Namespace(
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import logging
|
||||
import shlex
|
||||
|
||||
import borgmatic.borg.environment
|
||||
import borgmatic.borg.feature
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.execute
|
||||
from borgmatic.borg import flags
|
||||
from borgmatic.borg.pattern import write_patterns_file
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def diff(
|
||||
repository,
|
||||
archive,
|
||||
second_archive,
|
||||
config,
|
||||
local_borg_version,
|
||||
diff_arguments,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path=None,
|
||||
patterns=None,
|
||||
):
|
||||
'''
|
||||
Given a local or remote repository path, two archive names, a configuration dict, the local Borg
|
||||
version string, an argparse.Namespace of diff arguments, an argparse.Namespace of global
|
||||
arguments, optional local and remote Borg paths, executes the diff command with the given
|
||||
arguments.
|
||||
'''
|
||||
borgmatic.logger.add_custom_log_levels()
|
||||
|
||||
lock_wait = config.get('lock_wait', None)
|
||||
exclude_flags = flags.make_exclude_flags(config)
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get('diff', '')
|
||||
|
||||
if diff_arguments.only_patterns:
|
||||
# Write patterns to a temporary file and use that file with --patterns-from.
|
||||
patterns_file = write_patterns_file(
|
||||
patterns,
|
||||
borgmatic.config.paths.get_working_directory(config),
|
||||
)
|
||||
else:
|
||||
patterns_file = None
|
||||
|
||||
if borgmatic.borg.feature.available(
|
||||
borgmatic.borg.feature.Feature.NUMERIC_IDS, local_borg_version
|
||||
):
|
||||
numeric_ids_flags = ('--numeric-ids',) if config.get('numeric_ids') else ()
|
||||
else:
|
||||
numeric_ids_flags = ('--numeric-owner',) if config.get('numeric_ids') else ()
|
||||
|
||||
diff_command = (
|
||||
(local_path, 'diff')
|
||||
+ (('--remote-path', remote_path) if remote_path else ())
|
||||
+ ('--log-json',)
|
||||
+ (('--lock-wait', str(lock_wait)) if lock_wait is not None else ())
|
||||
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
|
||||
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
|
||||
+ (
|
||||
('--patterns-from', patterns_file.name)
|
||||
if patterns_file and diff_arguments.only_patterns
|
||||
else ()
|
||||
)
|
||||
+ exclude_flags
|
||||
+ numeric_ids_flags
|
||||
+ (('--same-chunker-params',) if diff_arguments.same_chunker_params else ())
|
||||
+ (('--sort-by', ','.join(diff_arguments.sort_keys)) if diff_arguments.sort_keys else ())
|
||||
+ (('--content-only',) if diff_arguments.content_only else ())
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ (
|
||||
(*flags.make_repository_flags(repository, local_borg_version), archive)
|
||||
if borgmatic.borg.feature.available(
|
||||
borgmatic.borg.feature.Feature.SEPARATE_REPOSITORY_ARCHIVE,
|
||||
local_borg_version,
|
||||
)
|
||||
else flags.make_repository_archive_flags(repository, archive, local_borg_version)
|
||||
)
|
||||
+ (second_archive,)
|
||||
)
|
||||
|
||||
borgmatic.execute.execute_command(
|
||||
full_command=diff_command,
|
||||
output_log_level=logging.ANSWER,
|
||||
environment=borgmatic.borg.environment.make_environment(config),
|
||||
working_directory=borgmatic.config.paths.get_working_directory(config),
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=config.get('borg_exit_codes'),
|
||||
)
|
||||
@@ -10,6 +10,7 @@ OPTION_TO_ENVIRONMENT_VARIABLE = {
|
||||
'borg_files_cache_ttl': 'BORG_FILES_CACHE_TTL',
|
||||
'borg_security_directory': 'BORG_SECURITY_DIR',
|
||||
'borg_keys_directory': 'BORG_KEYS_DIR',
|
||||
'borg_key_file': 'BORG_KEY_FILE',
|
||||
'ssh_command': 'BORG_RSH',
|
||||
'temporary_directory': 'TMPDIR',
|
||||
}
|
||||
@@ -24,6 +25,7 @@ DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE = {
|
||||
'relocated_repo_access_is_ok': 'BORG_RELOCATED_REPO_ACCESS_IS_OK',
|
||||
'unknown_unencrypted_repo_access_is_ok': 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK',
|
||||
'use_chunks_archive': 'BORG_USE_CHUNKS_ARCHIVE',
|
||||
'msgpack_version_check': 'BORG_MSGPACK_VERSION_CHECK',
|
||||
}
|
||||
|
||||
|
||||
@@ -64,7 +66,8 @@ def make_environment(config):
|
||||
environment.pop('BORG_PASSCOMMAND', None)
|
||||
|
||||
passphrase = borgmatic.hooks.credential.parse.resolve_credential(
|
||||
config.get('encryption_passphrase'), config
|
||||
config.get('encryption_passphrase'),
|
||||
config,
|
||||
)
|
||||
|
||||
if passphrase is None:
|
||||
@@ -88,7 +91,8 @@ def make_environment(config):
|
||||
) in DEFAULT_BOOL_OPTION_TO_ENVIRONMENT_VARIABLE.items():
|
||||
if os.environ.get(environment_variable_name) is None:
|
||||
value = config.get(option_name)
|
||||
environment[environment_variable_name] = 'YES' if value else 'NO'
|
||||
if value is not None:
|
||||
environment[environment_variable_name] = 'YES' if value else 'NO'
|
||||
|
||||
for (
|
||||
option_name,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
import os
|
||||
import shlex
|
||||
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.logger
|
||||
@@ -31,11 +32,12 @@ def export_key(
|
||||
umask = config.get('umask', None)
|
||||
lock_wait = config.get('lock_wait', None)
|
||||
working_directory = borgmatic.config.paths.get_working_directory(config)
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get('key_export', '')
|
||||
|
||||
if export_arguments.path and export_arguments.path != '-':
|
||||
if os.path.exists(os.path.join(working_directory or '', export_arguments.path)):
|
||||
raise FileExistsError(
|
||||
f'Destination path {export_arguments.path} already exists. Aborting.'
|
||||
f'Destination path {export_arguments.path} already exists. Aborting.',
|
||||
)
|
||||
|
||||
output_file = None
|
||||
@@ -46,12 +48,13 @@ def export_key(
|
||||
(local_path, 'key', 'export')
|
||||
+ (('--remote-path', remote_path) if remote_path else ())
|
||||
+ (('--umask', str(umask)) if umask else ())
|
||||
+ (('--log-json',) if config.get('log_json') else ())
|
||||
+ (('--log-json',) if output_file is None else ())
|
||||
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
||||
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
|
||||
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
|
||||
+ flags.make_flags('paper', export_arguments.paper)
|
||||
+ flags.make_flags('qr-html', export_arguments.qr_html)
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ flags.make_repository_flags(
|
||||
repository_path,
|
||||
local_borg_version,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
import shlex
|
||||
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.logger
|
||||
@@ -34,12 +35,13 @@ def export_tar_archive(
|
||||
borgmatic.logger.add_custom_log_levels()
|
||||
umask = config.get('umask', None)
|
||||
lock_wait = config.get('lock_wait', None)
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get('export_tar', '')
|
||||
|
||||
full_command = (
|
||||
(local_path, 'export-tar')
|
||||
+ (('--remote-path', remote_path) if remote_path else ())
|
||||
+ (('--umask', str(umask)) if umask else ())
|
||||
+ (('--log-json',) if config.get('log_json') else ())
|
||||
+ (('--log-json',) if destination_path != '-' else ())
|
||||
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
||||
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
|
||||
+ (('--list',) if config.get('list_details') else ())
|
||||
@@ -47,6 +49,7 @@ def export_tar_archive(
|
||||
+ (('--dry-run',) if dry_run else ())
|
||||
+ (('--tar-filter', tar_filter) if tar_filter else ())
|
||||
+ (('--strip-components', str(strip_components)) if strip_components else ())
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ flags.make_repository_archive_flags(
|
||||
repository_path,
|
||||
archive,
|
||||
@@ -56,13 +59,10 @@ def export_tar_archive(
|
||||
+ (tuple(paths) if paths else ())
|
||||
)
|
||||
|
||||
if config.get('list_details'):
|
||||
output_log_level = logging.ANSWER
|
||||
else:
|
||||
output_log_level = logging.INFO
|
||||
output_log_level = logging.ANSWER if config.get('list_details') else logging.INFO
|
||||
|
||||
if dry_run:
|
||||
logging.info('Skipping export to tar file (dry run)')
|
||||
logger.info('Skipping export to tar file (dry run)')
|
||||
return
|
||||
|
||||
execute_command(
|
||||
|
||||
+42
-25
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
|
||||
import borgmatic.config.paths
|
||||
@@ -23,6 +24,7 @@ def extract_last_archive_dry_run(
|
||||
Perform an extraction dry-run of the most recent archive. If there are no archives, skip the
|
||||
dry-run.
|
||||
'''
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get('extract', '')
|
||||
verbosity_flags = ()
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
verbosity_flags = ('--debug', '--show-rc')
|
||||
@@ -47,22 +49,36 @@ def extract_last_archive_dry_run(
|
||||
full_extract_command = (
|
||||
(local_path, 'extract', '--dry-run')
|
||||
+ (('--remote-path', remote_path) if remote_path else ())
|
||||
+ (('--log-json',) if config.get('log_json') else ())
|
||||
+ (('--log-json',) if not config.get('progress') else ())
|
||||
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
||||
+ verbosity_flags
|
||||
+ (('--progress',) if config.get('progress') else ())
|
||||
+ list_flag
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ flags.make_repository_archive_flags(
|
||||
repository_path, last_archive_name, local_borg_version
|
||||
repository_path,
|
||||
last_archive_name,
|
||||
local_borg_version,
|
||||
)
|
||||
)
|
||||
|
||||
execute_command(
|
||||
full_extract_command,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=borgmatic.config.paths.get_working_directory(config),
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=config.get('borg_exit_codes'),
|
||||
)
|
||||
if config.get('progress'):
|
||||
execute_command(
|
||||
full_extract_command,
|
||||
output_file=DO_NOT_CAPTURE,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=borgmatic.config.paths.get_working_directory(config),
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=config.get('borg_exit_codes'),
|
||||
)
|
||||
else:
|
||||
execute_command(
|
||||
full_extract_command,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=borgmatic.config.paths.get_working_directory(config),
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=config.get('borg_exit_codes'),
|
||||
)
|
||||
|
||||
|
||||
def extract_archive(
|
||||
@@ -90,9 +106,7 @@ def extract_archive(
|
||||
'''
|
||||
umask = config.get('umask', None)
|
||||
lock_wait = config.get('lock_wait', None)
|
||||
|
||||
if config.get('progress') and extract_to_stdout:
|
||||
raise ValueError('progress and extract to stdout cannot both be set')
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get('extract', '')
|
||||
|
||||
if feature.available(feature.Feature.NUMERIC_IDS, local_borg_version):
|
||||
numeric_ids_flags = ('--numeric-ids',) if config.get('numeric_ids') else ()
|
||||
@@ -121,14 +135,15 @@ def extract_archive(
|
||||
+ (('--remote-path', remote_path) if remote_path else ())
|
||||
+ numeric_ids_flags
|
||||
+ (('--umask', str(umask)) if umask else ())
|
||||
+ (('--log-json',) if config.get('log_json') else ())
|
||||
+ (('--log-json',) if (config.get('log_json') or not config.get('progress')) else ())
|
||||
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
||||
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
|
||||
+ (('--debug', '--list', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
|
||||
+ (('--dry-run',) if dry_run else ())
|
||||
+ (('--strip-components', str(strip_components)) if strip_components else ())
|
||||
+ (('--progress',) if config.get('progress') else ())
|
||||
+ (('--progress',) if config.get('progress') and not extract_to_stdout else ())
|
||||
+ (('--stdout',) if extract_to_stdout else ())
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ flags.make_repository_archive_flags(
|
||||
# Make the repository path absolute so the destination directory used below via changing
|
||||
# the working directory doesn't prevent Borg from finding the repo. But also apply the
|
||||
@@ -145,6 +160,17 @@ def extract_archive(
|
||||
os.path.join(working_directory or '', destination_path) if destination_path else None
|
||||
)
|
||||
|
||||
if extract_to_stdout:
|
||||
return execute_command(
|
||||
full_command,
|
||||
output_file=subprocess.PIPE,
|
||||
run_to_completion=False,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=full_destination_path,
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
|
||||
# The progress output isn't compatible with captured and logged output, as progress messes with
|
||||
# the terminal directly.
|
||||
if config.get('progress'):
|
||||
@@ -158,17 +184,6 @@ def extract_archive(
|
||||
)
|
||||
return None
|
||||
|
||||
if extract_to_stdout:
|
||||
return execute_command(
|
||||
full_command,
|
||||
output_file=subprocess.PIPE,
|
||||
run_to_completion=False,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=full_destination_path,
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
|
||||
# Don't give Borg local path so as to error on warnings, as "borg extract" only gives a warning
|
||||
# if the restore paths don't exist in the archive.
|
||||
execute_command(
|
||||
@@ -178,3 +193,5 @@ def extract_archive(
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
@@ -36,7 +36,7 @@ FEATURE_TO_MINIMUM_BORG_VERSION = {
|
||||
Feature.EXCLUDED_FILES_MINUS: parse('2.0.0b5'), # --list --filter uses "-" for excludes
|
||||
Feature.ARCHIVE_SERIES: parse('2.0.0b11'), # identically named archives form a series
|
||||
Feature.NO_PRUNE_STATS: parse('2.0.0b10'), # prune --stats is not available
|
||||
Feature.DRY_RUN_COMPACT: parse('1.4.1'), # borg compact --dry-run support
|
||||
Feature.DRY_RUN_COMPACT: parse('1.2.9'), # borg compact --dry-run support
|
||||
}
|
||||
|
||||
|
||||
|
||||
+26
-36
@@ -1,5 +1,4 @@
|
||||
import itertools
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
@@ -34,7 +33,7 @@ def make_flags_from_arguments(arguments, excludes=()):
|
||||
make_flags(name, value=getattr(arguments, name))
|
||||
for name in sorted(vars(arguments))
|
||||
if name not in excludes and not name.startswith('_')
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -50,7 +49,7 @@ def make_repository_flags(repository_path, local_borg_version):
|
||||
) + (repository_path,)
|
||||
|
||||
|
||||
ARCHIVE_HASH_PATTERN = re.compile('[0-9a-fA-F]{8,}$')
|
||||
ARCHIVE_HASH_PATTERN = re.compile(r'[0-9a-fA-F]{8,}$')
|
||||
|
||||
|
||||
def make_repository_archive_flags(repository_path, archive, local_borg_version):
|
||||
@@ -76,8 +75,8 @@ def make_repository_archive_flags(repository_path, archive, local_borg_version):
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_ARCHIVE_NAME_FORMAT_WITHOUT_SERIES = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}' # noqa: FS003
|
||||
DEFAULT_ARCHIVE_NAME_FORMAT_WITH_SERIES = '{hostname}' # noqa: FS003
|
||||
DEFAULT_ARCHIVE_NAME_FORMAT_WITHOUT_SERIES = '{hostname}-{now:%Y-%m-%dT%H:%M:%S.%f}'
|
||||
DEFAULT_ARCHIVE_NAME_FORMAT_WITH_SERIES = '{hostname}'
|
||||
|
||||
|
||||
def get_default_archive_name_format(local_borg_version):
|
||||
@@ -90,7 +89,7 @@ def get_default_archive_name_format(local_borg_version):
|
||||
return DEFAULT_ARCHIVE_NAME_FORMAT_WITHOUT_SERIES
|
||||
|
||||
|
||||
def make_match_archives_flags(
|
||||
def make_match_archives_flags( # noqa: PLR0911
|
||||
match_archives,
|
||||
archive_name_format,
|
||||
local_borg_version,
|
||||
@@ -115,8 +114,8 @@ def make_match_archives_flags(
|
||||
return ('--match-archives', f'aid:{match_archives}')
|
||||
|
||||
return ('--match-archives', match_archives)
|
||||
else:
|
||||
return ('--glob-archives', re.sub(r'^sh:', '', match_archives))
|
||||
|
||||
return ('--glob-archives', re.sub(r'^sh:', '', match_archives))
|
||||
|
||||
derived_match_archives = re.sub(
|
||||
r'\{(now|utcnow|pid)([:%\w\.-]*)\}',
|
||||
@@ -131,31 +130,22 @@ def make_match_archives_flags(
|
||||
|
||||
if feature.available(feature.Feature.MATCH_ARCHIVES, local_borg_version):
|
||||
return ('--match-archives', f'sh:{derived_match_archives}')
|
||||
else:
|
||||
return ('--glob-archives', f'{derived_match_archives}')
|
||||
|
||||
return ('--glob-archives', f'{derived_match_archives}')
|
||||
|
||||
|
||||
def warn_for_aggressive_archive_flags(json_command, json_output):
|
||||
def warn_for_aggressive_archive_flags(command, output_lines):
|
||||
'''
|
||||
Given a JSON archives command and the resulting JSON string output from running it, parse the
|
||||
JSON and warn if the command used an archive flag but the output indicates zero archives were
|
||||
found.
|
||||
Given an archives command and the resulting output lines from running it, warn if the command
|
||||
used an archive flag but the output indicates zero archives were found.
|
||||
'''
|
||||
archive_flags_used = {'--glob-archives', '--match-archives'}.intersection(set(json_command))
|
||||
|
||||
if not archive_flags_used:
|
||||
return
|
||||
|
||||
try:
|
||||
if len(json.loads(json_output)['archives']) == 0:
|
||||
logger.warning('An archive filter was applied, but no matching archives were found.')
|
||||
logger.warning(
|
||||
'Try adding --match-archives "*" or adjusting archive_name_format/match_archives in configuration.'
|
||||
)
|
||||
except json.JSONDecodeError as error:
|
||||
logger.debug(f'Cannot parse JSON output from archive command: {error}')
|
||||
except (TypeError, KeyError):
|
||||
logger.debug('Cannot parse JSON output from archive command: No "archives" key found')
|
||||
if {'--glob-archives', '--match-archives'}.intersection(set(command)) and len(
|
||||
tuple(line for line in output_lines if not line.startswith('terminating with '))
|
||||
) == 0:
|
||||
logger.warning('An archive filter was applied, but no matching archives were found.')
|
||||
logger.warning(
|
||||
'Try adding --match-archives "*" or adjusting archive_name_format/match_archives in configuration.',
|
||||
)
|
||||
|
||||
|
||||
def omit_flag(arguments, flag):
|
||||
@@ -193,8 +183,8 @@ def omit_flag_and_value(arguments, flag):
|
||||
# its value.
|
||||
return tuple(
|
||||
argument
|
||||
for (previous_argument, argument) in zip((None,) + arguments, arguments)
|
||||
if flag not in (previous_argument, argument)
|
||||
for (previous_argument, argument) in zip((None, *arguments), arguments)
|
||||
if flag not in {previous_argument, argument}
|
||||
if not argument.startswith(f'{flag}=')
|
||||
)
|
||||
|
||||
@@ -209,7 +199,7 @@ def make_exclude_flags(config):
|
||||
itertools.chain.from_iterable(
|
||||
('--exclude-if-present', if_present)
|
||||
for if_present in config.get('exclude_if_present', ())
|
||||
)
|
||||
),
|
||||
)
|
||||
keep_exclude_tags_flags = ('--keep-exclude-tags',) if config.get('keep_exclude_tags') else ()
|
||||
exclude_nodump_flags = ('--exclude-nodump',) if config.get('exclude_nodump') else ()
|
||||
@@ -229,10 +219,10 @@ def make_list_filter_flags(local_borg_version, dry_run):
|
||||
if feature.available(feature.Feature.EXCLUDED_FILES_MINUS, local_borg_version):
|
||||
if show_excludes or dry_run:
|
||||
return f'{base_flags}+-'
|
||||
else:
|
||||
return base_flags
|
||||
|
||||
return base_flags
|
||||
|
||||
if show_excludes:
|
||||
return f'{base_flags}x-'
|
||||
else:
|
||||
return f'{base_flags}-'
|
||||
|
||||
return f'{base_flags}-'
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import logging
|
||||
import os
|
||||
import shlex
|
||||
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.logger
|
||||
from borgmatic.borg import environment, flags
|
||||
from borgmatic.execute import DO_NOT_CAPTURE, execute_command
|
||||
from borgmatic.execute import execute_command
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -30,29 +30,30 @@ def import_key(
|
||||
umask = config.get('umask', None)
|
||||
lock_wait = config.get('lock_wait', None)
|
||||
working_directory = borgmatic.config.paths.get_working_directory(config)
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get('key_import', '')
|
||||
|
||||
if import_arguments.path and import_arguments.path != '-':
|
||||
if not os.path.exists(os.path.join(working_directory or '', import_arguments.path)):
|
||||
raise ValueError(f'Path {import_arguments.path} does not exist. Aborting.')
|
||||
|
||||
input_file = None
|
||||
else:
|
||||
input_file = DO_NOT_CAPTURE
|
||||
if (
|
||||
import_arguments.path
|
||||
and import_arguments.path != '-'
|
||||
and not os.path.exists(os.path.join(working_directory or '', import_arguments.path))
|
||||
):
|
||||
raise ValueError(f'Path {import_arguments.path} does not exist. Aborting.')
|
||||
|
||||
full_command = (
|
||||
(local_path, 'key', 'import')
|
||||
+ (('--remote-path', remote_path) if remote_path else ())
|
||||
+ (('--umask', str(umask)) if umask else ())
|
||||
+ (('--log-json',) if config.get('log_json') else ())
|
||||
+ ('--log-json',)
|
||||
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
||||
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
|
||||
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
|
||||
+ flags.make_flags('paper', import_arguments.paper)
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ flags.make_repository_flags(
|
||||
repository_path,
|
||||
local_borg_version,
|
||||
)
|
||||
+ ((import_arguments.path,) if input_file is None else ())
|
||||
+ (import_arguments.path or '-',)
|
||||
)
|
||||
|
||||
if global_arguments.dry_run:
|
||||
@@ -61,7 +62,6 @@ def import_key(
|
||||
|
||||
execute_command(
|
||||
full_command,
|
||||
input_file=input_file,
|
||||
output_log_level=logging.INFO,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=working_directory,
|
||||
|
||||
+30
-21
@@ -1,10 +1,11 @@
|
||||
import argparse
|
||||
import logging
|
||||
import shlex
|
||||
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.logger
|
||||
from borgmatic.borg import environment, feature, flags
|
||||
from borgmatic.execute import execute_command, execute_command_and_capture_output
|
||||
from borgmatic.execute import execute_command_and_capture_output
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -23,6 +24,8 @@ def make_info_command(
|
||||
arguments to the info action as an argparse.Namespace, and global arguments, return a command
|
||||
as a tuple to display summary information for archives in the repository.
|
||||
'''
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get('info', '')
|
||||
|
||||
return (
|
||||
(local_path, 'info')
|
||||
+ (
|
||||
@@ -37,7 +40,7 @@ def make_info_command(
|
||||
)
|
||||
+ flags.make_flags('remote-path', remote_path)
|
||||
+ flags.make_flags('umask', config.get('umask'))
|
||||
+ flags.make_flags('log-json', config.get('log_json'))
|
||||
+ ('--log-json',)
|
||||
+ flags.make_flags('lock-wait', config.get('lock_wait'))
|
||||
+ (
|
||||
(
|
||||
@@ -55,8 +58,10 @@ def make_info_command(
|
||||
)
|
||||
)
|
||||
+ flags.make_flags_from_arguments(
|
||||
info_arguments, excludes=('repository', 'archive', 'prefix', 'match_archives')
|
||||
info_arguments,
|
||||
excludes=('repository', 'archive', 'prefix', 'match_archives'),
|
||||
)
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ flags.make_repository_flags(repository_path, local_borg_version)
|
||||
)
|
||||
|
||||
@@ -98,24 +103,28 @@ def display_archives_info(
|
||||
borg_exit_codes = config.get('borg_exit_codes')
|
||||
working_directory = borgmatic.config.paths.get_working_directory(config)
|
||||
|
||||
json_info = execute_command_and_capture_output(
|
||||
json_command,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=working_directory,
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
|
||||
if info_arguments.json:
|
||||
return json_info
|
||||
return '\n'.join(
|
||||
execute_command_and_capture_output(
|
||||
json_command,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=working_directory,
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
)
|
||||
|
||||
flags.warn_for_aggressive_archive_flags(json_command, json_info)
|
||||
|
||||
execute_command(
|
||||
main_command,
|
||||
output_log_level=logging.ANSWER,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=working_directory,
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
output_lines = tuple(
|
||||
execute_command_and_capture_output(
|
||||
main_command,
|
||||
output_log_level=logging.ANSWER,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=working_directory,
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
)
|
||||
|
||||
flags.warn_for_aggressive_archive_flags(main_command, output_lines)
|
||||
|
||||
return None
|
||||
|
||||
+44
-40
@@ -1,7 +1,9 @@
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import shlex
|
||||
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.logger
|
||||
@@ -17,7 +19,10 @@ MAKE_FLAGS_EXCLUDES = (
|
||||
'archive',
|
||||
'paths',
|
||||
'find_paths',
|
||||
) + ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST
|
||||
'format',
|
||||
'json',
|
||||
*ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST,
|
||||
)
|
||||
|
||||
|
||||
def make_list_command(
|
||||
@@ -34,6 +39,8 @@ def make_list_command(
|
||||
and local and remote Borg paths, return a command as a tuple to list archives or paths within an
|
||||
archive.
|
||||
'''
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get('list', '')
|
||||
|
||||
return (
|
||||
(local_path, 'list')
|
||||
+ (
|
||||
@@ -48,12 +55,17 @@ def make_list_command(
|
||||
)
|
||||
+ flags.make_flags('remote-path', remote_path)
|
||||
+ flags.make_flags('umask', config.get('umask'))
|
||||
+ flags.make_flags('log-json', config.get('log_json'))
|
||||
+ ('--log-json',)
|
||||
+ flags.make_flags('json-lines', list_arguments.json)
|
||||
+ flags.make_flags('lock-wait', config.get('lock_wait'))
|
||||
+ flags.make_flags('format', list_arguments.format or config.get('file_list_format'))
|
||||
+ flags.make_flags_from_arguments(list_arguments, excludes=MAKE_FLAGS_EXCLUDES)
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ (
|
||||
flags.make_repository_archive_flags(
|
||||
repository_path, list_arguments.archive, local_borg_version
|
||||
repository_path,
|
||||
list_arguments.archive,
|
||||
local_borg_version,
|
||||
)
|
||||
if list_arguments.archive
|
||||
else flags.make_repository_flags(repository_path, local_borg_version)
|
||||
@@ -100,14 +112,15 @@ def capture_archive_listing(
|
||||
remote_path=None,
|
||||
):
|
||||
'''
|
||||
Given a local or remote repository path, an archive name, a configuration
|
||||
dict, the local Borg version, global arguments as an argparse.Namespace,
|
||||
the archive paths (or Borg patterns) in which to list files, the Borg path
|
||||
format to use for the output, and local and remote Borg paths, capture the
|
||||
output of listing that archive and return it as a list of file paths.
|
||||
Given a local or remote repository path, an archive name, a configuration dict, the local Borg
|
||||
version, global arguments as an argparse.Namespace, the archive paths (or Borg patterns) in
|
||||
which to list files, the Borg path format indicating keys to include in the output, and local
|
||||
and remote Borg paths, capture the output of listing that archive and return it as a generator
|
||||
of dicts, one per path.
|
||||
'''
|
||||
return tuple(
|
||||
execute_command_and_capture_output(
|
||||
return (
|
||||
json.loads(entry)
|
||||
for entry in execute_command_and_capture_output(
|
||||
make_list_command(
|
||||
repository_path,
|
||||
config,
|
||||
@@ -115,10 +128,10 @@ def capture_archive_listing(
|
||||
argparse.Namespace(
|
||||
repository=repository_path,
|
||||
archive=archive,
|
||||
paths=[path for path in list_paths] if list_paths else None,
|
||||
paths=list(list_paths) if list_paths else None,
|
||||
find_paths=None,
|
||||
json=None,
|
||||
format=path_format or '{path}{NUL}', # noqa: FS003
|
||||
json=True,
|
||||
format=path_format or None,
|
||||
),
|
||||
global_arguments,
|
||||
local_path,
|
||||
@@ -129,8 +142,6 @@ def capture_archive_listing(
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=config.get('borg_exit_codes'),
|
||||
)
|
||||
.strip('\0')
|
||||
.split('\0')
|
||||
)
|
||||
|
||||
|
||||
@@ -156,7 +167,7 @@ def list_archive(
|
||||
if not list_arguments.archive and not list_arguments.find_paths:
|
||||
if feature.available(feature.Feature.REPO_LIST, local_borg_version):
|
||||
logger.warning(
|
||||
'Omitting the --archive flag on the list action is deprecated when using Borg 2.x+. Use the repo-list action instead.'
|
||||
'Omitting the --archive flag on the list action is deprecated when using Borg 2.x+. Use the repo-list action instead.',
|
||||
)
|
||||
|
||||
repo_list_arguments = argparse.Namespace(
|
||||
@@ -184,14 +195,9 @@ def list_archive(
|
||||
for name in ARCHIVE_FILTER_FLAGS_MOVED_TO_REPO_LIST:
|
||||
if getattr(list_arguments, name, None):
|
||||
logger.warning(
|
||||
f"The --{name.replace('_', '-')} flag on the list action is ignored when using the --archive flag."
|
||||
f"The --{name.replace('_', '-')} flag on the list action is ignored when using the --archive flag.",
|
||||
)
|
||||
|
||||
if list_arguments.json:
|
||||
raise ValueError(
|
||||
'The --json flag on the list action is not supported when using the --archive/--find flags.'
|
||||
)
|
||||
|
||||
borg_exit_codes = config.get('borg_exit_codes')
|
||||
|
||||
# If there are any paths to find (and there's not a single archive already selected), start by
|
||||
@@ -210,24 +216,20 @@ def list_archive(
|
||||
)
|
||||
|
||||
# Ask Borg to list archives. Capture its output for use below.
|
||||
archive_lines = tuple(
|
||||
execute_command_and_capture_output(
|
||||
repo_list.make_repo_list_command(
|
||||
repository_path,
|
||||
config,
|
||||
local_borg_version,
|
||||
repo_list_arguments,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
),
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=borgmatic.config.paths.get_working_directory(config),
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
.strip('\n')
|
||||
.splitlines()
|
||||
archive_lines = execute_command_and_capture_output(
|
||||
repo_list.make_repo_list_command(
|
||||
repository_path,
|
||||
config,
|
||||
local_borg_version,
|
||||
repo_list_arguments,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
),
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=borgmatic.config.paths.get_working_directory(config),
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
else:
|
||||
archive_lines = (list_arguments.archive,)
|
||||
@@ -262,3 +264,5 @@ def list_archive(
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
import shlex
|
||||
|
||||
import borgmatic.config.paths
|
||||
from borgmatic.borg import environment, feature, flags
|
||||
@@ -25,12 +26,13 @@ def mount_archive(
|
||||
'''
|
||||
umask = config.get('umask', None)
|
||||
lock_wait = config.get('lock_wait', None)
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get('mount', '')
|
||||
|
||||
full_command = (
|
||||
(local_path, 'mount')
|
||||
+ (('--remote-path', remote_path) if remote_path else ())
|
||||
+ (('--umask', str(umask)) if umask else ())
|
||||
+ (('--log-json',) if config.get('log_json') else ())
|
||||
+ (('--log-json',) if not mount_arguments.foreground else ())
|
||||
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
||||
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
|
||||
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
|
||||
@@ -39,6 +41,7 @@ def mount_archive(
|
||||
excludes=('repository', 'archive', 'mount_point', 'paths', 'options'),
|
||||
)
|
||||
+ (('-o', mount_arguments.options) if mount_arguments.options else ())
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ (
|
||||
(
|
||||
flags.make_repository_flags(repository_path, local_borg_version)
|
||||
|
||||
@@ -17,9 +17,11 @@ def run_passcommand(passcommand, working_directory):
|
||||
Cache the results so that the passcommand only needs to run—and potentially prompt the user—once
|
||||
per borgmatic invocation.
|
||||
'''
|
||||
return borgmatic.execute.execute_command_and_capture_output(
|
||||
shlex.split(passcommand),
|
||||
working_directory=working_directory,
|
||||
return '\n'.join(
|
||||
borgmatic.execute.execute_command_and_capture_output(
|
||||
shlex.split(passcommand),
|
||||
working_directory=working_directory,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
|
||||
+12
-12
@@ -4,8 +4,6 @@ import logging
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import borgmatic.borg.pattern
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -59,9 +57,9 @@ Pattern = collections.namedtuple(
|
||||
|
||||
def write_patterns_file(patterns, borgmatic_runtime_directory, patterns_file=None):
|
||||
'''
|
||||
Given a sequence of patterns as borgmatic.borg.pattern.Pattern instances, write them to a named
|
||||
temporary file in the given borgmatic runtime directory and return the file object so it can
|
||||
continue to exist on disk as long as the caller needs it.
|
||||
Given a sequence of patterns as Pattern instances, write them to a named temporary file in the
|
||||
given borgmatic runtime directory and return the file object so it can continue to exist on disk
|
||||
as long as the caller needs it.
|
||||
|
||||
If an optional open pattern file is given, append to it instead of making a new temporary file.
|
||||
Return None if no patterns are provided.
|
||||
@@ -70,7 +68,9 @@ def write_patterns_file(patterns, borgmatic_runtime_directory, patterns_file=Non
|
||||
return None
|
||||
|
||||
if patterns_file is None:
|
||||
patterns_file = tempfile.NamedTemporaryFile('w', dir=borgmatic_runtime_directory)
|
||||
patterns_file = tempfile.NamedTemporaryFile(
|
||||
'w', dir=borgmatic_runtime_directory, encoding='utf-8'
|
||||
)
|
||||
operation_name = 'Writing'
|
||||
else:
|
||||
patterns_file.write('\n')
|
||||
@@ -88,19 +88,19 @@ def write_patterns_file(patterns, borgmatic_runtime_directory, patterns_file=Non
|
||||
return patterns_file
|
||||
|
||||
|
||||
def check_all_root_patterns_exist(patterns):
|
||||
def check_all_root_patterns_exist(patterns, working_directory):
|
||||
'''
|
||||
Given a sequence of borgmatic.borg.pattern.Pattern instances, check that all root pattern
|
||||
paths exist. If any don't, raise an exception.
|
||||
Given a sequence of Pattern instances and the current working directory, check that all root
|
||||
pattern paths exist. If any don't, raise an exception.
|
||||
'''
|
||||
missing_paths = [
|
||||
pattern.path
|
||||
for pattern in patterns
|
||||
if pattern.type == borgmatic.borg.pattern.Pattern_type.ROOT
|
||||
if not os.path.exists(pattern.path)
|
||||
if pattern.type == Pattern_type.ROOT
|
||||
if not os.path.exists(os.path.join(working_directory or '', pattern.path))
|
||||
]
|
||||
|
||||
if missing_paths:
|
||||
raise ValueError(
|
||||
f"Source directories or root pattern paths do not exist: {', '.join(missing_paths)}"
|
||||
f"Source directories or root pattern paths do not exist: {', '.join(missing_paths)}",
|
||||
)
|
||||
|
||||
@@ -72,9 +72,9 @@ def prune_archives(
|
||||
full_command = (
|
||||
(local_path, 'prune')
|
||||
+ make_prune_flags(config, prune_arguments, local_borg_version)
|
||||
+ ('--log-json',)
|
||||
+ (('--remote-path', remote_path) if remote_path else ())
|
||||
+ (('--umask', str(umask)) if umask else ())
|
||||
+ (('--log-json',) if config.get('log_json') else ())
|
||||
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
||||
+ (
|
||||
('--stats',)
|
||||
|
||||
@@ -32,19 +32,21 @@ def recreate_archive(
|
||||
exclude_flags = flags.make_exclude_flags(config)
|
||||
compression = config.get('compression', None)
|
||||
chunker_params = config.get('chunker_params', None)
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get('recreate', '')
|
||||
|
||||
# Available recompress MODES: "if-different", "always", "never" (default)
|
||||
recompress = config.get('recompress', None)
|
||||
|
||||
# Write patterns to a temporary file and use that file with --patterns-from.
|
||||
patterns_file = write_patterns_file(
|
||||
patterns, borgmatic.config.paths.get_working_directory(config)
|
||||
patterns,
|
||||
borgmatic.config.paths.get_working_directory(config),
|
||||
)
|
||||
|
||||
recreate_command = (
|
||||
(local_path, 'recreate')
|
||||
+ (('--remote-path', remote_path) if remote_path else ())
|
||||
+ (('--log-json',) if config.get('log_json') else ())
|
||||
+ ('--log-json',)
|
||||
+ (('--lock-wait', str(lock_wait)) if lock_wait is not None else ())
|
||||
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
|
||||
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
|
||||
@@ -70,6 +72,8 @@ def recreate_archive(
|
||||
+ (('--chunker-params', chunker_params) if chunker_params else ())
|
||||
+ (('--recompress', recompress) if recompress else ())
|
||||
+ exclude_flags
|
||||
+ (('--dry-run',) if global_arguments.dry_run else ())
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ (
|
||||
(
|
||||
flags.make_repository_flags(repository, local_borg_version)
|
||||
@@ -80,7 +84,8 @@ def recreate_archive(
|
||||
)
|
||||
)
|
||||
if borgmatic.borg.feature.available(
|
||||
borgmatic.borg.feature.Feature.SEPARATE_REPOSITORY_ARCHIVE, local_borg_version
|
||||
borgmatic.borg.feature.Feature.SEPARATE_REPOSITORY_ARCHIVE,
|
||||
local_borg_version,
|
||||
)
|
||||
else (
|
||||
flags.make_repository_archive_flags(repository, archive, local_borg_version)
|
||||
@@ -90,10 +95,6 @@ def recreate_archive(
|
||||
)
|
||||
)
|
||||
|
||||
if global_arguments.dry_run:
|
||||
logger.info('Skipping the archive recreation (dry run)')
|
||||
return
|
||||
|
||||
borgmatic.execute.execute_command(
|
||||
full_command=recreate_command,
|
||||
output_log_level=logging.INFO,
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import logging
|
||||
import shlex
|
||||
|
||||
import borgmatic.borg.environment
|
||||
import borgmatic.borg.flags
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -15,6 +17,8 @@ def make_rename_command(
|
||||
local_path,
|
||||
remote_path,
|
||||
):
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get('rename', '')
|
||||
|
||||
return (
|
||||
(local_path, 'rename')
|
||||
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
|
||||
@@ -22,10 +26,13 @@ def make_rename_command(
|
||||
+ borgmatic.borg.flags.make_flags('dry-run', dry_run)
|
||||
+ borgmatic.borg.flags.make_flags('remote-path', remote_path)
|
||||
+ borgmatic.borg.flags.make_flags('umask', config.get('umask'))
|
||||
+ borgmatic.borg.flags.make_flags('log-json', config.get('log_json'))
|
||||
+ ('--log-json',)
|
||||
+ borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ borgmatic.borg.flags.make_repository_archive_flags(
|
||||
repository_name, old_archive_name, local_borg_version
|
||||
repository_name,
|
||||
old_archive_name,
|
||||
local_borg_version,
|
||||
)
|
||||
+ (new_archive_name,)
|
||||
)
|
||||
|
||||
@@ -49,13 +49,13 @@ def create_repository(
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
),
|
||||
)
|
||||
repository_encryption_mode = info_data.get('encryption', {}).get('mode')
|
||||
|
||||
if repository_encryption_mode != encryption_mode:
|
||||
raise ValueError(
|
||||
f'Requested encryption mode "{encryption_mode}" does not match existing repository encryption mode "{repository_encryption_mode}"'
|
||||
f'Requested encryption mode "{encryption_mode}" does not match existing repository encryption mode "{repository_encryption_mode}"',
|
||||
)
|
||||
|
||||
logger.info('Repository already exists. Skipping creation.')
|
||||
@@ -66,8 +66,14 @@ def create_repository(
|
||||
|
||||
lock_wait = config.get('lock_wait')
|
||||
umask = config.get('umask')
|
||||
extra_borg_options_from_init = config.get('extra_borg_options', {}).get('init', '')
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get('repo-create', '')
|
||||
|
||||
if extra_borg_options_from_init:
|
||||
logger.warning(
|
||||
'The "init" option in "extra_borg_options" is deprecated and will be removed from a future release; use "repo_create" instead.'
|
||||
)
|
||||
|
||||
repo_create_command = (
|
||||
(local_path,)
|
||||
+ (
|
||||
@@ -83,16 +89,16 @@ def create_repository(
|
||||
+ (('--make-parent-dirs',) if make_parent_directories else ())
|
||||
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
|
||||
+ (('--debug',) if logger.isEnabledFor(logging.DEBUG) else ())
|
||||
+ (('--log-json',) if config.get('log_json') else ())
|
||||
+ (('--lock-wait', str(lock_wait)) if lock_wait else ())
|
||||
+ (('--remote-path', remote_path) if remote_path else ())
|
||||
+ (('--umask', str(umask)) if umask else ())
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ (tuple(shlex.split(extra_borg_options_from_init)) if extra_borg_options_from_init else ())
|
||||
+ flags.make_repository_flags(repository_path, local_borg_version)
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
logging.info('Skipping repository creation (dry run)')
|
||||
logger.info('Skipping repository creation (dry run)')
|
||||
return
|
||||
|
||||
# Do not capture output here, so as to support interactive prompts.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
import shlex
|
||||
|
||||
import borgmatic.borg.environment
|
||||
import borgmatic.borg.feature
|
||||
@@ -9,6 +10,9 @@ import borgmatic.execute
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
FORCE_HARDER_FLAG_COUNT = 2
|
||||
|
||||
|
||||
def make_repo_delete_command(
|
||||
repository,
|
||||
config,
|
||||
@@ -17,18 +21,30 @@ def make_repo_delete_command(
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
output_file,
|
||||
):
|
||||
'''
|
||||
Given a local or remote repository dict, a configuration dict, the local Borg version, the
|
||||
arguments to the repo_delete action as an argparse.Namespace, and global arguments, return a command
|
||||
as a tuple to repo_delete the entire repository.
|
||||
arguments to the repo_delete action as an argparse.Namespace, and global arguments, the Borg
|
||||
local path, the Borg remote path, and an optional output file, return a command as a tuple to
|
||||
repo_delete the entire repository.
|
||||
'''
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get(
|
||||
'repo_delete'
|
||||
if borgmatic.borg.feature.available(
|
||||
borgmatic.borg.feature.Feature.REPO_DELETE, local_borg_version
|
||||
)
|
||||
else 'delete',
|
||||
'',
|
||||
)
|
||||
|
||||
return (
|
||||
(local_path,)
|
||||
+ (
|
||||
('repo-delete',)
|
||||
if borgmatic.borg.feature.available(
|
||||
borgmatic.borg.feature.Feature.REPO_DELETE, local_borg_version
|
||||
borgmatic.borg.feature.Feature.REPO_DELETE,
|
||||
local_borg_version,
|
||||
)
|
||||
else ('delete',)
|
||||
)
|
||||
@@ -37,17 +53,22 @@ def make_repo_delete_command(
|
||||
+ borgmatic.borg.flags.make_flags('dry-run', global_arguments.dry_run)
|
||||
+ borgmatic.borg.flags.make_flags('remote-path', remote_path)
|
||||
+ borgmatic.borg.flags.make_flags('umask', config.get('umask'))
|
||||
+ borgmatic.borg.flags.make_flags('log-json', config.get('log_json'))
|
||||
+ (('--log-json',) if output_file is None else ())
|
||||
+ borgmatic.borg.flags.make_flags('lock-wait', config.get('lock_wait'))
|
||||
+ borgmatic.borg.flags.make_flags('list', config.get('list_details'))
|
||||
+ (
|
||||
(('--force',) + (('--force',) if repo_delete_arguments.force >= 2 else ()))
|
||||
(
|
||||
('--force',)
|
||||
+ (('--force',) if repo_delete_arguments.force >= FORCE_HARDER_FLAG_COUNT else ())
|
||||
)
|
||||
if repo_delete_arguments.force
|
||||
else ()
|
||||
)
|
||||
+ borgmatic.borg.flags.make_flags_from_arguments(
|
||||
repo_delete_arguments, excludes=('list_details', 'force', 'repository')
|
||||
repo_delete_arguments,
|
||||
excludes=('list_details', 'force', 'repository'),
|
||||
)
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ borgmatic.borg.flags.make_repository_flags(repository['path'], local_borg_version)
|
||||
)
|
||||
|
||||
@@ -68,6 +89,14 @@ def delete_repository(
|
||||
'''
|
||||
borgmatic.logger.add_custom_log_levels()
|
||||
|
||||
# Don't capture output when Borg is expected to prompt for interactive confirmation, or the
|
||||
# prompt won't work.
|
||||
output_file = (
|
||||
None
|
||||
if repo_delete_arguments.force or repo_delete_arguments.cache_only
|
||||
else borgmatic.execute.DO_NOT_CAPTURE
|
||||
)
|
||||
|
||||
command = make_repo_delete_command(
|
||||
repository,
|
||||
config,
|
||||
@@ -76,18 +105,13 @@ def delete_repository(
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
output_file,
|
||||
)
|
||||
|
||||
borgmatic.execute.execute_command(
|
||||
command,
|
||||
output_log_level=logging.ANSWER,
|
||||
# Don't capture output when Borg is expected to prompt for interactive confirmation, or the
|
||||
# prompt won't work.
|
||||
output_file=(
|
||||
None
|
||||
if repo_delete_arguments.force or repo_delete_arguments.cache_only
|
||||
else borgmatic.execute.DO_NOT_CAPTURE
|
||||
),
|
||||
output_file=output_file,
|
||||
environment=borgmatic.borg.environment.make_environment(config),
|
||||
working_directory=borgmatic.config.paths.get_working_directory(config),
|
||||
borg_local_path=local_path,
|
||||
|
||||
+27
-16
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
import shlex
|
||||
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.logger
|
||||
@@ -24,6 +25,10 @@ def display_repository_info(
|
||||
'''
|
||||
borgmatic.logger.add_custom_log_levels()
|
||||
lock_wait = config.get('lock_wait', None)
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get(
|
||||
'repo_info' if feature.available(feature.Feature.REPO_INFO, local_borg_version) else 'info',
|
||||
'',
|
||||
)
|
||||
|
||||
full_command = (
|
||||
(local_path,)
|
||||
@@ -32,6 +37,7 @@ def display_repository_info(
|
||||
if feature.available(feature.Feature.REPO_INFO, local_borg_version)
|
||||
else ('info',)
|
||||
)
|
||||
+ (('--critical',) if repo_info_arguments.json else ())
|
||||
+ (
|
||||
('--info',)
|
||||
if logger.getEffectiveLevel() == logging.INFO and not repo_info_arguments.json
|
||||
@@ -44,9 +50,10 @@ def display_repository_info(
|
||||
)
|
||||
+ flags.make_flags('remote-path', remote_path)
|
||||
+ flags.make_flags('umask', config.get('umask'))
|
||||
+ flags.make_flags('log-json', config.get('log_json'))
|
||||
+ flags.make_flags('lock-wait', lock_wait)
|
||||
+ ('--log-json',)
|
||||
+ (('--json',) if repo_info_arguments.json else ())
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ flags.make_repository_flags(repository_path, local_borg_version)
|
||||
)
|
||||
|
||||
@@ -54,19 +61,23 @@ def display_repository_info(
|
||||
borg_exit_codes = config.get('borg_exit_codes')
|
||||
|
||||
if repo_info_arguments.json:
|
||||
return execute_command_and_capture_output(
|
||||
full_command,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=working_directory,
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
else:
|
||||
execute_command(
|
||||
full_command,
|
||||
output_log_level=logging.ANSWER,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=working_directory,
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
return '\n'.join(
|
||||
execute_command_and_capture_output(
|
||||
full_command,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=working_directory,
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
)
|
||||
|
||||
execute_command(
|
||||
full_command,
|
||||
output_log_level=logging.ANSWER,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=working_directory,
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
+84
-48
@@ -1,11 +1,12 @@
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import shlex
|
||||
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.logger
|
||||
from borgmatic.borg import environment, feature, flags
|
||||
from borgmatic.execute import execute_command, execute_command_and_capture_output
|
||||
from borgmatic.execute import execute_command_and_capture_output
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -23,7 +24,9 @@ def resolve_archive_name(
|
||||
Given a local or remote repository path, an archive name, a configuration dict, the local Borg
|
||||
version, global arguments as an argparse.Namespace, a local Borg path, and a remote Borg path,
|
||||
return the archive name. But if the archive name is "latest", then instead introspect the
|
||||
repository for the latest archive and return its name.
|
||||
repository for the latest archive and return its name or ID, depending on whether the version of
|
||||
Borg in use supports archive series—different archives that share the same name but have unique
|
||||
IDs.
|
||||
|
||||
Raise ValueError if "latest" is given but there are no archives in the repository.
|
||||
'''
|
||||
@@ -39,7 +42,11 @@ def resolve_archive_name(
|
||||
remote_path=remote_path,
|
||||
)
|
||||
|
||||
return latest_archive['name']
|
||||
return (
|
||||
latest_archive['id']
|
||||
if feature.available(feature.Feature.ARCHIVE_SERIES, local_borg_version)
|
||||
else latest_archive['name']
|
||||
)
|
||||
|
||||
|
||||
def get_latest_archive(
|
||||
@@ -56,32 +63,48 @@ def get_latest_archive(
|
||||
|
||||
Raises ValueError if there are no archives in the repository.
|
||||
'''
|
||||
|
||||
full_command = (
|
||||
(
|
||||
local_path,
|
||||
(
|
||||
'repo-list'
|
||||
if feature.available(feature.Feature.REPO_LIST, local_borg_version)
|
||||
else 'list'
|
||||
),
|
||||
)
|
||||
+ flags.make_flags('remote-path', remote_path)
|
||||
+ flags.make_flags('umask', config.get('umask'))
|
||||
+ flags.make_flags('log-json', config.get('log_json'))
|
||||
+ flags.make_flags('lock-wait', config.get('lock_wait'))
|
||||
+ flags.make_flags('consider-checkpoints', consider_checkpoints)
|
||||
+ flags.make_flags('last', 1)
|
||||
+ ('--json',)
|
||||
+ flags.make_repository_flags(repository_path, local_borg_version)
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get(
|
||||
'repo_list' if feature.available(feature.Feature.REPO_LIST, local_borg_version) else 'list',
|
||||
'',
|
||||
)
|
||||
|
||||
json_output = execute_command_and_capture_output(
|
||||
full_command,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=borgmatic.config.paths.get_working_directory(config),
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=config.get('borg_exit_codes'),
|
||||
full_command = (
|
||||
local_path,
|
||||
(
|
||||
'repo-list'
|
||||
if feature.available(feature.Feature.REPO_LIST, local_borg_version)
|
||||
else 'list'
|
||||
),
|
||||
*flags.make_flags('remote-path', remote_path),
|
||||
*flags.make_flags('umask', config.get('umask')),
|
||||
*('--log-json',),
|
||||
*flags.make_flags('lock-wait', config.get('lock_wait')),
|
||||
*(
|
||||
flags.make_match_archives_flags(
|
||||
config.get('match_archives'),
|
||||
config.get('archive_name_format'),
|
||||
local_borg_version,
|
||||
)
|
||||
),
|
||||
*(
|
||||
flags.make_flags('consider-checkpoints', consider_checkpoints)
|
||||
if not feature.available(feature.Feature.REPO_LIST, local_borg_version)
|
||||
else ()
|
||||
),
|
||||
*flags.make_flags('last', 1),
|
||||
'--json',
|
||||
*(tuple(shlex.split(extra_borg_options)) if extra_borg_options else ()),
|
||||
*flags.make_repository_flags(repository_path, local_borg_version),
|
||||
)
|
||||
|
||||
json_output = '\n'.join(
|
||||
execute_command_and_capture_output(
|
||||
full_command,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=borgmatic.config.paths.get_working_directory(config),
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=config.get('borg_exit_codes'),
|
||||
)
|
||||
)
|
||||
|
||||
archives = json.loads(json_output)['archives']
|
||||
@@ -91,12 +114,12 @@ def get_latest_archive(
|
||||
except IndexError:
|
||||
raise ValueError('No archives found in the repository')
|
||||
|
||||
logger.debug(f'Latest archive is {latest_archive["name"]}')
|
||||
logger.debug(f'Latest archive is {latest_archive["name"]} ({latest_archive["id"]})')
|
||||
|
||||
return latest_archive
|
||||
|
||||
|
||||
MAKE_FLAGS_EXCLUDES = ('repository', 'prefix', 'match_archives')
|
||||
MAKE_FLAGS_EXCLUDES = ('repository', 'format', 'prefix', 'match_archives')
|
||||
|
||||
|
||||
def make_repo_list_command(
|
||||
@@ -113,6 +136,11 @@ def make_repo_list_command(
|
||||
arguments to the repo_list action, global arguments as an argparse.Namespace instance, and local and
|
||||
remote Borg paths, return a command as a tuple to list archives with a repository.
|
||||
'''
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get(
|
||||
'repo_list' if feature.available(feature.Feature.REPO_LIST, local_borg_version) else 'list',
|
||||
'',
|
||||
)
|
||||
|
||||
return (
|
||||
(
|
||||
local_path,
|
||||
@@ -134,7 +162,7 @@ def make_repo_list_command(
|
||||
)
|
||||
+ flags.make_flags('remote-path', remote_path)
|
||||
+ flags.make_flags('umask', config.get('umask'))
|
||||
+ flags.make_flags('log-json', config.get('log_json'))
|
||||
+ ('--log-json',)
|
||||
+ flags.make_flags('lock-wait', config.get('lock_wait'))
|
||||
+ (
|
||||
(
|
||||
@@ -151,7 +179,11 @@ def make_repo_list_command(
|
||||
)
|
||||
)
|
||||
)
|
||||
+ flags.make_flags(
|
||||
'format', repo_list_arguments.format or config.get('archive_list_format')
|
||||
)
|
||||
+ flags.make_flags_from_arguments(repo_list_arguments, excludes=MAKE_FLAGS_EXCLUDES)
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ flags.make_repository_flags(repository_path, local_borg_version)
|
||||
)
|
||||
|
||||
@@ -194,24 +226,28 @@ def list_repository(
|
||||
working_directory = borgmatic.config.paths.get_working_directory(config)
|
||||
borg_exit_codes = config.get('borg_exit_codes')
|
||||
|
||||
json_listing = execute_command_and_capture_output(
|
||||
json_command,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=working_directory,
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
|
||||
if repo_list_arguments.json:
|
||||
return json_listing
|
||||
return '\n'.join(
|
||||
execute_command_and_capture_output(
|
||||
json_command,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=working_directory,
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
)
|
||||
|
||||
flags.warn_for_aggressive_archive_flags(json_command, json_listing)
|
||||
|
||||
execute_command(
|
||||
main_command,
|
||||
output_log_level=logging.ANSWER,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=working_directory,
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
output_lines = tuple(
|
||||
execute_command_and_capture_output(
|
||||
main_command,
|
||||
output_log_level=logging.ANSWER,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=working_directory,
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=borg_exit_codes,
|
||||
)
|
||||
)
|
||||
|
||||
flags.warn_for_aggressive_archive_flags(main_command, output_lines)
|
||||
|
||||
return None
|
||||
|
||||
@@ -1 +1 @@
|
||||
DEFAULT_BORGMATIC_SOURCE_DIRECTORY = '~/.borgmatic'
|
||||
DEFAULT_BORGMATIC_SOURCE_DIRECTORY = '~/.borgmatic' # pragma: nocover
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
import shlex
|
||||
|
||||
import borgmatic.config.paths
|
||||
import borgmatic.logger
|
||||
@@ -24,6 +25,7 @@ def transfer_archives(
|
||||
instance, transfer archives to the given repository.
|
||||
'''
|
||||
borgmatic.logger.add_custom_log_levels()
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get('transfer', '')
|
||||
|
||||
full_command = (
|
||||
(local_path, 'transfer')
|
||||
@@ -31,7 +33,7 @@ def transfer_archives(
|
||||
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
|
||||
+ flags.make_flags('remote-path', remote_path)
|
||||
+ flags.make_flags('umask', config.get('umask'))
|
||||
+ flags.make_flags('log-json', config.get('log_json'))
|
||||
+ (('--log-json',) if (config.get('log_json') or not config.get('progress')) else ())
|
||||
+ flags.make_flags('lock-wait', config.get('lock_wait'))
|
||||
+ flags.make_flags('progress', config.get('progress'))
|
||||
+ (
|
||||
@@ -53,6 +55,7 @@ def transfer_archives(
|
||||
)
|
||||
)
|
||||
)
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ flags.make_repository_flags(repository_path, local_borg_version)
|
||||
+ flags.make_flags('other-repo', transfer_arguments.source_repository)
|
||||
+ flags.make_flags('dry-run', dry_run)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
import shlex
|
||||
|
||||
import borgmatic.config.paths
|
||||
from borgmatic.execute import execute_command
|
||||
@@ -11,10 +12,13 @@ def unmount_archive(config, mount_point, local_path='borg'):
|
||||
Given a mounted filesystem mount point, and an optional local Borg paths, umount the filesystem
|
||||
from the mount point.
|
||||
'''
|
||||
extra_borg_options = config.get('extra_borg_options', {}).get('umount', '')
|
||||
full_command = (
|
||||
(local_path, 'umount')
|
||||
+ ('--log-json',)
|
||||
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
|
||||
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
|
||||
+ (tuple(shlex.split(extra_borg_options)) if extra_borg_options else ())
|
||||
+ (mount_point,)
|
||||
)
|
||||
|
||||
|
||||
@@ -16,15 +16,19 @@ def local_borg_version(config, local_path='borg'):
|
||||
'''
|
||||
full_command = (
|
||||
(local_path, '--version')
|
||||
+ ('--log-json',)
|
||||
+ (('--info',) if logger.getEffectiveLevel() == logging.INFO else ())
|
||||
+ (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ())
|
||||
)
|
||||
output = execute_command_and_capture_output(
|
||||
full_command,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=borgmatic.config.paths.get_working_directory(config),
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=config.get('borg_exit_codes'),
|
||||
|
||||
output = '\n'.join(
|
||||
execute_command_and_capture_output(
|
||||
full_command,
|
||||
environment=environment.make_environment(config),
|
||||
working_directory=borgmatic.config.paths.get_working_directory(config),
|
||||
borg_local_path=local_path,
|
||||
borg_exit_codes=config.get('borg_exit_codes'),
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
|
||||
+347
-98
@@ -31,8 +31,10 @@ ACTION_ALIASES = {
|
||||
'transfer': [],
|
||||
'break-lock': [],
|
||||
'key': [],
|
||||
'borg': [],
|
||||
'recreate': [],
|
||||
'diff': [],
|
||||
'browse': [],
|
||||
'borg': [],
|
||||
}
|
||||
|
||||
|
||||
@@ -60,7 +62,7 @@ def get_subactions_for_actions(action_parsers):
|
||||
action: tuple(
|
||||
subaction_name
|
||||
for group_action in action_parser._subparsers._group_actions
|
||||
for subaction_name in group_action.choices.keys()
|
||||
for subaction_name in group_action.choices
|
||||
)
|
||||
for action, action_parser in action_parsers.items()
|
||||
if action_parser._subparsers
|
||||
@@ -77,21 +79,25 @@ def omit_values_colliding_with_action_names(unparsed_arguments, parsed_arguments
|
||||
'''
|
||||
remaining_arguments = list(unparsed_arguments)
|
||||
|
||||
for action_name, parsed in parsed_arguments.items():
|
||||
for parsed in parsed_arguments.values():
|
||||
for value in vars(parsed).values():
|
||||
if isinstance(value, str):
|
||||
if value in ACTION_ALIASES.keys() and value in remaining_arguments:
|
||||
if value in ACTION_ALIASES and value in remaining_arguments:
|
||||
remaining_arguments.remove(value)
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
if item in ACTION_ALIASES.keys() and item in remaining_arguments:
|
||||
if item in ACTION_ALIASES and item in remaining_arguments:
|
||||
remaining_arguments.remove(item)
|
||||
|
||||
return tuple(remaining_arguments)
|
||||
|
||||
|
||||
def parse_and_record_action_arguments(
|
||||
unparsed_arguments, parsed_arguments, action_parser, action_name, canonical_name=None
|
||||
unparsed_arguments,
|
||||
parsed_arguments,
|
||||
action_parser,
|
||||
action_name,
|
||||
canonical_name=None,
|
||||
):
|
||||
'''
|
||||
Given unparsed arguments as a sequence of strings, parsed arguments as a dict from action name
|
||||
@@ -102,7 +108,8 @@ def parse_and_record_action_arguments(
|
||||
given action doesn't apply to the given unparsed arguments.
|
||||
'''
|
||||
filtered_arguments = omit_values_colliding_with_action_names(
|
||||
unparsed_arguments, parsed_arguments
|
||||
unparsed_arguments,
|
||||
parsed_arguments,
|
||||
)
|
||||
|
||||
if action_name not in filtered_arguments:
|
||||
@@ -186,12 +193,12 @@ def get_unparsable_arguments(remaining_action_arguments):
|
||||
itertools.chain.from_iterable(
|
||||
argument_group
|
||||
for argument_group in dict.fromkeys(
|
||||
itertools.chain.from_iterable(grouped_action_arguments)
|
||||
).keys()
|
||||
itertools.chain.from_iterable(grouped_action_arguments),
|
||||
)
|
||||
if all(
|
||||
argument_group in action_arguments for action_arguments in grouped_action_arguments
|
||||
)
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -244,7 +251,7 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse
|
||||
subaction_name,
|
||||
)
|
||||
if argument != action_name
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
if subaction_name in arguments:
|
||||
@@ -256,14 +263,18 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse
|
||||
sys.exit(0)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Missing sub-action after {action_name} action. Expected one of: {', '.join(get_subactions_for_actions(action_parsers)[action_name])}"
|
||||
f"Missing sub-action after {action_name} action. Expected one of: {', '.join(get_subactions_for_actions(action_parsers)[action_name])}",
|
||||
)
|
||||
# Otherwise, parse with the main action parser.
|
||||
else:
|
||||
remaining_action_arguments.append(
|
||||
parse_and_record_action_arguments(
|
||||
unparsed_arguments, arguments, action_parser, action_name, canonical_name
|
||||
)
|
||||
unparsed_arguments,
|
||||
arguments,
|
||||
action_parser,
|
||||
action_name,
|
||||
canonical_name,
|
||||
),
|
||||
)
|
||||
|
||||
# If no actions were explicitly requested, assume defaults.
|
||||
@@ -272,11 +283,11 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse
|
||||
default_action_parser = action_parsers[default_action_name]
|
||||
remaining_action_arguments.append(
|
||||
parse_and_record_action_arguments(
|
||||
tuple(unparsed_arguments) + (default_action_name,),
|
||||
(*unparsed_arguments, default_action_name),
|
||||
arguments,
|
||||
default_action_parser,
|
||||
default_action_name,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
arguments['global'], remaining = global_parser.parse_known_args(unparsed_arguments)
|
||||
@@ -288,7 +299,14 @@ def parse_arguments_for_actions(unparsed_arguments, action_parsers, global_parse
|
||||
)
|
||||
|
||||
|
||||
OMITTED_FLAG_NAMES = {'match-archives', 'progress', 'statistics', 'list-details'}
|
||||
OMITTED_FLAG_NAMES = {
|
||||
'match-archives',
|
||||
'progress',
|
||||
'statistics',
|
||||
'list-details',
|
||||
'file-list-format',
|
||||
'archive-list-format',
|
||||
}
|
||||
|
||||
|
||||
def make_argument_description(schema, flag_name):
|
||||
@@ -304,7 +322,7 @@ def make_argument_description(schema, flag_name):
|
||||
|
||||
if '[0]' in flag_name:
|
||||
pieces.append(
|
||||
' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).'
|
||||
' To specify a different list element, replace the "[0]" with another array index ("[1]", "[2]", etc.).',
|
||||
)
|
||||
|
||||
if example and schema_type in ('array', 'object'):
|
||||
@@ -387,7 +405,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name):
|
||||
if not pattern.match(unparsed_flag_name) or unparsed_flag_name == existing_flag_name:
|
||||
continue
|
||||
|
||||
if action_registry_name in ('store_true', 'store_false'):
|
||||
if action_registry_name in {'store_true', 'store_false'}:
|
||||
arguments_group.add_argument(
|
||||
unparsed_flag_name,
|
||||
action=action_registry_name,
|
||||
@@ -408,7 +426,7 @@ def add_array_element_arguments(arguments_group, unparsed_arguments, flag_name):
|
||||
)
|
||||
|
||||
|
||||
def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None):
|
||||
def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names=None): # noqa: PLR0912
|
||||
'''
|
||||
Given an argparse._ArgumentGroup instance, a configuration schema dict, and a sequence of
|
||||
unparsed argument strings, convert the entire schema into corresponding command-line flags and
|
||||
@@ -466,7 +484,10 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
|
||||
if properties:
|
||||
for name, child in properties.items():
|
||||
add_arguments_from_schema(
|
||||
arguments_group, child, unparsed_arguments, names + (name,)
|
||||
arguments_group,
|
||||
child,
|
||||
unparsed_arguments,
|
||||
(*names, name),
|
||||
)
|
||||
|
||||
return
|
||||
@@ -483,12 +504,15 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
|
||||
arguments_group,
|
||||
child,
|
||||
unparsed_arguments,
|
||||
names[:-1] + (f'{names[-1]}[0]',) + (name,),
|
||||
(*names[:-1], f'{names[-1]}[0]', name),
|
||||
)
|
||||
# If there aren't any children, then this is an array of scalars. Recurse accordingly.
|
||||
else:
|
||||
add_arguments_from_schema(
|
||||
arguments_group, items, unparsed_arguments, names[:-1] + (f'{names[-1]}[0]',)
|
||||
arguments_group,
|
||||
items,
|
||||
unparsed_arguments,
|
||||
(*names[:-1], f'{names[-1]}[0]'),
|
||||
)
|
||||
|
||||
flag_name = '.'.join(names).replace('_', '-')
|
||||
@@ -515,9 +539,9 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
|
||||
)
|
||||
|
||||
if names[-1].startswith('no_'):
|
||||
no_flag_name = '.'.join(names[:-1] + (names[-1][len('no_') :],)).replace('_', '-')
|
||||
no_flag_name = '.'.join((*names[:-1], names[-1][len('no_') :])).replace('_', '-')
|
||||
else:
|
||||
no_flag_name = '.'.join(names[:-1] + ('no-' + names[-1],)).replace('_', '-')
|
||||
no_flag_name = '.'.join((*names[:-1], 'no-' + names[-1])).replace('_', '-')
|
||||
|
||||
arguments_group.add_argument(
|
||||
f'--{no_flag_name}',
|
||||
@@ -545,7 +569,7 @@ def add_arguments_from_schema(arguments_group, schema, unparsed_arguments, names
|
||||
add_array_element_arguments(arguments_group, unparsed_arguments, flag_name)
|
||||
|
||||
|
||||
def make_parsers(schema, unparsed_arguments):
|
||||
def make_parsers(schema, unparsed_arguments): # noqa: PLR0915
|
||||
'''
|
||||
Given a configuration schema dict and unparsed arguments as a sequence of strings, build a
|
||||
global arguments parser, individual action parsers, and a combined parser containing both.
|
||||
@@ -553,8 +577,7 @@ def make_parsers(schema, unparsed_arguments):
|
||||
ignoring actions, and the combined parser is handy for displaying help that includes everything:
|
||||
global flags, a list of actions, etc.
|
||||
'''
|
||||
config_paths = collect.get_default_config_paths(expand_home=True)
|
||||
unexpanded_config_paths = collect.get_default_config_paths(expand_home=False)
|
||||
config_paths = collect.get_default_config_paths()
|
||||
|
||||
# Using allow_abbrev=False here prevents the global parser from erroring about "ambiguous"
|
||||
# options like --encryption. Such options are intended for an action parser rather than the
|
||||
@@ -567,7 +590,7 @@ def make_parsers(schema, unparsed_arguments):
|
||||
'--config',
|
||||
dest='config_paths',
|
||||
action='append',
|
||||
help=f"Configuration filename or directory, can specify flag multiple times, defaults to: -c {' -c '.join(unexpanded_config_paths)}",
|
||||
help='Configuration filename or directory, can specify flag multiple times, defaults to /etc/borgmatic/config.yaml, /etc/borgmatic.d, $XDG_CONFIG_HOME/borgmatic/config.yaml, and $XDG_CONFIG_HOME/borgmatic.d, where $XDG_CONFIG_HOME defaults to $HOME/.config',
|
||||
)
|
||||
global_group.add_argument(
|
||||
'-n',
|
||||
@@ -626,8 +649,8 @@ def make_parsers(schema, unparsed_arguments):
|
||||
repo_create_parser = action_parsers.add_parser(
|
||||
'repo-create',
|
||||
aliases=ACTION_ALIASES['repo-create'],
|
||||
help='Create a new, empty Borg repository',
|
||||
description='Create a new, empty Borg repository',
|
||||
help='Create a new, empty Borg repository (also known as "init")',
|
||||
description='Create a new, empty Borg repository (also known as "init")',
|
||||
add_help=False,
|
||||
)
|
||||
repo_create_group = repo_create_parser.add_argument_group('repo-create arguments')
|
||||
@@ -670,7 +693,10 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Create any missing parent directories of the repository directory [Borg 1.x only]',
|
||||
)
|
||||
repo_create_group.add_argument(
|
||||
'-h', '--help', action='help', help='Show this help message and exit'
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
transfer_parser = action_parsers.add_parser(
|
||||
@@ -712,7 +738,9 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Only transfer archives with names, hashes, or series matching this pattern',
|
||||
)
|
||||
transfer_group.add_argument(
|
||||
'--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
|
||||
'--sort-by',
|
||||
metavar='KEYS',
|
||||
help='Comma-separated list of sorting keys',
|
||||
)
|
||||
transfer_group.add_argument(
|
||||
'--first',
|
||||
@@ -720,7 +748,9 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Only transfer first N archives after other filters are applied',
|
||||
)
|
||||
transfer_group.add_argument(
|
||||
'--last', metavar='N', help='Only transfer last N archives after other filters are applied'
|
||||
'--last',
|
||||
metavar='N',
|
||||
help='Only transfer last N archives after other filters are applied',
|
||||
)
|
||||
transfer_group.add_argument(
|
||||
'--oldest',
|
||||
@@ -743,7 +773,10 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Transfer archives that are newer than the specified time range (e.g. 7d or 12m) from the current time [Borg 2.x+ only]',
|
||||
)
|
||||
transfer_group.add_argument(
|
||||
'-h', '--help', action='help', help='Show this help message and exit'
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
prune_parser = action_parsers.add_parser(
|
||||
@@ -833,7 +866,10 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Minimum saved space percentage threshold for compacting a segment, defaults to 10',
|
||||
)
|
||||
compact_group.add_argument(
|
||||
'-h', '--help', action='help', help='Show this help message and exit'
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
create_parser = action_parsers.add_parser(
|
||||
@@ -861,6 +897,13 @@ def make_parsers(schema, unparsed_arguments):
|
||||
action='store_true',
|
||||
help='Display statistics of archive',
|
||||
)
|
||||
create_group.add_argument(
|
||||
'--quick-stats',
|
||||
dest='quick_statistics',
|
||||
default=None,
|
||||
action='store_true',
|
||||
help='Display statistics of archive, skipping repository-wide "All archives" and chunk index statistics [Borg 1.4.5+ only]',
|
||||
)
|
||||
create_group.add_argument(
|
||||
'--list',
|
||||
'--files',
|
||||
@@ -870,7 +913,11 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Show per-file details',
|
||||
)
|
||||
create_group.add_argument(
|
||||
'--json', dest='json', default=False, action='store_true', help='Output results as JSON'
|
||||
'--json',
|
||||
dest='json',
|
||||
default=False,
|
||||
action='store_true',
|
||||
help='Output results as JSON',
|
||||
)
|
||||
create_group.add_argument(
|
||||
'--comment',
|
||||
@@ -996,13 +1043,19 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Only delete archives with names, hashes, or series matching this pattern',
|
||||
)
|
||||
delete_group.add_argument(
|
||||
'--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
|
||||
'--sort-by',
|
||||
metavar='KEYS',
|
||||
help='Comma-separated list of sorting keys',
|
||||
)
|
||||
delete_group.add_argument(
|
||||
'--first', metavar='N', help='Delete first N archives after other filters are applied'
|
||||
'--first',
|
||||
metavar='N',
|
||||
help='Delete first N archives after other filters are applied',
|
||||
)
|
||||
delete_group.add_argument(
|
||||
'--last', metavar='N', help='Delete last N archives after other filters are applied'
|
||||
'--last',
|
||||
metavar='N',
|
||||
help='Delete last N archives after other filters are applied',
|
||||
)
|
||||
delete_group.add_argument(
|
||||
'--oldest',
|
||||
@@ -1039,7 +1092,9 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Path of repository to extract, defaults to the configured repository if there is only one, quoted globs supported',
|
||||
)
|
||||
extract_group.add_argument(
|
||||
'--archive', help='Name or hash of a single archive to extract (or "latest")', required=True
|
||||
'--archive',
|
||||
help='Name or hash of a single archive to extract (or "latest")',
|
||||
required=True,
|
||||
)
|
||||
extract_group.add_argument(
|
||||
'--path',
|
||||
@@ -1068,7 +1123,10 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Display progress for each file as it is extracted',
|
||||
)
|
||||
extract_group.add_argument(
|
||||
'-h', '--help', action='help', help='Show this help message and exit'
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
config_parser = action_parsers.add_parser(
|
||||
@@ -1093,7 +1151,7 @@ def make_parsers(schema, unparsed_arguments):
|
||||
add_help=False,
|
||||
)
|
||||
config_bootstrap_group = config_bootstrap_parser.add_argument_group(
|
||||
'config bootstrap arguments'
|
||||
'config bootstrap arguments',
|
||||
)
|
||||
config_bootstrap_group.add_argument(
|
||||
'--repository',
|
||||
@@ -1148,7 +1206,10 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Command to use instead of "ssh"',
|
||||
)
|
||||
config_bootstrap_group.add_argument(
|
||||
'-h', '--help', action='help', help='Show this help message and exit'
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
config_generate_parser = config_parsers.add_parser(
|
||||
@@ -1167,9 +1228,9 @@ def make_parsers(schema, unparsed_arguments):
|
||||
config_generate_group.add_argument(
|
||||
'-d',
|
||||
'--destination',
|
||||
dest='destination_filename',
|
||||
dest='destination_path',
|
||||
default=config_paths[0],
|
||||
help=f'Destination configuration file, default: {unexpanded_config_paths[0]}',
|
||||
help='Destination configuration file (or directory if using --split), default: /etc/borgmatic/config.yaml',
|
||||
)
|
||||
config_generate_group.add_argument(
|
||||
'--overwrite',
|
||||
@@ -1178,7 +1239,15 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Whether to overwrite any existing destination file, defaults to false',
|
||||
)
|
||||
config_generate_group.add_argument(
|
||||
'-h', '--help', action='help', help='Show this help message and exit'
|
||||
'--split',
|
||||
action='store_true',
|
||||
help='Assuming the destination is a directory instead of a file, split the configuration into separate files within it, one per option, useful for documentation',
|
||||
)
|
||||
config_generate_group.add_argument(
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
config_validate_parser = config_parsers.add_parser(
|
||||
@@ -1195,7 +1264,33 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Show the validated configuration after all include merging has occurred',
|
||||
)
|
||||
config_validate_group.add_argument(
|
||||
'-h', '--help', action='help', help='Show this help message and exit'
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
config_show_parser = config_parsers.add_parser(
|
||||
'show',
|
||||
help='Show the computed configuration for each file specified with --config (see borgmatic --help)',
|
||||
description='Show the computed configuration for each file specified with --config (see borgmatic --help)',
|
||||
add_help=False,
|
||||
)
|
||||
config_show_group = config_show_parser.add_argument_group('config show arguments')
|
||||
config_show_group.add_argument(
|
||||
'--option',
|
||||
help='Show the value of a single named configuration option instead of the entire configuration',
|
||||
)
|
||||
config_show_group.add_argument(
|
||||
'--json',
|
||||
action='store_true',
|
||||
help='Show the configuration as JSON with one array element per configuration file',
|
||||
)
|
||||
config_show_group.add_argument(
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
export_tar_parser = action_parsers.add_parser(
|
||||
@@ -1211,7 +1306,9 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Path of repository to export from, defaults to the configured repository if there is only one, quoted globs supported',
|
||||
)
|
||||
export_tar_group.add_argument(
|
||||
'--archive', help='Name or hash of a single archive to export (or "latest")', required=True
|
||||
'--archive',
|
||||
help='Name or hash of a single archive to export (or "latest")',
|
||||
required=True,
|
||||
)
|
||||
export_tar_group.add_argument(
|
||||
'--path',
|
||||
@@ -1228,7 +1325,8 @@ def make_parsers(schema, unparsed_arguments):
|
||||
required=True,
|
||||
)
|
||||
export_tar_group.add_argument(
|
||||
'--tar-filter', help='Name of filter program to pipe data through'
|
||||
'--tar-filter',
|
||||
help='Name of filter program to pipe data through',
|
||||
)
|
||||
export_tar_group.add_argument(
|
||||
'--list',
|
||||
@@ -1246,7 +1344,10 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Number of leading path components to remove from each exported path. Skip paths with fewer elements',
|
||||
)
|
||||
export_tar_group.add_argument(
|
||||
'-h', '--help', action='help', help='Show this help message and exit'
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
mount_parser = action_parsers.add_parser(
|
||||
@@ -1262,7 +1363,8 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Path of repository to use, defaults to the configured repository if there is only one, quoted globs supported',
|
||||
)
|
||||
mount_group.add_argument(
|
||||
'--archive', help='Name or hash of a single archive to mount (or "latest")'
|
||||
'--archive',
|
||||
help='Name or hash of a single archive to mount (or "latest")',
|
||||
)
|
||||
mount_group.add_argument(
|
||||
'--mount-point',
|
||||
@@ -1291,7 +1393,9 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Mount first N archives after other filters are applied',
|
||||
)
|
||||
mount_group.add_argument(
|
||||
'--last', metavar='N', help='Mount last N archives after other filters are applied'
|
||||
'--last',
|
||||
metavar='N',
|
||||
help='Mount last N archives after other filters are applied',
|
||||
)
|
||||
mount_group.add_argument(
|
||||
'--oldest',
|
||||
@@ -1368,7 +1472,10 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Do not delete the local security info when deleting a repository',
|
||||
)
|
||||
repo_delete_group.add_argument(
|
||||
'-h', '--help', action='help', help='Show this help message and exit'
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
restore_parser = action_parsers.add_parser(
|
||||
@@ -1411,6 +1518,10 @@ def make_parsers(schema, unparsed_arguments):
|
||||
'--port',
|
||||
help='Database port to restore to. Defaults to the "restore_port" option in borgmatic\'s configuration',
|
||||
)
|
||||
restore_group.add_argument(
|
||||
'--container',
|
||||
help='Container to restore to. Defaults to the "restore_container" option in borgmatic\'s configuration',
|
||||
)
|
||||
restore_group.add_argument(
|
||||
'--username',
|
||||
help='Username with which to connect to the database. Defaults to the "restore_username" option in borgmatic\'s configuration',
|
||||
@@ -1423,10 +1534,18 @@ def make_parsers(schema, unparsed_arguments):
|
||||
'--restore-path',
|
||||
help='Path to restore SQLite database dumps to. Defaults to the "restore_path" option in borgmatic\'s configuration',
|
||||
)
|
||||
restore_group.add_argument(
|
||||
'--original-label',
|
||||
help='The label where the dump to restore came from, only necessary if you need to disambiguate dumps',
|
||||
)
|
||||
restore_group.add_argument(
|
||||
'--original-hostname',
|
||||
help='The hostname where the dump to restore came from, only necessary if you need to disambiguate dumps',
|
||||
)
|
||||
restore_group.add_argument(
|
||||
'--original-container',
|
||||
help='The container where the dump to restore came from, only necessary if you need to disambiguate dumps',
|
||||
)
|
||||
restore_group.add_argument(
|
||||
'--original-port',
|
||||
type=int,
|
||||
@@ -1437,7 +1556,10 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='The name of the data source hook for the dump to restore, only necessary if you need to disambiguate dumps',
|
||||
)
|
||||
restore_group.add_argument(
|
||||
'-h', '--help', action='help', help='Show this help message and exit'
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
repo_list_parser = action_parsers.add_parser(
|
||||
@@ -1453,14 +1575,22 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Path of repository to list, defaults to the configured repositories, quoted globs supported',
|
||||
)
|
||||
repo_list_group.add_argument(
|
||||
'--short', default=False, action='store_true', help='Output only archive names'
|
||||
'--short',
|
||||
default=False,
|
||||
action='store_true',
|
||||
help='Output only archive names',
|
||||
)
|
||||
repo_list_group.add_argument('--format', help='Format for archive listing')
|
||||
repo_list_group.add_argument('--format', help='Borg format for the archive listing')
|
||||
repo_list_group.add_argument(
|
||||
'--json', default=False, action='store_true', help='Output results as JSON'
|
||||
'--json',
|
||||
default=False,
|
||||
action='store_true',
|
||||
help='Output results as JSON',
|
||||
)
|
||||
repo_list_group.add_argument(
|
||||
'-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix'
|
||||
'-P',
|
||||
'--prefix',
|
||||
help='Deprecated. Only list archive names starting with this prefix',
|
||||
)
|
||||
repo_list_group.add_argument(
|
||||
'-a',
|
||||
@@ -1470,13 +1600,19 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Only list archive names, hashes, or series matching this pattern',
|
||||
)
|
||||
repo_list_group.add_argument(
|
||||
'--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
|
||||
'--sort-by',
|
||||
metavar='KEYS',
|
||||
help='Comma-separated list of sorting keys',
|
||||
)
|
||||
repo_list_group.add_argument(
|
||||
'--first', metavar='N', help='List first N archives after other filters are applied'
|
||||
'--first',
|
||||
metavar='N',
|
||||
help='List first N archives after other filters are applied',
|
||||
)
|
||||
repo_list_group.add_argument(
|
||||
'--last', metavar='N', help='List last N archives after other filters are applied'
|
||||
'--last',
|
||||
metavar='N',
|
||||
help='List last N archives after other filters are applied',
|
||||
)
|
||||
repo_list_group.add_argument(
|
||||
'--oldest',
|
||||
@@ -1505,7 +1641,10 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help="List only deleted archives that haven't yet been compacted [Borg 2.x+ only]",
|
||||
)
|
||||
repo_list_group.add_argument(
|
||||
'-h', '--help', action='help', help='Show this help message and exit'
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
list_parser = action_parsers.add_parser(
|
||||
@@ -1521,7 +1660,8 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Path of repository containing archive to list, defaults to the configured repositories, quoted globs supported',
|
||||
)
|
||||
list_group.add_argument(
|
||||
'--archive', help='Name or hash of a single archive to list (or "latest")'
|
||||
'--archive',
|
||||
help='Name or hash of a single archive to list (or "latest")',
|
||||
)
|
||||
list_group.add_argument(
|
||||
'--path',
|
||||
@@ -1538,14 +1678,22 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Partial path or pattern to search for and list across multiple archives, can specify flag multiple times',
|
||||
)
|
||||
list_group.add_argument(
|
||||
'--short', default=False, action='store_true', help='Output only path names'
|
||||
'--short',
|
||||
default=False,
|
||||
action='store_true',
|
||||
help='Output only path names',
|
||||
)
|
||||
list_group.add_argument('--format', help='Format for file listing')
|
||||
list_group.add_argument('--format', help='Borg format for the file listing')
|
||||
list_group.add_argument(
|
||||
'--json', default=False, action='store_true', help='Output results as JSON'
|
||||
'--json',
|
||||
default=False,
|
||||
action='store_true',
|
||||
help='Output results as JSON',
|
||||
)
|
||||
list_group.add_argument(
|
||||
'-P', '--prefix', help='Deprecated. Only list archive names starting with this prefix'
|
||||
'-P',
|
||||
'--prefix',
|
||||
help='Deprecated. Only list archive names starting with this prefix',
|
||||
)
|
||||
list_group.add_argument(
|
||||
'-a',
|
||||
@@ -1555,19 +1703,30 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Only list archive names matching this pattern',
|
||||
)
|
||||
list_group.add_argument(
|
||||
'--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
|
||||
'--sort-by',
|
||||
metavar='KEYS',
|
||||
help='Comma-separated list of sorting keys',
|
||||
)
|
||||
list_group.add_argument(
|
||||
'--first', metavar='N', help='List first N archives after other filters are applied'
|
||||
'--first',
|
||||
metavar='N',
|
||||
help='List first N archives after other filters are applied',
|
||||
)
|
||||
list_group.add_argument(
|
||||
'--last', metavar='N', help='List last N archives after other filters are applied'
|
||||
'--last',
|
||||
metavar='N',
|
||||
help='List last N archives after other filters are applied',
|
||||
)
|
||||
list_group.add_argument(
|
||||
'-e', '--exclude', metavar='PATTERN', help='Exclude paths matching the pattern'
|
||||
'-e',
|
||||
'--exclude',
|
||||
metavar='PATTERN',
|
||||
help='Exclude paths matching the pattern',
|
||||
)
|
||||
list_group.add_argument(
|
||||
'--exclude-from', metavar='FILENAME', help='Exclude paths from exclude file, one per line'
|
||||
'--exclude-from',
|
||||
metavar='FILENAME',
|
||||
help='Exclude paths from exclude file, one per line',
|
||||
)
|
||||
list_group.add_argument('--pattern', help='Include or exclude paths matching a pattern')
|
||||
list_group.add_argument(
|
||||
@@ -1590,10 +1749,17 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Path of repository to show info for, defaults to the configured repository if there is only one, quoted globs supported',
|
||||
)
|
||||
repo_info_group.add_argument(
|
||||
'--json', dest='json', default=False, action='store_true', help='Output results as JSON'
|
||||
'--json',
|
||||
dest='json',
|
||||
default=False,
|
||||
action='store_true',
|
||||
help='Output results as JSON',
|
||||
)
|
||||
repo_info_group.add_argument(
|
||||
'-h', '--help', action='help', help='Show this help message and exit'
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
info_parser = action_parsers.add_parser(
|
||||
@@ -1609,10 +1775,15 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Path of repository containing archive to show info for, defaults to the configured repository if there is only one, quoted globs supported',
|
||||
)
|
||||
info_group.add_argument(
|
||||
'--archive', help='Archive name, hash, or series to show info for (or "latest")'
|
||||
'--archive',
|
||||
help='Archive name, hash, or series to show info for (or "latest")',
|
||||
)
|
||||
info_group.add_argument(
|
||||
'--json', dest='json', default=False, action='store_true', help='Output results as JSON'
|
||||
'--json',
|
||||
dest='json',
|
||||
default=False,
|
||||
action='store_true',
|
||||
help='Output results as JSON',
|
||||
)
|
||||
info_group.add_argument(
|
||||
'-P',
|
||||
@@ -1627,7 +1798,9 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Only show info for archive names, hashes, or series matching this pattern',
|
||||
)
|
||||
info_group.add_argument(
|
||||
'--sort-by', metavar='KEYS', help='Comma-separated list of sorting keys'
|
||||
'--sort-by',
|
||||
metavar='KEYS',
|
||||
help='Comma-separated list of sorting keys',
|
||||
)
|
||||
info_group.add_argument(
|
||||
'--first',
|
||||
@@ -1635,7 +1808,9 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Show info for first N archives after other filters are applied',
|
||||
)
|
||||
info_group.add_argument(
|
||||
'--last', metavar='N', help='Show info for last N archives after other filters are applied'
|
||||
'--last',
|
||||
metavar='N',
|
||||
help='Show info for last N archives after other filters are applied',
|
||||
)
|
||||
info_group.add_argument(
|
||||
'--oldest',
|
||||
@@ -1672,7 +1847,10 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Path of repository to break the lock for, defaults to the configured repository if there is only one, quoted globs supported',
|
||||
)
|
||||
break_lock_group.add_argument(
|
||||
'-h', '--help', action='help', help='Show this help message and exit'
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
key_parser = action_parsers.add_parser(
|
||||
@@ -1717,7 +1895,10 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Path to export the key to, defaults to stdout (but be careful about dirtying the output with --verbosity)',
|
||||
)
|
||||
key_export_group.add_argument(
|
||||
'-h', '--help', action='help', help='Show this help message and exit'
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
key_import_parser = key_parsers.add_parser(
|
||||
@@ -1742,7 +1923,10 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Path to import the key from backup, defaults to stdin',
|
||||
)
|
||||
key_import_group.add_argument(
|
||||
'-h', '--help', action='help', help='Show this help message and exit'
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
key_change_passphrase_parser = key_parsers.add_parser(
|
||||
@@ -1752,14 +1936,17 @@ def make_parsers(schema, unparsed_arguments):
|
||||
add_help=False,
|
||||
)
|
||||
key_change_passphrase_group = key_change_passphrase_parser.add_argument_group(
|
||||
'key change-passphrase arguments'
|
||||
'key change-passphrase arguments',
|
||||
)
|
||||
key_change_passphrase_group.add_argument(
|
||||
'--repository',
|
||||
help='Path of repository to change the passphrase for, defaults to the configured repository if there is only one, quoted globs supported',
|
||||
)
|
||||
key_change_passphrase_group.add_argument(
|
||||
'-h', '--help', action='help', help='Show this help message and exit'
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
recreate_parser = action_parsers.add_parser(
|
||||
@@ -1776,7 +1963,7 @@ def make_parsers(schema, unparsed_arguments):
|
||||
)
|
||||
recreate_group.add_argument(
|
||||
'--archive',
|
||||
help='Archive name, hash, or series to recreate',
|
||||
help='Archive name, hash, or series to recreate, defaults to all archives in the repository (if specified), or all archives across all repositories',
|
||||
)
|
||||
recreate_group.add_argument(
|
||||
'--list',
|
||||
@@ -1809,9 +1996,66 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Only consider archive names, hashes, or series matching this pattern [Borg 2.x+ only]',
|
||||
)
|
||||
recreate_group.add_argument(
|
||||
'-h', '--help', action='help', help='Show this help message and exit'
|
||||
'-h',
|
||||
'--help',
|
||||
action='help',
|
||||
help='Show this help message and exit',
|
||||
)
|
||||
|
||||
diff_parser = action_parsers.add_parser(
|
||||
'diff',
|
||||
aliases=ACTION_ALIASES['diff'],
|
||||
help='Find differences (file contents, user/group/mode) between archives',
|
||||
description='Find differences (file contents, user/group/mode) between archives',
|
||||
add_help=False,
|
||||
)
|
||||
diff_group = diff_parser.add_argument_group('diff arguments')
|
||||
diff_group.add_argument(
|
||||
'--repository',
|
||||
help='Path of repository containing archive to diff, defaults to the configured repository if there is only one, quoted globs supported',
|
||||
)
|
||||
diff_group.add_argument(
|
||||
'--archive',
|
||||
help='Archive name, hash, or series to diff',
|
||||
required=True,
|
||||
)
|
||||
diff_group.add_argument(
|
||||
'--second-archive',
|
||||
help='Second archive name, hash, or series to diff',
|
||||
required=True,
|
||||
)
|
||||
diff_group.add_argument(
|
||||
'--same-chunker-params', action='store_true', help='Override check of chunker parameters'
|
||||
)
|
||||
diff_group.add_argument(
|
||||
'--sort-by',
|
||||
metavar='KEY',
|
||||
dest='sort_keys',
|
||||
action='append',
|
||||
help='Advanced sorting: specify field(s) to sort by. Prefix with > for descending or < for ascending (default)',
|
||||
)
|
||||
diff_group.add_argument(
|
||||
'--content-only',
|
||||
action='store_true',
|
||||
help='Only compare differences in content (exclude metadata differences)',
|
||||
)
|
||||
diff_group.add_argument(
|
||||
'--only-patterns',
|
||||
action='store_true',
|
||||
help='Run the diff according to borgmatic configured patterns (ie do not diff entire archives)',
|
||||
)
|
||||
diff_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
|
||||
|
||||
browse_parser = action_parsers.add_parser(
|
||||
'browse',
|
||||
aliases=ACTION_ALIASES['browse'],
|
||||
help='Browse repositories, archives, and files in a console UI',
|
||||
description='Browse repositories, archives, and files in a console UI',
|
||||
add_help=False,
|
||||
)
|
||||
browse_group = browse_parser.add_argument_group('browse arguments')
|
||||
browse_group.add_argument('-h', '--help', action='help', help='Show this help message and exit')
|
||||
|
||||
borg_parser = action_parsers.add_parser(
|
||||
'borg',
|
||||
aliases=ACTION_ALIASES['borg'],
|
||||
@@ -1825,7 +2069,8 @@ def make_parsers(schema, unparsed_arguments):
|
||||
help='Path of repository to pass to Borg, defaults to the configured repositories, quoted globs supported',
|
||||
)
|
||||
borg_group.add_argument(
|
||||
'--archive', help='Archive name, hash, or series to pass to Borg (or "latest")'
|
||||
'--archive',
|
||||
help='Archive name, hash, or series to pass to Borg (or "latest")',
|
||||
)
|
||||
borg_group.add_argument(
|
||||
'--',
|
||||
@@ -1839,6 +2084,9 @@ def make_parsers(schema, unparsed_arguments):
|
||||
return global_parser, action_parsers, global_plus_action_parser
|
||||
|
||||
|
||||
HIGHLANDER_ACTION_ARGUMENTS_COUNT = 2 # 1 for "global" + 1 for the action
|
||||
|
||||
|
||||
def parse_arguments(schema, *unparsed_arguments):
|
||||
'''
|
||||
Given a configuration schema dict and the command-line arguments with which this script was
|
||||
@@ -1849,21 +2097,22 @@ def parse_arguments(schema, *unparsed_arguments):
|
||||
Raise SystemExit with an error code of 0 if "--help" was requested.
|
||||
'''
|
||||
global_parser, action_parsers, global_plus_action_parser = make_parsers(
|
||||
schema, unparsed_arguments
|
||||
schema,
|
||||
unparsed_arguments,
|
||||
)
|
||||
arguments, remaining_action_arguments = parse_arguments_for_actions(
|
||||
unparsed_arguments, action_parsers.choices, global_parser
|
||||
unparsed_arguments,
|
||||
action_parsers.choices,
|
||||
global_parser,
|
||||
)
|
||||
|
||||
if not arguments['global'].config_paths:
|
||||
arguments['global'].config_paths = collect.get_default_config_paths(expand_home=True)
|
||||
arguments['global'].config_paths = collect.get_default_config_paths()
|
||||
|
||||
for action_name in ('bootstrap', 'generate', 'validate'):
|
||||
if (
|
||||
action_name in arguments.keys() and len(arguments.keys()) > 2
|
||||
): # 2 = 1 for 'global' + 1 for the action
|
||||
if action_name in arguments and len(arguments) > HIGHLANDER_ACTION_ARGUMENTS_COUNT:
|
||||
raise ValueError(
|
||||
f'The {action_name} action cannot be combined with other actions. Please run it separately.'
|
||||
f'The {action_name} action cannot be combined with other actions. Please run it separately.',
|
||||
)
|
||||
|
||||
unknown_arguments = get_unparsable_arguments(remaining_action_arguments)
|
||||
@@ -1875,7 +2124,7 @@ def parse_arguments(schema, *unparsed_arguments):
|
||||
|
||||
global_plus_action_parser.print_usage()
|
||||
raise ValueError(
|
||||
f"Unrecognized argument{'s' if len(unknown_arguments) > 1 else ''}: {' '.join(unknown_arguments)}"
|
||||
f"Unrecognized argument{'s' if len(unknown_arguments) > 1 else ''}: {' '.join(unknown_arguments)}",
|
||||
)
|
||||
|
||||
if (
|
||||
@@ -1887,14 +2136,14 @@ def parse_arguments(schema, *unparsed_arguments):
|
||||
|
||||
if 'list' in arguments and (arguments['list'].prefix and arguments['list'].match_archives):
|
||||
raise ValueError(
|
||||
'With the list action, only one of --prefix or --match-archives flags can be used.'
|
||||
'With the list action, only one of --prefix or --match-archives flags can be used.',
|
||||
)
|
||||
|
||||
if 'repo-list' in arguments and (
|
||||
arguments['repo-list'].prefix and arguments['repo-list'].match_archives
|
||||
):
|
||||
raise ValueError(
|
||||
'With the repo-list action, only one of --prefix or --match-archives flags can be used.'
|
||||
'With the repo-list action, only one of --prefix or --match-archives flags can be used.',
|
||||
)
|
||||
|
||||
if 'info' in arguments and (
|
||||
@@ -1903,7 +2152,7 @@ def parse_arguments(schema, *unparsed_arguments):
|
||||
or (arguments['info'].prefix and arguments['info'].match_archives)
|
||||
):
|
||||
raise ValueError(
|
||||
'With the info action, only one of --archive, --prefix, or --match-archives flags can be used.'
|
||||
'With the info action, only one of --archive, --prefix, or --match-archives flags can be used.',
|
||||
)
|
||||
|
||||
if 'borg' in arguments and arguments['global'].dry_run:
|
||||
|
||||
+249
-175
@@ -12,14 +12,17 @@ import ruamel.yaml
|
||||
|
||||
import borgmatic.actions.borg
|
||||
import borgmatic.actions.break_lock
|
||||
import borgmatic.actions.browse.run
|
||||
import borgmatic.actions.change_passphrase
|
||||
import borgmatic.actions.check
|
||||
import borgmatic.actions.compact
|
||||
import borgmatic.actions.config.bootstrap
|
||||
import borgmatic.actions.config.generate
|
||||
import borgmatic.actions.config.show
|
||||
import borgmatic.actions.config.validate
|
||||
import borgmatic.actions.create
|
||||
import borgmatic.actions.delete
|
||||
import borgmatic.actions.diff
|
||||
import borgmatic.actions.export_key
|
||||
import borgmatic.actions.export_tar
|
||||
import borgmatic.actions.extract
|
||||
@@ -96,7 +99,7 @@ class Monitoring_hooks:
|
||||
self.config = config
|
||||
self.dry_run = global_arguments.dry_run
|
||||
self.monitoring_log_level = verbosity_to_log_level(
|
||||
get_verbosity({config_filename: config}, 'monitoring_verbosity')
|
||||
get_verbosity({config_filename: config}, 'monitoring_verbosity'),
|
||||
)
|
||||
self.monitoring_hooks_are_activated = (
|
||||
using_primary_action and self.monitoring_log_level != DISABLED
|
||||
@@ -182,7 +185,7 @@ class Monitoring_hooks:
|
||||
)
|
||||
|
||||
|
||||
def run_configuration(config_filename, config, config_paths, arguments):
|
||||
def run_configuration(config_filename, config, config_paths, arguments): # noqa: PLR0912, PLR0915
|
||||
'''
|
||||
Given a config filename, the corresponding parsed config dict, a sequence of loaded
|
||||
configuration paths, and command-line arguments as a dict from subparser name to a namespace of
|
||||
@@ -206,12 +209,13 @@ def run_configuration(config_filename, config, config_paths, arguments):
|
||||
|
||||
if skip_actions:
|
||||
logger.debug(
|
||||
f"Skipping {'/'.join(skip_actions)} action{'s' if len(skip_actions) > 1 else ''} due to configured skip_actions"
|
||||
f"Skipping {'/'.join(skip_actions)} action{'s' if len(skip_actions) > 1 else ''} due to configured skip_actions",
|
||||
)
|
||||
|
||||
try:
|
||||
with Monitoring_hooks(config_filename, config, arguments, global_arguments):
|
||||
with borgmatic.hooks.command.Before_after_hooks(
|
||||
with (
|
||||
Monitoring_hooks(config_filename, config, arguments, global_arguments),
|
||||
borgmatic.hooks.command.Before_after_hooks(
|
||||
command_hooks=config.get('commands'),
|
||||
before_after='configuration',
|
||||
umask=config.get('umask'),
|
||||
@@ -220,73 +224,77 @@ def run_configuration(config_filename, config, config_paths, arguments):
|
||||
action_names=arguments.keys(),
|
||||
configuration_filename=config_filename,
|
||||
log_file=config.get('log_file', ''),
|
||||
):
|
||||
try:
|
||||
local_borg_version = borg_version.local_borg_version(config, local_path)
|
||||
logger.debug(f'Borg {local_borg_version}')
|
||||
except (OSError, CalledProcessError, ValueError) as error:
|
||||
yield from log_error_records(
|
||||
f'{config_filename}: Error getting local Borg version', error
|
||||
)
|
||||
raise
|
||||
),
|
||||
):
|
||||
try:
|
||||
local_borg_version = borg_version.local_borg_version(config, local_path)
|
||||
logger.debug(f'Borg {local_borg_version}')
|
||||
except (OSError, CalledProcessError, ValueError) as error:
|
||||
yield from log_error_records(
|
||||
f'{config_filename}: Error getting local Borg version',
|
||||
error,
|
||||
)
|
||||
raise
|
||||
|
||||
for repo in config['repositories']:
|
||||
repo_queue.put(
|
||||
(repo, 0),
|
||||
)
|
||||
for repo in config['repositories']:
|
||||
repo_queue.put(
|
||||
(repo, 0),
|
||||
)
|
||||
|
||||
while not repo_queue.empty():
|
||||
repository, retry_num = repo_queue.get()
|
||||
while not repo_queue.empty():
|
||||
repository, retry_num = repo_queue.get()
|
||||
|
||||
with Log_prefix(repository.get('label', repository['path'])):
|
||||
logger.debug('Running actions for repository')
|
||||
timeout = retry_num * retry_wait
|
||||
if timeout:
|
||||
logger.warning(f'Sleeping {timeout}s before next retry')
|
||||
time.sleep(timeout)
|
||||
try:
|
||||
yield from run_actions(
|
||||
arguments=arguments,
|
||||
config_filename=config_filename,
|
||||
config=config,
|
||||
config_paths=config_paths,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
local_borg_version=local_borg_version,
|
||||
repository=repository,
|
||||
with Log_prefix(repository.get('label', repository['path'])):
|
||||
logger.debug('Running actions for repository')
|
||||
timeout = retry_num * retry_wait
|
||||
|
||||
if timeout:
|
||||
logger.warning(f'Sleeping {timeout}s before next retry')
|
||||
time.sleep(timeout)
|
||||
|
||||
try:
|
||||
yield from run_actions(
|
||||
arguments=arguments,
|
||||
config_filename=config_filename,
|
||||
config=config,
|
||||
config_paths=config_paths,
|
||||
local_path=local_path,
|
||||
remote_path=remote_path,
|
||||
local_borg_version=local_borg_version,
|
||||
repository=repository,
|
||||
)
|
||||
except (OSError, CalledProcessError, ValueError) as error:
|
||||
if retry_num < retries:
|
||||
repo_queue.put(
|
||||
(repository, retry_num + 1),
|
||||
)
|
||||
except (OSError, CalledProcessError, ValueError) as error:
|
||||
if retry_num < retries:
|
||||
repo_queue.put(
|
||||
(repository, retry_num + 1),
|
||||
)
|
||||
tuple( # Consume the generator so as to trigger logging.
|
||||
log_error_records(
|
||||
'Error running actions for repository',
|
||||
error,
|
||||
levelno=logging.WARNING,
|
||||
log_command_error_output=True,
|
||||
)
|
||||
)
|
||||
logger.warning(f'Retrying... attempt {retry_num + 1}/{retries}')
|
||||
continue
|
||||
|
||||
if command.considered_soft_failure(error):
|
||||
continue
|
||||
|
||||
yield from log_error_records(
|
||||
'Error running actions for repository',
|
||||
error,
|
||||
tuple( # Consume the generator so as to trigger logging.
|
||||
log_error_records(
|
||||
'Error running actions for repository',
|
||||
error,
|
||||
levelno=logging.WARNING,
|
||||
log_command_error_output=True,
|
||||
),
|
||||
)
|
||||
encountered_error = error
|
||||
error_repository = repository
|
||||
logger.warning(f'Retrying... attempt {retry_num + 1}/{retries}')
|
||||
continue
|
||||
|
||||
# Re-raise any error, so that the Monitoring_hooks context manager wrapping this
|
||||
# code can see the error and act accordingly. Do this here rather than as soon as
|
||||
# the error is encountered so that an error with one repository doesn't prevent
|
||||
# other repositories from running.
|
||||
if encountered_error:
|
||||
raise encountered_error
|
||||
if command.considered_soft_failure(error):
|
||||
continue
|
||||
|
||||
yield from log_error_records(
|
||||
'Error running actions for repository',
|
||||
error,
|
||||
)
|
||||
encountered_error = error
|
||||
error_repository = repository
|
||||
|
||||
# Re-raise any error, so that the Monitoring_hooks context manager wrapping this
|
||||
# code can see the error and act accordingly. Do this here rather than as soon as
|
||||
# the error is encountered so that an error with one repository doesn't prevent
|
||||
# other repositories from running.
|
||||
if encountered_error:
|
||||
raise encountered_error
|
||||
|
||||
except (OSError, CalledProcessError, ValueError) as error:
|
||||
yield from log_error_records('Error running configuration')
|
||||
@@ -321,7 +329,7 @@ def run_configuration(config_filename, config, config_paths, arguments):
|
||||
yield from log_error_records(f'{config_filename}: Error running after error hook', error)
|
||||
|
||||
|
||||
def run_actions(
|
||||
def run_actions( # noqa: PLR0912, PLR0915
|
||||
*,
|
||||
arguments,
|
||||
config_filename,
|
||||
@@ -335,7 +343,7 @@ def run_actions(
|
||||
'''
|
||||
Given parsed command-line arguments as an argparse.ArgumentParser instance, the configuration
|
||||
filename, a configuration dict, a sequence of loaded configuration paths, local and remote paths
|
||||
to Borg, a local Borg version string, and a repository name, run all actions from the
|
||||
to Borg, a local Borg version string, and a repository dict, run all actions from the
|
||||
command-line arguments on the given repository.
|
||||
|
||||
Yield JSON output strings from executing any actions that produce JSON.
|
||||
@@ -357,6 +365,22 @@ def run_actions(
|
||||
'repository': repository_path,
|
||||
}
|
||||
skip_actions = set(get_skip_actions(config, arguments))
|
||||
requested_repository = next(
|
||||
(
|
||||
repository
|
||||
for action_arguments in arguments.values()
|
||||
for repository in (getattr(action_arguments, 'repository', None),)
|
||||
if repository is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
if requested_repository and not borgmatic.config.validate.repositories_match(
|
||||
repository,
|
||||
requested_repository,
|
||||
):
|
||||
logger.debug('Skipping actions because the requested --repository does not match')
|
||||
return
|
||||
|
||||
with borgmatic.hooks.command.Before_after_hooks(
|
||||
command_hooks=config.get('commands'),
|
||||
@@ -420,6 +444,7 @@ def run_actions(
|
||||
local_borg_version,
|
||||
action_arguments,
|
||||
global_arguments,
|
||||
dry_run_label,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
@@ -600,6 +625,16 @@ def run_actions(
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
elif action_name == 'diff':
|
||||
borgmatic.actions.diff.run_diff(
|
||||
repository,
|
||||
config,
|
||||
local_borg_version,
|
||||
action_arguments,
|
||||
global_arguments,
|
||||
local_path,
|
||||
remote_path,
|
||||
)
|
||||
elif action_name == 'borg':
|
||||
borgmatic.actions.borg.run_borg(
|
||||
repository,
|
||||
@@ -639,9 +674,10 @@ def load_configurations(config_filenames, arguments, overrides=None, resolve_env
|
||||
levelno=logging.DEBUG,
|
||||
levelname='DEBUG',
|
||||
msg=f'{config_filename}: Loading configuration file',
|
||||
)
|
||||
name=logger.name,
|
||||
),
|
||||
),
|
||||
]
|
||||
],
|
||||
)
|
||||
try:
|
||||
configs[config_filename], paths, parse_logs = validate.parse_configuration(
|
||||
@@ -658,12 +694,13 @@ def load_configurations(config_filenames, arguments, overrides=None, resolve_env
|
||||
[
|
||||
logging.makeLogRecord(
|
||||
dict(
|
||||
levelno=logging.WARNING,
|
||||
levelname='WARNING',
|
||||
levelno=logging.CRITICAL,
|
||||
levelname='CRITICAL',
|
||||
msg=f'{config_filename}: Insufficient permissions to read configuration file',
|
||||
)
|
||||
name=logger.name,
|
||||
),
|
||||
),
|
||||
]
|
||||
],
|
||||
)
|
||||
except (ValueError, OSError, validate.Validation_error) as error:
|
||||
logs.extend(
|
||||
@@ -673,12 +710,18 @@ def load_configurations(config_filenames, arguments, overrides=None, resolve_env
|
||||
levelno=logging.CRITICAL,
|
||||
levelname='CRITICAL',
|
||||
msg=f'{config_filename}: Error parsing configuration file',
|
||||
)
|
||||
name=logger.name,
|
||||
),
|
||||
),
|
||||
logging.makeLogRecord(
|
||||
dict(levelno=logging.CRITICAL, levelname='CRITICAL', msg=str(error))
|
||||
dict(
|
||||
levelno=logging.CRITICAL,
|
||||
levelname='CRITICAL',
|
||||
msg=str(error),
|
||||
name=logger.name,
|
||||
),
|
||||
),
|
||||
]
|
||||
],
|
||||
)
|
||||
|
||||
return (configs, sorted(config_paths), logs)
|
||||
@@ -689,7 +732,7 @@ def log_record(suppress_log=False, **kwargs):
|
||||
Create a log record based on the given makeLogRecord() arguments, one of which must be
|
||||
named "levelno". Log the record (unless suppress log is set) and return it.
|
||||
'''
|
||||
record = logging.makeLogRecord(kwargs)
|
||||
record = logging.makeLogRecord(dict(kwargs, name=logger.name))
|
||||
if suppress_log:
|
||||
return record
|
||||
|
||||
@@ -701,7 +744,10 @@ BORG_REPOSITORY_ACCESS_ABORTED_EXIT_CODE = 62
|
||||
|
||||
|
||||
def log_error_records(
|
||||
message, error=None, levelno=logging.CRITICAL, log_command_error_output=False
|
||||
message,
|
||||
error=None,
|
||||
levelno=logging.CRITICAL,
|
||||
log_command_error_output=False,
|
||||
):
|
||||
'''
|
||||
Given error message text, an optional exception object, an optional log level, and whether to
|
||||
@@ -719,14 +765,14 @@ def log_error_records(
|
||||
|
||||
try:
|
||||
raise error
|
||||
except CalledProcessError as error:
|
||||
except CalledProcessError as called_process_error:
|
||||
yield log_record(levelno=levelno, levelname=level_name, msg=str(message))
|
||||
|
||||
if error.output:
|
||||
if called_process_error.output:
|
||||
try:
|
||||
output = error.output.decode('utf-8')
|
||||
output = called_process_error.output.decode('utf-8')
|
||||
except (UnicodeDecodeError, AttributeError):
|
||||
output = error.output
|
||||
output = called_process_error.output
|
||||
|
||||
# Suppress these logs for now and save the error output for the log summary at the end.
|
||||
# Log a separate record per line, as some errors can be really verbose and overflow the
|
||||
@@ -739,17 +785,17 @@ def log_error_records(
|
||||
suppress_log=True,
|
||||
)
|
||||
|
||||
yield log_record(levelno=levelno, levelname=level_name, msg=str(error))
|
||||
yield log_record(levelno=levelno, levelname=level_name, msg=str(called_process_error))
|
||||
|
||||
if error.returncode == BORG_REPOSITORY_ACCESS_ABORTED_EXIT_CODE:
|
||||
if called_process_error.returncode == BORG_REPOSITORY_ACCESS_ABORTED_EXIT_CODE:
|
||||
yield log_record(
|
||||
levelno=levelno,
|
||||
levelname=level_name,
|
||||
msg='\nTo work around this, set either the "relocated_repo_access_is_ok" or "unknown_unencrypted_repo_access_is_ok" option to "true", as appropriate.',
|
||||
)
|
||||
except (ValueError, OSError) as error:
|
||||
except (ValueError, OSError) as other_error:
|
||||
yield log_record(levelno=levelno, levelname=level_name, msg=str(message))
|
||||
yield log_record(levelno=levelno, levelname=level_name, msg=str(error))
|
||||
yield log_record(levelno=levelno, levelname=level_name, msg=str(other_error))
|
||||
except: # noqa: E722, S110
|
||||
# Raising above only as a means of determining the error type. Swallow the exception here
|
||||
# because we don't want the exception to propagate out of this function.
|
||||
@@ -777,70 +823,63 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
|
||||
'''
|
||||
add_custom_log_levels()
|
||||
|
||||
if 'bootstrap' in arguments:
|
||||
try:
|
||||
# No configuration file is needed for bootstrap.
|
||||
local_borg_version = borg_version.local_borg_version(
|
||||
{}, arguments['bootstrap'].local_path
|
||||
)
|
||||
except (OSError, CalledProcessError, ValueError) as error:
|
||||
yield from log_error_records('Error getting local Borg version', error)
|
||||
return
|
||||
try:
|
||||
if 'bootstrap' in arguments:
|
||||
try:
|
||||
local_borg_version = borg_version.local_borg_version(
|
||||
# No configuration file is needed for bootstrap.
|
||||
{},
|
||||
arguments['bootstrap'].local_path,
|
||||
)
|
||||
except (OSError, CalledProcessError, ValueError) as error:
|
||||
yield from log_error_records('Error getting local Borg version', error)
|
||||
return
|
||||
|
||||
try:
|
||||
borgmatic.actions.config.bootstrap.run_bootstrap(
|
||||
arguments['bootstrap'], arguments['global'], local_borg_version
|
||||
arguments['bootstrap'],
|
||||
arguments['global'],
|
||||
local_borg_version,
|
||||
)
|
||||
yield logging.makeLogRecord(
|
||||
dict(
|
||||
levelno=logging.ANSWER,
|
||||
levelname='ANSWER',
|
||||
msg='Bootstrap successful',
|
||||
)
|
||||
name=logger.name,
|
||||
),
|
||||
)
|
||||
except (
|
||||
CalledProcessError,
|
||||
ValueError,
|
||||
OSError,
|
||||
) as error:
|
||||
yield from log_error_records(error)
|
||||
|
||||
return
|
||||
return
|
||||
|
||||
if 'generate' in arguments:
|
||||
try:
|
||||
if 'generate' in arguments:
|
||||
borgmatic.actions.config.generate.run_generate(
|
||||
arguments['generate'], arguments['global']
|
||||
arguments['generate'],
|
||||
arguments['global'],
|
||||
)
|
||||
yield logging.makeLogRecord(
|
||||
dict(
|
||||
levelno=logging.ANSWER,
|
||||
levelname='ANSWER',
|
||||
msg='Generate successful',
|
||||
)
|
||||
)
|
||||
except (
|
||||
CalledProcessError,
|
||||
ValueError,
|
||||
OSError,
|
||||
) as error:
|
||||
yield from log_error_records(error)
|
||||
|
||||
return
|
||||
|
||||
if 'validate' in arguments:
|
||||
if configuration_parse_errors:
|
||||
yield logging.makeLogRecord(
|
||||
dict(
|
||||
levelno=logging.CRITICAL,
|
||||
levelname='CRITICAL',
|
||||
msg='Configuration validation failed',
|
||||
)
|
||||
name=logger.name,
|
||||
),
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
try:
|
||||
if 'validate' in arguments:
|
||||
if configuration_parse_errors:
|
||||
yield logging.makeLogRecord(
|
||||
dict(
|
||||
levelno=logging.CRITICAL,
|
||||
levelname='CRITICAL',
|
||||
msg='Configuration validation failed',
|
||||
name=logger.name,
|
||||
),
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
borgmatic.actions.config.validate.run_validate(arguments['validate'], configs)
|
||||
|
||||
yield logging.makeLogRecord(
|
||||
@@ -848,19 +887,33 @@ def collect_highlander_action_summary_logs(configs, arguments, configuration_par
|
||||
levelno=logging.ANSWER,
|
||||
levelname='ANSWER',
|
||||
msg='All configuration files are valid',
|
||||
)
|
||||
name=logger.name,
|
||||
),
|
||||
)
|
||||
except (
|
||||
CalledProcessError,
|
||||
ValueError,
|
||||
OSError,
|
||||
) as error:
|
||||
yield from log_error_records(error)
|
||||
|
||||
return
|
||||
return
|
||||
|
||||
if 'show' in arguments:
|
||||
borgmatic.actions.config.show.run_show(arguments['show'], configs)
|
||||
|
||||
return
|
||||
|
||||
if 'browse' in arguments:
|
||||
borgmatic.actions.browse.run.run_browse(
|
||||
arguments['browse'],
|
||||
arguments['global'],
|
||||
configs,
|
||||
)
|
||||
|
||||
except (
|
||||
CalledProcessError,
|
||||
ValueError,
|
||||
OSError,
|
||||
) as error:
|
||||
yield from log_error_records(error)
|
||||
|
||||
|
||||
def collect_configuration_run_summary_logs(configs, config_paths, arguments, log_file_path):
|
||||
def collect_configuration_run_summary_logs(configs, config_paths, arguments, log_file_path): # noqa: PLR0912
|
||||
'''
|
||||
Given a dict of configuration filename to corresponding parsed configuration, a sequence of
|
||||
loaded configuration paths, parsed command-line arguments as a dict from subparser name to a
|
||||
@@ -873,9 +926,9 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments, log
|
||||
# Run cross-file validation checks.
|
||||
repository = None
|
||||
|
||||
for action_name, action_arguments in arguments.items():
|
||||
for action_arguments in arguments.values():
|
||||
if hasattr(action_arguments, 'repository'):
|
||||
repository = getattr(action_arguments, 'repository')
|
||||
repository = action_arguments.repository
|
||||
break
|
||||
|
||||
try:
|
||||
@@ -940,7 +993,8 @@ def collect_configuration_run_summary_logs(configs, config_paths, arguments, log
|
||||
levelno=logging.INFO,
|
||||
levelname='INFO',
|
||||
msg=f'{config_filename}: Successfully ran configuration file',
|
||||
)
|
||||
name=logger.name,
|
||||
),
|
||||
)
|
||||
if results:
|
||||
json_results.extend(results)
|
||||
@@ -998,16 +1052,17 @@ def exit_with_help_link(): # pragma: no cover
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def check_and_show_help_on_no_args(configs):
|
||||
def check_and_show_help_on_no_args(configs, schema):
|
||||
'''
|
||||
Given a dict of configuration filename to corresponding parsed configuration, check if the
|
||||
borgmatic command is run without any arguments. If the configuration option "default_actions" is
|
||||
set to False, show the help message. Otherwise, trigger the default backup behavior.
|
||||
Given a dict of configuration filename to corresponding parsed configuration and the
|
||||
configuration schema as a dict, check if the borgmatic command was run without any arguments. If
|
||||
the configuration option "default_actions" is set to False, then show the help message an exit.
|
||||
'''
|
||||
if len(sys.argv) == 1: # No arguments provided
|
||||
if len(sys.argv) == 1: # No arguments provided.
|
||||
default_actions = any(config.get('default_actions', True) for config in configs.values())
|
||||
if not default_actions:
|
||||
parse_arguments('--help')
|
||||
|
||||
if configs and not default_actions:
|
||||
parse_arguments(schema, '--help')
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
@@ -1029,17 +1084,34 @@ def get_singular_option_value(configs, option_name):
|
||||
configure_logging(logging.CRITICAL)
|
||||
joined_values = ', '.join(str(value) for value in distinct_values)
|
||||
logger.critical(
|
||||
f'The {option_name} option has conflicting values across configuration files: {joined_values}'
|
||||
f'The {option_name} option has conflicting values across configuration files: {joined_values}',
|
||||
)
|
||||
exit_with_help_link()
|
||||
|
||||
try:
|
||||
return tuple(distinct_values)[0]
|
||||
except IndexError:
|
||||
return next(iter(distinct_values))
|
||||
except StopIteration:
|
||||
return None
|
||||
|
||||
|
||||
def main(extra_summary_logs=[]): # pragma: no cover
|
||||
def display_summary(summary_logs, log_json): # pragma: no cover
|
||||
summary_logs_max_level = max(log.levelno for log in summary_logs)
|
||||
|
||||
for message in ('summary:',) if log_json else ('', 'summary:'):
|
||||
log_record(
|
||||
levelno=summary_logs_max_level,
|
||||
levelname=logging.getLevelName(summary_logs_max_level),
|
||||
msg=message,
|
||||
)
|
||||
|
||||
for log in summary_logs:
|
||||
logger.handle(log)
|
||||
|
||||
if summary_logs_max_level >= logging.CRITICAL:
|
||||
exit_with_help_link()
|
||||
|
||||
|
||||
def main(extra_summary_logs=()): # pragma: no cover
|
||||
configure_signals()
|
||||
configure_delayed_logging()
|
||||
schema_filename = validate.schema_filename()
|
||||
@@ -1059,20 +1131,24 @@ def main(extra_summary_logs=[]): # pragma: no cover
|
||||
exit_with_help_link()
|
||||
except SystemExit as error:
|
||||
if error.code == 0:
|
||||
raise error
|
||||
raise
|
||||
|
||||
configure_logging(logging.CRITICAL)
|
||||
logger.critical(f"Error parsing arguments: {' '.join(sys.argv)}")
|
||||
exit_with_help_link()
|
||||
|
||||
global_arguments = arguments['global']
|
||||
|
||||
if global_arguments.version:
|
||||
print(importlib.metadata.version('borgmatic'))
|
||||
print(importlib.metadata.version('borgmatic')) # noqa: T201
|
||||
sys.exit(0)
|
||||
|
||||
if global_arguments.bash_completion:
|
||||
print(borgmatic.commands.completion.bash.bash_completion())
|
||||
print(borgmatic.commands.completion.bash.bash_completion()) # noqa: T201
|
||||
sys.exit(0)
|
||||
|
||||
if global_arguments.fish_completion:
|
||||
print(borgmatic.commands.completion.fish.fish_completion())
|
||||
print(borgmatic.commands.completion.fish.fish_completion()) # noqa: T201
|
||||
sys.exit(0)
|
||||
|
||||
config_filenames = tuple(collect.collect_config_filenames(global_arguments.config_paths))
|
||||
@@ -1083,8 +1159,7 @@ def main(extra_summary_logs=[]): # pragma: no cover
|
||||
resolve_env=global_arguments.resolve_env and not arguments.get('validate'),
|
||||
)
|
||||
|
||||
# Use the helper function to check and show help on no arguments, passing the preloaded configs
|
||||
check_and_show_help_on_no_args(configs)
|
||||
check_and_show_help_on_no_args(configs, schema)
|
||||
|
||||
configuration_parse_errors = (
|
||||
(max(log.levelno for log in parse_logs) >= logging.CRITICAL) if parse_logs else False
|
||||
@@ -1094,6 +1169,7 @@ def main(extra_summary_logs=[]): # pragma: no cover
|
||||
getattr(sub_arguments, 'json', False) for sub_arguments in arguments.values()
|
||||
)
|
||||
log_file_path = get_singular_option_value(configs, 'log_file')
|
||||
log_json = get_singular_option_value(configs, 'log_json')
|
||||
|
||||
try:
|
||||
configure_logging(
|
||||
@@ -1103,7 +1179,8 @@ def main(extra_summary_logs=[]): # pragma: no cover
|
||||
verbosity_to_log_level(get_verbosity(configs, 'monitoring_verbosity')),
|
||||
log_file_path,
|
||||
get_singular_option_value(configs, 'log_file_format'),
|
||||
color_enabled=should_do_markup(configs, any_json_flags),
|
||||
log_json,
|
||||
color_enabled=should_do_markup(configs, any_json_flags or log_json),
|
||||
)
|
||||
except (FileNotFoundError, PermissionError) as error:
|
||||
configure_logging(logging.CRITICAL)
|
||||
@@ -1111,32 +1188,29 @@ def main(extra_summary_logs=[]): # pragma: no cover
|
||||
exit_with_help_link()
|
||||
|
||||
summary_logs = (
|
||||
extra_summary_logs
|
||||
list(extra_summary_logs)
|
||||
+ parse_logs
|
||||
+ (
|
||||
list(
|
||||
collect_highlander_action_summary_logs(
|
||||
configs, arguments, configuration_parse_errors
|
||||
)
|
||||
configs,
|
||||
arguments,
|
||||
configuration_parse_errors,
|
||||
),
|
||||
)
|
||||
or list(
|
||||
collect_configuration_run_summary_logs(
|
||||
configs, config_paths, arguments, log_file_path
|
||||
)
|
||||
configs,
|
||||
config_paths,
|
||||
arguments,
|
||||
log_file_path,
|
||||
),
|
||||
)
|
||||
)
|
||||
)
|
||||
summary_logs_max_level = max(log.levelno for log in summary_logs)
|
||||
|
||||
for message in ('', 'summary:'):
|
||||
log_record(
|
||||
levelno=summary_logs_max_level,
|
||||
levelname=logging.getLevelName(summary_logs_max_level),
|
||||
msg=message,
|
||||
)
|
||||
display_summary(summary_logs, log_json)
|
||||
|
||||
for log in summary_logs:
|
||||
logger.handle(log)
|
||||
|
||||
if summary_logs_max_level >= logging.CRITICAL:
|
||||
exit_with_help_link()
|
||||
if __name__ == '__main__': # pragma: no cover
|
||||
main()
|
||||
|
||||
@@ -22,15 +22,15 @@ def available_actions(subparsers, current_action=None):
|
||||
action of "config" but not "list".
|
||||
'''
|
||||
action_to_subactions = borgmatic.commands.arguments.get_subactions_for_actions(
|
||||
subparsers.choices
|
||||
subparsers.choices,
|
||||
)
|
||||
current_subactions = action_to_subactions.get(current_action)
|
||||
|
||||
if current_subactions:
|
||||
return current_subactions
|
||||
|
||||
all_subactions = set(
|
||||
all_subactions = {
|
||||
subaction for subactions in action_to_subactions.values() for subaction in subactions
|
||||
)
|
||||
}
|
||||
|
||||
return tuple(action for action in subparsers.choices.keys() if action not in all_subactions)
|
||||
return tuple(action for action in subparsers.choices if action not in all_subactions)
|
||||
|
||||
@@ -23,7 +23,7 @@ def bash_completion():
|
||||
borgmatic's command-line argument parsers.
|
||||
'''
|
||||
(
|
||||
unused_global_parser,
|
||||
_,
|
||||
action_parsers,
|
||||
global_plus_action_parser,
|
||||
) = borgmatic.commands.arguments.make_parsers(
|
||||
@@ -33,12 +33,13 @@ def bash_completion():
|
||||
global_flags = parser_flags(global_plus_action_parser)
|
||||
|
||||
# Avert your eyes.
|
||||
# fmt: off
|
||||
return '\n'.join(
|
||||
(
|
||||
'check_version() {',
|
||||
' local this_script="$(cat "$BASH_SOURCE" 2> /dev/null)"',
|
||||
' local installed_script="$(borgmatic --bash-completion 2> /dev/null)"',
|
||||
' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];'
|
||||
' if [ "$this_script" != "$installed_script" ] && [ "$installed_script" != "" ];',
|
||||
f''' then cat << EOF\n{borgmatic.commands.completion.actions.upgrade_message(
|
||||
'bash',
|
||||
'sudo sh -c "borgmatic --bash-completion > $BASH_SOURCE"',
|
||||
@@ -47,24 +48,22 @@ def bash_completion():
|
||||
' fi',
|
||||
'}',
|
||||
'complete_borgmatic() {',
|
||||
)
|
||||
+ tuple(
|
||||
*tuple(
|
||||
''' if [[ " ${COMP_WORDS[*]} " =~ " %s " ]]; then
|
||||
COMPREPLY=($(compgen -W "%s %s %s" -- "${COMP_WORDS[COMP_CWORD]}"))
|
||||
return 0
|
||||
fi'''
|
||||
fi''' # noqa: UP031
|
||||
% (
|
||||
action,
|
||||
parser_flags(action_parser),
|
||||
' '.join(
|
||||
borgmatic.commands.completion.actions.available_actions(action_parsers, action)
|
||||
borgmatic.commands.completion.actions.available_actions(action_parsers, action),
|
||||
),
|
||||
global_flags,
|
||||
)
|
||||
for action, action_parser in reversed(action_parsers.choices.items())
|
||||
)
|
||||
+ (
|
||||
' COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))' # noqa: FS003
|
||||
),
|
||||
' COMPREPLY=($(compgen -W "%s %s" -- "${COMP_WORDS[COMP_CWORD]}"))' # noqa: UP031
|
||||
% (
|
||||
' '.join(borgmatic.commands.completion.actions.available_actions(action_parsers)),
|
||||
global_flags,
|
||||
@@ -72,5 +71,5 @@ def bash_completion():
|
||||
' (check_version &)',
|
||||
'}',
|
||||
'\ncomplete -o bashdefault -o default -F complete_borgmatic borgmatic',
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
@@ -11,10 +11,10 @@ def has_file_options(action: Action):
|
||||
'''
|
||||
Given an argparse.Action instance, return True if it takes a file argument.
|
||||
'''
|
||||
return action.metavar in (
|
||||
return action.metavar in {
|
||||
'FILENAME',
|
||||
'PATH',
|
||||
) or action.dest in ('config_paths',)
|
||||
} or action.dest in {'config_paths'}
|
||||
|
||||
|
||||
def has_choice_options(action: Action):
|
||||
@@ -36,11 +36,11 @@ def has_unknown_required_param_options(action: Action):
|
||||
return (
|
||||
action.required is True
|
||||
or action.nargs
|
||||
in (
|
||||
in {
|
||||
'+',
|
||||
'*',
|
||||
)
|
||||
or action.metavar in ('PATTERN', 'KEYS', 'N')
|
||||
}
|
||||
or action.metavar in {'PATTERN', 'KEYS', 'N'}
|
||||
or (action.type is not None and action.default is None)
|
||||
)
|
||||
|
||||
@@ -77,7 +77,7 @@ def exact_options_completion(action: Action):
|
||||
return f'''\ncomplete -c borgmatic -x -n "__borgmatic_current_arg {args}"'''
|
||||
|
||||
raise ValueError(
|
||||
f'Unexpected action: {action} passes has_exact_options but has no choices produced'
|
||||
f'Unexpected action: {action} passes has_exact_options but has no choices produced',
|
||||
)
|
||||
|
||||
|
||||
@@ -96,7 +96,7 @@ def fish_completion():
|
||||
borgmatic's command-line argument parsers.
|
||||
'''
|
||||
(
|
||||
unused_global_parser,
|
||||
_,
|
||||
action_parsers,
|
||||
global_plus_action_parser,
|
||||
) = borgmatic.commands.arguments.make_parsers(
|
||||
@@ -104,7 +104,7 @@ def fish_completion():
|
||||
unparsed_arguments=(),
|
||||
)
|
||||
|
||||
all_action_parsers = ' '.join(action for action in action_parsers.choices.keys())
|
||||
all_action_parsers = ' '.join(action for action in action_parsers.choices)
|
||||
|
||||
exact_option_args = tuple(
|
||||
' '.join(action.option_strings)
|
||||
@@ -119,8 +119,9 @@ def fish_completion():
|
||||
)
|
||||
|
||||
# Avert your eyes.
|
||||
return '\n'.join(
|
||||
dedent_strip_as_tuple(
|
||||
# fmt: off
|
||||
return '\n'.join((
|
||||
*dedent_strip_as_tuple(
|
||||
f'''
|
||||
function __borgmatic_check_version
|
||||
set -fx this_filename (status current-filename)
|
||||
@@ -157,27 +158,27 @@ def fish_completion():
|
||||
|
||||
set --local action_parser_condition "not __fish_seen_subcommand_from {all_action_parsers}"
|
||||
set --local exact_option_condition "not __borgmatic_current_arg {' '.join(exact_option_args)}"
|
||||
'''
|
||||
)
|
||||
+ ('\n# action_parser completions',)
|
||||
+ tuple(
|
||||
''',
|
||||
),
|
||||
'\n# action_parser completions',
|
||||
*tuple(
|
||||
f'''complete -c borgmatic -f -n "$action_parser_condition" -n "$exact_option_condition" -a '{action_name}' -d {shlex.quote(action_parser.description)}'''
|
||||
for action_name, action_parser in action_parsers.choices.items()
|
||||
)
|
||||
+ ('\n# global flags',)
|
||||
+ tuple(
|
||||
),
|
||||
'\n# global flags',
|
||||
*tuple(
|
||||
# -n is checked in order, so put faster / more likely to be true checks first
|
||||
f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)}{exact_options_completion(action)}'''
|
||||
for action in global_plus_action_parser._actions
|
||||
# ignore the noargs action, as this is an impossible completion for fish
|
||||
if len(action.option_strings) > 0
|
||||
if 'Deprecated' not in action.help
|
||||
)
|
||||
+ ('\n# action_parser flags',)
|
||||
+ tuple(
|
||||
),
|
||||
'\n# action_parser flags',
|
||||
*tuple(
|
||||
f'''complete -c borgmatic -f -n "$exact_option_condition" -a '{' '.join(action.option_strings)}' -d {shlex.quote(action.help)} -n "__fish_seen_subcommand_from {action_name}"{exact_options_completion(action)}'''
|
||||
for action_name, action_parser in action_parsers.choices.items()
|
||||
for action in action_parser._actions
|
||||
if 'Deprecated' not in (action.help or ())
|
||||
)
|
||||
)
|
||||
),
|
||||
))
|
||||
|
||||
@@ -5,7 +5,7 @@ def variants(flag_name):
|
||||
"--foo[9].bar".
|
||||
'''
|
||||
if '[0]' in flag_name:
|
||||
for index in range(0, 10):
|
||||
for index in range(10):
|
||||
yield flag_name.replace('[0]', f'[{index}]')
|
||||
|
||||
return
|
||||
|
||||
@@ -10,8 +10,8 @@ def main():
|
||||
levelno=logging.WARNING,
|
||||
levelname='WARNING',
|
||||
msg='generate-borgmatic-config is deprecated and will be removed from a future release. Please use "borgmatic config generate" instead.',
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
sys.argv = ['borgmatic', 'config', 'generate'] + sys.argv[1:]
|
||||
sys.argv = ['borgmatic', 'config', 'generate', *sys.argv[1:]]
|
||||
borgmatic.commands.borgmatic.main([warning_log])
|
||||
|
||||
@@ -10,8 +10,8 @@ def main():
|
||||
levelno=logging.WARNING,
|
||||
levelname='WARNING',
|
||||
msg='validate-borgmatic-config is deprecated and will be removed from a future release. Please use "borgmatic config validate" instead.',
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
sys.argv = ['borgmatic', 'config', 'validate'] + sys.argv[1:]
|
||||
sys.argv = ['borgmatic', 'config', 'validate', *sys.argv[1:]]
|
||||
borgmatic.commands.borgmatic.main([warning_log])
|
||||
|
||||
@@ -146,16 +146,16 @@ def prepare_arguments_for_config(global_arguments, schema):
|
||||
keys = tuple(argument_name.split('.'))
|
||||
option_type = type_for_option(schema, keys)
|
||||
|
||||
# The argument doesn't correspond to any option in the schema, so ignore it. It's
|
||||
# probably a flag that borgmatic has on the command-line but not in configuration.
|
||||
if option_type is None:
|
||||
# The argument doesn't correspond to any option in the schema, or it is a complex argument, so ignore it.
|
||||
# It's probably a flag that borgmatic has on the command-line but not in configuration.
|
||||
if option_type in {'object', None}:
|
||||
continue
|
||||
|
||||
prepared_values.append(
|
||||
(
|
||||
keys,
|
||||
convert_value_type(value, option_type),
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
return tuple(prepared_values)
|
||||
|
||||
@@ -1,17 +1,14 @@
|
||||
import os
|
||||
|
||||
|
||||
def get_default_config_paths(expand_home=True):
|
||||
def get_default_config_paths():
|
||||
'''
|
||||
Based on the value of the XDG_CONFIG_HOME and HOME environment variables, return a list of
|
||||
default configuration paths. This includes both system-wide configuration and configuration in
|
||||
the current user's home directory.
|
||||
|
||||
Don't expand the home directory ($HOME) if the expand home flag is False.
|
||||
Return a list of default configuration paths. This includes both system-wide
|
||||
configuration and configuration in the current user's home directory.
|
||||
'''
|
||||
user_config_directory = os.getenv('XDG_CONFIG_HOME') or os.path.join('$HOME', '.config')
|
||||
if expand_home:
|
||||
user_config_directory = os.path.expandvars(user_config_directory)
|
||||
user_config_directory = os.path.expandvars(
|
||||
os.getenv('XDG_CONFIG_HOME') or os.path.join('$HOME', '.config')
|
||||
)
|
||||
|
||||
return [
|
||||
'/etc/borgmatic/config.yaml',
|
||||
@@ -49,6 +46,7 @@ def collect_config_filenames(config_paths):
|
||||
|
||||
for filename in sorted(os.listdir(path)):
|
||||
full_filename = os.path.join(path, filename)
|
||||
matching_filetype = full_filename.endswith('.yaml') or full_filename.endswith('.yml')
|
||||
matching_filetype = full_filename.endswith(('.yaml', '.yml'))
|
||||
|
||||
if matching_filetype and not os.path.isdir(full_filename):
|
||||
yield os.path.abspath(full_filename)
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import contextlib
|
||||
import functools
|
||||
import re
|
||||
import shlex
|
||||
|
||||
|
||||
@@ -6,20 +9,58 @@ def coerce_scalar(value):
|
||||
Given a configuration value, coerce it to an integer or a boolean as appropriate and return the
|
||||
result.
|
||||
'''
|
||||
try:
|
||||
with contextlib.suppress(TypeError, ValueError):
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
if value == 'true' or value == 'True':
|
||||
return True
|
||||
if value == 'false' or value == 'False':
|
||||
return False
|
||||
|
||||
return value
|
||||
try:
|
||||
return {
|
||||
'true': True,
|
||||
'True': True,
|
||||
'false': False,
|
||||
'False': False,
|
||||
}.get(value, value)
|
||||
except TypeError: # e.g. for an unhashable type
|
||||
return value
|
||||
|
||||
|
||||
def apply_constants(value, constants, shell_escape=False):
|
||||
CONSTANT_PATTERN = re.compile(r'(?P<left_escape>\\)?\{(?P<name>[\w]+)(?P<right_escape>\\)?\}')
|
||||
|
||||
|
||||
def resolve_constant(match, constants, command_hook):
|
||||
'''
|
||||
Given a re.Match instance of CONSTANT_PATTERN representing a matched constant name to be
|
||||
interpolated, a constants dict, and whether this is for a command hook, lookup the matched
|
||||
constant name within the given constants and return its value.
|
||||
|
||||
If the match is escaped with backslashes, then instead of resolving the variable's value, strip
|
||||
off the backslashing and return the literal value.
|
||||
|
||||
If the variable name isn't found in the given constants, then return the literal value.
|
||||
'''
|
||||
name = match.group('name')
|
||||
|
||||
# The would-be variable is escaped, so strip off the escaping and return the result without
|
||||
# resolving the name—unless this is for a command hook, in which case just return the literal
|
||||
# string. That way, subsequent variable interpolation will still see the string as escaped
|
||||
# instead of trying to interpolate it.
|
||||
if match.group('left_escape') and match.group('right_escape'):
|
||||
if command_hook:
|
||||
return match.group(0)
|
||||
|
||||
return '{' + name + '}'
|
||||
|
||||
value = constants.get(name)
|
||||
|
||||
# The matched variable is in the constants, so return its value. And if this is for a command
|
||||
# hook, then shell escape the value so as to prevent shell injection attacks.
|
||||
if value is not None:
|
||||
return shlex.quote(str(value)) if command_hook else str(value)
|
||||
|
||||
# The matched variable name isn't in the constants. Return the whole string unaltered.
|
||||
return match.group(0)
|
||||
|
||||
|
||||
def apply_constants(value, constants, command_hook=False):
|
||||
'''
|
||||
Given a configuration value (bool, dict, int, list, or string) and a dict of named constants,
|
||||
replace any configuration string values of the form "{constant}" (or containing it) with the
|
||||
@@ -37,27 +78,25 @@ def apply_constants(value, constants, shell_escape=False):
|
||||
return value
|
||||
|
||||
if isinstance(value, str):
|
||||
for constant_name, constant_value in constants.items():
|
||||
value = value.replace(
|
||||
'{' + constant_name + '}',
|
||||
shlex.quote(str(constant_value)) if shell_escape else str(constant_value),
|
||||
)
|
||||
|
||||
# Support constants within non-string scalars by coercing the value to its appropriate type.
|
||||
value = coerce_scalar(value)
|
||||
value = coerce_scalar(
|
||||
CONSTANT_PATTERN.sub(
|
||||
functools.partial(resolve_constant, constants=constants, command_hook=command_hook),
|
||||
value,
|
||||
)
|
||||
)
|
||||
elif isinstance(value, list):
|
||||
for index, list_value in enumerate(value):
|
||||
value[index] = apply_constants(list_value, constants, shell_escape)
|
||||
value[index] = apply_constants(list_value, constants, command_hook)
|
||||
elif isinstance(value, dict):
|
||||
for option_name, option_value in value.items():
|
||||
value[option_name] = apply_constants(
|
||||
option_value,
|
||||
constants,
|
||||
shell_escape=(
|
||||
shell_escape
|
||||
or option_name.startswith('before_')
|
||||
or option_name.startswith('after_')
|
||||
or option_name == 'on_error'
|
||||
command_hook=(
|
||||
command_hook
|
||||
or option_name.startswith(('before_', 'after_'))
|
||||
or option_name in {'on_error', 'run'}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import os
|
||||
import re
|
||||
|
||||
VARIABLE_PATTERN = re.compile(
|
||||
r'(?P<escape>\\)?(?P<variable>\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\})'
|
||||
r'(?P<escape>\\)?(?P<variable>\$\{(?P<name>[A-Za-z0-9_]+)((:?-)(?P<default>[^}]+))?\})',
|
||||
)
|
||||
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user