upstream/borgbackup
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2026-05-28 04:03:21 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'master' into lrucache

Browse source
This commit is contained in:
Alan Jenkins 2015-08-14 10:59:21 +01:00
commit 5e0013c5db
84 changed files with 5792 additions and 2588 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
.coveragerc Normal file
View file

@ -0,0 +1,17 @@
[run]
branch = True
source = borg
omit =
borg/__init__.py
borg/__main__.py
borg/_version.py
[report]
exclude_lines =
pragma: no cover
def __repr__
raise AssertionError
raise NotImplementedError
if 0:
if __name__ == .__main__.:
ignore_errors = True

2
.gitattributes vendored
View file

@ -1 +1 @@
attic/_version.py export-subst
borg/_version.py export-subst

10
.gitignore vendored
View file

@ -6,8 +6,18 @@ env
.tox
hashindex.c
chunker.c
crypto.c
platform_darwin.c
platform_freebsd.c
platform_linux.c
*.egg-info
*.pyc
*.pyo
*.so
docs/usage/*.inc
.idea/
.cache/
borg.build/
borg.dist/
borg.exe
.coverage

55
.travis.yml
View file

@ -1,12 +1,47 @@
sudo: required
language: python
python:
- "3.2"
- "3.3"
- "3.4"
# command to install dependencies
cache:
directories:
- $HOME/.cache/pip
matrix:
include:
- python: 3.2
os: linux
env: TOXENV=py32
- python: 3.3
os: linux
env: TOXENV=py33
- python: 3.4
os: linux
env: TOXENV=py34
- language: generic
os: osx
osx_image: xcode6.4
env: TOXENV=py32
- language: generic
os: osx
osx_image: xcode6.4
env: TOXENV=py33
- language: generic
os: osx
osx_image: xcode6.4
env: TOXENV=py34
install:
- "sudo apt-get install -y libacl1-dev"
- "pip install --use-mirrors Cython"
- "pip install -e ."
# command to run tests
script: fakeroot -u python -m attic.testsuite.run -vb
- ./.travis/install.sh
script:
- ./.travis/run.sh
after_success:
- ./.travis/upload_coverage.sh
notifications:
irc:
channels:
- "irc.freenode.org#borgbackup"
use_notice: true
skip_join: true

43
.travis/install.sh Executable file
View file

@ -0,0 +1,43 @@
#!/bin/bash
set -e
set -x
if [[ "$(uname -s)" == 'Darwin' ]]; then
brew update || brew update
if [[ "${OPENSSL}" != "0.9.8" ]]; then
brew outdated openssl || brew upgrade openssl
fi
if which pyenv > /dev/null; then
eval "$(pyenv init -)"
fi
brew outdated pyenv || brew upgrade pyenv
case "${TOXENV}" in
py32)
pyenv install 3.2.6
pyenv global 3.2.6
;;
py33)
pyenv install 3.3.6
pyenv global 3.3.6
;;
py34)
pyenv install 3.4.3
pyenv global 3.4.3
;;
esac
pyenv rehash
python -m pip install --user virtualenv
else
pip install virtualenv
sudo apt-get install -y libacl1-dev
fi
python -m virtualenv ~/.venv
source ~/.venv/bin/activate
pip install tox pytest pytest-cov codecov Cython
pip install -e .

23
.travis/run.sh Executable file
View file

@ -0,0 +1,23 @@
#!/bin/bash
set -e
set -x
if [[ "$(uname -s)" == "Darwin" ]]; then
eval "$(pyenv init -)"
if [[ "${OPENSSL}" != "0.9.8" ]]; then
# set our flags to use homebrew openssl
export ARCHFLAGS="-arch x86_64"
export LDFLAGS="-L/usr/local/opt/openssl/lib"
export CFLAGS="-I/usr/local/opt/openssl/include"
fi
fi
source ~/.venv/bin/activate
if [[ "$(uname -s)" == "Darwin" ]]; then
# no fakeroot on OS X
sudo tox -e $TOXENV
else
fakeroot -u tox
fi

13
.travis/upload_coverage.sh Executable file
View file

@ -0,0 +1,13 @@
#!/bin/bash
set -e
set -x
NO_COVERAGE_TOXENVS=(pep8)
if ! [[ "${NO_COVERAGE_TOXENVS[*]}" =~ "${TOXENV}" ]]; then
source ~/.venv/bin/activate
ln .tox/.coverage .coverage
# on osx, tests run as root, need access to .coverage
sudo chmod 666 .coverage
codecov -e TRAVIS_OS_NAME TOXENV
fi

12
AUTHORS
View file

@ -1,9 +1,15 @@
Attic is written and maintained by Jonas Borgström and
various contributors:
Borg Developers / Contributors ("The Borg Collective")
``````````````````````````````````````````````````````
- Thomas Waldmann <tw@waldmann-edv.de>
- Antoine Beaupré
- Radek Podgorny <radek@podgorny.cz>
- Yuri D'Elia
Borg is a fork of Attic. Attic is written and maintained
by Jonas Borgström and various contributors:
Development Lead
````````````````
- Jonas Borgström <jonas@borgstrom.se>
Patches and Suggestions

163
CHANGES
View file

@ -1,163 +0,0 @@
Attic Changelog
===============
Here you can see the full list of changes between each Attic release.
Version 0.17
------------
(bugfix release, released on X)
- Fix hashindex ARM memory alignment issue (#309)
- Improve hashindex error messages (#298)
Version 0.16
------------
(bugfix release, released on May 16, 2015)
- Fix typo preventing the security confirmation prompt from working (#303)
- Improve handling of systems with improperly configured file system encoding (#289)
- Fix "All archives" output for attic info. (#183)
- More user friendly error message when repository key file is not found (#236)
- Fix parsing of iso 8601 timestamps with zero microseconds (#282)
Version 0.15
------------
(bugfix release, released on Apr 15, 2015)
- xattr: Be less strict about unknown/unsupported platforms (#239)
- Reduce repository listing memory usage (#163).
- Fix BrokenPipeError for remote repositories (#233)
- Fix incorrect behavior with two character directory names (#265, #268)
- Require approval before accessing relocated/moved repository (#271)
- Require approval before accessing previously unknown unencrypted repositories (#271)
- Fix issue with hash index files larger than 2GB.
- Fix Python 3.2 compatibility issue with noatime open() (#164)
- Include missing pyx files in dist files (#168)
Version 0.14
------------
(feature release, released on Dec 17, 2014)
- Added support for stripping leading path segments (#95)
"attic extract --strip-segments X"
- Add workaround for old Linux systems without acl_extended_file_no_follow (#96)
- Add MacPorts' path to the default openssl search path (#101)
- HashIndex improvements, eliminates unnecessary IO on low memory systems.
- Fix "Number of files" output for attic info. (#124)
- limit create file permissions so files aren't read while restoring
- Fix issue with empty xattr values (#106)
Version 0.13
------------
(feature release, released on Jun 29, 2014)
- Fix sporadic "Resource temporarily unavailable" when using remote repositories
- Reduce file cache memory usage (#90)
- Faster AES encryption (utilizing AES-NI when available)
- Experimental Linux, OS X and FreeBSD ACL support (#66)
- Added support for backup and restore of BSDFlags (OSX, FreeBSD) (#56)
- Fix bug where xattrs on symlinks were not correctly restored
- Added cachedir support. CACHEDIR.TAG compatible cache directories
can now be excluded using ``--exclude-caches`` (#74)
- Fix crash on extreme mtime timestamps (year 2400+) (#81)
- Fix Python 3.2 specific lockf issue (EDEADLK)
Version 0.12
------------
(feature release, released on April 7, 2014)
- Python 3.4 support (#62)
- Various documentation improvements a new style
- ``attic mount`` now supports mounting an entire repository not only
individual archives (#59)
- Added option to restrict remote repository access to specific path(s):
``attic serve --restrict-to-path X`` (#51)
- Include "all archives" size information in "--stats" output. (#54)
- Added ``--stats`` option to ``attic delete`` and ``attic prune``
- Fixed bug where ``attic prune`` used UTC instead of the local time zone
when determining which archives to keep.
- Switch to SI units (Power of 1000 instead 1024) when printing file sizes
Version 0.11
------------
(feature release, released on March 7, 2014)
- New "check" command for repository consistency checking (#24)
- Documentation improvements
- Fix exception during "attic create" with repeated files (#39)
- New "--exclude-from" option for attic create/extract/verify.
- Improved archive metadata deduplication.
- "attic verify" has been deprecated. Use "attic extract --dry-run" instead.
- "attic prune --hourly|daily|..." has been deprecated.
Use "attic prune --keep-hourly|daily|..." instead.
- Ignore xattr errors during "extract" if not supported by the filesystem. (#46)
Version 0.10
------------
(bugfix release, released on Jan 30, 2014)
- Fix deadlock when extracting 0 sized files from remote repositories
- "--exclude" wildcard patterns are now properly applied to the full path
not just the file name part (#5).
- Make source code endianness agnostic (#1)
Version 0.9
-----------
(feature release, released on Jan 23, 2014)
- Remote repository speed and reliability improvements.
- Fix sorting of segment names to ignore NFS left over files. (#17)
- Fix incorrect display of time (#13)
- Improved error handling / reporting. (#12)
- Use fcntl() instead of flock() when locking repository/cache. (#15)
- Let ssh figure out port/user if not specified so we don't override .ssh/config (#9)
- Improved libcrypto path detection (#23).
Version 0.8.1
-------------
(bugfix release, released on Oct 4, 2013)
- Fix segmentation fault issue.
Version 0.8
-----------
(feature release, released on Oct 3, 2013)
- Fix xattr issue when backing up sshfs filesystems (#4)
- Fix issue with excessive index file size (#6)
- Support access of read only repositories.
- New syntax to enable repository encryption:
attic init --encryption="none|passphrase|keyfile".
- Detect and abort if repository is older than the cache.
Version 0.7
-----------
(feature release, released on Aug 5, 2013)
- Ported to FreeBSD
- Improved documentation
- Experimental: Archives mountable as fuse filesystems.
- The "user." prefix is no longer stripped from xattrs on Linux
Version 0.6.1
-------------
(bugfix release, released on July 19, 2013)
- Fixed an issue where mtime was not always correctly restored.
Version 0.6
-----------
First public release on July 9, 2013

359
CHANGES.rst Normal file
View file

@ -0,0 +1,359 @@
Borg Changelog
==============
Version 0.25.0 (not released yet)
---------------------------------
Incompatible changes (compared to 0.24):
- none yet
Deprecations:
- none yet
New features:
- honor the nodump flag (UF_NODUMP) and do not backup such items
Bug fixes:
- close fds of segments we delete (during compaction)
Other changes:
- none yet
Version 0.24.0
--------------
Incompatible changes (compared to 0.23):
- borg now always issues --umask NNN option when invoking another borg via ssh
on the repository server. By that, it's making sure it uses the same umask
for remote repos as for local ones. Because of this, you must upgrade both
server and client(s) to 0.24.
- the default umask is 077 now (if you do not specify via --umask) which might
be a different one as you used previously. The default umask avoids that
you accidentally give access permissions for group and/or others to files
created by borg (e.g. the repository).
Deprecations:
- "--encryption passphrase" mode is deprecated, see #85 and #97.
See the new "--encryption repokey" mode for a replacement.
New features:
- borg create --chunker-params ... to configure the chunker, fixes #16
(attic #302, attic #300, and somehow also #41).
This can be used to reduce memory usage caused by chunk management overhead,
so borg does not create a huge chunks index/repo index and eats all your RAM
if you back up lots of data in huge files (like VM disk images).
See docs/misc/create_chunker-params.txt for more information.
- borg info now reports chunk counts in the chunk index.
- borg create --compression 0..9 to select zlib compression level, fixes #66
(attic #295).
- borg init --encryption repokey (to store the encryption key into the repo),
fixes #85
- improve at-end error logging, always log exceptions and set exit_code=1
- LoggedIO: better error checks / exceptions / exception handling
- implement --remote-path to allow non-default-path borg locations, #125
- implement --umask M and use 077 as default umask for better security, #117
- borg check: give a named single archive to it, fixes #139
- cache sync: show progress indication
- cache sync: reimplement the chunk index merging in C
Bug fixes:
- fix segfault that happened for unreadable files (chunker: n needs to be a
signed size_t), #116
- fix the repair mode, #144
- repo delete: add destroy to allowed rpc methods, fixes issue #114
- more compatible repository locking code (based on mkdir), maybe fixes #92
(attic #317, attic #201).
- better Exception msg if no Borg is installed on the remote repo server, #56
- create a RepositoryCache implementation that can cope with >2GiB,
fixes attic #326.
- fix Traceback when running check --repair, attic #232
- clarify help text, fixes #73.
- add help string for --no-files-cache, fixes #140
Other changes:
- improved docs:
- added docs/misc directory for misc. writeups that won't be included
"as is" into the html docs.
- document environment variables and return codes (attic #324, attic #52)
- web site: add related projects, fix web site url, IRC #borgbackup
- Fedora/Fedora-based install instructions added to docs
- Cygwin-based install instructions added to docs
- updated AUTHORS
- add FAQ entries about redundancy / integrity
- clarify that borg extract uses the cwd as extraction target
- update internals doc about chunker params, memory usage and compression
- added docs about development
- add some words about resource usage in general
- document how to backup a raw disk
- add note about how to run borg from virtual env
- add solutions for (ll)fuse installation problems
- document what borg check does, fixes #138
- reorganize borgbackup.github.io sidebar, prev/next at top
- deduplicate and refactor the docs / README.rst
- use borg-tmp as prefix for temporary files / directories
- short prune options without "keep-" are deprecated, do not suggest them
- improved tox configuration
- remove usage of unittest.mock, always use mock from pypi
- use entrypoints instead of scripts, for better use of the wheel format and
modern installs
- add requirements.d/development.txt and modify tox.ini
- use travis-ci for testing based on Linux and (new) OS X
- use coverage.py, pytest-cov and codecov.io for test coverage support
I forgot to list some stuff already implemented in 0.23.0, here they are:
New features:
- efficient archive list from manifest, meaning a big speedup for slow
repo connections and "list <repo>", "delete <repo>", "prune" (attic #242,
attic #167)
- big speedup for chunks cache sync (esp. for slow repo connections), fixes #18
- hashindex: improve error messages
Other changes:
- explicitly specify binary mode to open binary files
- some easy micro optimizations
Version 0.23.0
--------------
Incompatible changes (compared to attic, fork related):
- changed sw name and cli command to "borg", updated docs
- package name (and name in urls) uses "borgbackup" to have less collisions
- changed repo / cache internal magic strings from ATTIC* to BORG*,
changed cache location to .cache/borg/ - this means that it currently won't
accept attic repos (see issue #21 about improving that)
Bug fixes:
- avoid defect python-msgpack releases, fixes attic #171, fixes attic #185
- fix traceback when trying to do unsupported passphrase change, fixes attic #189
- datetime does not like the year 10.000, fixes attic #139
- fix "info" all archives stats, fixes attic #183
- fix parsing with missing microseconds, fixes attic #282
- fix misleading hint the fuse ImportError handler gave, fixes attic #237
- check unpacked data from RPC for tuple type and correct length, fixes attic #127
- fix Repository._active_txn state when lock upgrade fails
- give specific path to xattr.is_enabled(), disable symlink setattr call that
always fails
- fix test setup for 32bit platforms, partial fix for attic #196
- upgraded versioneer, PEP440 compliance, fixes attic #257
New features:
- less memory usage: add global option --no-cache-files
- check --last N (only check the last N archives)
- check: sort archives in reverse time order
- rename repo::oldname newname (rename repository)
- create -v output more informative
- create --progress (backup progress indicator)
- create --timestamp (utc string or reference file/dir)
- create: if "-" is given as path, read binary from stdin
- extract: if --stdout is given, write all extracted binary data to stdout
- extract --sparse (simple sparse file support)
- extra debug information for 'fread failed'
- delete <repo> (deletes whole repo + local cache)
- FUSE: reflect deduplication in allocated blocks
- only allow whitelisted RPC calls in server mode
- normalize source/exclude paths before matching
- use posix_fadvise to not spoil the OS cache, fixes attic #252
- toplevel error handler: show tracebacks for better error analysis
- sigusr1 / sigint handler to print current file infos - attic PR #286
- RPCError: include the exception args we get from remote
Other changes:
- source: misc. cleanups, pep8, style
- docs and faq improvements, fixes, updates
- cleanup crypto.pyx, make it easier to adapt to other AES modes
- do os.fsync like recommended in the python docs
- source: Let chunker optionally work with os-level file descriptor.
- source: Linux: remove duplicate os.fsencode calls
- source: refactor _open_rb code a bit, so it is more consistent / regular
- source: refactor indicator (status) and item processing
- source: use py.test for better testing, flake8 for code style checks
- source: fix tox >=2.0 compatibility (test runner)
- pypi package: add python version classifiers, add FreeBSD to platforms
Attic Changelog
===============
Here you can see the full list of changes between each Attic release until Borg
forked from Attic:
Version 0.17
------------
(bugfix release, released on X)
- Fix hashindex ARM memory alignment issue (#309)
- Improve hashindex error messages (#298)
Version 0.16
------------
(bugfix release, released on May 16, 2015)
- Fix typo preventing the security confirmation prompt from working (#303)
- Improve handling of systems with improperly configured file system encoding (#289)
- Fix "All archives" output for attic info. (#183)
- More user friendly error message when repository key file is not found (#236)
- Fix parsing of iso 8601 timestamps with zero microseconds (#282)
Version 0.15
------------
(bugfix release, released on Apr 15, 2015)
- xattr: Be less strict about unknown/unsupported platforms (#239)
- Reduce repository listing memory usage (#163).
- Fix BrokenPipeError for remote repositories (#233)
- Fix incorrect behavior with two character directory names (#265, #268)
- Require approval before accessing relocated/moved repository (#271)
- Require approval before accessing previously unknown unencrypted repositories (#271)
- Fix issue with hash index files larger than 2GB.
- Fix Python 3.2 compatibility issue with noatime open() (#164)
- Include missing pyx files in dist files (#168)
Version 0.14
------------
(feature release, released on Dec 17, 2014)
- Added support for stripping leading path segments (#95)
"attic extract --strip-segments X"
- Add workaround for old Linux systems without acl_extended_file_no_follow (#96)
- Add MacPorts' path to the default openssl search path (#101)
- HashIndex improvements, eliminates unnecessary IO on low memory systems.
- Fix "Number of files" output for attic info. (#124)
- limit create file permissions so files aren't read while restoring
- Fix issue with empty xattr values (#106)
Version 0.13
------------
(feature release, released on Jun 29, 2014)
- Fix sporadic "Resource temporarily unavailable" when using remote repositories
- Reduce file cache memory usage (#90)
- Faster AES encryption (utilizing AES-NI when available)
- Experimental Linux, OS X and FreeBSD ACL support (#66)
- Added support for backup and restore of BSDFlags (OSX, FreeBSD) (#56)
- Fix bug where xattrs on symlinks were not correctly restored
- Added cachedir support. CACHEDIR.TAG compatible cache directories
can now be excluded using ``--exclude-caches`` (#74)
- Fix crash on extreme mtime timestamps (year 2400+) (#81)
- Fix Python 3.2 specific lockf issue (EDEADLK)
Version 0.12
------------
(feature release, released on April 7, 2014)
- Python 3.4 support (#62)
- Various documentation improvements a new style
- ``attic mount`` now supports mounting an entire repository not only
individual archives (#59)
- Added option to restrict remote repository access to specific path(s):
``attic serve --restrict-to-path X`` (#51)
- Include "all archives" size information in "--stats" output. (#54)
- Added ``--stats`` option to ``attic delete`` and ``attic prune``
- Fixed bug where ``attic prune`` used UTC instead of the local time zone
when determining which archives to keep.
- Switch to SI units (Power of 1000 instead 1024) when printing file sizes
Version 0.11
------------
(feature release, released on March 7, 2014)
- New "check" command for repository consistency checking (#24)
- Documentation improvements
- Fix exception during "attic create" with repeated files (#39)
- New "--exclude-from" option for attic create/extract/verify.
- Improved archive metadata deduplication.
- "attic verify" has been deprecated. Use "attic extract --dry-run" instead.
- "attic prune --hourly|daily|..." has been deprecated.
Use "attic prune --keep-hourly|daily|..." instead.
- Ignore xattr errors during "extract" if not supported by the filesystem. (#46)
Version 0.10
------------
(bugfix release, released on Jan 30, 2014)
- Fix deadlock when extracting 0 sized files from remote repositories
- "--exclude" wildcard patterns are now properly applied to the full path
not just the file name part (#5).
- Make source code endianness agnostic (#1)
Version 0.9
-----------
(feature release, released on Jan 23, 2014)
- Remote repository speed and reliability improvements.
- Fix sorting of segment names to ignore NFS left over files. (#17)
- Fix incorrect display of time (#13)
- Improved error handling / reporting. (#12)
- Use fcntl() instead of flock() when locking repository/cache. (#15)
- Let ssh figure out port/user if not specified so we don't override .ssh/config (#9)
- Improved libcrypto path detection (#23).
Version 0.8.1
-------------
(bugfix release, released on Oct 4, 2013)
- Fix segmentation fault issue.
Version 0.8
-----------
(feature release, released on Oct 3, 2013)
- Fix xattr issue when backing up sshfs filesystems (#4)
- Fix issue with excessive index file size (#6)
- Support access of read only repositories.
- New syntax to enable repository encryption:
attic init --encryption="none|passphrase|keyfile".
- Detect and abort if repository is older than the cache.
Version 0.7
-----------
(feature release, released on Aug 5, 2013)
- Ported to FreeBSD
- Improved documentation
- Experimental: Archives mountable as fuse filesystems.
- The "user." prefix is no longer stripped from xattrs on Linux
Version 0.6.1
-------------
(bugfix release, released on July 19, 2013)
- Fixed an issue where mtime was not always correctly restored.
Version 0.6
-----------
First public release on July 9, 2013

1
LICENSE
View file

@ -1,3 +1,4 @@
Copyright (C) 2015 The Borg Collective (see AUTHORS file)
Copyright (C) 2010-2014 Jonas Borgström <jonas@borgstrom.se>
All rights reserved.

5
MANIFEST.in
View file

@ -1,6 +1,7 @@
include README.rst LICENSE CHANGES MANIFEST.in versioneer.py
recursive-include attic *.pyx
include README.rst AUTHORS LICENSE CHANGES.rst MANIFEST.in versioneer.py
recursive-include borg *.pyx
recursive-include docs *
recursive-exclude docs *.pyc
recursive-exclude docs *.pyo
prune docs/_build
include borg/_version.py

158
README.rst
View file

@ -1,57 +1,129 @@
What is Attic?
--------------
Attic is a deduplicating backup program. The main goal of Attic is to provide
an efficient and secure way to backup data. The data deduplication
technique used makes Attic suitable for daily backups since only changes
are stored.
What is BorgBackup?
-------------------
BorgBackup (short: Borg) is a deduplicating backup program.
Optionally, it supports compression and authenticated encryption.
Easy to use
~~~~~~~~~~~
Initialize backup repository and create a backup archive::
The main goal of Borg is to provide an efficient and secure way to backup data.
The data deduplication technique used makes Borg suitable for daily backups
since only changes are stored.
The authenticated encryption technique makes it suitable for backups to not
fully trusted targets.
`Borg Installation docs <http://borgbackup.github.io/borgbackup/installation.html>`_
$ attic init /usbdrive/my-backup.attic
$ attic create -v /usbdrive/my-backup.attic::documents ~/Documents
Main features
~~~~~~~~~~~~~
Space efficient storage
Variable block size deduplication is used to reduce the number of bytes
stored by detecting redundant data. Each file is split into a number of
variable length chunks and only chunks that have never been seen before are
compressed and added to the repository.
**Space efficient storage**
Deduplication based on content-defined chunking is used to reduce the number
of bytes stored: each file is split into a number of variable length chunks
and only chunks that have never been seen before are added to the repository.
Optional data encryption
All data can be protected using 256-bit AES encryption and data integrity
and authenticity is verified using HMAC-SHA256.
To deduplicate, all the chunks in the same repository are considered, no
matter whether they come from different machines, from previous backups,
from the same backup or even from the same single file.
Off-site backups
Attic can store data on any remote host accessible over SSH. This is
most efficient if Attic is also installed on the remote host.
Compared to other deduplication approaches, this method does NOT depend on:
Backups mountable as filesystems
Backup archives are mountable as userspace filesystems for easy backup
verification and restores.
* file/directory names staying the same
What do I need?
---------------
Attic requires Python 3.2 or above to work. Besides Python, Attic also requires
msgpack-python and sufficiently recent OpenSSL (>= 1.0.0).
In order to mount archives as filesystems, llfuse is required.
So you can move your stuff around without killing the deduplication,
even between machines sharing a repo.
How do I install it?
--------------------
::
* complete files or time stamps staying the same
$ pip install Attic
If a big file changes a little, only a few new chunks will be stored -
this is great for VMs or raw disks.
Where are the docs?
-------------------
Go to https://attic-backup.org/ for a prebuilt version of the documentation.
You can also build it yourself from the docs folder.
* the absolute position of a data chunk inside a file
Where are the tests?
--------------------
The tests are in the attic/testsuite package. To run the test suite use the
following command::
Stuff may get shifted and will still be found by the deduplication
algorithm.
$ fakeroot -u python -m attic.testsuite.run
**Speed**
* performance critical code (chunking, compression, encryption) is
implemented in C/Cython
* local caching of files/chunks index data
* quick detection of unmodified files
**Data encryption**
All data can be protected using 256-bit AES encryption, data integrity and
authenticity is verified using HMAC-SHA256.
**Compression**
All data can be compressed by zlib, level 0-9.
**Off-site backups**
Borg can store data on any remote host accessible over SSH. If Borg is
installed on the remote host, big performance gains can be achieved
compared to using a network filesystem (sshfs, nfs, ...).
**Backups mountable as filesystems**
Backup archives are mountable as userspace filesystems for easy interactive
backup examination and restores (e.g. by using a regular file manager).
**Platforms Borg works on**
* Linux
* FreeBSD
* Mac OS X
* Cygwin (unsupported)
**Free and Open Source Software**
* security and functionality can be audited independently
* licensed under the BSD (3-clause) license
Easy to use
~~~~~~~~~~~
Initialize a new backup repository and create a backup archive::
$ borg init /mnt/backup
$ borg create /mnt/backup::Monday ~/Documents
Now doing another backup, just to show off the great deduplication::
$ borg create --stats /mnt/backup::Tuesday ~/Documents
Archive name: Tuesday
Archive fingerprint: 387a5e3f9b0e792e91c...
Start time: Tue Mar 25 12:00:10 2014
End time: Tue Mar 25 12:00:10 2014
Duration: 0.08 seconds
Number of files: 358
Original size Compressed size Deduplicated size
This archive: 57.16 MB 46.78 MB 151.67 kB <--- !
All archives: 114.02 MB 93.46 MB 44.81 MB
For a graphical frontend refer to our complementary project
`BorgWeb <https://github.com/borgbackup/borgweb>`_.
Notes
-----
Borg is a fork of `Attic <https://github.com/jborg/attic>`_ and maintained by
"`The Borg Collective <https://github.com/borgbackup/borg/blob/master/AUTHORS>`_".
Read `issue #1 <https://github.com/borgbackup/borg/issues/1>`_ about the initial
considerations regarding project goals and policy of the Borg project.
BORG IS NOT COMPATIBLE WITH ORIGINAL ATTIC.
EXPECT THAT WE WILL BREAK COMPATIBILITY REPEATEDLY WHEN MAJOR RELEASE NUMBER
CHANGES (like when going from 0.x.y to 1.0.0). Please read CHANGES document.
NOT RELEASED DEVELOPMENT VERSIONS HAVE UNKNOWN COMPATIBILITY PROPERTIES.
THIS IS SOFTWARE IN DEVELOPMENT, DECIDE YOURSELF WHETHER IT FITS YOUR NEEDS.
For more information, please also see the
`LICENSE <https://github.com/borgbackup/borg/blob/master/LICENSE>`_.
|build| |coverage|
.. |build| image:: https://travis-ci.org/borgbackup/borg.svg
:alt: Build Status
:target: https://travis-ci.org/borgbackup/borg
.. |coverage| image:: http://codecov.io/github/borgbackup/borg/coverage.svg?branch=master
:alt: Test Coverage
:target: http://codecov.io/github/borgbackup/borg?branch=master

197
attic/_version.py
View file

@ -1,197 +0,0 @@
IN_LONG_VERSION_PY = True
# This file helps to compute a version number in source trees obtained from
# git-archive tarball (such as those provided by githubs download-from-tag
# feature). Distribution tarballs (build by setup.py sdist) and build
# directories (produced by setup.py build) will contain a much shorter file
# that just contains the computed version number.
# This file is released into the public domain. Generated by
# versioneer-0.7+ (https://github.com/warner/python-versioneer)
# these strings will be replaced by git during git-archive
git_refnames = "$Format:%d$"
git_full = "$Format:%H$"
import subprocess
import sys
def run_command(args, cwd=None, verbose=False):
try:
# remember shell=False, so use git.cmd on windows, not just git
p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd)
except EnvironmentError:
e = sys.exc_info()[1]
if verbose:
print("unable to run %s" % args[0])
print(e)
return None
stdout = p.communicate()[0].strip()
if sys.version >= '3':
stdout = stdout.decode()
if p.returncode != 0:
if verbose:
print("unable to run %s (error)" % args[0])
return None
return stdout
import sys
import re
import os.path
def get_expanded_variables(versionfile_source):
# the code embedded in _version.py can just fetch the value of these
# variables. When used from setup.py, we don't want to import
# _version.py, so we do it with a regexp instead. This function is not
# used from _version.py.
variables = {}
try:
for line in open(versionfile_source,"r").readlines():
if line.strip().startswith("git_refnames ="):
mo = re.search(r'=\s*"(.*)"', line)
if mo:
variables["refnames"] = mo.group(1)
if line.strip().startswith("git_full ="):
mo = re.search(r'=\s*"(.*)"', line)
if mo:
variables["full"] = mo.group(1)
except EnvironmentError:
pass
return variables
def versions_from_expanded_variables(variables, tag_prefix, verbose=False):
refnames = variables["refnames"].strip()
if refnames.startswith("$Format"):
if verbose:
print("variables are unexpanded, not using")
return {} # unexpanded, so not in an unpacked git-archive tarball
refs = set([r.strip() for r in refnames.strip("()").split(",")])
for ref in list(refs):
if not re.search(r'\d', ref):
if verbose:
print("discarding '%s', no digits" % ref)
refs.discard(ref)
# Assume all version tags have a digit. git's %d expansion
# behaves like git log --decorate=short and strips out the
# refs/heads/ and refs/tags/ prefixes that would let us
# distinguish between branches and tags. By ignoring refnames
# without digits, we filter out many common branch names like
# "release" and "stabilization", as well as "HEAD" and "master".
if verbose:
print("remaining refs: %s" % ",".join(sorted(refs)))
for ref in sorted(refs):
# sorting will prefer e.g. "2.0" over "2.0rc1"
if ref.startswith(tag_prefix):
r = ref[len(tag_prefix):]
if verbose:
print("picking %s" % r)
return { "version": r,
"full": variables["full"].strip() }
# no suitable tags, so we use the full revision id
if verbose:
print("no suitable tags, using full revision id")
return { "version": variables["full"].strip(),
"full": variables["full"].strip() }
def versions_from_vcs(tag_prefix, versionfile_source, verbose=False):
# this runs 'git' from the root of the source tree. That either means
# someone ran a setup.py command (and this code is in versioneer.py, so
# IN_LONG_VERSION_PY=False, thus the containing directory is the root of
# the source tree), or someone ran a project-specific entry point (and
# this code is in _version.py, so IN_LONG_VERSION_PY=True, thus the
# containing directory is somewhere deeper in the source tree). This only
# gets called if the git-archive 'subst' variables were *not* expanded,
# and _version.py hasn't already been rewritten with a short version
# string, meaning we're inside a checked out source tree.
try:
here = os.path.abspath(__file__)
except NameError:
# some py2exe/bbfreeze/non-CPython implementations don't do __file__
return {} # not always correct
# versionfile_source is the relative path from the top of the source tree
# (where the .git directory might live) to this file. Invert this to find
# the root from __file__.
root = here
if IN_LONG_VERSION_PY:
for i in range(len(versionfile_source.split("/"))):
root = os.path.dirname(root)
else:
root = os.path.dirname(here)
if not os.path.exists(os.path.join(root, ".git")):
if verbose:
print("no .git in %s" % root)
return {}
GIT = "git"
if sys.platform == "win32":
GIT = "git.cmd"
stdout = run_command([GIT, "describe", "--tags", "--dirty", "--always"],
cwd=root)
if stdout is None:
return {}
if not stdout.startswith(tag_prefix):
if verbose:
print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix))
return {}
tag = stdout[len(tag_prefix):]
stdout = run_command([GIT, "rev-parse", "HEAD"], cwd=root)
if stdout is None:
return {}
full = stdout.strip()
if tag.endswith("-dirty"):
full += "-dirty"
return {"version": tag, "full": full}
def versions_from_parentdir(parentdir_prefix, versionfile_source, verbose=False):
if IN_LONG_VERSION_PY:
# We're running from _version.py. If it's from a source tree
# (execute-in-place), we can work upwards to find the root of the
# tree, and then check the parent directory for a version string. If
# it's in an installed application, there's no hope.
try:
here = os.path.abspath(__file__)
except NameError:
# py2exe/bbfreeze/non-CPython don't have __file__
return {} # without __file__, we have no hope
# versionfile_source is the relative path from the top of the source
# tree to _version.py. Invert this to find the root from __file__.
root = here
for i in range(len(versionfile_source.split("/"))):
root = os.path.dirname(root)
else:
# we're running from versioneer.py, which means we're running from
# the setup.py in a source tree. sys.argv[0] is setup.py in the root.
here = os.path.abspath(sys.argv[0])
root = os.path.dirname(here)
# Source tarballs conventionally unpack into a directory that includes
# both the project name and a version string.
dirname = os.path.basename(root)
if not dirname.startswith(parentdir_prefix):
if verbose:
print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" %
(root, dirname, parentdir_prefix))
return None
return {"version": dirname[len(parentdir_prefix):], "full": ""}
tag_prefix = ""
parentdir_prefix = "Attic-"
versionfile_source = "attic/_version.py"
def get_versions(default={"version": "unknown", "full": ""}, verbose=False):
variables = { "refnames": git_refnames, "full": git_full }
ver = versions_from_expanded_variables(variables, tag_prefix, verbose)
if not ver:
ver = versions_from_vcs(tag_prefix, versionfile_source, verbose)
if not ver:
ver = versions_from_parentdir(parentdir_prefix, versionfile_source,
verbose)
if not ver:
ver = default
return ver

125
attic/crypto.pyx
View file

@ -1,125 +0,0 @@
"""A thin OpenSSL wrapper
This could be replaced by PyCrypto or something similar when the performance
of their PBKDF2 implementation is comparable to the OpenSSL version.
"""
from libc.stdlib cimport malloc, free
API_VERSION = 2
cdef extern from "openssl/rand.h":
int RAND_bytes(unsigned char *buf,int num)
cdef extern from "openssl/evp.h":
ctypedef struct EVP_MD:
pass
ctypedef struct EVP_CIPHER:
pass
ctypedef struct EVP_CIPHER_CTX:
unsigned char *iv
pass
ctypedef struct ENGINE:
pass
const EVP_MD *EVP_sha256()
const EVP_CIPHER *EVP_aes_256_ctr()
void EVP_CIPHER_CTX_init(EVP_CIPHER_CTX *a)
void EVP_CIPHER_CTX_cleanup(EVP_CIPHER_CTX *a)
int EVP_EncryptInit_ex(EVP_CIPHER_CTX *ctx,const EVP_CIPHER *cipher, ENGINE *impl,
const unsigned char *key, const unsigned char *iv)
int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out,
int *outl, const unsigned char *in_, int inl)
int PKCS5_PBKDF2_HMAC(const char *password, int passwordlen,
const unsigned char *salt, int saltlen, int iter,
const EVP_MD *digest,
int keylen, unsigned char *out)
import struct
_int = struct.Struct('>I')
_long = struct.Struct('>Q')
bytes_to_int = lambda x, offset=0: _int.unpack_from(x, offset)[0]
bytes_to_long = lambda x, offset=0: _long.unpack_from(x, offset)[0]
long_to_bytes = lambda x: _long.pack(x)
def num_aes_blocks(length):
"""Return the number of AES blocks required to encrypt/decrypt *length* bytes of data
"""
return (length + 15) // 16
def pbkdf2_sha256(password, salt, iterations, size):
"""Password based key derivation function 2 (RFC2898)
"""
cdef unsigned char *key = <unsigned char *>malloc(size)
if not key:
raise MemoryError
try:
rv = PKCS5_PBKDF2_HMAC(password, len(password), salt, len(salt), iterations, EVP_sha256(), size, key)
if not rv:
raise Exception('PKCS5_PBKDF2_HMAC failed')
return key[:size]
finally:
free(key)
def get_random_bytes(n):
"""Return n cryptographically strong pseudo-random bytes
"""
cdef unsigned char *buf = <unsigned char *>malloc(n)
if not buf:
raise MemoryError
try:
if RAND_bytes(buf, n) < 1:
raise Exception('RAND_bytes failed')
return buf[:n]
finally:
free(buf)
cdef class AES:
"""A thin wrapper around the OpenSSL EVP cipher API
"""
cdef EVP_CIPHER_CTX ctx
def __cinit__(self, key, iv=None):
EVP_CIPHER_CTX_init(&self.ctx)
if not EVP_EncryptInit_ex(&self.ctx, EVP_aes_256_ctr(), NULL, NULL, NULL):
raise Exception('EVP_EncryptInit_ex failed')
self.reset(key, iv)
def __dealloc__(self):
EVP_CIPHER_CTX_cleanup(&self.ctx)
def reset(self, key=None, iv=None):
cdef const unsigned char *key2 = NULL
cdef const unsigned char *iv2 = NULL
if key:
key2 = key
if iv:
iv2 = iv
if not EVP_EncryptInit_ex(&self.ctx, NULL, NULL, key2, iv2):
raise Exception('EVP_EncryptInit_ex failed')
@property
def iv(self):
return self.ctx.iv[:16]
def encrypt(self, data):
cdef int inl = len(data)
cdef int outl
cdef unsigned char *out = <unsigned char *>malloc(inl)
if not out:
raise MemoryError
try:
if not EVP_EncryptUpdate(&self.ctx, out, &outl, data, inl):
raise Exception('EVP_EncryptUpdate failed')
return out[:inl]
finally:
free(out)
decrypt = encrypt

528
attic/testsuite/archiver.py
View file

@ -1,528 +0,0 @@
from binascii import hexlify
from configparser import RawConfigParser
import os
from io import StringIO
import stat
import subprocess
import sys
import shutil
import tempfile
import time
import unittest
from hashlib import sha256
from attic import xattr
from attic.archive import Archive, ChunkBuffer
from attic.archiver import Archiver
from attic.cache import Cache
from attic.crypto import bytes_to_long, num_aes_blocks
from attic.helpers import Manifest
from attic.remote import RemoteRepository, PathNotAllowed
from attic.repository import Repository
from attic.testsuite import AtticTestCase
from attic.testsuite.mock import patch
try:
import llfuse
has_llfuse = True
except ImportError:
has_llfuse = False
has_lchflags = hasattr(os, 'lchflags')
src_dir = os.path.join(os.getcwd(), os.path.dirname(__file__), '..')
class changedir:
def __init__(self, dir):
self.dir = dir
def __enter__(self):
self.old = os.getcwd()
os.chdir(self.dir)
def __exit__(self, *args, **kw):
os.chdir(self.old)
class environment_variable:
def __init__(self, **values):
self.values = values
self.old_values = {}
def __enter__(self):
for k, v in self.values.items():
self.old_values[k] = os.environ.get(k)
os.environ[k] = v
def __exit__(self, *args, **kw):
for k, v in self.old_values.items():
if v is not None:
os.environ[k] = v
class ArchiverTestCaseBase(AtticTestCase):
prefix = ''
def setUp(self):
os.environ['ATTIC_CHECK_I_KNOW_WHAT_I_AM_DOING'] = '1'
self.archiver = Archiver()
self.tmpdir = tempfile.mkdtemp()
self.repository_path = os.path.join(self.tmpdir, 'repository')
self.repository_location = self.prefix + self.repository_path
self.input_path = os.path.join(self.tmpdir, 'input')
self.output_path = os.path.join(self.tmpdir, 'output')
self.keys_path = os.path.join(self.tmpdir, 'keys')
self.cache_path = os.path.join(self.tmpdir, 'cache')
self.exclude_file_path = os.path.join(self.tmpdir, 'excludes')
os.environ['ATTIC_KEYS_DIR'] = self.keys_path
os.environ['ATTIC_CACHE_DIR'] = self.cache_path
os.mkdir(self.input_path)
os.mkdir(self.output_path)
os.mkdir(self.keys_path)
os.mkdir(self.cache_path)
with open(self.exclude_file_path, 'wb') as fd:
fd.write(b'input/file2\n# A commment line, then a blank line\n\n')
self._old_wd = os.getcwd()
os.chdir(self.tmpdir)
def tearDown(self):
shutil.rmtree(self.tmpdir)
os.chdir(self._old_wd)
def attic(self, *args, **kw):
exit_code = kw.get('exit_code', 0)
fork = kw.get('fork', False)
if fork:
try:
output = subprocess.check_output((sys.executable, '-m', 'attic.archiver') + args)
ret = 0
except subprocess.CalledProcessError as e:
output = e.output
ret = e.returncode
output = os.fsdecode(output)
if ret != exit_code:
print(output)
self.assert_equal(exit_code, ret)
return output
args = list(args)
stdin, stdout, stderr = sys.stdin, sys.stdout, sys.stderr
try:
sys.stdin = StringIO()
output = StringIO()
sys.stdout = sys.stderr = output
ret = self.archiver.run(args)
sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr
if ret != exit_code:
print(output.getvalue())
self.assert_equal(exit_code, ret)
return output.getvalue()
finally:
sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr
def create_src_archive(self, name):
self.attic('create', self.repository_location + '::' + name, src_dir)
class ArchiverTestCase(ArchiverTestCaseBase):
def create_regular_file(self, name, size=0, contents=None):
filename = os.path.join(self.input_path, name)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
with open(filename, 'wb') as fd:
if contents is None:
contents = b'X' * size
fd.write(contents)
def create_test_files(self):
"""Create a minimal test case including all supported file types
"""
# File
self.create_regular_file('empty', size=0)
# 2600-01-01 > 2**64 ns
os.utime('input/empty', (19880895600, 19880895600))
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('flagfile', size=1024)
# Directory
self.create_regular_file('dir2/file2', size=1024 * 80)
# File owner
os.chown('input/file1', 100, 200)
# File mode
os.chmod('input/file1', 0o7755)
os.chmod('input/dir2', 0o555)
# Block device
os.mknod('input/bdev', 0o600 | stat.S_IFBLK, os.makedev(10, 20))
# Char device
os.mknod('input/cdev', 0o600 | stat.S_IFCHR, os.makedev(30, 40))
# Hard link
os.link(os.path.join(self.input_path, 'file1'),
os.path.join(self.input_path, 'hardlink'))
# Symlink
os.symlink('somewhere', os.path.join(self.input_path, 'link1'))
if xattr.is_enabled():
xattr.setxattr(os.path.join(self.input_path, 'file1'), 'user.foo', b'bar')
xattr.setxattr(os.path.join(self.input_path, 'link1'), 'user.foo_symlink', b'bar_symlink', follow_symlinks=False)
# FIFO node
os.mkfifo(os.path.join(self.input_path, 'fifo1'))
if has_lchflags:
os.lchflags(os.path.join(self.input_path, 'flagfile'), stat.UF_NODUMP)
def test_basic_functionality(self):
self.create_test_files()
self.attic('init', self.repository_location)
self.attic('create', self.repository_location + '::test', 'input')
self.attic('create', self.repository_location + '::test.2', 'input')
with changedir('output'):
self.attic('extract', self.repository_location + '::test')
self.assert_equal(len(self.attic('list', self.repository_location).splitlines()), 2)
self.assert_equal(len(self.attic('list', self.repository_location + '::test').splitlines()), 11)
self.assert_dirs_equal('input', 'output/input')
info_output = self.attic('info', self.repository_location + '::test')
self.assert_in('Number of files: 4', info_output)
shutil.rmtree(self.cache_path)
with environment_variable(ATTIC_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK='1'):
info_output2 = self.attic('info', self.repository_location + '::test')
# info_output2 starts with some "initializing cache" text but should
# end the same way as info_output
assert info_output2.endswith(info_output)
def _extract_repository_id(self, path):
return Repository(self.repository_path).id
def _set_repository_id(self, path, id):
config = RawConfigParser()
config.read(os.path.join(path, 'config'))
config.set('repository', 'id', hexlify(id).decode('ascii'))
with open(os.path.join(path, 'config'), 'w') as fd:
config.write(fd)
return Repository(self.repository_path).id
def test_repository_swap_detection(self):
self.create_test_files()
os.environ['ATTIC_PASSPHRASE'] = 'passphrase'
self.attic('init', '--encryption=passphrase', self.repository_location)
repository_id = self._extract_repository_id(self.repository_path)
self.attic('create', self.repository_location + '::test', 'input')
shutil.rmtree(self.repository_path)
self.attic('init', '--encryption=none', self.repository_location)
self._set_repository_id(self.repository_path, repository_id)
self.assert_equal(repository_id, self._extract_repository_id(self.repository_path))
self.assert_raises(Cache.EncryptionMethodMismatch, lambda :self.attic('create', self.repository_location + '::test.2', 'input'))
def test_repository_swap_detection2(self):
self.create_test_files()
self.attic('init', '--encryption=none', self.repository_location + '_unencrypted')
os.environ['ATTIC_PASSPHRASE'] = 'passphrase'
self.attic('init', '--encryption=passphrase', self.repository_location + '_encrypted')
self.attic('create', self.repository_location + '_encrypted::test', 'input')
shutil.rmtree(self.repository_path + '_encrypted')
os.rename(self.repository_path + '_unencrypted', self.repository_path + '_encrypted')
self.assert_raises(Cache.RepositoryAccessAborted, lambda :self.attic('create', self.repository_location + '_encrypted::test.2', 'input'))
def test_strip_components(self):
self.attic('init', self.repository_location)
self.create_regular_file('dir/file')
self.attic('create', self.repository_location + '::test', 'input')
with changedir('output'):
self.attic('extract', self.repository_location + '::test', '--strip-components', '3')
self.assert_true(not os.path.exists('file'))
with self.assert_creates_file('file'):
self.attic('extract', self.repository_location + '::test', '--strip-components', '2')
with self.assert_creates_file('dir/file'):
self.attic('extract', self.repository_location + '::test', '--strip-components', '1')
with self.assert_creates_file('input/dir/file'):
self.attic('extract', self.repository_location + '::test', '--strip-components', '0')
def test_extract_include_exclude(self):
self.attic('init', self.repository_location)
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('file2', size=1024 * 80)
self.create_regular_file('file3', size=1024 * 80)
self.create_regular_file('file4', size=1024 * 80)
self.attic('create', '--exclude=input/file4', self.repository_location + '::test', 'input')
with changedir('output'):
self.attic('extract', self.repository_location + '::test', 'input/file1', )
self.assert_equal(sorted(os.listdir('output/input')), ['file1'])
with changedir('output'):
self.attic('extract', '--exclude=input/file2', self.repository_location + '::test')
self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file3'])
with changedir('output'):
self.attic('extract', '--exclude-from=' + self.exclude_file_path, self.repository_location + '::test')
self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file3'])
def test_exclude_caches(self):
self.attic('init', self.repository_location)
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('cache1/CACHEDIR.TAG', contents = b'Signature: 8a477f597d28d172789f06886806bc55 extra stuff')
self.create_regular_file('cache2/CACHEDIR.TAG', contents = b'invalid signature')
self.attic('create', '--exclude-caches', self.repository_location + '::test', 'input')
with changedir('output'):
self.attic('extract', self.repository_location + '::test')
self.assert_equal(sorted(os.listdir('output/input')), ['cache2', 'file1'])
self.assert_equal(sorted(os.listdir('output/input/cache2')), ['CACHEDIR.TAG'])
def test_path_normalization(self):
self.attic('init', self.repository_location)
self.create_regular_file('dir1/dir2/file', size=1024 * 80)
with changedir('input/dir1/dir2'):
self.attic('create', self.repository_location + '::test', '../../../input/dir1/../dir1/dir2/..')
output = self.attic('list', self.repository_location + '::test')
self.assert_not_in('..', output)
self.assert_in(' input/dir1/dir2/file', output)
def test_repeated_files(self):
self.create_regular_file('file1', size=1024 * 80)
self.attic('init', self.repository_location)
self.attic('create', self.repository_location + '::test', 'input', 'input')
def test_overwrite(self):
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('dir2/file2', size=1024 * 80)
self.attic('init', self.repository_location)
self.attic('create', self.repository_location + '::test', 'input')
# Overwriting regular files and directories should be supported
os.mkdir('output/input')
os.mkdir('output/input/file1')
os.mkdir('output/input/dir2')
with changedir('output'):
self.attic('extract', self.repository_location + '::test')
self.assert_dirs_equal('input', 'output/input')
# But non-empty dirs should fail
os.unlink('output/input/file1')
os.mkdir('output/input/file1')
os.mkdir('output/input/file1/dir')
with changedir('output'):
self.attic('extract', self.repository_location + '::test', exit_code=1)
def test_delete(self):
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('dir2/file2', size=1024 * 80)
self.attic('init', self.repository_location)
self.attic('create', self.repository_location + '::test', 'input')
self.attic('create', self.repository_location + '::test.2', 'input')
self.attic('extract', '--dry-run', self.repository_location + '::test')
self.attic('extract', '--dry-run', self.repository_location + '::test.2')
self.attic('delete', self.repository_location + '::test')
self.attic('extract', '--dry-run', self.repository_location + '::test.2')
self.attic('delete', self.repository_location + '::test.2')
# Make sure all data except the manifest has been deleted
repository = Repository(self.repository_path)
self.assert_equal(len(repository), 1)
def test_corrupted_repository(self):
self.attic('init', self.repository_location)
self.create_src_archive('test')
self.attic('extract', '--dry-run', self.repository_location + '::test')
self.attic('check', self.repository_location)
name = sorted(os.listdir(os.path.join(self.tmpdir, 'repository', 'data', '0')), reverse=True)[0]
fd = open(os.path.join(self.tmpdir, 'repository', 'data', '0', name), 'r+')
fd.seek(100)
fd.write('XXXX')
fd.close()
self.attic('check', self.repository_location, exit_code=1)
def test_readonly_repository(self):
self.attic('init', self.repository_location)
self.create_src_archive('test')
os.system('chmod -R ugo-w ' + self.repository_path)
try:
self.attic('extract', '--dry-run', self.repository_location + '::test')
finally:
# Restore permissions so shutil.rmtree is able to delete it
os.system('chmod -R u+w ' + self.repository_path)
def test_cmdline_compatibility(self):
self.create_regular_file('file1', size=1024 * 80)
self.attic('init', self.repository_location)
self.attic('create', self.repository_location + '::test', 'input')
output = self.attic('verify', '-v', self.repository_location + '::test')
self.assert_in('"attic verify" has been deprecated', output)
output = self.attic('prune', self.repository_location, '--hourly=1')
self.assert_in('"--hourly" has been deprecated. Use "--keep-hourly" instead', output)
def test_prune_repository(self):
self.attic('init', self.repository_location)
self.attic('create', self.repository_location + '::test1', src_dir)
self.attic('create', self.repository_location + '::test2', src_dir)
output = self.attic('prune', '-v', '--dry-run', self.repository_location, '--keep-daily=2')
self.assert_in('Keeping archive: test2', output)
self.assert_in('Would prune: test1', output)
output = self.attic('list', self.repository_location)
self.assert_in('test1', output)
self.assert_in('test2', output)
self.attic('prune', self.repository_location, '--keep-daily=2')
output = self.attic('list', self.repository_location)
self.assert_not_in('test1', output)
self.assert_in('test2', output)
def test_usage(self):
self.assert_raises(SystemExit, lambda: self.attic())
self.assert_raises(SystemExit, lambda: self.attic('-h'))
@unittest.skipUnless(has_llfuse, 'llfuse not installed')
def test_fuse_mount_repository(self):
mountpoint = os.path.join(self.tmpdir, 'mountpoint')
os.mkdir(mountpoint)
self.attic('init', self.repository_location)
self.create_test_files()
self.attic('create', self.repository_location + '::archive', 'input')
self.attic('create', self.repository_location + '::archive2', 'input')
try:
self.attic('mount', self.repository_location, mountpoint, fork=True)
self.wait_for_mount(mountpoint)
self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive', 'input'))
self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive2', 'input'))
finally:
if sys.platform.startswith('linux'):
os.system('fusermount -u ' + mountpoint)
else:
os.system('umount ' + mountpoint)
os.rmdir(mountpoint)
# Give the daemon some time to exit
time.sleep(.2)
@unittest.skipUnless(has_llfuse, 'llfuse not installed')
def test_fuse_mount_archive(self):
mountpoint = os.path.join(self.tmpdir, 'mountpoint')
os.mkdir(mountpoint)
self.attic('init', self.repository_location)
self.create_test_files()
self.attic('create', self.repository_location + '::archive', 'input')
try:
self.attic('mount', self.repository_location + '::archive', mountpoint, fork=True)
self.wait_for_mount(mountpoint)
self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'input'))
finally:
if sys.platform.startswith('linux'):
os.system('fusermount -u ' + mountpoint)
else:
os.system('umount ' + mountpoint)
os.rmdir(mountpoint)
# Give the daemon some time to exit
time.sleep(.2)
def verify_aes_counter_uniqueness(self, method):
seen = set() # Chunks already seen
used = set() # counter values already used
def verify_uniqueness():
repository = Repository(self.repository_path)
for key, _ in repository.open_index(repository.get_transaction_id()).iteritems():
data = repository.get(key)
hash = sha256(data).digest()
if not hash in seen:
seen.add(hash)
num_blocks = num_aes_blocks(len(data) - 41)
nonce = bytes_to_long(data[33:41])
for counter in range(nonce, nonce + num_blocks):
self.assert_not_in(counter, used)
used.add(counter)
self.create_test_files()
os.environ['ATTIC_PASSPHRASE'] = 'passphrase'
self.attic('init', '--encryption=' + method, self.repository_location)
verify_uniqueness()
self.attic('create', self.repository_location + '::test', 'input')
verify_uniqueness()
self.attic('create', self.repository_location + '::test.2', 'input')
verify_uniqueness()
self.attic('delete', self.repository_location + '::test.2')
verify_uniqueness()
self.assert_equal(used, set(range(len(used))))
def test_aes_counter_uniqueness_keyfile(self):
self.verify_aes_counter_uniqueness('keyfile')
def test_aes_counter_uniqueness_passphrase(self):
self.verify_aes_counter_uniqueness('passphrase')
class ArchiverCheckTestCase(ArchiverTestCaseBase):
def setUp(self):
super(ArchiverCheckTestCase, self).setUp()
with patch.object(ChunkBuffer, 'BUFFER_SIZE', 10):
self.attic('init', self.repository_location)
self.create_src_archive('archive1')
self.create_src_archive('archive2')
def open_archive(self, name):
repository = Repository(self.repository_path)
manifest, key = Manifest.load(repository)
archive = Archive(repository, key, manifest, name)
return archive, repository
def test_check_usage(self):
output = self.attic('check', self.repository_location, exit_code=0)
self.assert_in('Starting repository check', output)
self.assert_in('Starting archive consistency check', output)
output = self.attic('check', '--repository-only', self.repository_location, exit_code=0)
self.assert_in('Starting repository check', output)
self.assert_not_in('Starting archive consistency check', output)
output = self.attic('check', '--archives-only', self.repository_location, exit_code=0)
self.assert_not_in('Starting repository check', output)
self.assert_in('Starting archive consistency check', output)
def test_missing_file_chunk(self):
archive, repository = self.open_archive('archive1')
for item in archive.iter_items():
if item[b'path'].endswith('testsuite/archiver.py'):
repository.delete(item[b'chunks'][-1][0])
break
repository.commit()
self.attic('check', self.repository_location, exit_code=1)
self.attic('check', '--repair', self.repository_location, exit_code=0)
self.attic('check', self.repository_location, exit_code=0)
def test_missing_archive_item_chunk(self):
archive, repository = self.open_archive('archive1')
repository.delete(archive.metadata[b'items'][-5])
repository.commit()
self.attic('check', self.repository_location, exit_code=1)
self.attic('check', '--repair', self.repository_location, exit_code=0)
self.attic('check', self.repository_location, exit_code=0)
def test_missing_archive_metadata(self):
archive, repository = self.open_archive('archive1')
repository.delete(archive.id)
repository.commit()
self.attic('check', self.repository_location, exit_code=1)
self.attic('check', '--repair', self.repository_location, exit_code=0)
self.attic('check', self.repository_location, exit_code=0)
def test_missing_manifest(self):
archive, repository = self.open_archive('archive1')
repository.delete(Manifest.MANIFEST_ID)
repository.commit()
self.attic('check', self.repository_location, exit_code=1)
output = self.attic('check', '--repair', self.repository_location, exit_code=0)
self.assert_in('archive1', output)
self.assert_in('archive2', output)
self.attic('check', self.repository_location, exit_code=0)
def test_extra_chunks(self):
self.attic('check', self.repository_location, exit_code=0)
repository = Repository(self.repository_location)
repository.put(b'01234567890123456789012345678901', b'xxxx')
repository.commit()
repository.close()
self.attic('check', self.repository_location, exit_code=1)
self.attic('check', self.repository_location, exit_code=1)
self.attic('check', '--repair', self.repository_location, exit_code=0)
self.attic('check', self.repository_location, exit_code=0)
self.attic('extract', '--dry-run', self.repository_location + '::archive1', exit_code=0)
class RemoteArchiverTestCase(ArchiverTestCase):
prefix = '__testsuite__:'
def test_remote_repo_restrict_to_path(self):
self.attic('init', self.repository_location)
path_prefix = os.path.dirname(self.repository_path)
with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-path', '/foo']):
self.assert_raises(PathNotAllowed, lambda: self.attic('init', self.repository_location + '_1'))
with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-path', path_prefix]):
self.attic('init', self.repository_location + '_2')
with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-path', '/foo', '--restrict-to-path', path_prefix]):
self.attic('init', self.repository_location + '_3')

29
attic/testsuite/chunker.py
View file

@ -1,29 +0,0 @@
from attic.chunker import Chunker, buzhash, buzhash_update
from attic.testsuite import AtticTestCase
from io import BytesIO
class ChunkerTestCase(AtticTestCase):
def test_chunkify(self):
data = b'0' * 1024 * 1024 * 15 + b'Y'
parts = [bytes(c) for c in Chunker(2, 0x3, 2, 0).chunkify(BytesIO(data))]
self.assert_equal(len(parts), 2)
self.assert_equal(b''.join(parts), data)
self.assert_equal([bytes(c) for c in Chunker(2, 0x3, 2, 0).chunkify(BytesIO(b''))], [])
self.assert_equal([bytes(c) for c in Chunker(2, 0x3, 2, 0).chunkify(BytesIO(b'foobarboobaz' * 3))], [b'fooba', b'rboobaz', b'fooba', b'rboobaz', b'fooba', b'rboobaz'])
self.assert_equal([bytes(c) for c in Chunker(2, 0x3, 2, 1).chunkify(BytesIO(b'foobarboobaz' * 3))], [b'fo', b'obarb', b'oob', b'azf', b'oobarb', b'oob', b'azf', b'oobarb', b'oobaz'])
self.assert_equal([bytes(c) for c in Chunker(2, 0x3, 2, 2).chunkify(BytesIO(b'foobarboobaz' * 3))], [b'foob', b'ar', b'boobazfoob', b'ar', b'boobazfoob', b'ar', b'boobaz'])
self.assert_equal([bytes(c) for c in Chunker(3, 0x3, 3, 0).chunkify(BytesIO(b'foobarboobaz' * 3))], [b'foobarboobaz' * 3])
self.assert_equal([bytes(c) for c in Chunker(3, 0x3, 3, 1).chunkify(BytesIO(b'foobarboobaz' * 3))], [b'foobar', b'boo', b'bazfo', b'obar', b'boo', b'bazfo', b'obar', b'boobaz'])
self.assert_equal([bytes(c) for c in Chunker(3, 0x3, 3, 2).chunkify(BytesIO(b'foobarboobaz' * 3))], [b'foo', b'barboobaz', b'foo', b'barboobaz', b'foo', b'barboobaz'])
self.assert_equal([bytes(c) for c in Chunker(3, 0x3, 4, 0).chunkify(BytesIO(b'foobarboobaz' * 3))], [b'foobarboobaz' * 3])
self.assert_equal([bytes(c) for c in Chunker(3, 0x3, 4, 1).chunkify(BytesIO(b'foobarboobaz' * 3))], [b'foobar', b'boobazfo', b'obar', b'boobazfo', b'obar', b'boobaz'])
self.assert_equal([bytes(c) for c in Chunker(3, 0x3, 4, 2).chunkify(BytesIO(b'foobarboobaz' * 3))], [b'foob', b'arboobaz', b'foob', b'arboobaz', b'foob', b'arboobaz'])
def test_buzhash(self):
self.assert_equal(buzhash(b'abcdefghijklmnop', 0), 3795437769)
self.assert_equal(buzhash(b'abcdefghijklmnop', 1), 3795400502)
self.assert_equal(buzhash(b'abcdefghijklmnop', 1), buzhash_update(buzhash(b'Xabcdefghijklmno', 1), ord('X'), ord('p'), 16, 1))
# Test with more than 31 bytes to make sure our barrel_shift macro works correctly
self.assert_equal(buzhash(b'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz', 0), 566521248)

5
attic/testsuite/mock.py
View file

@ -1,5 +0,0 @@
try:
# Only available in python 3.3+
from unittest.mock import *
except ImportError:
from mock import *

10
attic/testsuite/run.py
View file

@ -1,10 +0,0 @@
import unittest
from attic.testsuite import TestLoader
def main():
unittest.main(testLoader=TestLoader(), defaultTest='')
if __name__ == '__main__':
main()

0
attic/__init__.py → borg/__init__.py
View file

3
borg/__main__.py Normal file
View file

@ -0,0 +1,3 @@
from borg.archiver import main
main()

60
attic/_chunker.c → borg/_chunker.c
View file

@ -1,4 +1,5 @@
#include <Python.h>
#include <fcntl.h>
/* Cyclic polynomial / buzhash: https://en.wikipedia.org/wiki/Rolling_hash */
@ -80,29 +81,31 @@ typedef struct {
uint32_t *table;
uint8_t *data;
PyObject *fd;
int fh;
int done, eof;
size_t remaining, bytes_read, bytes_yielded, position, last;
} Chunker;
static Chunker *
chunker_init(int window_size, int chunk_mask, int min_size, uint32_t seed)
chunker_init(int window_size, int chunk_mask, int min_size, int max_size, uint32_t seed)
{
Chunker *c = calloc(sizeof(Chunker), 1);
c->window_size = window_size;
c->chunk_mask = chunk_mask;
c->min_size = min_size;
c->table = buzhash_init_table(seed);
c->buf_size = 10 * 1024 * 1024;
c->buf_size = max_size;
c->data = malloc(c->buf_size);
return c;
}
static void
chunker_set_fd(Chunker *c, PyObject *fd)
chunker_set_fd(Chunker *c, PyObject *fd, int fh)
{
Py_XDECREF(c->fd);
c->fd = fd;
Py_INCREF(fd);
c->fh = fh;
c->done = 0;
c->remaining = 0;
c->bytes_read = 0;
@ -124,7 +127,7 @@ chunker_free(Chunker *c)
static int
chunker_fill(Chunker *c)
{
size_t n;
ssize_t n;
PyObject *data;
memmove(c->data, c->data + c->last, c->position + c->remaining - c->last);
c->position -= c->last;
@ -133,20 +136,47 @@ chunker_fill(Chunker *c)
if(c->eof || n == 0) {
return 1;
}
data = PyObject_CallMethod(c->fd, "read", "i", n);
if(!data) {
return 0;
}
n = PyBytes_Size(data);
if(n) {
memcpy(c->data + c->position + c->remaining, PyBytes_AsString(data), n);
c->remaining += n;
c->bytes_read += n;
if(c->fh >= 0) {
// if we have a os-level file descriptor, use os-level API
n = read(c->fh, c->data + c->position + c->remaining, n);
if(n > 0) {
c->remaining += n;
c->bytes_read += n;
}
else
if(n == 0) {
c->eof = 1;
}
else {
// some error happened
return 0;
}
#if ( _XOPEN_SOURCE >= 600 || _POSIX_C_SOURCE >= 200112L )
// We tell the OS that we do not need the data of this file any more
// that it maybe has in the cache. This avoids that we spoil the
// complete cache with data that we only read once and (due to cache
// size limit) kick out data from the cache that might be still useful
// for the OS or other processes.
posix_fadvise(c->fh, (off_t) 0, (off_t) 0, POSIX_FADV_DONTNEED);
#endif
}
else {
c->eof = 1;
// no os-level file descriptor, use Python file object API
data = PyObject_CallMethod(c->fd, "read", "i", n);
if(!data) {
return 0;
}
n = PyBytes_Size(data);
if(n) {
memcpy(c->data + c->position + c->remaining, PyBytes_AsString(data), n);
c->remaining += n;
c->bytes_read += n;
}
else {
c->eof = 1;
}
Py_DECREF(data);
}
Py_DECREF(data);
return 1;
}

83
attic/_hashindex.c → borg/_hashindex.c
View file

@ -18,8 +18,11 @@
#error Unknown byte order
#endif
#define MAGIC "BORG_IDX"
#define MAGIC_LEN 8
typedef struct {
char magic[8];
char magic[MAGIC_LEN];
int32_t num_entries;
int32_t num_buckets;
int8_t key_size;
@ -37,7 +40,6 @@ typedef struct {
int upper_limit;
} HashIndex;
#define MAGIC "ATTICIDX"
#define EMPTY _htole32(0xffffffff)
#define DELETED _htole32(0xfffffffe)
#define MAX_BUCKET_SIZE 512
@ -132,20 +134,23 @@ static HashIndex *
hashindex_read(const char *path)
{
FILE *fd;
off_t length, buckets_length;
off_t length, buckets_length, bytes_read;
HashHeader header;
HashIndex *index = NULL;
if((fd = fopen(path, "r")) == NULL) {
EPRINTF_PATH(path, "fopen failed");
if((fd = fopen(path, "rb")) == NULL) {
EPRINTF_PATH(path, "fopen for reading failed");
return NULL;
}
if(fread(&header, 1, sizeof(HashHeader), fd) != sizeof(HashHeader)) {
bytes_read = fread(&header, 1, sizeof(HashHeader), fd);
if(bytes_read != sizeof(HashHeader)) {
if(ferror(fd)) {
EPRINTF_PATH(path, "fread failed");
EPRINTF_PATH(path, "fread header failed (expected %ju, got %ju)",
(uintmax_t) sizeof(HashHeader), (uintmax_t) bytes_read);
}
else {
EPRINTF_MSG_PATH(path, "failed to read %ld bytes", sizeof(HashHeader));
EPRINTF_MSG_PATH(path, "fread header failed (expected %ju, got %ju)",
(uintmax_t) sizeof(HashHeader), (uintmax_t) bytes_read);
}
goto fail;
}
@ -161,31 +166,35 @@ hashindex_read(const char *path)
EPRINTF_PATH(path, "fseek failed");
goto fail;
}
if(memcmp(header.magic, MAGIC, 8)) {
EPRINTF_MSG_PATH(path, "Unknown file header");
if(memcmp(header.magic, MAGIC, MAGIC_LEN)) {
EPRINTF_MSG_PATH(path, "Unknown MAGIC in header");
goto fail;
}
buckets_length = (off_t)_le32toh(header.num_buckets) * (header.key_size + header.value_size);
if(length != sizeof(HashHeader) + buckets_length) {
EPRINTF_MSG_PATH(path, "Incorrect file length");
EPRINTF_MSG_PATH(path, "Incorrect file length (expected %ju, got %ju)",
(uintmax_t) sizeof(HashHeader) + buckets_length, (uintmax_t) length);
goto fail;
}
if(!(index = malloc(sizeof(HashIndex)))) {
EPRINTF_PATH(path, "malloc failed");
EPRINTF_PATH(path, "malloc header failed");
goto fail;
}
if(!(index->buckets = malloc(buckets_length))) {
EPRINTF_PATH(path, "malloc failed");
EPRINTF_PATH(path, "malloc buckets failed");
free(index);
index = NULL;
goto fail;
}
if(fread(index->buckets, 1, buckets_length, fd) != buckets_length) {
bytes_read = fread(index->buckets, 1, buckets_length, fd);
if(bytes_read != buckets_length) {
if(ferror(fd)) {
EPRINTF_PATH(path, "fread failed");
EPRINTF_PATH(path, "fread buckets failed (expected %ju, got %ju)",
(uintmax_t) buckets_length, (uintmax_t) bytes_read);
}
else {
EPRINTF_MSG_PATH(path, "failed to read %ld bytes", length);
EPRINTF_MSG_PATH(path, "fread buckets failed (expected %ju, got %ju)",
(uintmax_t) buckets_length, (uintmax_t) bytes_read);
}
free(index->buckets);
free(index);
@ -215,12 +224,12 @@ hashindex_init(int capacity, int key_size, int value_size)
capacity = MAX(MIN_BUCKETS, capacity);
if(!(index = malloc(sizeof(HashIndex)))) {
EPRINTF("malloc failed");
EPRINTF("malloc header failed");
return NULL;
}
buckets_length = (off_t)capacity * (key_size + value_size);
if(!(index->buckets = calloc(buckets_length, 1))) {
EPRINTF("malloc failed");
EPRINTF("malloc buckets failed");
free(index);
return NULL;
}
@ -258,16 +267,16 @@ hashindex_write(HashIndex *index, const char *path)
};
int ret = 1;
if((fd = fopen(path, "w")) == NULL) {
EPRINTF_PATH(path, "open failed");
if((fd = fopen(path, "wb")) == NULL) {
EPRINTF_PATH(path, "fopen for writing failed");
return 0;
}
if(fwrite(&header, 1, sizeof(header), fd) != sizeof(header)) {
EPRINTF_PATH(path, "fwrite failed");
EPRINTF_PATH(path, "fwrite header failed");
ret = 0;
}
if(fwrite(index->buckets, 1, buckets_length, fd) != buckets_length) {
EPRINTF_PATH(path, "fwrite failed");
EPRINTF_PATH(path, "fwrite buckets failed");
ret = 0;
}
if(fclose(fd) < 0) {
@ -357,14 +366,18 @@ hashindex_get_size(HashIndex *index)
}
static void
hashindex_summarize(HashIndex *index, long long *total_size, long long *total_csize, long long *total_unique_size, long long *total_unique_csize)
hashindex_summarize(HashIndex *index, long long *total_size, long long *total_csize,
long long *total_unique_size, long long *total_unique_csize,
long long *total_unique_chunks, long long *total_chunks)
{
int64_t size = 0, csize = 0, unique_size = 0, unique_csize = 0;
int64_t size = 0, csize = 0, unique_size = 0, unique_csize = 0, chunks = 0, unique_chunks = 0;
const int32_t *values;
void *key = NULL;
while((key = hashindex_next_key(index, key))) {
values = key + 32;
values = key + index->key_size;
unique_chunks++;
chunks += values[0];
unique_size += values[1];
unique_csize += values[2];
size += values[0] * values[1];
@ -374,5 +387,25 @@ hashindex_summarize(HashIndex *index, long long *total_size, long long *total_cs
*total_csize = csize;
*total_unique_size = unique_size;
*total_unique_csize = unique_csize;
*total_unique_chunks = unique_chunks;
*total_chunks = chunks;
}
static void
hashindex_merge(HashIndex *index, HashIndex *other)
{
int32_t key_size = index->key_size;
const int32_t *other_values;
int32_t *my_values;
void *key = NULL;
while((key = hashindex_next_key(other, key))) {
other_values = key + key_size;
my_values = (int32_t *)hashindex_get(index, key);
if(my_values == NULL) {
hashindex_set(index, key, other_values);
} else {
*my_values += *other_values;
}
}
}

239
borg/_version.py Normal file
View file

@ -0,0 +1,239 @@
# This file helps to compute a version number in source trees obtained from
# git-archive tarball (such as those provided by githubs download-from-tag
# feature). Distribution tarballs (built by setup.py sdist) and build
# directories (produced by setup.py build) will contain a much shorter file
# that just contains the computed version number.
# This file is released into the public domain. Generated by
# versioneer-0.14 (https://github.com/warner/python-versioneer)
import errno
import os
import re
import subprocess
import sys
# these strings will be replaced by git during git-archive
git_refnames = "$Format:%d$"
git_full = "$Format:%H$"
# these strings are filled in when 'setup.py versioneer' creates _version.py
tag_prefix = ""
parentdir_prefix = "borgbackup-"
versionfile_source = "borg/_version.py"
def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False):
assert isinstance(commands, list)
p = None
for c in commands:
try:
# remember shell=False, so use git.cmd on windows, not just git
p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE,
stderr=(subprocess.PIPE if hide_stderr
else None))
break
except EnvironmentError:
e = sys.exc_info()[1]
if e.errno == errno.ENOENT:
continue
if verbose:
print("unable to run %s" % args[0])
print(e)
return None
else:
if verbose:
print("unable to find command, tried %s" % (commands,))
return None
stdout = p.communicate()[0].strip()
if sys.version_info[0] >= 3:
stdout = stdout.decode()
if p.returncode != 0:
if verbose:
print("unable to run %s (error)" % args[0])
return None
return stdout
def versions_from_parentdir(parentdir_prefix, root, verbose=False):
# Source tarballs conventionally unpack into a directory that includes
# both the project name and a version string.
dirname = os.path.basename(root)
if not dirname.startswith(parentdir_prefix):
if verbose:
print("guessing rootdir is '%s', but '%s' doesn't start with "
"prefix '%s'" % (root, dirname, parentdir_prefix))
return None
return {"version": dirname[len(parentdir_prefix):], "full": ""}
def git_get_keywords(versionfile_abs):
# the code embedded in _version.py can just fetch the value of these
# keywords. When used from setup.py, we don't want to import _version.py,
# so we do it with a regexp instead. This function is not used from
# _version.py.
keywords = {}
try:
f = open(versionfile_abs, "r")
for line in f.readlines():
if line.strip().startswith("git_refnames ="):
mo = re.search(r'=\s*"(.*)"', line)
if mo:
keywords["refnames"] = mo.group(1)
if line.strip().startswith("git_full ="):
mo = re.search(r'=\s*"(.*)"', line)
if mo:
keywords["full"] = mo.group(1)
f.close()
except EnvironmentError:
pass
return keywords
def git_versions_from_keywords(keywords, tag_prefix, verbose=False):
if not keywords:
return {} # keyword-finding function failed to find keywords
refnames = keywords["refnames"].strip()
if refnames.startswith("$Format"):
if verbose:
print("keywords are unexpanded, not using")
return {} # unexpanded, so not in an unpacked git-archive tarball
refs = set([r.strip() for r in refnames.strip("()").split(",")])
# starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
# just "foo-1.0". If we see a "tag: " prefix, prefer those.
TAG = "tag: "
tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
if not tags:
# Either we're using git < 1.8.3, or there really are no tags. We use
# a heuristic: assume all version tags have a digit. The old git %d
# expansion behaves like git log --decorate=short and strips out the
# refs/heads/ and refs/tags/ prefixes that would let us distinguish
# between branches and tags. By ignoring refnames without digits, we
# filter out many common branch names like "release" and
# "stabilization", as well as "HEAD" and "master".
tags = set([r for r in refs if re.search(r'\d', r)])
if verbose:
print("discarding '%s', no digits" % ",".join(refs-tags))
if verbose:
print("likely tags: %s" % ",".join(sorted(tags)))
for ref in sorted(tags):
# sorting will prefer e.g. "2.0" over "2.0rc1"
if ref.startswith(tag_prefix):
r = ref[len(tag_prefix):]
if verbose:
print("picking %s" % r)
return {"version": r,
"full": keywords["full"].strip()}
# no suitable tags, so version is "0+unknown", but full hex is still there
if verbose:
print("no suitable tags, using unknown + full revision id")
return {"version": "0+unknown",
"full": keywords["full"].strip()}
def git_parse_vcs_describe(git_describe, tag_prefix, verbose=False):
# TAG-NUM-gHEX[-dirty] or HEX[-dirty] . TAG might have hyphens.
# dirty
dirty = git_describe.endswith("-dirty")
if dirty:
git_describe = git_describe[:git_describe.rindex("-dirty")]
dirty_suffix = ".dirty" if dirty else ""
# now we have TAG-NUM-gHEX or HEX
if "-" not in git_describe: # just HEX
return "0+untagged.g"+git_describe+dirty_suffix, dirty
# just TAG-NUM-gHEX
mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
if not mo:
# unparseable. Maybe git-describe is misbehaving?
return "0+unparseable"+dirty_suffix, dirty
# tag
full_tag = mo.group(1)
if not full_tag.startswith(tag_prefix):
if verbose:
fmt = "tag '%s' doesn't start with prefix '%s'"
print(fmt % (full_tag, tag_prefix))
return None, dirty
tag = full_tag[len(tag_prefix):]
# distance: number of commits since tag
distance = int(mo.group(2))
# commit: short hex revision ID
commit = mo.group(3)
# now build up version string, with post-release "local version
# identifier". Our goal: TAG[+NUM.gHEX[.dirty]] . Note that if you get a
# tagged build and then dirty it, you'll get TAG+0.gHEX.dirty . So you
# can always test version.endswith(".dirty").
version = tag
if distance or dirty:
version += "+%d.g%s" % (distance, commit) + dirty_suffix
return version, dirty
def git_versions_from_vcs(tag_prefix, root, verbose=False):
# this runs 'git' from the root of the source tree. This only gets called
# if the git-archive 'subst' keywords were *not* expanded, and
# _version.py hasn't already been rewritten with a short version string,
# meaning we're inside a checked out source tree.
if not os.path.exists(os.path.join(root, ".git")):
if verbose:
print("no .git in %s" % root)
return {} # get_versions() will try next method
GITS = ["git"]
if sys.platform == "win32":
GITS = ["git.cmd", "git.exe"]
# if there is a tag, this yields TAG-NUM-gHEX[-dirty]
# if there are no tags, this yields HEX[-dirty] (no NUM)
stdout = run_command(GITS, ["describe", "--tags", "--dirty",
"--always", "--long"],
cwd=root)
# --long was added in git-1.5.5
if stdout is None:
return {} # try next method
version, dirty = git_parse_vcs_describe(stdout, tag_prefix, verbose)
# build "full", which is FULLHEX[.dirty]
stdout = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
if stdout is None:
return {}
full = stdout.strip()
if dirty:
full += ".dirty"
return {"version": version, "full": full}
def get_versions(default={"version": "0+unknown", "full": ""}, verbose=False):
# I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
# __file__, we can work backwards from there to the root. Some
# py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
# case we can only use expanded keywords.
keywords = {"refnames": git_refnames, "full": git_full}
ver = git_versions_from_keywords(keywords, tag_prefix, verbose)
if ver:
return ver
try:
root = os.path.realpath(__file__)
# versionfile_source is the relative path from the top of the source
# tree (where the .git directory might live) to this file. Invert
# this to find the root from __file__.
for i in versionfile_source.split('/'):
root = os.path.dirname(root)
except NameError:
return default
return (git_versions_from_vcs(tag_prefix, root, verbose)
or versions_from_parentdir(parentdir_prefix, root, verbose)
or default)

286
attic/archive.py → borg/archive.py
View file

@ -2,10 +2,8 @@ from datetime import datetime
from getpass import getuser
from itertools import groupby
import errno
import shutil
import tempfile
from attic.key import key_factory
from attic.remote import cache_if_remote
from .key import key_factory
from .remote import cache_if_remote
import msgpack
import os
import socket
@ -13,17 +11,22 @@ import stat
import sys
import time
from io import BytesIO
from attic import xattr
from attic.platform import acl_get, acl_set
from attic.chunker import Chunker
from attic.hashindex import ChunkIndex
from attic.helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \
from . import xattr
from .platform import acl_get, acl_set
from .chunker import Chunker
from .hashindex import ChunkIndex
from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \
Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int
ITEMS_BUFFER = 1024 * 1024
CHUNK_MIN = 1024
WINDOW_SIZE = 0xfff
CHUNK_MASK = 0xffff
CHUNK_MIN_EXP = 10 # 2**10 == 1kiB
CHUNK_MAX_EXP = 23 # 2**23 == 8MiB
HASH_WINDOW_SIZE = 0xfff # 4095B
HASH_MASK_BITS = 16 # results in ~64kiB chunks statistically
# defaults, use --chunker-params to override
CHUNKER_PARAMS = (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE)
utime_supports_fd = os.utime in getattr(os, 'supports_fd', {})
utime_supports_follow_symlinks = os.utime in getattr(os, 'supports_follow_symlinks', {})
@ -66,12 +69,12 @@ class DownloadPipeline:
class ChunkBuffer:
BUFFER_SIZE = 1 * 1024 * 1024
def __init__(self, key):
def __init__(self, key, chunker_params=CHUNKER_PARAMS):
self.buffer = BytesIO()
self.packer = msgpack.Packer(unicode_errors='surrogateescape')
self.chunks = []
self.key = key
self.chunker = Chunker(WINDOW_SIZE, CHUNK_MASK, CHUNK_MIN, self.key.chunk_seed)
self.chunker = Chunker(self.key.chunk_seed, *chunker_params)
def add(self, item):
self.buffer.write(self.packer.pack(StableDict(item)))
@ -101,8 +104,8 @@ class ChunkBuffer:
class CacheChunkBuffer(ChunkBuffer):
def __init__(self, cache, key, stats):
super(CacheChunkBuffer, self).__init__(key)
def __init__(self, cache, key, stats, chunker_params=CHUNKER_PARAMS):
super().__init__(key, chunker_params)
self.cache = cache
self.stats = stats
@ -122,9 +125,9 @@ class Archive:
class IncompatibleFilesystemEncodingError(Error):
"""Failed to encode filename "{}" into file system encoding "{}". Consider configuring the LANG environment variable."""
def __init__(self, repository, key, manifest, name, cache=None, create=False,
checkpoint_interval=300, numeric_owner=False):
checkpoint_interval=300, numeric_owner=False, progress=False,
chunker_params=CHUNKER_PARAMS):
self.cwd = os.getcwd()
self.key = key
self.repository = repository
@ -132,20 +135,22 @@ class Archive:
self.manifest = manifest
self.hard_links = {}
self.stats = Statistics()
self.show_progress = progress
self.last_progress = time.time()
self.name = name
self.checkpoint_interval = checkpoint_interval
self.numeric_owner = numeric_owner
self.pipeline = DownloadPipeline(self.repository, self.key)
if create:
self.items_buffer = CacheChunkBuffer(self.cache, self.key, self.stats)
self.chunker = Chunker(WINDOW_SIZE, CHUNK_MASK, CHUNK_MIN, self.key.chunk_seed)
self.items_buffer = CacheChunkBuffer(self.cache, self.key, self.stats, chunker_params)
self.chunker = Chunker(self.key.chunk_seed, *chunker_params)
if name in manifest.archives:
raise self.AlreadyExists(name)
self.last_checkpoint = time.time()
i = 0
while True:
self.checkpoint_name = '%s.checkpoint%s' % (name, i and ('.%d' % i) or '')
if not self.checkpoint_name in manifest.archives:
if self.checkpoint_name not in manifest.archives:
break
i += 1
else:
@ -153,13 +158,18 @@ class Archive:
raise self.DoesNotExist(name)
info = self.manifest.archives[name]
self.load(info[b'id'])
self.zeros = b'\0' * (1 << chunker_params[1])
def _load_meta(self, id):
data = self.key.decrypt(id, self.repository.get(id))
metadata = msgpack.unpackb(data)
if metadata[b'version'] != 1:
raise Exception('Unknown archive metadata version')
return metadata
def load(self, id):
self.id = id
data = self.key.decrypt(self.id, self.repository.get(self.id))
self.metadata = msgpack.unpackb(data)
if self.metadata[b'version'] != 1:
raise Exception('Unknown archive metadata version')
self.metadata = self._load_meta(self.id)
decode_dict(self.metadata, (b'name', b'hostname', b'username', b'time'))
self.metadata[b'cmdline'] = [arg.decode('utf-8', 'surrogateescape') for arg in self.metadata[b'cmdline']]
self.name = self.metadata[b'name']
@ -177,6 +187,9 @@ class Archive:
yield item
def add_item(self, item):
if self.show_progress and time.time() - self.last_progress > 0.2:
self.stats.show_progress(item=item)
self.last_progress = time.time()
self.items_buffer.add(item)
if time.time() - self.last_checkpoint > self.checkpoint_interval:
self.write_checkpoint()
@ -187,11 +200,13 @@ class Archive:
del self.manifest.archives[self.checkpoint_name]
self.cache.chunk_decref(self.id, self.stats)
def save(self, name=None):
def save(self, name=None, timestamp=None):
name = name or self.name
if name in self.manifest.archives:
raise self.AlreadyExists(name)
self.items_buffer.flush(flush=True)
if timestamp is None:
timestamp = datetime.utcnow()
metadata = StableDict({
'version': 1,
'name': name,
@ -199,7 +214,7 @@ class Archive:
'cmdline': sys.argv,
'hostname': socket.gethostname(),
'username': getuser(),
'time': datetime.utcnow().isoformat(),
'time': timestamp.isoformat(),
})
data = msgpack.packb(metadata, unicode_errors='surrogateescape')
self.id = self.key.id_hash(data)
@ -214,9 +229,11 @@ class Archive:
count, size, csize = cache.chunks[id]
stats.update(size, csize, count == 1)
cache.chunks[id] = count - 1, size, csize
def add_file_chunks(chunks):
for id, _, _ in chunks:
add(id)
# This function is a bit evil since it abuses the cache to calculate
# the stats. The cache transaction must be rolled back afterwards
unpacker = msgpack.Unpacker(use_list=False)
@ -233,11 +250,14 @@ class Archive:
cache.rollback()
return stats
def extract_item(self, item, restore_attrs=True, dry_run=False):
if dry_run:
def extract_item(self, item, restore_attrs=True, dry_run=False, stdout=False, sparse=False):
if dry_run or stdout:
if b'chunks' in item:
for _ in self.pipeline.fetch_many([c[0] for c in item[b'chunks']], is_preloaded=True):
pass
for data in self.pipeline.fetch_many([c[0] for c in item[b'chunks']], is_preloaded=True):
if stdout:
sys.stdout.buffer.write(data)
if stdout:
sys.stdout.buffer.flush()
return
dest = self.cwd
@ -256,12 +276,7 @@ class Archive:
except OSError:
pass
mode = item[b'mode']
if stat.S_ISDIR(mode):
if not os.path.exists(path):
os.makedirs(path)
if restore_attrs:
self.restore_attrs(path, item)
elif stat.S_ISREG(mode):
if stat.S_ISREG(mode):
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
# Hard link?
@ -274,14 +289,20 @@ class Archive:
with open(path, 'wb') as fd:
ids = [c[0] for c in item[b'chunks']]
for data in self.pipeline.fetch_many(ids, is_preloaded=True):
fd.write(data)
if sparse and self.zeros.startswith(data):
# all-zero chunk: create a hole in a sparse file
fd.seek(len(data), 1)
else:
fd.write(data)
pos = fd.tell()
fd.truncate(pos)
fd.flush()
self.restore_attrs(path, item, fd=fd.fileno())
elif stat.S_ISFIFO(mode):
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
os.mkfifo(path)
self.restore_attrs(path, item)
elif stat.S_ISDIR(mode):
if not os.path.exists(path):
os.makedirs(path)
if restore_attrs:
self.restore_attrs(path, item)
elif stat.S_ISLNK(mode):
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
@ -290,6 +311,11 @@ class Archive:
os.unlink(path)
os.symlink(source, path)
self.restore_attrs(path, item, symlink=True)
elif stat.S_ISFIFO(mode):
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
os.mkfifo(path)
self.restore_attrs(path, item)
elif stat.S_ISCHR(mode) or stat.S_ISBLK(mode):
os.mknod(path, item[b'mode'], item[b'rdev'])
self.restore_attrs(path, item)
@ -340,6 +366,18 @@ class Archive:
except OSError:
pass
def rename(self, name):
if name in self.manifest.archives:
raise self.AlreadyExists(name)
metadata = StableDict(self._load_meta(self.id))
metadata[b'name'] = name
data = msgpack.packb(metadata, unicode_errors='surrogateescape')
new_id = self.key.id_hash(data)
self.cache.add_chunk(new_id, data, self.stats)
self.manifest.archives[name] = {'id': new_id, 'time': metadata[b'time']}
self.cache.chunk_decref(self.id, self.stats)
del self.manifest.archives[self.name]
def delete(self, stats):
unpacker = msgpack.Unpacker(use_list=False)
for items_id, data in zip(self.metadata[b'items'], self.repository.get_many(self.metadata[b'items'])):
@ -370,23 +408,53 @@ class Archive:
acl_get(path, item, st, self.numeric_owner)
return item
def process_item(self, path, st):
def process_dir(self, path, st):
item = {b'path': make_path_safe(path)}
item.update(self.stat_attrs(st, path))
self.add_item(item)
return 'd' # directory
def process_fifo(self, path, st):
item = {b'path': make_path_safe(path)}
item.update(self.stat_attrs(st, path))
self.add_item(item)
return 'f' # fifo
def process_dev(self, path, st):
item = {b'path': make_path_safe(path), b'rdev': st.st_rdev}
item.update(self.stat_attrs(st, path))
self.add_item(item)
if stat.S_ISCHR(st.st_mode):
return 'c' # char device
elif stat.S_ISBLK(st.st_mode):
return 'b' # block device
def process_symlink(self, path, st):
source = os.readlink(path)
item = {b'path': make_path_safe(path), b'source': source}
item.update(self.stat_attrs(st, path))
self.add_item(item)
return 's' # symlink
def process_stdin(self, path, cache):
uid, gid = 0, 0
fd = sys.stdin.buffer # binary
chunks = []
for chunk in self.chunker.chunkify(fd):
chunks.append(cache.add_chunk(self.key.id_hash(chunk), chunk, self.stats))
self.stats.nfiles += 1
item = {
b'path': path,
b'chunks': chunks,
b'mode': 0o100660, # regular file, ug=rw
b'uid': uid, b'user': uid2user(uid),
b'gid': gid, b'group': gid2group(gid),
b'mtime': int_to_bigint(int(time.time()) * 1000000000)
}
self.add_item(item)
def process_file(self, path, st, cache):
status = None
safe_path = make_path_safe(path)
# Is it a hard link?
if st.st_nlink > 1:
@ -395,7 +463,8 @@ class Archive:
item = self.stat_attrs(st, path)
item.update({b'path': safe_path, b'source': source})
self.add_item(item)
return
status = 'h' # regular file, hardlink (to already seen inodes)
return status
else:
self.hard_links[st.st_ino, st.st_dev] = safe_path
path_hash = self.key.id_hash(os.path.join(self.cwd, path).encode('utf-8', 'surrogateescape'))
@ -408,70 +477,80 @@ class Archive:
break
else:
chunks = [cache.chunk_incref(id_, self.stats) for id_ in ids]
status = 'U' # regular file, unchanged
else:
status = 'A' # regular file, added
# Only chunkify the file if needed
if chunks is None:
with Archive._open_rb(path, st) as fd:
fh = Archive._open_rb(path, st)
with os.fdopen(fh, 'rb') as fd:
chunks = []
for chunk in self.chunker.chunkify(fd):
for chunk in self.chunker.chunkify(fd, fh):
chunks.append(cache.add_chunk(self.key.id_hash(chunk), chunk, self.stats))
cache.memorize_file(path_hash, st, [c[0] for c in chunks])
status = status or 'M' # regular file, modified (if not 'A' already)
item = {b'path': safe_path, b'chunks': chunks}
item.update(self.stat_attrs(st, path))
self.stats.nfiles += 1
self.add_item(item)
return status
@staticmethod
def list_archives(repository, key, manifest, cache=None):
# expensive! see also Manifest.list_archive_infos.
for name, info in manifest.archives.items():
yield Archive(repository, key, manifest, name, cache=cache)
@staticmethod
def _open_rb(path, st):
flags_noatime = None
flags_normal = os.O_RDONLY | getattr(os, 'O_BINARY', 0)
flags_noatime = flags_normal | getattr(os, 'NO_ATIME', 0)
euid = None
def open_simple(p, s):
return open(p, 'rb')
def open_noatime_if_owner(p, s):
if s.st_uid == euid:
return os.fdopen(os.open(p, flags_noatime), 'rb')
else:
return open(p, 'rb')
return os.open(p, flags_normal)
def open_noatime(p, s):
return os.open(p, flags_noatime)
def open_noatime_if_owner(p, s):
if euid == 0 or s.st_uid == euid:
# we are root or owner of file
return open_noatime(p, s)
else:
return open_simple(p, s)
def open_noatime_with_fallback(p, s):
try:
fd = os.open(p, flags_noatime)
except PermissionError:
# Was this EPERM due to the O_NOATIME flag?
fo = open(p, 'rb')
fd = os.open(p, flags_normal)
# Yes, it was -- otherwise the above line would have thrown
# another exception.
nonlocal euid
euid = os.geteuid()
# So in future, let's check whether the file is owned by us
# before attempting to use O_NOATIME.
Archive._open_rb = open_noatime_if_owner
return fo
return os.fdopen(fd, 'rb')
return fd
o_noatime = getattr(os, 'O_NOATIME', None)
if o_noatime is not None:
flags_noatime = os.O_RDONLY | getattr(os, 'O_BINARY', 0) | o_noatime
if flags_noatime != flags_normal:
# Always use O_NOATIME version.
Archive._open_rb = open_noatime
Archive._open_rb = open_noatime_with_fallback
else:
# Always use non-O_NOATIME version.
Archive._open_rb = open_simple
return Archive._open_rb(path, st)
class RobustUnpacker():
class RobustUnpacker:
"""A restartable/robust version of the streaming msgpack unpacker
"""
item_keys = [msgpack.packb(name) for name in ('path', 'mode', 'source', 'chunks', 'rdev', 'xattrs', 'user', 'group', 'uid', 'gid', 'mtime')]
def __init__(self, validator):
super(RobustUnpacker, self).__init__()
super().__init__()
self.validator = validator
self._buffered_data = []
self._resync = False
@ -529,23 +608,21 @@ class ArchiveChecker:
def __init__(self):
self.error_found = False
self.possibly_superseded = set()
self.tmpdir = tempfile.mkdtemp()
def __del__(self):
shutil.rmtree(self.tmpdir)
def check(self, repository, repair=False):
def check(self, repository, repair=False, archive=None, last=None):
self.report_progress('Starting archive consistency check...')
self.check_all = archive is None and last is None
self.repair = repair
self.repository = repository
self.init_chunks()
self.key = self.identify_key(repository)
if not Manifest.MANIFEST_ID in self.chunks:
if Manifest.MANIFEST_ID not in self.chunks:
self.manifest = self.rebuild_manifest()
else:
self.manifest, _ = Manifest.load(repository, key=self.key)
self.rebuild_refcounts()
self.verify_chunks()
self.rebuild_refcounts(archive=archive, last=last)
self.orphan_chunks_check()
self.finish()
if not self.error_found:
self.report_progress('Archive consistency check complete, no problems found.')
return self.repair or not self.error_found
@ -553,7 +630,7 @@ class ArchiveChecker:
def init_chunks(self):
"""Fetch a list of all object keys from repository
"""
# Explicity set the initial hash table capacity to avoid performance issues
# Explicitly set the initial hash table capacity to avoid performance issues
# due to hash table "resonance"
capacity = int(len(self.repository) * 1.2)
self.chunks = ChunkIndex(capacity)
@ -588,11 +665,13 @@ class ArchiveChecker:
# Some basic sanity checks of the payload before feeding it into msgpack
if len(data) < 2 or ((data[0] & 0xf0) != 0x80) or ((data[1] & 0xe0) != 0xa0):
continue
if not b'cmdline' in data or not b'\xa7version\x01' in data:
if b'cmdline' not in data or b'\xa7version\x01' not in data:
continue
try:
archive = msgpack.unpackb(data)
except:
# Ignore exceptions that might be raised when feeding
# msgpack with invalid data
except (TypeError, ValueError, StopIteration):
continue
if isinstance(archive, dict) and b'items' in archive and b'cmdline' in archive:
self.report_progress('Found archive ' + archive[b'name'].decode('utf-8'), error=True)
@ -600,7 +679,7 @@ class ArchiveChecker:
self.report_progress('Manifest rebuild complete', error=True)
return manifest
def rebuild_refcounts(self):
def rebuild_refcounts(self, archive=None, last=None):
"""Rebuild object reference counts by walking the metadata
Missing and/or incorrect data is repaired when detected
@ -637,7 +716,7 @@ class ArchiveChecker:
offset = 0
chunk_list = []
for chunk_id, size, csize in item[b'chunks']:
if not chunk_id in self.chunks:
if chunk_id not in self.chunks:
# If a file chunk is missing, create an all empty replacement chunk
self.report_progress('{}: Missing file chunk detected (Byte {}-{})'.format(item[b'path'].decode('utf-8', 'surrogateescape'), offset, offset + size), error=True)
data = bytes(size)
@ -658,11 +737,13 @@ class ArchiveChecker:
"""
unpacker = RobustUnpacker(lambda item: isinstance(item, dict) and b'path' in item)
_state = 0
def missing_chunk_detector(chunk_id):
nonlocal _state
if _state % 2 != int(not chunk_id in self.chunks):
if _state % 2 != int(chunk_id not in self.chunks):
_state += 1
return _state
for state, items in groupby(archive[b'items'], missing_chunk_detector):
items = list(items)
if state % 2:
@ -673,14 +754,28 @@ class ArchiveChecker:
for chunk_id, cdata in zip(items, repository.get_many(items)):
unpacker.feed(self.key.decrypt(chunk_id, cdata))
for item in unpacker:
if not isinstance(item, dict):
self.report_progress('Did not get expected metadata dict - archive corrupted!',
error=True)
continue
yield item
repository = cache_if_remote(self.repository)
num_archives = len(self.manifest.archives)
for i, (name, info) in enumerate(list(self.manifest.archives.items()), 1):
self.report_progress('Analyzing archive {} ({}/{})'.format(name, i, num_archives))
if archive is None:
# we need last N or all archives
archive_items = sorted(self.manifest.archives.items(), reverse=True,
key=lambda name_info: name_info[1][b'time'])
num_archives = len(self.manifest.archives)
end = None if last is None else min(num_archives, last)
else:
# we only want one specific archive
archive_items = [item for item in self.manifest.archives.items() if item[0] == archive]
num_archives = 1
end = 1
for i, (name, info) in enumerate(archive_items[:end]):
self.report_progress('Analyzing archive {} ({}/{})'.format(name, num_archives - i, num_archives))
archive_id = info[b'id']
if not archive_id in self.chunks:
if archive_id not in self.chunks:
self.report_progress('Archive metadata block is missing', error=True)
del self.manifest.archives[name]
continue
@ -707,17 +802,22 @@ class ArchiveChecker:
add_reference(new_archive_id, len(data), len(cdata), cdata)
info[b'id'] = new_archive_id
def verify_chunks(self):
unused = set()
for id_, (count, size, csize) in self.chunks.iteritems():
if count == 0:
unused.add(id_)
orphaned = unused - self.possibly_superseded
if orphaned:
self.report_progress('{} orphaned objects found'.format(len(orphaned)), error=True)
def orphan_chunks_check(self):
if self.check_all:
unused = set()
for id_, (count, size, csize) in self.chunks.iteritems():
if count == 0:
unused.add(id_)
orphaned = unused - self.possibly_superseded
if orphaned:
self.report_progress('{} orphaned objects found'.format(len(orphaned)), error=True)
if self.repair:
for id_ in unused:
self.repository.delete(id_)
else:
self.report_progress('Orphaned objects check skipped (needs all archives checked)')
def finish(self):
if self.repair:
for id_ in unused:
self.repository.delete(id_)
self.manifest.write()
self.repository.commit()

361
attic/archiver.py → borg/archiver.py
View file

@ -3,23 +3,28 @@ from binascii import hexlify
from datetime import datetime
from operator import attrgetter
import functools
import inspect
import io
import os
import signal
import stat
import sys
import textwrap
import traceback
from attic import __version__
from attic.archive import Archive, ArchiveChecker
from attic.repository import Repository
from attic.cache import Cache
from attic.key import key_creator
from attic.helpers import Error, location_validator, format_time, \
format_file_mode, ExcludePattern, exclude_path, adjust_patterns, to_localtime, \
from . import __version__
from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS
from .repository import Repository
from .cache import Cache
from .key import key_creator
from .helpers import Error, location_validator, format_time, format_file_size, \
format_file_mode, ExcludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \
get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \
Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
is_cachedir, bigint_to_int
from attic.remote import RepositoryServer, RemoteRepository
is_cachedir, bigint_to_int, ChunkerParams
from .remote import RepositoryServer, RemoteRepository
has_lchflags = hasattr(os, 'lchflags')
class Archiver:
@ -38,7 +43,7 @@ class Archiver:
def print_error(self, msg, *args):
msg = args and msg % args or msg
self.exit_code = 1
print('attic: ' + msg, file=sys.stderr)
print('borg: ' + msg, file=sys.stderr)
def print_verbose(self, msg, *args, **kw):
if self.verbose:
@ -49,7 +54,7 @@ class Archiver:
print(msg, end=' ')
def do_serve(self, args):
"""Start Attic in server mode. This command is usually not used manually.
"""Start in server mode. This command is usually not used manually.
"""
return RepositoryServer(restrict_to_paths=args.restrict_to_paths).serve()
@ -69,7 +74,7 @@ class Archiver:
"""Check repository consistency"""
repository = self.open_repository(args.repository, exclusive=args.repair)
if args.repair:
while not os.environ.get('ATTIC_CHECK_I_KNOW_WHAT_I_AM_DOING'):
while not os.environ.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'):
self.print_error("""Warning: 'check --repair' is an experimental feature that might result
in data loss.
@ -82,8 +87,9 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
print('Repository check complete, no problems found.')
else:
return 1
if not args.repo_only and not ArchiveChecker().check(repository, repair=args.repair):
return 1
if not args.repo_only and not ArchiveChecker().check(
repository, repair=args.repair, archive=args.repository.archive, last=args.last):
return 1
return 0
def do_change_passphrase(self, args):
@ -98,11 +104,13 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
t0 = datetime.now()
repository = self.open_repository(args.archive, exclusive=True)
manifest, key = Manifest.load(repository)
cache = Cache(repository, key, manifest)
key.compression_level = args.compression
cache = Cache(repository, key, manifest, do_files=args.cache_files)
archive = Archive(repository, key, manifest, args.archive.archive, cache=cache,
create=True, checkpoint_interval=args.checkpoint_interval,
numeric_owner=args.numeric_owner)
# Add Attic cache dir to inode_skip list
numeric_owner=args.numeric_owner, progress=args.progress,
chunker_params=args.chunker_params)
# Add cache dir to inode_skip list
skip_inodes = set()
try:
st = os.stat(get_cache_dir())
@ -117,6 +125,14 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
except IOError:
pass
for path in args.paths:
if path == '-': # stdin
path = 'stdin'
self.print_verbose(path)
try:
archive.process_stdin(path, cache)
except IOError as e:
self.print_error('%s: %s', path, e)
continue
path = os.path.normpath(path)
if args.dontcross:
try:
@ -127,7 +143,9 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
else:
restrict_dev = None
self._process(archive, cache, args.excludes, args.exclude_caches, skip_inodes, path, restrict_dev)
archive.save()
archive.save(timestamp=args.timestamp)
if args.progress:
archive.stats.show_progress(final=True)
if args.stats:
t = datetime.now()
diff = t - t0
@ -155,48 +173,67 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
# Entering a new filesystem?
if restrict_dev and st.st_dev != restrict_dev:
return
# Ignore unix sockets
if stat.S_ISSOCK(st.st_mode):
status = None
# Ignore if nodump flag is set
if has_lchflags and (st.st_flags & stat.UF_NODUMP):
return
self.print_verbose(remove_surrogates(path))
if stat.S_ISREG(st.st_mode):
try:
archive.process_file(path, st, cache)
status = archive.process_file(path, st, cache)
except IOError as e:
self.print_error('%s: %s', path, e)
elif stat.S_ISDIR(st.st_mode):
if exclude_caches and is_cachedir(path):
return
archive.process_item(path, st)
status = archive.process_dir(path, st)
try:
entries = os.listdir(path)
except OSError as e:
self.print_error('%s: %s', path, e)
else:
for filename in sorted(entries):
entry_path = os.path.normpath(os.path.join(path, filename))
self._process(archive, cache, excludes, exclude_caches, skip_inodes,
os.path.join(path, filename), restrict_dev)
entry_path, restrict_dev)
elif stat.S_ISLNK(st.st_mode):
archive.process_symlink(path, st)
status = archive.process_symlink(path, st)
elif stat.S_ISFIFO(st.st_mode):
archive.process_item(path, st)
status = archive.process_fifo(path, st)
elif stat.S_ISCHR(st.st_mode) or stat.S_ISBLK(st.st_mode):
archive.process_dev(path, st)
status = archive.process_dev(path, st)
elif stat.S_ISSOCK(st.st_mode):
# Ignore unix sockets
return
else:
self.print_error('Unknown file type: %s', path)
return
# Status output
# A lowercase character means a file type other than a regular file,
# borg usually just stores them. E.g. (d)irectory.
# Hardlinks to already seen content are indicated by (h).
# A uppercase character means a regular file that was (A)dded,
# (M)odified or was (U)nchanged.
# Note: A/M/U is relative to the "files" cache, not to the repo.
# This would be an issue if the files cache is not used.
if status is None:
status = '?' # need to add a status code somewhere
# output ALL the stuff - it can be easily filtered using grep.
# even stuff considered unchanged might be interesting.
self.print_verbose("%1s %s", status, remove_surrogates(path))
def do_extract(self, args):
"""Extract archive contents"""
# be restrictive when restoring files, restore permissions later
if sys.getfilesystemencoding() == 'ascii':
print('Warning: File system encoding is "ascii", extracting non-ascii filenames will not be supported.')
os.umask(0o077)
repository = self.open_repository(args.archive)
manifest, key = Manifest.load(repository)
archive = Archive(repository, key, manifest, args.archive.archive,
numeric_owner=args.numeric_owner)
patterns = adjust_patterns(args.paths, args.excludes)
dry_run = args.dry_run
stdout = args.stdout
sparse = args.sparse
strip_components = args.strip_components
dirs = []
for item in archive.iter_items(lambda item: not exclude_path(item[b'path'], patterns), preload=True):
@ -207,7 +244,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
continue
if not args.dry_run:
while dirs and not item[b'path'].startswith(dirs[-1][b'path']):
archive.extract_item(dirs.pop(-1))
archive.extract_item(dirs.pop(-1), stdout=stdout)
self.print_verbose(remove_surrogates(orig_path))
try:
if dry_run:
@ -217,7 +254,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
dirs.append(item)
archive.extract_item(item, restore_attrs=False)
else:
archive.extract_item(item)
archive.extract_item(item, stdout=stdout, sparse=sparse)
except IOError as e:
self.print_error('%s: %s', remove_surrogates(orig_path), e)
@ -226,27 +263,51 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
archive.extract_item(dirs.pop(-1))
return self.exit_code
def do_delete(self, args):
"""Delete an existing archive"""
def do_rename(self, args):
"""Rename an existing archive"""
repository = self.open_repository(args.archive, exclusive=True)
manifest, key = Manifest.load(repository)
cache = Cache(repository, key, manifest)
archive = Archive(repository, key, manifest, args.archive.archive, cache=cache)
stats = Statistics()
archive.delete(stats)
archive.rename(args.name)
manifest.write()
repository.commit()
cache.commit()
if args.stats:
stats.print_('Deleted data:', cache)
return self.exit_code
def do_delete(self, args):
"""Delete an existing repository or archive"""
repository = self.open_repository(args.target, exclusive=True)
manifest, key = Manifest.load(repository)
cache = Cache(repository, key, manifest, do_files=args.cache_files)
if args.target.archive:
archive = Archive(repository, key, manifest, args.target.archive, cache=cache)
stats = Statistics()
archive.delete(stats)
manifest.write()
repository.commit()
cache.commit()
if args.stats:
stats.print_('Deleted data:', cache)
else:
print("You requested to completely DELETE the repository *including* all archives it contains:")
for archive_info in manifest.list_archive_infos(sort_by='ts'):
print(format_archive(archive_info))
while not os.environ.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'):
print("""Type "YES" if you understand this and want to continue.\n""")
if input('Do you want to continue? ') == 'YES':
break
repository.destroy()
cache.destroy()
print("Repository and corresponding cache were deleted.")
return self.exit_code
def do_mount(self, args):
"""Mount archive or an entire repository as a FUSE fileystem"""
try:
from attic.fuse import AtticOperations
except ImportError:
self.print_error('the "llfuse" module is required to use this feature')
from .fuse import FuseOperations
except ImportError as e:
self.print_error('loading fuse support failed [ImportError: %s]' % str(e))
return self.exit_code
if not os.path.isdir(args.mountpoint) or not os.access(args.mountpoint, os.R_OK | os.W_OK | os.X_OK):
@ -259,7 +320,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
archive = Archive(repository, key, manifest, args.src.archive)
else:
archive = None
operations = AtticOperations(key, repository, manifest, archive)
operations = FuseOperations(key, repository, manifest, archive)
self.print_verbose("Mounting filesystem")
try:
operations.mount(args.mountpoint, args.options, args.foreground)
@ -284,7 +345,11 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
size = sum(size for _, size, _ in item[b'chunks'])
except KeyError:
pass
mtime = format_time(datetime.fromtimestamp(bigint_to_int(item[b'mtime']) / 1e9))
try:
mtime = datetime.fromtimestamp(bigint_to_int(item[b'mtime']) / 1e9)
except ValueError:
# likely a broken mtime and datetime did not want to go beyond year 9999
mtime = datetime(9999, 12, 31, 23, 59, 59)
if b'source' in item:
if type == 'l':
extra = ' -> %s' % item[b'source']
@ -293,19 +358,20 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
extra = ' link to %s' % item[b'source']
else:
extra = ''
print('%s%s %-6s %-6s %8d %s %s%s' % (type, mode, item[b'user'] or item[b'uid'],
item[b'group'] or item[b'gid'], size, mtime,
remove_surrogates(item[b'path']), extra))
print('%s%s %-6s %-6s %8d %s %s%s' % (
type, mode, item[b'user'] or item[b'uid'],
item[b'group'] or item[b'gid'], size, format_time(mtime),
remove_surrogates(item[b'path']), extra))
else:
for archive in sorted(Archive.list_archives(repository, key, manifest), key=attrgetter('ts')):
print(format_archive(archive))
for archive_info in manifest.list_archive_infos(sort_by='ts'):
print(format_archive(archive_info))
return self.exit_code
def do_info(self, args):
"""Show archive details such as disk space used"""
repository = self.open_repository(args.archive)
manifest, key = Manifest.load(repository)
cache = Cache(repository, key, manifest)
cache = Cache(repository, key, manifest, do_files=args.cache_files)
archive = Archive(repository, key, manifest, args.archive.archive, cache=cache)
stats = archive.calc_stats(cache)
print('Name:', archive.name)
@ -322,12 +388,11 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
"""Prune repository archives according to specified rules"""
repository = self.open_repository(args.repository, exclusive=True)
manifest, key = Manifest.load(repository)
cache = Cache(repository, key, manifest)
archives = list(sorted(Archive.list_archives(repository, key, manifest, cache),
key=attrgetter('ts'), reverse=True))
cache = Cache(repository, key, manifest, do_files=args.cache_files)
archives = manifest.list_archive_infos(sort_by='ts', reverse=True) # just a ArchiveInfo list
if args.hourly + args.daily + args.weekly + args.monthly + args.yearly == 0 and args.within is None:
self.print_error('At least one of the "within", "hourly", "daily", "weekly", "monthly" or "yearly" '
'settings must be specified')
self.print_error('At least one of the "within", "keep-hourly", "keep-daily", "keep-weekly", '
'"keep-monthly" or "keep-yearly" settings must be specified')
return 1
if args.prefix:
archives = [archive for archive in archives if archive.name.startswith(args.prefix)]
@ -355,7 +420,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
self.print_verbose('Would prune: %s' % format_archive(archive))
else:
self.print_verbose('Pruning archive: %s' % format_archive(archive))
archive.delete(stats)
Archive(repository, key, manifest, archive.name, cache).delete(stats)
if to_delete and not args.dry_run:
manifest.write()
repository.commit()
@ -381,17 +446,17 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
Examples:
# Exclude '/home/user/file.o' but not '/home/user/file.odt':
$ attic create -e '*.o' repo.attic /
$ borg create -e '*.o' backup /
# Exclude '/home/user/junk' and '/home/user/subdir/junk' but
# not '/home/user/importantjunk' or '/etc/junk':
$ attic create -e '/home/*/junk' repo.attic /
$ borg create -e '/home/*/junk' backup /
# Exclude the contents of '/home/user/cache' but not the directory itself:
$ attic create -e /home/user/cache/ repo.attic /
$ borg create -e /home/user/cache/ backup /
# The file '/home/user/cache/important' is *not* backed up:
$ attic create -e /home/user/cache/ repo.attic / /home/user/cache/important
$ borg create -e /home/user/cache/ backup / /home/user/cache/important
'''
def do_help(self, parser, commands, args):
@ -420,7 +485,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
('--yearly', '--keep-yearly', 'Warning: "--yearly" has been deprecated. Use "--keep-yearly" instead.')
]
if args and args[0] == 'verify':
print('Warning: "attic verify" has been deprecated. Use "attic extract --dry-run" instead.')
print('Warning: "borg verify" has been deprecated. Use "borg extract --dry-run" instead.')
args = ['extract', '--dry-run'] + args[1:]
for i, arg in enumerate(args[:]):
for old_name, new_name, warning in deprecations:
@ -442,24 +507,34 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
with open(os.path.join(cache_dir, 'CACHEDIR.TAG'), 'w') as fd:
fd.write(textwrap.dedent("""
Signature: 8a477f597d28d172789f06886806bc55
# This file is a cache directory tag created by Attic.
# This file is a cache directory tag created by Borg.
# For information about cache directory tags, see:
# http://www.brynosaurus.com/cachedir/
""").lstrip())
common_parser = argparse.ArgumentParser(add_help=False)
common_parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
default=False,
help='verbose output')
default=False,
help='verbose output')
common_parser.add_argument('--no-files-cache', dest='cache_files', action='store_false',
help='do not load/update the file metadata cache used to detect unchanged files')
common_parser.add_argument('--umask', dest='umask', type=lambda s: int(s, 8), default=0o077, metavar='M',
help='set umask to M (local and remote, default: 0o077)')
common_parser.add_argument('--remote-path', dest='remote_path', default='borg', metavar='PATH',
help='set remote path to executable (default: "borg")')
# We can't use argparse for "serve" since we don't want it to show up in "Available commands"
if args:
args = self.preprocess_args(args)
parser = argparse.ArgumentParser(description='Attic %s - Deduplicated Backups' % __version__)
parser = argparse.ArgumentParser(description='Borg %s - Deduplicated Backups' % __version__)
subparsers = parser.add_subparsers(title='Available commands')
serve_epilog = textwrap.dedent("""
This command starts a repository server process. This command is usually not used manually.
""")
subparser = subparsers.add_parser('serve', parents=[common_parser],
description=self.do_serve.__doc__)
description=self.do_serve.__doc__, epilog=serve_epilog,
formatter_class=argparse.RawDescriptionHelpFormatter)
subparser.set_defaults(func=self.do_serve)
subparser.add_argument('--restrict-to-path', dest='restrict_to_paths', action='append',
metavar='PATH', help='restrict repository access to PATH')
@ -467,6 +542,8 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
This command initializes an empty repository. A repository is a filesystem
directory containing the deduplicated data from zero or more archives.
Encryption can be enabled at repository init time.
Please note that the 'passphrase' encryption mode is DEPRECATED (instead of it,
consider using 'repokey').
""")
subparser = subparsers.add_parser('init', parents=[common_parser],
description=self.do_init.__doc__, epilog=init_epilog,
@ -476,27 +553,51 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
type=location_validator(archive=False),
help='repository to create')
subparser.add_argument('-e', '--encryption', dest='encryption',
choices=('none', 'passphrase', 'keyfile'), default='none',
help='select encryption method')
choices=('none', 'keyfile', 'repokey', 'passphrase'), default='none',
help='select encryption key mode')
check_epilog = textwrap.dedent("""
The check command verifies the consistency of a repository and the corresponding
archives. The underlying repository data files are first checked to detect bit rot
and other types of damage. After that the consistency and correctness of the archive
metadata is verified.
The check command verifies the consistency of a repository and the corresponding archives.
The archive metadata checks can be time consuming and requires access to the key
file and/or passphrase if encryption is enabled. These checks can be skipped using
the --repository-only option.
First, the underlying repository data files are checked:
- For all segments the segment magic (header) is checked
- For all objects stored in the segments, all metadata (e.g. crc and size) and
all data is read. The read data is checked by size and CRC. Bit rot and other
types of accidental damage can be detected this way.
- If we are in repair mode and a integrity error is detected for a segment,
we try to recover as many objects from the segment as possible.
- In repair mode, it makes sure that the index is consistent with the data
stored in the segments.
- If you use a remote repo server via ssh:, the repo check is executed on the
repo server without causing significant network traffic.
- The repository check can be skipped using the --archives-only option.
Second, the consistency and correctness of the archive metadata is verified:
- Is the repo manifest present? If not, it is rebuilt from archive metadata
chunks (this requires reading and decrypting of all metadata and data).
- Check if archive metadata chunk is present. if not, remove archive from
manifest.
- For all files (items) in the archive, for all chunks referenced by these
files, check if chunk is present (if not and we are in repair mode, replace
it with a same-size chunk of zeros). This requires reading of archive and
file metadata, but not data.
- If we are in repair mode and we checked all the archives: delete orphaned
chunks from the repo.
- if you use a remote repo server via ssh:, the archive check is executed on
the client machine (because if encryption is enabled, the checks will require
decryption and this is always done client-side, because key access will be
required).
- The archive checks can be time consuming, they can be skipped using the
--repository-only option.
""")
subparser = subparsers.add_parser('check', parents=[common_parser],
description=self.do_check.__doc__,
epilog=check_epilog,
formatter_class=argparse.RawDescriptionHelpFormatter)
subparser.set_defaults(func=self.do_check)
subparser.add_argument('repository', metavar='REPOSITORY',
type=location_validator(archive=False),
help='repository to check consistency of')
subparser.add_argument('repository', metavar='REPOSITORY_OR_ARCHIVE',
type=location_validator(),
help='repository or archive to check consistency of')
subparser.add_argument('--repository-only', dest='repo_only', action='store_true',
default=False,
help='only perform repository checks')
@ -506,6 +607,9 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
subparser.add_argument('--repair', dest='repair', action='store_true',
default=False,
help='attempt to repair any inconsistencies found')
subparser.add_argument('--last', dest='last',
type=int, default=None, metavar='N',
help='only check last N archives (Default: all)')
change_passphrase_epilog = textwrap.dedent("""
The key files used for repository encryption are optionally passphrase
@ -524,7 +628,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
traversing all paths specified. The archive will consume almost no disk space for
files or parts of files that have already been stored in other archives.
See "attic help patterns" for more help on exclude patterns.
See the output of the "borg help patterns" command for more help on exclude patterns.
""")
subparser = subparsers.add_parser('create', parents=[common_parser],
@ -535,6 +639,9 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
subparser.add_argument('-s', '--stats', dest='stats',
action='store_true', default=False,
help='print statistics for the created archive')
subparser.add_argument('-p', '--progress', dest='progress',
action='store_true', default=False,
help='print progress while creating the archive')
subparser.add_argument('-e', '--exclude', dest='excludes',
type=ExcludePattern, action='append',
metavar="PATTERN", help='exclude paths matching PATTERN')
@ -553,6 +660,19 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
subparser.add_argument('--numeric-owner', dest='numeric_owner',
action='store_true', default=False,
help='only store numeric user and group identifiers')
subparser.add_argument('--timestamp', dest='timestamp',
type=timestamp, default=None,
metavar='yyyy-mm-ddThh:mm:ss',
help='manually specify the archive creation date/time (UTC). '
'alternatively, give a reference file/directory.')
subparser.add_argument('--chunker-params', dest='chunker_params',
type=ChunkerParams, default=CHUNKER_PARAMS,
metavar='CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE',
help='specify the chunker parameters. default: %d,%d,%d,%d' % CHUNKER_PARAMS)
subparser.add_argument('-C', '--compression', dest='compression',
type=int, default=0, metavar='N',
help='select compression algorithm and level. 0..9 is supported and means zlib '
'level 0 (no compression, fast, default) .. zlib level 9 (high compression, slow).')
subparser.add_argument('archive', metavar='ARCHIVE',
type=location_validator(archive=True),
help='archive to create')
@ -565,7 +685,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
by passing a list of ``PATHs`` as arguments. The file selection can further
be restricted by using the ``--exclude`` option.
See "attic help patterns" for more help on exclude patterns.
See the output of the "borg help patterns" command for more help on exclude patterns.
""")
subparser = subparsers.add_parser('extract', parents=[common_parser],
description=self.do_extract.__doc__,
@ -587,15 +707,36 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
subparser.add_argument('--strip-components', dest='strip_components',
type=int, default=0, metavar='NUMBER',
help='Remove the specified number of leading path elements. Pathnames with fewer elements will be silently skipped.')
subparser.add_argument('--stdout', dest='stdout',
action='store_true', default=False,
help='write all extracted data to stdout')
subparser.add_argument('--sparse', dest='sparse',
action='store_true', default=False,
help='create holes in output sparse file from all-zero chunks')
subparser.add_argument('archive', metavar='ARCHIVE',
type=location_validator(archive=True),
help='archive to extract')
subparser.add_argument('paths', metavar='PATH', nargs='*', type=str,
help='paths to extract')
rename_epilog = textwrap.dedent("""
This command renames an archive in the repository.
""")
subparser = subparsers.add_parser('rename', parents=[common_parser],
description=self.do_rename.__doc__,
epilog=rename_epilog,
formatter_class=argparse.RawDescriptionHelpFormatter)
subparser.set_defaults(func=self.do_rename)
subparser.add_argument('archive', metavar='ARCHIVE',
type=location_validator(archive=True),
help='archive to rename')
subparser.add_argument('name', metavar='NEWNAME', type=str,
help='the new archive name to use')
delete_epilog = textwrap.dedent("""
This command deletes an archive from the repository. Any disk space not
shared with any other existing archive is also reclaimed.
This command deletes an archive from the repository or the complete repository.
Disk space is reclaimed accordingly. If you delete the complete repository, the
local cache for it (if any) is also deleted.
""")
subparser = subparsers.add_parser('delete', parents=[common_parser],
description=self.do_delete.__doc__,
@ -605,9 +746,9 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
subparser.add_argument('-s', '--stats', dest='stats',
action='store_true', default=False,
help='print statistics for the deleted archive')
subparser.add_argument('archive', metavar='ARCHIVE',
type=location_validator(archive=True),
help='archive to delete')
subparser.add_argument('target', metavar='TARGET',
type=location_validator(),
help='archive or repository to delete')
list_epilog = textwrap.dedent("""
This command lists the contents of a repository or an archive.
@ -716,27 +857,69 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
args = parser.parse_args(args or ['-h'])
self.verbose = args.verbose
os.umask(args.umask)
RemoteRepository.remote_path = args.remote_path
RemoteRepository.umask = args.umask
update_excludes(args)
return args.func(args)
def main():
def sig_info_handler(signum, stack): # pragma: no cover
"""search the stack for infos about the currently processed file and print them"""
for frame in inspect.getouterframes(stack):
func, loc = frame[3], frame[0].f_locals
if func in ('process_file', '_process', ): # create op
path = loc['path']
try:
pos = loc['fd'].tell()
total = loc['st'].st_size
except Exception:
pos, total = 0, 0
print("{0} {1}/{2}".format(path, format_file_size(pos), format_file_size(total)))
break
if func in ('extract_item', ): # extract op
path = loc['item'][b'path']
try:
pos = loc['fd'].tell()
except Exception:
pos = 0
print("{0} {1}/???".format(path, format_file_size(pos)))
break
def setup_signal_handlers(): # pragma: no cover
sigs = []
if hasattr(signal, 'SIGUSR1'):
sigs.append(signal.SIGUSR1) # kill -USR1 pid
if hasattr(signal, 'SIGINFO'):
sigs.append(signal.SIGINFO) # kill -INFO pid (or ctrl-t)
for sig in sigs:
signal.signal(sig, sig_info_handler)
def main(): # pragma: no cover
# Make sure stdout and stderr have errors='replace') to avoid unicode
# issues when print()-ing unicode file names
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, sys.stdout.encoding, 'replace', line_buffering=True)
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, sys.stderr.encoding, 'replace', line_buffering=True)
setup_signal_handlers()
archiver = Archiver()
try:
exit_code = archiver.run(sys.argv[1:])
except Error as e:
archiver.print_error(e.get_message())
archiver.print_error(e.get_message() + "\n%s" % traceback.format_exc())
exit_code = e.exit_code
except KeyboardInterrupt:
archiver.print_error('Error: Keyboard interrupt')
except RemoteRepository.RPCError as e:
archiver.print_error('Error: Remote Exception.\n%s' % str(e))
exit_code = 1
else:
if exit_code:
archiver.print_error('Exiting with failure status due to previous errors')
except Exception:
archiver.print_error('Error: Local Exception.\n%s' % traceback.format_exc())
exit_code = 1
except KeyboardInterrupt:
archiver.print_error('Error: Keyboard interrupt.\n%s' % traceback.format_exc())
exit_code = 1
if exit_code:
archiver.print_error('Exiting with failure status due to previous errors')
sys.exit(exit_code)
if __name__ == '__main__':

192
attic/cache.py → borg/cache.py
View file

@ -1,18 +1,22 @@
from configparser import RawConfigParser
from attic.remote import cache_if_remote
from .remote import cache_if_remote
import errno
import msgpack
import os
import sys
from binascii import hexlify
import shutil
import tarfile
import tempfile
from .key import PlaintextKey
from .helpers import Error, get_cache_dir, decode_dict, st_mtime_ns, unhexlify, UpgradableLock, int_to_bigint, \
from .helpers import Error, get_cache_dir, decode_dict, st_mtime_ns, unhexlify, int_to_bigint, \
bigint_to_int
from .locking import UpgradableLock
from .hashindex import ChunkIndex
class Cache(object):
class Cache:
"""Client Side cache
"""
class RepositoryReplay(Error):
@ -28,26 +32,28 @@ class Cache(object):
"""Repository encryption method changed since last acccess, refusing to continue
"""
def __init__(self, repository, key, manifest, path=None, sync=True, warn_if_unencrypted=True):
def __init__(self, repository, key, manifest, path=None, sync=True, do_files=False, warn_if_unencrypted=True):
self.lock = None
self.timestamp = None
self.lock = None
self.txn_active = False
self.repository = repository
self.key = key
self.manifest = manifest
self.path = path or os.path.join(get_cache_dir(), hexlify(repository.id).decode('ascii'))
self.do_files = do_files
# Warn user before sending data to a never seen before unencrypted repository
if not os.path.exists(self.path):
if warn_if_unencrypted and isinstance(key, PlaintextKey):
if not self._confirm('Warning: Attempting to access a previously unknown unencrypted repository',
'ATTIC_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'):
'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'):
raise self.CacheInitAbortedError()
self.create()
self.open()
# Warn user before sending data to a relocated repository
if self.previous_location and self.previous_location != repository._location.canonical_path():
msg = 'Warning: The repository at location {} was previously located at {}'.format(repository._location.canonical_path(), self.previous_location)
if not self._confirm(msg, 'ATTIC_RELOCATED_REPO_ACCESS_IS_OK'):
if not self._confirm(msg, 'BORG_RELOCATED_REPO_ACCESS_IS_OK'):
raise self.RepositoryAccessAborted()
if sync and self.manifest.id != self.manifest_id:
@ -77,11 +83,11 @@ class Cache(object):
return answer and answer in 'Yy'
def create(self):
"""Create a new empty cache at `path`
"""Create a new empty cache at `self.path`
"""
os.makedirs(self.path)
with open(os.path.join(self.path, 'README'), 'w') as fd:
fd.write('This is an Attic cache')
fd.write('This is a Borg cache')
config = RawConfigParser()
config.add_section('cache')
config.set('cache', 'version', '1')
@ -90,14 +96,23 @@ class Cache(object):
with open(os.path.join(self.path, 'config'), 'w') as fd:
config.write(fd)
ChunkIndex().write(os.path.join(self.path, 'chunks').encode('utf-8'))
with open(os.path.join(self.path, 'files'), 'w') as fd:
with open(os.path.join(self.path, 'chunks.archive'), 'wb') as fd:
pass # empty file
with open(os.path.join(self.path, 'files'), 'wb') as fd:
pass # empty file
def destroy(self):
"""destroy the cache at `self.path`
"""
self.close()
os.remove(os.path.join(self.path, 'config')) # kill config first
shutil.rmtree(self.path)
def _do_open(self):
self.config = RawConfigParser()
self.config.read(os.path.join(self.path, 'config'))
if self.config.getint('cache', 'version') != 1:
raise Exception('%s Does not look like an Attic cache')
raise Exception('%s Does not look like a Borg cache')
self.id = self.config.get('cache', 'repository')
self.manifest_id = unhexlify(self.config.get('cache', 'manifest'))
self.timestamp = self.config.get('cache', 'timestamp', fallback=None)
@ -108,8 +123,8 @@ class Cache(object):
def open(self):
if not os.path.isdir(self.path):
raise Exception('%s Does not look like an Attic cache' % self.path)
self.lock = UpgradableLock(os.path.join(self.path, 'config'), exclusive=True)
raise Exception('%s Does not look like a Borg cache' % self.path)
self.lock = UpgradableLock(os.path.join(self.path, 'lock'), exclusive=True).acquire()
self.rollback()
def close(self):
@ -129,6 +144,7 @@ class Cache(object):
u.feed(data)
for path_hash, item in u:
item[0] += 1
# in the end, this takes about 240 Bytes per file
self.files[path_hash] = msgpack.packb(item)
def begin_txn(self):
@ -137,6 +153,7 @@ class Cache(object):
os.mkdir(txn_dir)
shutil.copy(os.path.join(self.path, 'config'), txn_dir)
shutil.copy(os.path.join(self.path, 'chunks'), txn_dir)
shutil.copy(os.path.join(self.path, 'chunks.archive'), txn_dir)
shutil.copy(os.path.join(self.path, 'files'), txn_dir)
os.rename(os.path.join(self.path, 'txn.tmp'),
os.path.join(self.path, 'txn.active'))
@ -178,6 +195,7 @@ class Cache(object):
if os.path.exists(txn_dir):
shutil.copy(os.path.join(txn_dir, 'config'), self.path)
shutil.copy(os.path.join(txn_dir, 'chunks'), self.path)
shutil.copy(os.path.join(txn_dir, 'chunks.archive'), self.path)
shutil.copy(os.path.join(txn_dir, 'files'), self.path)
os.rename(txn_dir, os.path.join(self.path, 'txn.tmp'))
if os.path.exists(os.path.join(self.path, 'txn.tmp')):
@ -186,37 +204,145 @@ class Cache(object):
self._do_open()
def sync(self):
"""Initializes cache by fetching and reading all archive indicies
"""Re-synchronize chunks cache with repository.
If present, uses a compressed tar archive of known backup archive
indices, so it only needs to fetch infos from repo and build a chunk
index once per backup archive.
If out of sync, the tar gets rebuilt from known + fetched chunk infos,
so it has complete and current information about all backup archives.
Finally, it builds the master chunks index by merging all indices from
the tar.
Note: compression (esp. xz) is very effective in keeping the tar
relatively small compared to the files it contains.
"""
def add(id, size, csize):
in_archive_path = os.path.join(self.path, 'chunks.archive')
out_archive_path = os.path.join(self.path, 'chunks.archive.tmp')
def open_in_archive():
try:
count, size, csize = self.chunks[id]
self.chunks[id] = count + 1, size, csize
tf = tarfile.open(in_archive_path, 'r')
except OSError as e:
if e.errno != errno.ENOENT:
raise
# file not found
tf = None
except tarfile.ReadError:
# empty file?
tf = None
return tf
def open_out_archive():
for compression in ('xz', 'bz2', 'gz'):
# xz needs py 3.3, bz2 and gz also work on 3.2
try:
tf = tarfile.open(out_archive_path, 'w:'+compression, format=tarfile.PAX_FORMAT)
break
except tarfile.CompressionError:
continue
else: # shouldn't happen
tf = None
return tf
def close_archive(tf):
if tf:
tf.close()
def delete_in_archive():
os.unlink(in_archive_path)
def rename_out_archive():
os.rename(out_archive_path, in_archive_path)
def add(chunk_idx, id, size, csize, incr=1):
try:
count, size, csize = chunk_idx[id]
chunk_idx[id] = count + incr, size, csize
except KeyError:
self.chunks[id] = 1, size, csize
self.begin_txn()
print('Initializing cache...')
self.chunks.clear()
unpacker = msgpack.Unpacker()
repository = cache_if_remote(self.repository)
for name, info in self.manifest.archives.items():
archive_id = info[b'id']
chunk_idx[id] = incr, size, csize
def transfer_known_idx(archive_id, tf_in, tf_out):
archive_id_hex = hexlify(archive_id).decode('ascii')
tarinfo = tf_in.getmember(archive_id_hex)
archive_name = tarinfo.pax_headers['archive_name']
print('Already known archive:', archive_name)
f_in = tf_in.extractfile(archive_id_hex)
tf_out.addfile(tarinfo, f_in)
return archive_name
def fetch_and_build_idx(archive_id, repository, key, tmp_dir, tf_out):
chunk_idx = ChunkIndex()
cdata = repository.get(archive_id)
data = self.key.decrypt(archive_id, cdata)
add(archive_id, len(data), len(cdata))
data = key.decrypt(archive_id, cdata)
add(chunk_idx, archive_id, len(data), len(cdata))
archive = msgpack.unpackb(data)
if archive[b'version'] != 1:
raise Exception('Unknown archive metadata version')
decode_dict(archive, (b'name',))
print('Analyzing archive:', archive[b'name'])
for key, chunk in zip(archive[b'items'], repository.get_many(archive[b'items'])):
data = self.key.decrypt(key, chunk)
add(key, len(data), len(chunk))
print('Analyzing new archive:', archive[b'name'])
unpacker = msgpack.Unpacker()
for item_id, chunk in zip(archive[b'items'], repository.get_many(archive[b'items'])):
data = key.decrypt(item_id, chunk)
add(chunk_idx, item_id, len(data), len(chunk))
unpacker.feed(data)
for item in unpacker:
if not isinstance(item, dict):
print('Error: Did not get expected metadata dict - archive corrupted!')
continue
if b'chunks' in item:
for chunk_id, size, csize in item[b'chunks']:
add(chunk_id, size, csize)
add(chunk_idx, chunk_id, size, csize)
archive_id_hex = hexlify(archive_id).decode('ascii')
file_tmp = os.path.join(tmp_dir, archive_id_hex).encode('utf-8')
chunk_idx.write(file_tmp)
tarinfo = tf_out.gettarinfo(file_tmp, archive_id_hex)
tarinfo.pax_headers['archive_name'] = archive[b'name']
with open(file_tmp, 'rb') as f:
tf_out.addfile(tarinfo, f)
os.unlink(file_tmp)
def create_master_idx(chunk_idx, tf_in, tmp_dir):
chunk_idx.clear()
for tarinfo in tf_in:
archive_id_hex = tarinfo.name
archive_name = tarinfo.pax_headers['archive_name']
print("- extracting archive %s ..." % archive_name)
tf_in.extract(archive_id_hex, tmp_dir)
chunk_idx_path = os.path.join(tmp_dir, archive_id_hex).encode('utf-8')
print("- reading archive ...")
archive_chunk_idx = ChunkIndex.read(chunk_idx_path)
print("- merging archive ...")
chunk_idx.merge(archive_chunk_idx)
os.unlink(chunk_idx_path)
self.begin_txn()
print('Synchronizing chunks cache...')
# XXX we have to do stuff on disk due to lacking ChunkIndex api
with tempfile.TemporaryDirectory(prefix='borg-tmp') as tmp_dir:
repository = cache_if_remote(self.repository)
out_archive = open_out_archive()
in_archive = open_in_archive()
if in_archive:
known_ids = set(unhexlify(hexid) for hexid in in_archive.getnames())
else:
known_ids = set()
archive_ids = set(info[b'id'] for info in self.manifest.archives.values())
print('Rebuilding archive collection. Known: %d Repo: %d Unknown: %d' % (
len(known_ids), len(archive_ids), len(archive_ids - known_ids), ))
for archive_id in archive_ids & known_ids:
transfer_known_idx(archive_id, in_archive, out_archive)
close_archive(in_archive)
delete_in_archive() # free disk space
for archive_id in archive_ids - known_ids:
fetch_and_build_idx(archive_id, repository, self.key, tmp_dir, out_archive)
close_archive(out_archive)
rename_out_archive()
print('Merging collection into master chunks cache...')
in_archive = open_in_archive()
create_master_idx(self.chunks, in_archive, tmp_dir)
close_archive(in_archive)
print('Done.')
def add_chunk(self, id, data, stats):
if not self.txn_active:
@ -255,6 +381,8 @@ class Cache(object):
stats.update(-size, -csize, False)
def file_known_and_unchanged(self, path_hash, st):
if not self.do_files:
return None
if self.files is None:
self._read_files()
entry = self.files.get(path_hash)
@ -270,6 +398,8 @@ class Cache(object):
return None
def memorize_file(self, path_hash, st, ids):
if not self.do_files:
return
# Entry: Age, inode, size, mtime, chunk ids
mtime_ns = st_mtime_ns(st)
self.files[path_hash] = msgpack.packb((0, st.st_ino, st.st_size, int_to_bigint(mtime_ns), ids))

24
attic/chunker.pyx → borg/chunker.pyx
View file

@ -8,8 +8,8 @@ cdef extern from "_chunker.c":
ctypedef int uint32_t
ctypedef struct _Chunker "Chunker":
pass
_Chunker *chunker_init(int window_size, int chunk_mask, int min_size, uint32_t seed)
void chunker_set_fd(_Chunker *chunker, object fd)
_Chunker *chunker_init(int window_size, int chunk_mask, int min_size, int max_size, uint32_t seed)
void chunker_set_fd(_Chunker *chunker, object f, int fd)
void chunker_free(_Chunker *chunker)
object chunker_process(_Chunker *chunker)
uint32_t *buzhash_init_table(uint32_t seed)
@ -20,11 +20,21 @@ cdef extern from "_chunker.c":
cdef class Chunker:
cdef _Chunker *chunker
def __cinit__(self, window_size, chunk_mask, min_size, seed):
self.chunker = chunker_init(window_size, chunk_mask, min_size, seed & 0xffffffff)
def __cinit__(self, seed, chunk_min_exp, chunk_max_exp, hash_mask_bits, hash_window_size):
min_size = 1 << chunk_min_exp
max_size = 1 << chunk_max_exp
hash_mask = (1 << hash_mask_bits) - 1
self.chunker = chunker_init(hash_window_size, hash_mask, min_size, max_size, seed & 0xffffffff)
def chunkify(self, fd):
chunker_set_fd(self.chunker, fd)
def chunkify(self, fd, fh=-1):
"""
Cut a file into chunks.
:param fd: Python file object
:param fh: OS-level file handle (if available),
defaults to -1 which means not to use OS-level fd.
"""
chunker_set_fd(self.chunker, fd, fh)
return self
def __dealloc__(self):
@ -52,4 +62,4 @@ def buzhash_update(uint32_t sum, unsigned char remove, unsigned char add, size_t
table = buzhash_init_table(seed & 0xffffffff)
sum = c_buzhash_update(sum, remove, add, len, table)
free(table)
return sum
return sum

173
borg/crypto.pyx Normal file
View file

@ -0,0 +1,173 @@
"""A thin OpenSSL wrapper
This could be replaced by PyCrypto or something similar when the performance
of their PBKDF2 implementation is comparable to the OpenSSL version.
"""
from libc.stdlib cimport malloc, free
API_VERSION = 2
cdef extern from "openssl/rand.h":
int RAND_bytes(unsigned char *buf, int num)
cdef extern from "openssl/evp.h":
ctypedef struct EVP_MD:
pass
ctypedef struct EVP_CIPHER:
pass
ctypedef struct EVP_CIPHER_CTX:
unsigned char *iv
pass
ctypedef struct ENGINE:
pass
const EVP_MD *EVP_sha256()
const EVP_CIPHER *EVP_aes_256_ctr()
void EVP_CIPHER_CTX_init(EVP_CIPHER_CTX *a)
void EVP_CIPHER_CTX_cleanup(EVP_CIPHER_CTX *a)
int EVP_EncryptInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ENGINE *impl,
const unsigned char *key, const unsigned char *iv)
int EVP_DecryptInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ENGINE *impl,
const unsigned char *key, const unsigned char *iv)
int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
const unsigned char *in_, int inl)
int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
const unsigned char *in_, int inl)
int EVP_EncryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
int EVP_DecryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
int PKCS5_PBKDF2_HMAC(const char *password, int passwordlen,
const unsigned char *salt, int saltlen, int iter,
const EVP_MD *digest,
int keylen, unsigned char *out)
import struct
_int = struct.Struct('>I')
_long = struct.Struct('>Q')
bytes_to_int = lambda x, offset=0: _int.unpack_from(x, offset)[0]
bytes_to_long = lambda x, offset=0: _long.unpack_from(x, offset)[0]
long_to_bytes = lambda x: _long.pack(x)
def num_aes_blocks(length):
"""Return the number of AES blocks required to encrypt/decrypt *length* bytes of data.
Note: this is only correct for modes without padding, like AES-CTR.
"""
return (length + 15) // 16
def pbkdf2_sha256(password, salt, iterations, size):
"""Password based key derivation function 2 (RFC2898)
"""
cdef unsigned char *key = <unsigned char *>malloc(size)
if not key:
raise MemoryError
try:
rv = PKCS5_PBKDF2_HMAC(password, len(password), salt, len(salt), iterations, EVP_sha256(), size, key)
if not rv:
raise Exception('PKCS5_PBKDF2_HMAC failed')
return key[:size]
finally:
free(key)
def get_random_bytes(n):
"""Return n cryptographically strong pseudo-random bytes
"""
cdef unsigned char *buf = <unsigned char *>malloc(n)
if not buf:
raise MemoryError
try:
if RAND_bytes(buf, n) < 1:
raise Exception('RAND_bytes failed')
return buf[:n]
finally:
free(buf)
cdef class AES:
"""A thin wrapper around the OpenSSL EVP cipher API
"""
cdef EVP_CIPHER_CTX ctx
cdef int is_encrypt
def __cinit__(self, is_encrypt, key, iv=None):
EVP_CIPHER_CTX_init(&self.ctx)
self.is_encrypt = is_encrypt
# Set cipher type and mode
cipher_mode = EVP_aes_256_ctr()
if self.is_encrypt:
if not EVP_EncryptInit_ex(&self.ctx, cipher_mode, NULL, NULL, NULL):
raise Exception('EVP_EncryptInit_ex failed')
else: # decrypt
if not EVP_DecryptInit_ex(&self.ctx, cipher_mode, NULL, NULL, NULL):
raise Exception('EVP_DecryptInit_ex failed')
self.reset(key, iv)
def __dealloc__(self):
EVP_CIPHER_CTX_cleanup(&self.ctx)
def reset(self, key=None, iv=None):
cdef const unsigned char *key2 = NULL
cdef const unsigned char *iv2 = NULL
if key:
key2 = key
if iv:
iv2 = iv
# Initialise key and IV
if self.is_encrypt:
if not EVP_EncryptInit_ex(&self.ctx, NULL, NULL, key2, iv2):
raise Exception('EVP_EncryptInit_ex failed')
else: # decrypt
if not EVP_DecryptInit_ex(&self.ctx, NULL, NULL, key2, iv2):
raise Exception('EVP_DecryptInit_ex failed')
@property
def iv(self):
return self.ctx.iv[:16]
def encrypt(self, data):
cdef int inl = len(data)
cdef int ctl = 0
cdef int outl = 0
# note: modes that use padding, need up to one extra AES block (16b)
cdef unsigned char *out = <unsigned char *>malloc(inl+16)
if not out:
raise MemoryError
try:
if not EVP_EncryptUpdate(&self.ctx, out, &outl, data, inl):
raise Exception('EVP_EncryptUpdate failed')
ctl = outl
if not EVP_EncryptFinal_ex(&self.ctx, out+ctl, &outl):
raise Exception('EVP_EncryptFinal failed')
ctl += outl
return out[:ctl]
finally:
free(out)
def decrypt(self, data):
cdef int inl = len(data)
cdef int ptl = 0
cdef int outl = 0
# note: modes that use padding, need up to one extra AES block (16b).
# This is what the openssl docs say. I am not sure this is correct,
# but OTOH it will not cause any harm if our buffer is a little bigger.
cdef unsigned char *out = <unsigned char *>malloc(inl+16)
if not out:
raise MemoryError
try:
if not EVP_DecryptUpdate(&self.ctx, out, &outl, data, inl):
raise Exception('EVP_DecryptUpdate failed')
ptl = outl
if EVP_DecryptFinal_ex(&self.ctx, out+ptl, &outl) <= 0:
# this error check is very important for modes with padding or
# authentication. for them, a failure here means corrupted data.
# CTR mode does not use padding nor authentication.
raise Exception('EVP_DecryptFinal failed')
ptl += outl
return out[:ptl]
finally:
free(out)

30
attic/fuse.py → borg/fuse.py
View file

@ -7,9 +7,9 @@ import os
import stat
import tempfile
import time
from attic.archive import Archive
from attic.helpers import daemonize
from attic.remote import cache_if_remote
from .archive import Archive
from .helpers import daemonize
from .remote import cache_if_remote
# Does this version of llfuse support ns precision?
have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns')
@ -17,7 +17,7 @@ have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns')
class ItemCache:
def __init__(self):
self.fd = tempfile.TemporaryFile()
self.fd = tempfile.TemporaryFile(prefix='borg-tmp')
self.offset = 1000000
def add(self, item):
@ -30,11 +30,11 @@ class ItemCache:
return next(msgpack.Unpacker(self.fd))
class AtticOperations(llfuse.Operations):
"""Export Attic archive as a fuse filesystem
class FuseOperations(llfuse.Operations):
"""Export archive as a fuse filesystem
"""
def __init__(self, key, repository, manifest, archive):
super(AtticOperations, self).__init__()
super().__init__()
self._inode_count = 0
self.key = key
self.repository = cache_if_remote(repository)
@ -43,6 +43,7 @@ class AtticOperations(llfuse.Operations):
self.contents = defaultdict(dict)
self.default_dir = {b'mode': 0o40755, b'mtime': int(time.time() * 1e9), b'uid': os.getuid(), b'gid': os.getgid()}
self.pending_archives = {}
self.accounted_chunks = {}
self.cache = ItemCache()
if archive:
self.process_archive(archive)
@ -130,8 +131,13 @@ class AtticOperations(llfuse.Operations):
def getattr(self, inode):
item = self.get_item(inode)
size = 0
dsize = 0
try:
size = sum(size for _, size, _ in item[b'chunks'])
for key, chunksize, _ in item[b'chunks']:
size += chunksize
if self.accounted_chunks.get(key, inode) == inode:
self.accounted_chunks[key] = inode
dsize += chunksize
except KeyError:
pass
entry = llfuse.EntryAttributes()
@ -146,7 +152,7 @@ class AtticOperations(llfuse.Operations):
entry.st_rdev = item.get(b'rdev', 0)
entry.st_size = size
entry.st_blksize = 512
entry.st_blocks = 1
entry.st_blocks = dsize / 512
if have_fuse_mtime_ns:
entry.st_atime_ns = item[b'mtime']
entry.st_mtime_ns = item[b'mtime']
@ -220,7 +226,7 @@ class AtticOperations(llfuse.Operations):
return os.fsencode(item[b'source'])
def mount(self, mountpoint, extra_options, foreground=False):
options = ['fsname=atticfs', 'ro']
options = ['fsname=borgfs', 'ro']
if extra_options:
options.extend(extra_options.split(','))
llfuse.init(self, mountpoint, options)
@ -228,7 +234,5 @@ class AtticOperations(llfuse.Operations):
daemonize()
try:
llfuse.main(single=True)
except:
finally:
llfuse.close()
raise
llfuse.close()

25
attic/hashindex.pyx → borg/hashindex.pyx
View file

@ -11,7 +11,10 @@ cdef extern from "_hashindex.c":
HashIndex *hashindex_read(char *path)
HashIndex *hashindex_init(int capacity, int key_size, int value_size)
void hashindex_free(HashIndex *index)
void hashindex_summarize(HashIndex *index, long long *total_size, long long *total_csize, long long *unique_size, long long *unique_csize)
void hashindex_summarize(HashIndex *index, long long *total_size, long long *total_csize,
long long *unique_size, long long *unique_csize,
long long *total_unique_chunks, long long *total_chunks)
void hashindex_merge(HashIndex *index, HashIndex *other)
int hashindex_get_size(HashIndex *index)
int hashindex_write(HashIndex *index, char *path)
void *hashindex_get(HashIndex *index, void *key)
@ -22,15 +25,18 @@ cdef extern from "_hashindex.c":
int _le32toh(int v)
_NoDefault = object()
cdef _NoDefault = object()
cimport cython
@cython.internal
cdef class IndexBase:
cdef HashIndex *index
key_size = 32
def __cinit__(self, capacity=0, path=None):
if path:
self.index = hashindex_read(<bytes>os.fsencode(path))
self.index = hashindex_read(os.fsencode(path))
if not self.index:
raise Exception('hashindex_read failed')
else:
@ -47,7 +53,7 @@ cdef class IndexBase:
return cls(path=path)
def write(self, path):
if not hashindex_write(self.index, <bytes>os.fsencode(path)):
if not hashindex_write(self.index, os.fsencode(path)):
raise Exception('hashindex_write failed')
def clear(self):
@ -179,9 +185,14 @@ cdef class ChunkIndex(IndexBase):
return iter
def summarize(self):
cdef long long total_size, total_csize, unique_size, unique_csize
hashindex_summarize(self.index, &total_size, &total_csize, &unique_size, &unique_csize)
return total_size, total_csize, unique_size, unique_csize
cdef long long total_size, total_csize, unique_size, unique_csize, total_unique_chunks, total_chunks
hashindex_summarize(self.index, &total_size, &total_csize,
&unique_size, &unique_csize,
&total_unique_chunks, &total_chunks)
return total_size, total_csize, unique_size, unique_csize, total_unique_chunks, total_chunks
def merge(self, ChunkIndex other):
hashindex_merge(self.index, other.index)
cdef class ChunkKeyIterator:

150
attic/helpers.py → borg/helpers.py
View file

@ -1,7 +1,7 @@
import argparse
import binascii
from collections import namedtuple
import grp
import msgpack
import os
import pwd
import re
@ -10,11 +10,12 @@ import time
from datetime import datetime, timezone, timedelta
from fnmatch import translate
from operator import attrgetter
import fcntl
import attic.hashindex
import attic.chunker
import attic.crypto
import msgpack
from . import hashindex
from . import chunker
from . import crypto
class Error(Exception):
@ -27,55 +28,18 @@ class Error(Exception):
class ExtensionModuleError(Error):
"""The Attic binary extension modules does not seem to be properly installed"""
class UpgradableLock:
class ReadLockFailed(Error):
"""Failed to acquire read lock on {}"""
class WriteLockFailed(Error):
"""Failed to acquire write lock on {}"""
def __init__(self, path, exclusive=False):
self.path = path
try:
self.fd = open(path, 'r+')
except IOError:
self.fd = open(path, 'r')
try:
if exclusive:
fcntl.lockf(self.fd, fcntl.LOCK_EX)
else:
fcntl.lockf(self.fd, fcntl.LOCK_SH)
# Python 3.2 raises IOError, Python3.3+ raises OSError
except (IOError, OSError):
if exclusive:
raise self.WriteLockFailed(self.path)
else:
raise self.ReadLockFailed(self.path)
self.is_exclusive = exclusive
def upgrade(self):
try:
fcntl.lockf(self.fd, fcntl.LOCK_EX)
# Python 3.2 raises IOError, Python3.3+ raises OSError
except (IOError, OSError):
raise self.WriteLockFailed(self.path)
self.is_exclusive = True
def release(self):
fcntl.lockf(self.fd, fcntl.LOCK_UN)
self.fd.close()
"""The Borg binary extension modules do not seem to be properly installed"""
def check_extension_modules():
import attic.platform
if (attic.hashindex.API_VERSION != 2 or
attic.chunker.API_VERSION != 2 or
attic.crypto.API_VERSION != 2 or
attic.platform.API_VERSION != 2):
from . import platform
if hashindex.API_VERSION != 2:
raise ExtensionModuleError
if chunker.API_VERSION != 2:
raise ExtensionModuleError
if crypto.API_VERSION != 2:
raise ExtensionModuleError
if platform.API_VERSION != 2:
raise ExtensionModuleError
@ -101,7 +65,7 @@ class Manifest:
m = msgpack.unpackb(data)
if not m.get(b'version') == 1:
raise ValueError('Invalid manifest version')
manifest.archives = dict((k.decode('utf-8'), v) for k,v in m[b'archives'].items())
manifest.archives = dict((k.decode('utf-8'), v) for k, v in m[b'archives'].items())
manifest.timestamp = m.get(b'timestamp')
if manifest.timestamp:
manifest.timestamp = manifest.timestamp.decode('ascii')
@ -119,6 +83,18 @@ class Manifest:
self.id = self.key.id_hash(data)
self.repository.put(self.MANIFEST_ID, self.key.encrypt(data))
def list_archive_infos(self, sort_by=None, reverse=False):
# inexpensive Archive.list_archives replacement if we just need .name, .id, .ts
ArchiveInfo = namedtuple('ArchiveInfo', 'name id ts')
archives = []
for name, values in self.archives.items():
ts = parse_timestamp(values[b'time'].decode('utf-8'))
id = values[b'id']
archives.append(ArchiveInfo(name=name, id=id, ts=ts))
if sort_by is not None:
archives = sorted(archives, key=attrgetter(sort_by), reverse=reverse)
return archives
def prune_within(archives, within):
multiplier = {'H': 1, 'd': 24, 'w': 24*7, 'm': 24*31, 'y': 24*365}
@ -144,7 +120,8 @@ def prune_split(archives, pattern, n, skip=[]):
last = period
if a not in skip:
keep.append(a)
if len(keep) == n: break
if len(keep) == n:
break
return keep
@ -160,23 +137,38 @@ class Statistics:
self.usize += csize
def print_(self, label, cache):
total_size, total_csize, unique_size, unique_csize = cache.chunks.summarize()
total_size, total_csize, unique_size, unique_csize, total_unique_chunks, total_chunks = cache.chunks.summarize()
print()
print(' Original size Compressed size Deduplicated size')
print('%-15s %20s %20s %20s' % (label, format_file_size(self.osize), format_file_size(self.csize), format_file_size(self.usize)))
print('All archives: %20s %20s %20s' % (format_file_size(total_size), format_file_size(total_csize), format_file_size(unique_csize)))
print()
print(' Unique chunks Total chunks')
print('Chunk index: %20d %20d' % (total_unique_chunks, total_chunks))
def show_progress(self, item=None, final=False):
if not final:
path = remove_surrogates(item[b'path']) if item else ''
if len(path) > 43:
path = '%s...%s' % (path[:20], path[-20:])
msg = '%9s O %9s C %9s D %-43s' % (
format_file_size(self.osize), format_file_size(self.csize), format_file_size(self.usize), path)
else:
msg = ' ' * 79
print(msg, end='\r')
sys.stdout.flush()
def get_keys_dir():
"""Determine where to repository keys and cache"""
return os.environ.get('ATTIC_KEYS_DIR',
os.path.join(os.path.expanduser('~'), '.attic', 'keys'))
return os.environ.get('BORG_KEYS_DIR',
os.path.join(os.path.expanduser('~'), '.borg', 'keys'))
def get_cache_dir():
"""Determine where to repository keys and cache"""
return os.environ.get('ATTIC_CACHE_DIR',
os.path.join(os.path.expanduser('~'), '.cache', 'attic'))
return os.environ.get('BORG_CACHE_DIR',
os.path.join(os.path.expanduser('~'), '.cache', 'borg'))
def to_localtime(ts):
@ -235,7 +227,7 @@ class IncludePattern:
path match as well. A trailing slash makes no difference.
"""
def __init__(self, pattern):
self.pattern = pattern.rstrip(os.path.sep)+os.path.sep
self.pattern = os.path.normpath(pattern).rstrip(os.path.sep)+os.path.sep
def match(self, path):
return (path+os.path.sep).startswith(self.pattern)
@ -250,9 +242,9 @@ class ExcludePattern(IncludePattern):
"""
def __init__(self, pattern):
if pattern.endswith(os.path.sep):
self.pattern = pattern+'*'+os.path.sep
self.pattern = os.path.normpath(pattern).rstrip(os.path.sep)+os.path.sep+'*'+os.path.sep
else:
self.pattern = pattern+os.path.sep+'*'
self.pattern = os.path.normpath(pattern)+os.path.sep+'*'
# fnmatch and re.match both cache compiled regular expressions.
# Nevertheless, this is about 10 times faster.
self.regex = re.compile(translate(self.pattern))
@ -264,6 +256,31 @@ class ExcludePattern(IncludePattern):
return '%s(%s)' % (type(self), self.pattern)
def timestamp(s):
"""Convert a --timestamp=s argument to a datetime object"""
try:
# is it pointing to a file / directory?
ts = os.stat(s).st_mtime
return datetime.utcfromtimestamp(ts)
except OSError:
# didn't work, try parsing as timestamp. UTC, no TZ, no microsecs support.
for format in ('%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%dT%H:%M:%S+00:00',
'%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S',
'%Y-%m-%dT%H:%M', '%Y-%m-%d %H:%M',
'%Y-%m-%d', '%Y-%j',
):
try:
return datetime.strptime(s, format)
except ValueError:
continue
raise ValueError
def ChunkerParams(s):
window_size, chunk_mask, chunk_min, chunk_max = s.split(',')
return int(window_size), int(chunk_mask), int(chunk_min), int(chunk_max)
def is_cachedir(path):
"""Determines whether the specified path is a cache directory (and
therefore should potentially be excluded from the backup) according to
@ -475,9 +492,9 @@ class Location:
else:
path = self.path
return 'ssh://{}{}{}{}'.format('{}@'.format(self.user) if self.user else '',
self.host,
':{}'.format(self.port) if self.port else '',
path)
self.host,
':{}'.format(self.port) if self.port else '',
path)
def location_validator(archive=None):
@ -503,7 +520,7 @@ def write_msgpack(filename, d):
with open(filename + '.tmp', 'wb') as fd:
msgpack.pack(d, fd)
fd.flush()
os.fsync(fd)
os.fsync(fd.fileno())
os.rename(filename + '.tmp', filename)
@ -552,7 +569,7 @@ def daemonize():
class StableDict(dict):
"""A dict subclass with stable items() ordering"""
def items(self):
return sorted(super(StableDict, self).items())
return sorted(super().items())
if sys.version < '3.3':
@ -588,4 +605,3 @@ def int_to_bigint(value):
if value.bit_length() > 63:
return value.to_bytes((value.bit_length() + 9) // 8, 'little', signed=True)
return value

300
attic/key.py → borg/key.py
View file

@ -1,5 +1,6 @@
from binascii import hexlify, a2b_base64, b2a_base64
from getpass import getpass
import configparser
import getpass
import os
import msgpack
import textwrap
@ -7,8 +8,8 @@ import hmac
from hashlib import sha256
import zlib
from attic.crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks
from attic.helpers import IntegrityError, get_keys_dir, Error
from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks
from .helpers import IntegrityError, get_keys_dir, Error
PREFIX = b'\0' * 8
@ -17,11 +18,17 @@ class UnsupportedPayloadError(Error):
"""Unsupported payload type {}. A newer version is required to access this repository.
"""
class KeyfileNotFoundError(Error):
"""No key file for repository {} found in {}.
"""
class RepoKeyNotFoundError(Error):
"""No key entry found in the config of repository {}.
"""
class HMAC(hmac.HMAC):
"""Workaround a bug in Python < 3.4 Where HMAC does not accept memoryviews
"""
@ -32,27 +39,36 @@ class HMAC(hmac.HMAC):
def key_creator(repository, args):
if args.encryption == 'keyfile':
return KeyfileKey.create(repository, args)
elif args.encryption == 'passphrase':
elif args.encryption == 'repokey':
return RepoKey.create(repository, args)
elif args.encryption == 'passphrase': # deprecated, kill in 1.x
return PassphraseKey.create(repository, args)
else:
return PlaintextKey.create(repository, args)
def key_factory(repository, manifest_data):
if manifest_data[0] == KeyfileKey.TYPE:
key_type = manifest_data[0]
if key_type == KeyfileKey.TYPE:
return KeyfileKey.detect(repository, manifest_data)
elif manifest_data[0] == PassphraseKey.TYPE:
elif key_type == RepoKey.TYPE:
return RepoKey.detect(repository, manifest_data)
elif key_type == PassphraseKey.TYPE: # deprecated, kill in 1.x
return PassphraseKey.detect(repository, manifest_data)
elif manifest_data[0] == PlaintextKey.TYPE:
elif key_type == PlaintextKey.TYPE:
return PlaintextKey.detect(repository, manifest_data)
else:
raise UnsupportedPayloadError(manifest_data[0])
raise UnsupportedPayloadError(key_type)
class KeyBase(object):
class KeyBase:
TYPE = None # override in subclasses
def __init__(self):
def __init__(self, repository):
self.TYPE_STR = bytes([self.TYPE])
self.repository = repository
self.target = None # key location file path / repo obj
self.compression_level = 0
def id_hash(self, data):
"""Return HMAC hash using the "id" HMAC key
@ -72,18 +88,18 @@ class PlaintextKey(KeyBase):
@classmethod
def create(cls, repository, args):
print('Encryption NOT enabled.\nUse the "--encryption=passphrase|keyfile" to enable encryption.')
return cls()
print('Encryption NOT enabled.\nUse the "--encryption=repokey|keyfile|passphrase" to enable encryption.')
return cls(repository)
@classmethod
def detect(cls, repository, manifest_data):
return cls()
return cls(repository)
def id_hash(self, data):
return sha256(data).digest()
def encrypt(self, data):
return b''.join([self.TYPE_STR, zlib.compress(data)])
return b''.join([self.TYPE_STR, zlib.compress(data, self.compression_level)])
def decrypt(self, id, data):
if data[0] != self.TYPE:
@ -115,7 +131,7 @@ class AESKeyBase(KeyBase):
return HMAC(self.id_key, data, sha256).digest()
def encrypt(self, data):
data = zlib.compress(data)
data = zlib.compress(data, self.compression_level)
self.enc_cipher.reset()
data = b''.join((self.enc_cipher.iv[8:], self.enc_cipher.encrypt(data)))
hmac = HMAC(self.enc_hmac_key, data, sha256).digest()
@ -149,42 +165,69 @@ class AESKeyBase(KeyBase):
self.chunk_seed = self.chunk_seed - 0xffffffff - 1
def init_ciphers(self, enc_iv=b''):
self.enc_cipher = AES(self.enc_key, enc_iv)
self.dec_cipher = AES(self.enc_key)
self.enc_cipher = AES(is_encrypt=True, key=self.enc_key, iv=enc_iv)
self.dec_cipher = AES(is_encrypt=False, key=self.enc_key)
class Passphrase(str):
@classmethod
def env_passphrase(cls, default=None):
passphrase = os.environ.get('BORG_PASSPHRASE', default)
if passphrase is not None:
return cls(passphrase)
@classmethod
def getpass(cls, prompt):
return cls(getpass.getpass(prompt))
@classmethod
def new(cls, allow_empty=False):
passphrase = cls.env_passphrase()
if passphrase is not None:
return passphrase
while True:
passphrase = cls.getpass('Enter new passphrase: ')
if allow_empty or passphrase:
passphrase2 = cls.getpass('Enter same passphrase again: ')
if passphrase == passphrase2:
print('Remember your passphrase. Your data will be inaccessible without it.')
return passphrase
else:
print('Passphrases do not match')
else:
print('Passphrase must not be blank')
def __repr__(self):
return '<Passphrase "***hidden***">'
def kdf(self, salt, iterations, length):
return pbkdf2_sha256(self.encode('utf-8'), salt, iterations, length)
class PassphraseKey(AESKeyBase):
# This mode is DEPRECATED and will be killed at 1.0 release.
# With this mode:
# - you can never ever change your passphrase for existing repos.
# - you can never ever use a different iterations count for existing repos.
TYPE = 0x01
iterations = 100000
iterations = 100000 # must not be changed ever!
@classmethod
def create(cls, repository, args):
key = cls()
passphrase = os.environ.get('ATTIC_PASSPHRASE')
if passphrase is not None:
passphrase2 = passphrase
else:
passphrase, passphrase2 = 1, 2
while passphrase != passphrase2:
passphrase = getpass('Enter passphrase: ')
if not passphrase:
print('Passphrase must not be blank')
continue
passphrase2 = getpass('Enter same passphrase again: ')
if passphrase != passphrase2:
print('Passphrases do not match')
key = cls(repository)
print('WARNING: "passphrase" mode is deprecated and will be removed in 1.0.')
print('If you want something similar (but with less issues), use "repokey" mode.')
passphrase = Passphrase.new(allow_empty=False)
key.init(repository, passphrase)
if passphrase:
print('Remember your passphrase. Your data will be inaccessible without it.')
return key
@classmethod
def detect(cls, repository, manifest_data):
prompt = 'Enter passphrase for %s: ' % repository._location.orig
key = cls()
passphrase = os.environ.get('ATTIC_PASSPHRASE')
key = cls(repository)
passphrase = Passphrase.env_passphrase()
if passphrase is None:
passphrase = getpass(prompt)
passphrase = Passphrase.getpass(prompt)
while True:
key.init(repository, passphrase)
try:
@ -193,44 +236,40 @@ class PassphraseKey(AESKeyBase):
key.init_ciphers(PREFIX + long_to_bytes(key.extract_nonce(manifest_data) + num_blocks))
return key
except IntegrityError:
passphrase = getpass(prompt)
passphrase = Passphrase.getpass(prompt)
def change_passphrase(self):
class ImmutablePassphraseError(Error):
"""The passphrase for this encryption key type can't be changed."""
raise ImmutablePassphraseError
def init(self, repository, passphrase):
self.init_from_random_data(pbkdf2_sha256(passphrase.encode('utf-8'), repository.id, self.iterations, 100))
self.init_from_random_data(passphrase.kdf(repository.id, self.iterations, 100))
self.init_ciphers()
class KeyfileKey(AESKeyBase):
FILE_ID = 'ATTIC KEY'
TYPE = 0x00
class KeyfileKeyBase(AESKeyBase):
@classmethod
def detect(cls, repository, manifest_data):
key = cls()
path = cls.find_key_file(repository)
prompt = 'Enter passphrase for key file %s: ' % path
passphrase = os.environ.get('ATTIC_PASSPHRASE', '')
while not key.load(path, passphrase):
passphrase = getpass(prompt)
key = cls(repository)
target = key.find_key()
prompt = 'Enter passphrase for key %s: ' % target
passphrase = Passphrase.env_passphrase(default='')
while not key.load(target, passphrase):
passphrase = Passphrase.getpass(prompt)
num_blocks = num_aes_blocks(len(manifest_data) - 41)
key.init_ciphers(PREFIX + long_to_bytes(key.extract_nonce(manifest_data) + num_blocks))
return key
@classmethod
def find_key_file(cls, repository):
id = hexlify(repository.id).decode('ascii')
keys_dir = get_keys_dir()
for name in os.listdir(keys_dir):
filename = os.path.join(keys_dir, name)
with open(filename, 'r') as fd:
line = fd.readline().strip()
if line and line.startswith(cls.FILE_ID) and line[10:] == id:
return filename
raise KeyfileNotFoundError(repository._location.canonical_path(), get_keys_dir())
def find_key(self):
raise NotImplementedError
def load(self, filename, passphrase):
with open(filename, 'r') as fd:
cdata = a2b_base64(''.join(fd.readlines()[1:]).encode('ascii')) # .encode needed for Python 3.[0-2]
def load(self, target, passphrase):
raise NotImplementedError
def _load(self, key_data, passphrase):
cdata = a2b_base64(key_data.encode('ascii')) # .encode needed for Python 3.[0-2]
data = self.decrypt_key_file(cdata, passphrase)
if data:
key = msgpack.unpackb(data)
@ -241,25 +280,24 @@ class KeyfileKey(AESKeyBase):
self.enc_hmac_key = key[b'enc_hmac_key']
self.id_key = key[b'id_key']
self.chunk_seed = key[b'chunk_seed']
self.path = filename
return True
return False
def decrypt_key_file(self, data, passphrase):
d = msgpack.unpackb(data)
assert d[b'version'] == 1
assert d[b'algorithm'] == b'sha256'
key = pbkdf2_sha256(passphrase.encode('utf-8'), d[b'salt'], d[b'iterations'], 32)
data = AES(key).decrypt(d[b'data'])
if HMAC(key, data, sha256).digest() != d[b'hash']:
return None
return data
key = passphrase.kdf(d[b'salt'], d[b'iterations'], 32)
data = AES(is_encrypt=False, key=key).decrypt(d[b'data'])
if HMAC(key, data, sha256).digest() == d[b'hash']:
return data
def encrypt_key_file(self, data, passphrase):
salt = get_random_bytes(32)
iterations = 100000
key = pbkdf2_sha256(passphrase.encode('utf-8'), salt, iterations, 32)
key = passphrase.kdf(salt, iterations, 32)
hash = HMAC(key, data, sha256).digest()
cdata = AES(key).encrypt(data)
cdata = AES(is_encrypt=True, key=key).encrypt(data)
d = {
'version': 1,
'salt': salt,
@ -270,7 +308,7 @@ class KeyfileKey(AESKeyBase):
}
return msgpack.packb(d)
def save(self, path, passphrase):
def _save(self, passphrase):
key = {
'version': 1,
'repository_id': self.repository_id,
@ -280,45 +318,101 @@ class KeyfileKey(AESKeyBase):
'chunk_seed': self.chunk_seed,
}
data = self.encrypt_key_file(msgpack.packb(key), passphrase)
with open(path, 'w') as fd:
fd.write('%s %s\n' % (self.FILE_ID, hexlify(self.repository_id).decode('ascii')))
fd.write('\n'.join(textwrap.wrap(b2a_base64(data).decode('ascii'))))
fd.write('\n')
self.path = path
key_data = '\n'.join(textwrap.wrap(b2a_base64(data).decode('ascii')))
return key_data
def change_passphrase(self):
passphrase, passphrase2 = 1, 2
while passphrase != passphrase2:
passphrase = getpass('New passphrase: ')
passphrase2 = getpass('Enter same passphrase again: ')
if passphrase != passphrase2:
print('Passphrases do not match')
self.save(self.path, passphrase)
print('Key file "%s" updated' % self.path)
passphrase = Passphrase.new(allow_empty=True)
self.save(self.target, passphrase)
print('Key updated')
@classmethod
def create(cls, repository, args):
passphrase = Passphrase.new(allow_empty=True)
key = cls(repository)
key.repository_id = repository.id
key.init_from_random_data(get_random_bytes(100))
key.init_ciphers()
target = key.get_new_target(args)
key.save(target, passphrase)
print('Key in "%s" created.' % target)
print('Keep this key safe. Your data will be inaccessible without it.')
return key
def save(self, target, passphrase):
raise NotImplementedError
def get_new_target(self, args):
raise NotImplementedError
class KeyfileKey(KeyfileKeyBase):
TYPE = 0x00
FILE_ID = 'BORG_KEY'
def find_key(self):
id = hexlify(self.repository.id).decode('ascii')
keys_dir = get_keys_dir()
for name in os.listdir(keys_dir):
filename = os.path.join(keys_dir, name)
with open(filename, 'r') as fd:
line = fd.readline().strip()
if line.startswith(self.FILE_ID) and line[len(self.FILE_ID)+1:] == id:
return filename
raise KeyfileNotFoundError(self.repository._location.canonical_path(), get_keys_dir())
def get_new_target(self, args):
filename = args.repository.to_key_filename()
path = filename
i = 1
while os.path.exists(path):
i += 1
path = filename + '.%d' % i
passphrase = os.environ.get('ATTIC_PASSPHRASE')
if passphrase is not None:
passphrase2 = passphrase
else:
passphrase, passphrase2 = 1, 2
while passphrase != passphrase2:
passphrase = getpass('Enter passphrase (empty for no passphrase):')
passphrase2 = getpass('Enter same passphrase again: ')
if passphrase != passphrase2:
print('Passphrases do not match')
key = cls()
key.repository_id = repository.id
key.init_from_random_data(get_random_bytes(100))
key.init_ciphers()
key.save(path, passphrase)
print('Key file "%s" created.' % key.path)
print('Keep this file safe. Your data will be inaccessible without it.')
return key
return path
def load(self, target, passphrase):
with open(target, 'r') as fd:
key_data = ''.join(fd.readlines()[1:])
success = self._load(key_data, passphrase)
if success:
self.target = target
return success
def save(self, target, passphrase):
key_data = self._save(passphrase)
with open(target, 'w') as fd:
fd.write('%s %s\n' % (self.FILE_ID, hexlify(self.repository_id).decode('ascii')))
fd.write(key_data)
fd.write('\n')
self.target = target
class RepoKey(KeyfileKeyBase):
TYPE = 0x03
def find_key(self):
loc = self.repository._location.canonical_path()
try:
self.repository.load_key()
return loc
except configparser.NoOptionError:
raise RepoKeyNotFoundError(loc)
def get_new_target(self, args):
return self.repository
def load(self, target, passphrase):
# what we get in target is just a repo location, but we already have the repo obj:
target = self.repository
key_data = target.load_key()
key_data = key_data.decode('utf-8') # remote repo: msgpack issue #99, getting bytes
success = self._load(key_data, passphrase)
if success:
self.target = target
return success
def save(self, target, passphrase):
key_data = self._save(passphrase)
key_data = key_data.encode('utf-8') # remote repo: msgpack issue #99, giving bytes
target.save_key(key_data)
self.target = target

286
borg/locking.py Normal file
View file

@ -0,0 +1,286 @@
import errno
import json
import os
import socket
import threading
import time
from borg.helpers import Error
ADD, REMOVE = 'add', 'remove'
SHARED, EXCLUSIVE = 'shared', 'exclusive'
def get_id():
"""Get identification tuple for 'us'"""
hostname = socket.gethostname()
pid = os.getpid()
tid = threading.current_thread().ident & 0xffffffff
return hostname, pid, tid
class TimeoutTimer:
"""
A timer for timeout checks (can also deal with no timeout, give timeout=None [default]).
It can also compute and optionally execute a reasonable sleep time (e.g. to avoid
polling too often or to support thread/process rescheduling).
"""
def __init__(self, timeout=None, sleep=None):
"""
Initialize a timer.
:param timeout: time out interval [s] or None (no timeout)
:param sleep: sleep interval [s] (>= 0: do sleep call, <0: don't call sleep)
or None (autocompute: use 10% of timeout, or 1s for no timeout)
"""
if timeout is not None and timeout < 0:
raise ValueError("timeout must be >= 0")
self.timeout_interval = timeout
if sleep is None:
if timeout is None:
sleep = 1.0
else:
sleep = timeout / 10.0
self.sleep_interval = sleep
self.start_time = None
self.end_time = None
def __repr__(self):
return "<%s: start=%r end=%r timeout=%r sleep=%r>" % (
self.__class__.__name__, self.start_time, self.end_time,
self.timeout_interval, self.sleep_interval)
def start(self):
self.start_time = time.time()
if self.timeout_interval is not None:
self.end_time = self.start_time + self.timeout_interval
return self
def sleep(self):
if self.sleep_interval >= 0:
time.sleep(self.sleep_interval)
def timed_out(self):
return self.end_time is not None and time.time() >= self.end_time
def timed_out_or_sleep(self):
if self.timed_out():
return True
else:
self.sleep()
return False
class ExclusiveLock:
"""An exclusive Lock based on mkdir fs operation being atomic"""
class LockError(Error):
"""Failed to acquire the lock {}."""
class LockTimeout(LockError):
"""Failed to create/acquire the lock {} (timeout)."""
class LockFailed(LockError):
"""Failed to create/acquire the lock {} ({})."""
class UnlockError(Error):
"""Failed to release the lock {}."""
class NotLocked(UnlockError):
"""Failed to release the lock {} (was not locked)."""
class NotMyLock(UnlockError):
"""Failed to release the lock {} (was/is locked, but not by me)."""
def __init__(self, path, timeout=None, sleep=None, id=None):
self.timeout = timeout
self.sleep = sleep
self.path = os.path.abspath(path)
self.id = id or get_id()
self.unique_name = os.path.join(self.path, "%s.%d-%x" % self.id)
def __enter__(self):
return self.acquire()
def __exit__(self, *exc):
self.release()
def __repr__(self):
return "<%s: %r>" % (self.__class__.__name__, self.unique_name)
def acquire(self, timeout=None, sleep=None):
if timeout is None:
timeout = self.timeout
if sleep is None:
sleep = self.sleep
timer = TimeoutTimer(timeout, sleep).start()
while True:
try:
os.mkdir(self.path)
except OSError as err:
if err.errno == errno.EEXIST: # already locked
if self.by_me():
return self
if timer.timed_out_or_sleep():
raise self.LockTimeout(self.path)
else:
raise self.LockFailed(self.path, str(err))
else:
with open(self.unique_name, "wb"):
pass
return self
def release(self):
if not self.is_locked():
raise self.NotLocked(self.path)
if not self.by_me():
raise self.NotMyLock(self.path)
os.unlink(self.unique_name)
os.rmdir(self.path)
def is_locked(self):
return os.path.exists(self.path)
def by_me(self):
return os.path.exists(self.unique_name)
def break_lock(self):
if self.is_locked():
for name in os.listdir(self.path):
os.unlink(os.path.join(self.path, name))
os.rmdir(self.path)
class LockRoster:
"""
A Lock Roster to track shared/exclusive lockers.
Note: you usually should call the methods with an exclusive lock held,
to avoid conflicting access by multiple threads/processes/machines.
"""
def __init__(self, path, id=None):
self.path = path
self.id = id or get_id()
def load(self):
try:
with open(self.path) as f:
data = json.load(f)
except IOError as err:
if err.errno != errno.ENOENT:
raise
data = {}
return data
def save(self, data):
with open(self.path, "w") as f:
json.dump(data, f)
def remove(self):
os.unlink(self.path)
def get(self, key):
roster = self.load()
return set(tuple(e) for e in roster.get(key, []))
def modify(self, key, op):
roster = self.load()
try:
elements = set(tuple(e) for e in roster[key])
except KeyError:
elements = set()
if op == ADD:
elements.add(self.id)
elif op == REMOVE:
elements.remove(self.id)
else:
raise ValueError('Unknown LockRoster op %r' % op)
roster[key] = list(list(e) for e in elements)
self.save(roster)
class UpgradableLock:
"""
A Lock for a resource that can be accessed in a shared or exclusive way.
Typically, write access to a resource needs an exclusive lock (1 writer,
noone is allowed reading) and read access to a resource needs a shared
lock (multiple readers are allowed).
"""
class SharedLockFailed(Error):
"""Failed to acquire shared lock [{}]"""
class ExclusiveLockFailed(Error):
"""Failed to acquire write lock [{}]"""
def __init__(self, path, exclusive=False, sleep=None, id=None):
self.path = path
self.is_exclusive = exclusive
self.sleep = sleep
self.id = id or get_id()
# globally keeping track of shared and exclusive lockers:
self._roster = LockRoster(path + '.roster', id=id)
# an exclusive lock, used for:
# - holding while doing roster queries / updates
# - holding while the UpgradableLock itself is exclusive
self._lock = ExclusiveLock(path + '.exclusive', id=id)
def __enter__(self):
return self.acquire()
def __exit__(self, *exc):
self.release()
def __repr__(self):
return "<%s: %r>" % (self.__class__.__name__, self.id)
def acquire(self, exclusive=None, remove=None, sleep=None):
if exclusive is None:
exclusive = self.is_exclusive
sleep = sleep or self.sleep or 0.2
try:
if exclusive:
self._wait_for_readers_finishing(remove, sleep)
self._roster.modify(EXCLUSIVE, ADD)
else:
with self._lock:
if remove is not None:
self._roster.modify(remove, REMOVE)
self._roster.modify(SHARED, ADD)
self.is_exclusive = exclusive
return self
except ExclusiveLock.LockError as err:
msg = str(err)
if exclusive:
raise self.ExclusiveLockFailed(msg)
else:
raise self.SharedLockFailed(msg)
def _wait_for_readers_finishing(self, remove, sleep):
while True:
self._lock.acquire()
if remove is not None:
self._roster.modify(remove, REMOVE)
remove = None
if len(self._roster.get(SHARED)) == 0:
return # we are the only one and we keep the lock!
self._lock.release()
time.sleep(sleep)
def release(self):
if self.is_exclusive:
self._roster.modify(EXCLUSIVE, REMOVE)
self._lock.release()
else:
with self._lock:
self._roster.modify(SHARED, REMOVE)
def upgrade(self):
if not self.is_exclusive:
self.acquire(exclusive=True, remove=SHARED)
def downgrade(self):
if self.is_exclusive:
self.acquire(exclusive=False, remove=EXCLUSIVE)
def break_lock(self):
self._roster.remove()
self._lock.break_lock()

0
attic/lrucache.py → borg/lrucache.py
View file

7
attic/platform.py → borg/platform.py
View file

@ -1,15 +1,16 @@
import sys
if sys.platform.startswith('linux'):
from attic.platform_linux import acl_get, acl_set, API_VERSION
from .platform_linux import acl_get, acl_set, API_VERSION
elif sys.platform.startswith('freebsd'):
from attic.platform_freebsd import acl_get, acl_set, API_VERSION
from .platform_freebsd import acl_get, acl_set, API_VERSION
elif sys.platform == 'darwin':
from attic.platform_darwin import acl_get, acl_set, API_VERSION
from .platform_darwin import acl_get, acl_set, API_VERSION
else:
API_VERSION = 2
def acl_get(path, item, st, numeric_owner=False):
pass
def acl_set(path, item, numeric_owner=False):
pass

2
attic/platform_darwin.pyx → borg/platform_darwin.pyx
View file

@ -1,5 +1,5 @@
import os
from attic.helpers import user2uid, group2gid
from .helpers import user2uid, group2gid
API_VERSION = 2

2
attic/platform_freebsd.pyx → borg/platform_freebsd.pyx
View file

@ -1,5 +1,5 @@
import os
from attic.helpers import posix_acl_use_stored_uid_gid
from .helpers import posix_acl_use_stored_uid_gid
API_VERSION = 2

2
attic/platform_linux.pyx → borg/platform_linux.pyx
View file

@ -1,7 +1,7 @@
import os
import re
from stat import S_ISLNK
from attic.helpers import posix_acl_use_stored_uid_gid, user2uid, group2gid
from .helpers import posix_acl_use_stored_uid_gid, user2uid, group2gid
API_VERSION = 2

127
attic/remote.py → borg/remote.py
View file

@ -3,12 +3,13 @@ import fcntl
import msgpack
import os
import select
import shutil
from subprocess import Popen, PIPE
import sys
import tempfile
import traceback
from . import __version__
from .hashindex import NSIndex
from .helpers import Error, IntegrityError
from .repository import Repository
@ -23,7 +24,27 @@ class PathNotAllowed(Error):
"""Repository path not allowed"""
class RepositoryServer(object):
class InvalidRPCMethod(Error):
"""RPC method is not valid"""
class RepositoryServer: # pragma: no cover
rpc_methods = (
'__len__',
'check',
'commit',
'delete',
'destroy',
'get',
'list',
'negotiate',
'open',
'put',
'repair',
'rollback',
'save_key',
'load_key',
)
def __init__(self, restrict_to_paths):
self.repository = None
@ -46,16 +67,22 @@ class RepositoryServer(object):
if not data:
return
unpacker.feed(data)
for type, msgid, method, args in unpacker:
for unpacked in unpacker:
if not (isinstance(unpacked, tuple) and len(unpacked) == 4):
raise Exception("Unexpected RPC data format.")
type, msgid, method, args = unpacked
method = method.decode('ascii')
try:
if method not in self.rpc_methods:
raise InvalidRPCMethod(method)
try:
f = getattr(self, method)
except AttributeError:
f = getattr(self.repository, method)
res = f(*args)
except Exception as e:
os.write(stdout_fd, msgpack.packb((1, msgid, e.__class__.__name__, e.args)))
except BaseException as e:
exc = "Remote Traceback by Borg %s%s%s" % (__version__, os.linesep, traceback.format_exc())
os.write(stdout_fd, msgpack.packb((1, msgid, e.__class__.__name__, exc)))
else:
os.write(stdout_fd, msgpack.packb((1, msgid, None, res)))
if es:
@ -79,11 +106,12 @@ class RepositoryServer(object):
return self.repository.id
class RemoteRepository(object):
class RemoteRepository:
extra_test_args = []
remote_path = None
umask = None
class RPCError(Exception):
def __init__(self, name):
self.name = name
@ -97,9 +125,11 @@ class RemoteRepository(object):
self.responses = {}
self.unpacker = msgpack.Unpacker(use_list=False)
self.p = None
# use local umask also for the remote process
umask = ['--umask', '%03o' % self.umask]
if location.host == '__testsuite__':
args = [sys.executable, '-m', 'attic.archiver', 'serve'] + self.extra_test_args
else:
args = [sys.executable, '-m', 'borg.archiver', 'serve'] + umask + self.extra_test_args
else: # pragma: no cover
args = ['ssh']
if location.port:
args += ['-p', str(location.port)]
@ -107,7 +137,7 @@ class RemoteRepository(object):
args.append('%s@%s' % (location.user, location.host))
else:
args.append('%s' % location.host)
args += ['attic', 'serve']
args += [self.remote_path, 'serve'] + umask
self.p = Popen(args, bufsize=0, stdin=PIPE, stdout=PIPE)
self.stdin_fd = self.p.stdin.fileno()
self.stdout_fd = self.p.stdout.fileno()
@ -116,7 +146,10 @@ class RemoteRepository(object):
self.r_fds = [self.stdout_fd]
self.x_fds = [self.stdin_fd, self.stdout_fd]
version = self.call('negotiate', 1)
try:
version = self.call('negotiate', 1)
except ConnectionClosed:
raise Exception('Server immediately closed connection - is Borg installed and working on the server?')
if version != 1:
raise Exception('Server insisted on using unsupported protocol version %d' % version)
self.id = self.call('open', location.path, create)
@ -124,6 +157,9 @@ class RemoteRepository(object):
def __del__(self):
self.close()
def __repr__(self):
return '<%s %s>' % (self.__class__.__name__, self.location.canonical_path())
def call(self, cmd, *args, **kw):
for resp in self.call_many(cmd, [args], **kw):
return resp
@ -131,6 +167,7 @@ class RemoteRepository(object):
def call_many(self, cmd, calls, wait=True, is_preloaded=False):
if not calls:
return
def fetch_from_cache(args):
msgid = self.cache[args].pop(0)
if not self.cache[args]:
@ -156,9 +193,12 @@ class RemoteRepository(object):
raise IntegrityError(res)
elif error == b'PathNotAllowed':
raise PathNotAllowed(*res)
if error == b'ObjectNotFound':
elif error == b'ObjectNotFound':
raise Repository.ObjectNotFound(res[0], self.location.orig)
raise self.RPCError(error)
elif error == b'InvalidRPCMethod':
raise InvalidRPCMethod(*res)
else:
raise self.RPCError(res.decode('utf-8'))
else:
yield res
if not waiting_for and not calls:
@ -167,13 +207,16 @@ class RemoteRepository(object):
break
r, w, x = select.select(self.r_fds, w_fds, self.x_fds, 1)
if x:
raise Exception('FD exception occured')
raise Exception('FD exception occurred')
if r:
data = os.read(self.stdout_fd, BUFSIZE)
if not data:
raise ConnectionClosed()
self.unpacker.feed(data)
for type, msgid, error, res in self.unpacker:
for unpacked in self.unpacker:
if not (isinstance(unpacked, tuple) and len(unpacked) == 4):
raise Exception("Unexpected RPC data format.")
type, msgid, error, res = unpacked
if msgid in self.ignore_responses:
self.ignore_responses.remove(msgid)
else:
@ -219,6 +262,9 @@ class RemoteRepository(object):
def rollback(self, *args):
return self.call('rollback')
def destroy(self):
return self.call('destroy')
def __len__(self):
return self.call('__len__')
@ -239,6 +285,12 @@ class RemoteRepository(object):
def delete(self, id_, wait=True):
return self.call('delete', id_, wait=wait)
def save_key(self, keydata):
return self.call('save_key', keydata)
def load_key(self):
return self.call('load_key')
def close(self):
if self.p:
self.p.stdin.close()
@ -253,56 +305,29 @@ class RemoteRepository(object):
class RepositoryCache:
"""A caching Repository wrapper
Caches Repository GET operations using a temporary file
Caches Repository GET operations using a local temporary Repository.
"""
def __init__(self, repository):
self.tmppath = None
self.index = None
self.data_fd = None
self.repository = repository
self.entries = {}
self.initialize()
tmppath = tempfile.mkdtemp(prefix='borg-tmp')
self.caching_repo = Repository(tmppath, create=True, exclusive=True)
def __del__(self):
self.cleanup()
def initialize(self):
self.tmppath = tempfile.mkdtemp()
self.index = NSIndex()
self.data_fd = open(os.path.join(self.tmppath, 'data'), 'a+b')
def cleanup(self):
del self.index
if self.data_fd:
self.data_fd.close()
if self.tmppath:
shutil.rmtree(self.tmppath)
def load_object(self, offset, size):
self.data_fd.seek(offset)
data = self.data_fd.read(size)
assert len(data) == size
return data
def store_object(self, key, data):
self.data_fd.seek(0, os.SEEK_END)
self.data_fd.write(data)
offset = self.data_fd.tell()
self.index[key] = offset - len(data), len(data)
self.caching_repo.destroy()
def get(self, key):
return next(self.get_many([key]))
def get_many(self, keys):
unknown_keys = [key for key in keys if not key in self.index]
unknown_keys = [key for key in keys if key not in self.caching_repo]
repository_iterator = zip(unknown_keys, self.repository.get_many(unknown_keys))
for key in keys:
try:
yield self.load_object(*self.index[key])
except KeyError:
yield self.caching_repo.get(key)
except Repository.ObjectNotFound:
for key_, data in repository_iterator:
if key_ == key:
self.store_object(key, data)
self.caching_repo.put(key, data)
yield data
break
# Consume any pending requests

113
attic/repository.py → borg/repository.py
View file

@ -9,17 +9,19 @@ import sys
from zlib import crc32
from .hashindex import NSIndex
from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify, UpgradableLock
from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify
from .locking import UpgradableLock
from .lrucache import LRUCache
MAX_OBJECT_SIZE = 20 * 1024 * 1024
MAGIC = b'ATTICSEG'
MAGIC = b'BORG_SEG'
MAGIC_LEN = len(MAGIC)
TAG_PUT = 0
TAG_DELETE = 1
TAG_COMMIT = 2
class Repository(object):
class Repository:
"""Filesystem based transactional key value store
On disk layout:
@ -33,19 +35,19 @@ class Repository(object):
DEFAULT_SEGMENTS_PER_DIR = 10000
class DoesNotExist(Error):
"""Repository {} does not exist"""
"""Repository {} does not exist."""
class AlreadyExists(Error):
"""Repository {} already exists"""
"""Repository {} already exists."""
class InvalidRepository(Error):
"""{} is not a valid repository"""
"""{} is not a valid repository."""
class CheckNeeded(Error):
'''Inconsistency detected. Please run "attic check {}"'''
"""Inconsistency detected. Please run "borg check {}"."""
class ObjectNotFound(Error):
"""Object with key {} not found in repository {}"""
"""Object with key {} not found in repository {}."""
def __init__(self, path, create=False, exclusive=False):
self.path = path
@ -60,6 +62,9 @@ class Repository(object):
def __del__(self):
self.close()
def __repr__(self):
return '<%s %s>' % (self.__class__.__name__, self.path)
def create(self, path):
"""Create a new empty repository at `path`
"""
@ -68,7 +73,7 @@ class Repository(object):
if not os.path.exists(path):
os.mkdir(path)
with open(os.path.join(path, 'README'), 'w') as fd:
fd.write('This is an Attic repository\n')
fd.write('This is a Borg repository\n')
os.mkdir(os.path.join(path, 'data'))
config = RawConfigParser()
config.add_section('repository')
@ -76,13 +81,34 @@ class Repository(object):
config.set('repository', 'segments_per_dir', self.DEFAULT_SEGMENTS_PER_DIR)
config.set('repository', 'max_segment_size', self.DEFAULT_MAX_SEGMENT_SIZE)
config.set('repository', 'id', hexlify(os.urandom(32)).decode('ascii'))
with open(os.path.join(path, 'config'), 'w') as fd:
self.save_config(path, config)
def save_config(self, path, config):
config_path = os.path.join(path, 'config')
with open(config_path, 'w') as fd:
config.write(fd)
def save_key(self, keydata):
assert self.config
keydata = keydata.decode('utf-8') # remote repo: msgpack issue #99, getting bytes
self.config.set('repository', 'key', keydata)
self.save_config(self.path, self.config)
def load_key(self):
keydata = self.config.get('repository', 'key')
return keydata.encode('utf-8') # remote repo: msgpack issue #99, returning bytes
def destroy(self):
"""Destroy the repository at `self.path`
"""
self.close()
os.remove(os.path.join(self.path, 'config')) # kill config first
shutil.rmtree(self.path)
def get_index_transaction_id(self):
indicies = sorted((int(name[6:]) for name in os.listdir(self.path) if name.startswith('index.') and name[6:].isdigit()))
if indicies:
return indicies[-1]
indices = sorted((int(name[6:]) for name in os.listdir(self.path) if name.startswith('index.') and name[6:].isdigit()))
if indices:
return indices[-1]
else:
return None
@ -105,11 +131,11 @@ class Repository(object):
self.path = path
if not os.path.isdir(path):
raise self.DoesNotExist(path)
self.lock = UpgradableLock(os.path.join(path, 'lock'), exclusive).acquire()
self.config = RawConfigParser()
self.config.read(os.path.join(self.path, 'config'))
if not 'repository' in self.config.sections() or self.config.getint('repository', 'version') != 1:
if 'repository' not in self.config.sections() or self.config.getint('repository', 'version') != 1:
raise self.InvalidRepository(path)
self.lock = UpgradableLock(os.path.join(path, 'config'), exclusive)
self.max_segment_size = self.config.getint('repository', 'max_segment_size')
self.segments_per_dir = self.config.getint('repository', 'segments_per_dir')
self.id = unhexlify(self.config.get('repository', 'id').strip())
@ -138,7 +164,14 @@ class Repository(object):
def prepare_txn(self, transaction_id, do_cleanup=True):
self._active_txn = True
self.lock.upgrade()
try:
self.lock.upgrade()
except UpgradableLock.ExclusiveLockFailed:
# if upgrading the lock to exclusive fails, we do not have an
# active transaction. this is important for "serve" mode, where
# the repository instance lives on - even if exceptions happened.
self._active_txn = False
raise
if not self.index:
self.index = self.open_index(transaction_id)
if transaction_id is None:
@ -241,6 +274,7 @@ class Repository(object):
the index is consistent with the data stored in the segments.
"""
error_found = False
def report_error(msg):
nonlocal error_found
error_found = True
@ -266,8 +300,8 @@ class Repository(object):
continue
try:
objects = list(self.io.iter_objects(segment))
except (IntegrityError, struct.error):
report_error('Error reading segment {}'.format(segment))
except IntegrityError as err:
report_error('Error reading segment {}: {}'.format(segment, err))
objects = []
if repair:
self.io.recover_segment(segment, filename)
@ -324,6 +358,11 @@ class Repository(object):
self.index = self.open_index(self.get_transaction_id())
return len(self.index)
def __contains__(self, id):
if not self.index:
self.index = self.open_index(self.get_transaction_id())
return id in self.index
def list(self, limit=None, marker=None):
if not self.index:
self.index = self.open_index(self.get_transaction_id())
@ -373,11 +412,11 @@ class Repository(object):
self.segments.setdefault(segment, 0)
def preload(self, ids):
"""Preload objects (only applies to remote repositories
"""Preload objects (only applies to remote repositories)
"""
class LoggedIO(object):
class LoggedIO:
header_fmt = struct.Struct('<IIB')
assert header_fmt.size == 9
@ -445,7 +484,7 @@ class LoggedIO(object):
with open(filename, 'rb') as fd:
try:
fd.seek(-self.header_fmt.size, os.SEEK_END)
except Exception as e:
except OSError as e:
# return False if segment file is empty or too small
if e.errno == errno.EINVAL:
return False
@ -465,7 +504,7 @@ class LoggedIO(object):
os.mkdir(dirname)
self._write_fd = open(self.segment_filename(self.segment), 'ab')
self._write_fd.write(MAGIC)
self.offset = 8
self.offset = MAGIC_LEN
return self._write_fd
def get_fd(self, segment):
@ -490,19 +529,26 @@ class LoggedIO(object):
def iter_objects(self, segment, include_data=False):
fd = self.get_fd(segment)
fd.seek(0)
if fd.read(8) != MAGIC:
raise IntegrityError('Invalid segment header')
offset = 8
if fd.read(MAGIC_LEN) != MAGIC:
raise IntegrityError('Invalid segment magic')
offset = MAGIC_LEN
header = fd.read(self.header_fmt.size)
while header:
crc, size, tag = self.header_fmt.unpack(header)
try:
crc, size, tag = self.header_fmt.unpack(header)
except struct.error as err:
raise IntegrityError('Invalid segment entry header [offset {}]: {}'.format(offset, err))
if size > MAX_OBJECT_SIZE:
raise IntegrityError('Invalid segment object size')
rest = fd.read(size - self.header_fmt.size)
raise IntegrityError('Invalid segment entry size [offset {}]'.format(offset))
length = size - self.header_fmt.size
rest = fd.read(length)
if len(rest) != length:
raise IntegrityError('Segment entry data short read [offset {}]: expected: {}, got {} bytes'.format(
offset, length, len(rest)))
if crc32(rest, crc32(memoryview(header)[4:])) & 0xffffffff != crc:
raise IntegrityError('Segment checksum mismatch')
raise IntegrityError('Segment entry checksum mismatch [offset {}]'.format(offset))
if tag not in (TAG_PUT, TAG_DELETE, TAG_COMMIT):
raise IntegrityError('Invalid segment entry header')
raise IntegrityError('Invalid segment entry tag [offset {}]'.format(offset))
key = None
if tag in (TAG_PUT, TAG_DELETE):
key = rest[:32]
@ -579,6 +625,11 @@ class LoggedIO(object):
if self._write_fd:
self.segment += 1
self.offset = 0
os.fsync(self._write_fd)
self._write_fd.flush()
os.fsync(self._write_fd.fileno())
if hasattr(os, 'posix_fadvise'): # python >= 3.3, only on UNIX
# tell the OS that it does not need to cache what we just wrote,
# avoids spoiling the cache for the OS and other processes.
os.posix_fadvise(self._write_fd.fileno(), 0, 0, os.POSIX_FADV_DONTNEED)
self._write_fd.close()
self._write_fd = None

39
attic/testsuite/__init__.py → borg/testsuite/__init__.py
View file

@ -6,8 +6,8 @@ import sys
import sysconfig
import time
import unittest
from attic.helpers import st_mtime_ns
from attic.xattr import get_all
from ..helpers import st_mtime_ns
from ..xattr import get_all
try:
import llfuse
@ -19,7 +19,7 @@ except ImportError:
has_lchflags = hasattr(os, 'lchflags')
# The mtime get/set precison varies on different OS and Python versions
# The mtime get/set precision varies on different OS and Python versions
if 'HAVE_FUTIMENS' in getattr(posix, '_have_functions', []):
st_mtime_ns_round = 0
elif 'HAVE_UTIMES' in sysconfig.get_config_vars():
@ -32,7 +32,7 @@ has_mtime_ns = sys.version >= '3.3'
utime_supports_fd = os.utime in getattr(os, 'supports_fd', {})
class AtticTestCase(unittest.TestCase):
class BaseTestCase(unittest.TestCase):
"""
"""
assert_in = unittest.TestCase.assertIn
@ -68,12 +68,12 @@ class AtticTestCase(unittest.TestCase):
if has_lchflags:
attrs.append('st_flags')
if not fuse or not os.path.isdir(path1):
# dir nlink is always 1 on our fuse fileystem
# dir nlink is always 1 on our fuse filesystem
attrs.append('st_nlink')
d1 = [filename] + [getattr(s1, a) for a in attrs]
d2 = [filename] + [getattr(s2, a) for a in attrs]
if not os.path.islink(path1) or utime_supports_fd:
# Older versions of llfuse does not support ns precision properly
# Older versions of llfuse do not support ns precision properly
if fuse and not have_fuse_mtime_ns:
d1.append(round(st_mtime_ns(s1), -4))
d2.append(round(st_mtime_ns(s2), -4))
@ -94,30 +94,3 @@ class AtticTestCase(unittest.TestCase):
return
time.sleep(.1)
raise Exception('wait_for_mount(%s) timeout' % path)
def get_tests(suite):
"""Generates a sequence of tests from a test suite
"""
for item in suite:
try:
# TODO: This could be "yield from..." with Python 3.3+
for i in get_tests(item):
yield i
except TypeError:
yield item
class TestLoader(unittest.TestLoader):
"""A customzied test loader that properly detects and filters our test cases
"""
def loadTestsFromName(self, pattern, module=None):
suite = self.discover('attic.testsuite', '*.py')
tests = unittest.TestSuite()
for test in get_tests(suite):
if pattern.lower() in test.id().lower():
tests.addTest(test)
return tests

38
attic/testsuite/archive.py → borg/testsuite/archive.py
View file

@ -1,7 +1,12 @@
from datetime import datetime, timezone
import msgpack
from attic.testsuite import AtticTestCase
from attic.archive import CacheChunkBuffer, RobustUnpacker
from attic.key import PlaintextKey
from mock import Mock
from ..archive import Archive, CacheChunkBuffer, RobustUnpacker
from ..key import PlaintextKey
from ..helpers import Manifest
from . import BaseTestCase
class MockCache:
@ -14,12 +19,33 @@ class MockCache:
return id, len(data), len(data)
class ChunkBufferTestCase(AtticTestCase):
class ArchiveTimestampTestCase(BaseTestCase):
def _test_timestamp_parsing(self, isoformat, expected):
repository = Mock()
key = PlaintextKey(repository)
manifest = Manifest(repository, key)
a = Archive(repository, key, manifest, 'test', create=True)
a.metadata = {b'time': isoformat}
self.assert_equal(a.ts, expected)
def test_with_microseconds(self):
self._test_timestamp_parsing(
'1970-01-01T00:00:01.000001',
datetime(1970, 1, 1, 0, 0, 1, 1, timezone.utc))
def test_without_microseconds(self):
self._test_timestamp_parsing(
'1970-01-01T00:00:01',
datetime(1970, 1, 1, 0, 0, 1, 0, timezone.utc))
class ChunkBufferTestCase(BaseTestCase):
def test(self):
data = [{b'foo': 1}, {b'bar': 2}]
cache = MockCache()
key = PlaintextKey()
key = PlaintextKey(None)
chunks = CacheChunkBuffer(cache, key, None)
for d in data:
chunks.add(d)
@ -32,7 +58,7 @@ class ChunkBufferTestCase(AtticTestCase):
self.assert_equal(data, list(unpacker))
class RobustUnpackerTestCase(AtticTestCase):
class RobustUnpackerTestCase(BaseTestCase):
def make_chunks(self, items):
return b''.join(msgpack.packb({'path': item}) for item in items)

675
borg/testsuite/archiver.py Normal file
View file

@ -0,0 +1,675 @@
from binascii import hexlify
from configparser import RawConfigParser
import os
from io import StringIO
import stat
import subprocess
import sys
import shutil
import tempfile
import time
import unittest
from hashlib import sha256
from mock import patch
from .. import xattr
from ..archive import Archive, ChunkBuffer, CHUNK_MAX_EXP
from ..archiver import Archiver
from ..cache import Cache
from ..crypto import bytes_to_long, num_aes_blocks
from ..helpers import Manifest
from ..remote import RemoteRepository, PathNotAllowed
from ..repository import Repository
from . import BaseTestCase
try:
import llfuse
has_llfuse = True or llfuse # avoids "unused import"
except ImportError:
has_llfuse = False
has_lchflags = hasattr(os, 'lchflags')
src_dir = os.path.join(os.getcwd(), os.path.dirname(__file__), '..')
class changedir:
def __init__(self, dir):
self.dir = dir
def __enter__(self):
self.old = os.getcwd()
os.chdir(self.dir)
def __exit__(self, *args, **kw):
os.chdir(self.old)
class environment_variable:
def __init__(self, **values):
self.values = values
self.old_values = {}
def __enter__(self):
for k, v in self.values.items():
self.old_values[k] = os.environ.get(k)
os.environ[k] = v
def __exit__(self, *args, **kw):
for k, v in self.old_values.items():
if v is not None:
os.environ[k] = v
class ArchiverTestCaseBase(BaseTestCase):
prefix = ''
def setUp(self):
os.environ['BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'] = '1'
self.archiver = Archiver()
self.tmpdir = tempfile.mkdtemp()
self.repository_path = os.path.join(self.tmpdir, 'repository')
self.repository_location = self.prefix + self.repository_path
self.input_path = os.path.join(self.tmpdir, 'input')
self.output_path = os.path.join(self.tmpdir, 'output')
self.keys_path = os.path.join(self.tmpdir, 'keys')
self.cache_path = os.path.join(self.tmpdir, 'cache')
self.exclude_file_path = os.path.join(self.tmpdir, 'excludes')
os.environ['BORG_KEYS_DIR'] = self.keys_path
os.environ['BORG_CACHE_DIR'] = self.cache_path
os.mkdir(self.input_path)
os.mkdir(self.output_path)
os.mkdir(self.keys_path)
os.mkdir(self.cache_path)
with open(self.exclude_file_path, 'wb') as fd:
fd.write(b'input/file2\n# A comment line, then a blank line\n\n')
self._old_wd = os.getcwd()
os.chdir(self.tmpdir)
def tearDown(self):
shutil.rmtree(self.tmpdir)
os.chdir(self._old_wd)
def cmd(self, *args, **kw):
exit_code = kw.get('exit_code', 0)
fork = kw.get('fork', False)
if fork:
try:
output = subprocess.check_output((sys.executable, '-m', 'borg.archiver') + args)
ret = 0
except subprocess.CalledProcessError as e:
output = e.output
ret = e.returncode
output = os.fsdecode(output)
if ret != exit_code:
print(output)
self.assert_equal(exit_code, ret)
return output
args = list(args)
stdin, stdout, stderr = sys.stdin, sys.stdout, sys.stderr
try:
sys.stdin = StringIO()
output = StringIO()
sys.stdout = sys.stderr = output
ret = self.archiver.run(args)
sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr
if ret != exit_code:
print(output.getvalue())
self.assert_equal(exit_code, ret)
return output.getvalue()
finally:
sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr
def create_src_archive(self, name):
self.cmd('create', self.repository_location + '::' + name, src_dir)
class ArchiverTestCase(ArchiverTestCaseBase):
def create_regular_file(self, name, size=0, contents=None):
filename = os.path.join(self.input_path, name)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
with open(filename, 'wb') as fd:
if contents is None:
contents = b'X' * size
fd.write(contents)
def create_test_files(self):
"""Create a minimal test case including all supported file types
"""
# File
self.create_regular_file('empty', size=0)
# next code line raises OverflowError on 32bit cpu (raspberry pi 2):
# 2600-01-01 > 2**64 ns
# os.utime('input/empty', (19880895600, 19880895600))
# thus, we better test with something not that far in future:
# 2038-01-19 (1970 + 2^31 - 1 seconds) is the 32bit "deadline":
os.utime('input/empty', (2**31 - 1, 2**31 - 1))
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('flagfile', size=1024)
# Directory
self.create_regular_file('dir2/file2', size=1024 * 80)
# File owner
os.chown('input/file1', 100, 200)
# File mode
os.chmod('input/file1', 0o7755)
os.chmod('input/dir2', 0o555)
# Block device
os.mknod('input/bdev', 0o600 | stat.S_IFBLK, os.makedev(10, 20))
# Char device
os.mknod('input/cdev', 0o600 | stat.S_IFCHR, os.makedev(30, 40))
# Hard link
os.link(os.path.join(self.input_path, 'file1'),
os.path.join(self.input_path, 'hardlink'))
# Symlink
os.symlink('somewhere', os.path.join(self.input_path, 'link1'))
if xattr.is_enabled(self.input_path):
xattr.setxattr(os.path.join(self.input_path, 'file1'), 'user.foo', b'bar')
# XXX this always fails for me
# ubuntu 14.04, on a TMP dir filesystem with user_xattr, using fakeroot
# same for newer ubuntu and centos.
# if this is supported just on specific platform, platform should be checked first,
# so that the test setup for all tests using it does not fail here always for others.
# xattr.setxattr(os.path.join(self.input_path, 'link1'), 'user.foo_symlink', b'bar_symlink', follow_symlinks=False)
# FIFO node
os.mkfifo(os.path.join(self.input_path, 'fifo1'))
if has_lchflags:
os.lchflags(os.path.join(self.input_path, 'flagfile'), stat.UF_NODUMP)
def test_basic_functionality(self):
self.create_test_files()
self.cmd('init', self.repository_location)
self.cmd('create', self.repository_location + '::test', 'input')
self.cmd('create', '--stats', self.repository_location + '::test.2', 'input')
with changedir('output'):
self.cmd('extract', self.repository_location + '::test')
self.assert_equal(len(self.cmd('list', self.repository_location).splitlines()), 2)
item_count = 10 if has_lchflags else 11 # one file is UF_NODUMP
self.assert_equal(len(self.cmd('list', self.repository_location + '::test').splitlines()), item_count)
if has_lchflags:
# remove the file we did not backup, so input and output become equal
os.remove(os.path.join('input', 'flagfile'))
self.assert_dirs_equal('input', 'output/input')
info_output = self.cmd('info', self.repository_location + '::test')
item_count = 3 if has_lchflags else 4 # one file is UF_NODUMP
self.assert_in('Number of files: %d' % item_count, info_output)
shutil.rmtree(self.cache_path)
with environment_variable(BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK='1'):
info_output2 = self.cmd('info', self.repository_location + '::test')
# info_output2 starts with some "initializing cache" text but should
# end the same way as info_output
assert info_output2.endswith(info_output)
def _extract_repository_id(self, path):
return Repository(self.repository_path).id
def _set_repository_id(self, path, id):
config = RawConfigParser()
config.read(os.path.join(path, 'config'))
config.set('repository', 'id', hexlify(id).decode('ascii'))
with open(os.path.join(path, 'config'), 'w') as fd:
config.write(fd)
return Repository(self.repository_path).id
def test_sparse_file(self):
# no sparse file support on Mac OS X
sparse_support = sys.platform != 'darwin'
filename = os.path.join(self.input_path, 'sparse')
content = b'foobar'
hole_size = 5 * (1 << CHUNK_MAX_EXP) # 5 full chunker buffers
with open(filename, 'wb') as fd:
# create a file that has a hole at the beginning and end (if the
# OS and filesystem supports sparse files)
fd.seek(hole_size, 1)
fd.write(content)
fd.seek(hole_size, 1)
pos = fd.tell()
fd.truncate(pos)
total_len = hole_size + len(content) + hole_size
st = os.stat(filename)
self.assert_equal(st.st_size, total_len)
if sparse_support and hasattr(st, 'st_blocks'):
self.assert_true(st.st_blocks * 512 < total_len / 10) # is input sparse?
self.cmd('init', self.repository_location)
self.cmd('create', self.repository_location + '::test', 'input')
with changedir('output'):
self.cmd('extract', '--sparse', self.repository_location + '::test')
self.assert_dirs_equal('input', 'output/input')
filename = os.path.join(self.output_path, 'input', 'sparse')
with open(filename, 'rb') as fd:
# check if file contents are as expected
self.assert_equal(fd.read(hole_size), b'\0' * hole_size)
self.assert_equal(fd.read(len(content)), content)
self.assert_equal(fd.read(hole_size), b'\0' * hole_size)
st = os.stat(filename)
self.assert_equal(st.st_size, total_len)
if sparse_support and hasattr(st, 'st_blocks'):
self.assert_true(st.st_blocks * 512 < total_len / 10) # is output sparse?
def test_unusual_filenames(self):
filenames = ['normal', 'with some blanks', '(with_parens)', ]
for filename in filenames:
filename = os.path.join(self.input_path, filename)
with open(filename, 'wb') as fd:
pass
self.cmd('init', self.repository_location)
self.cmd('create', self.repository_location + '::test', 'input')
for filename in filenames:
with changedir('output'):
self.cmd('extract', self.repository_location + '::test', os.path.join('input', filename))
assert os.path.exists(os.path.join('output', 'input', filename))
def test_repository_swap_detection(self):
self.create_test_files()
os.environ['BORG_PASSPHRASE'] = 'passphrase'
self.cmd('init', '--encryption=passphrase', self.repository_location)
repository_id = self._extract_repository_id(self.repository_path)
self.cmd('create', self.repository_location + '::test', 'input')
shutil.rmtree(self.repository_path)
self.cmd('init', '--encryption=none', self.repository_location)
self._set_repository_id(self.repository_path, repository_id)
self.assert_equal(repository_id, self._extract_repository_id(self.repository_path))
self.assert_raises(Cache.EncryptionMethodMismatch, lambda: self.cmd('create', self.repository_location + '::test.2', 'input'))
def test_repository_swap_detection2(self):
self.create_test_files()
self.cmd('init', '--encryption=none', self.repository_location + '_unencrypted')
os.environ['BORG_PASSPHRASE'] = 'passphrase'
self.cmd('init', '--encryption=passphrase', self.repository_location + '_encrypted')
self.cmd('create', self.repository_location + '_encrypted::test', 'input')
shutil.rmtree(self.repository_path + '_encrypted')
os.rename(self.repository_path + '_unencrypted', self.repository_path + '_encrypted')
self.assert_raises(Cache.RepositoryAccessAborted, lambda: self.cmd('create', self.repository_location + '_encrypted::test.2', 'input'))
def test_strip_components(self):
self.cmd('init', self.repository_location)
self.create_regular_file('dir/file')
self.cmd('create', self.repository_location + '::test', 'input')
with changedir('output'):
self.cmd('extract', self.repository_location + '::test', '--strip-components', '3')
self.assert_true(not os.path.exists('file'))
with self.assert_creates_file('file'):
self.cmd('extract', self.repository_location + '::test', '--strip-components', '2')
with self.assert_creates_file('dir/file'):
self.cmd('extract', self.repository_location + '::test', '--strip-components', '1')
with self.assert_creates_file('input/dir/file'):
self.cmd('extract', self.repository_location + '::test', '--strip-components', '0')
def test_extract_include_exclude(self):
self.cmd('init', self.repository_location)
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('file2', size=1024 * 80)
self.create_regular_file('file3', size=1024 * 80)
self.create_regular_file('file4', size=1024 * 80)
self.cmd('create', '--exclude=input/file4', self.repository_location + '::test', 'input')
with changedir('output'):
self.cmd('extract', self.repository_location + '::test', 'input/file1', )
self.assert_equal(sorted(os.listdir('output/input')), ['file1'])
with changedir('output'):
self.cmd('extract', '--exclude=input/file2', self.repository_location + '::test')
self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file3'])
with changedir('output'):
self.cmd('extract', '--exclude-from=' + self.exclude_file_path, self.repository_location + '::test')
self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file3'])
def test_exclude_caches(self):
self.cmd('init', self.repository_location)
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('cache1/CACHEDIR.TAG', contents=b'Signature: 8a477f597d28d172789f06886806bc55 extra stuff')
self.create_regular_file('cache2/CACHEDIR.TAG', contents=b'invalid signature')
self.cmd('create', '--exclude-caches', self.repository_location + '::test', 'input')
with changedir('output'):
self.cmd('extract', self.repository_location + '::test')
self.assert_equal(sorted(os.listdir('output/input')), ['cache2', 'file1'])
self.assert_equal(sorted(os.listdir('output/input/cache2')), ['CACHEDIR.TAG'])
def test_path_normalization(self):
self.cmd('init', self.repository_location)
self.create_regular_file('dir1/dir2/file', size=1024 * 80)
with changedir('input/dir1/dir2'):
self.cmd('create', self.repository_location + '::test', '../../../input/dir1/../dir1/dir2/..')
output = self.cmd('list', self.repository_location + '::test')
self.assert_not_in('..', output)
self.assert_in(' input/dir1/dir2/file', output)
def test_exclude_normalization(self):
self.cmd('init', self.repository_location)
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('file2', size=1024 * 80)
with changedir('input'):
self.cmd('create', '--exclude=file1', self.repository_location + '::test1', '.')
with changedir('output'):
self.cmd('extract', self.repository_location + '::test1')
self.assert_equal(sorted(os.listdir('output')), ['file2'])
with changedir('input'):
self.cmd('create', '--exclude=./file1', self.repository_location + '::test2', '.')
with changedir('output'):
self.cmd('extract', self.repository_location + '::test2')
self.assert_equal(sorted(os.listdir('output')), ['file2'])
self.cmd('create', '--exclude=input/./file1', self.repository_location + '::test3', 'input')
with changedir('output'):
self.cmd('extract', self.repository_location + '::test3')
self.assert_equal(sorted(os.listdir('output/input')), ['file2'])
def test_repeated_files(self):
self.create_regular_file('file1', size=1024 * 80)
self.cmd('init', self.repository_location)
self.cmd('create', self.repository_location + '::test', 'input', 'input')
def test_overwrite(self):
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('dir2/file2', size=1024 * 80)
self.cmd('init', self.repository_location)
self.cmd('create', self.repository_location + '::test', 'input')
# Overwriting regular files and directories should be supported
os.mkdir('output/input')
os.mkdir('output/input/file1')
os.mkdir('output/input/dir2')
with changedir('output'):
self.cmd('extract', self.repository_location + '::test')
self.assert_dirs_equal('input', 'output/input')
# But non-empty dirs should fail
os.unlink('output/input/file1')
os.mkdir('output/input/file1')
os.mkdir('output/input/file1/dir')
with changedir('output'):
self.cmd('extract', self.repository_location + '::test', exit_code=1)
def test_rename(self):
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('dir2/file2', size=1024 * 80)
self.cmd('init', self.repository_location)
self.cmd('create', self.repository_location + '::test', 'input')
self.cmd('create', self.repository_location + '::test.2', 'input')
self.cmd('extract', '--dry-run', self.repository_location + '::test')
self.cmd('extract', '--dry-run', self.repository_location + '::test.2')
self.cmd('rename', self.repository_location + '::test', 'test.3')
self.cmd('extract', '--dry-run', self.repository_location + '::test.2')
self.cmd('rename', self.repository_location + '::test.2', 'test.4')
self.cmd('extract', '--dry-run', self.repository_location + '::test.3')
self.cmd('extract', '--dry-run', self.repository_location + '::test.4')
# Make sure both archives have been renamed
repository = Repository(self.repository_path)
manifest, key = Manifest.load(repository)
self.assert_equal(len(manifest.archives), 2)
self.assert_in('test.3', manifest.archives)
self.assert_in('test.4', manifest.archives)
def test_delete(self):
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('dir2/file2', size=1024 * 80)
self.cmd('init', self.repository_location)
self.cmd('create', self.repository_location + '::test', 'input')
self.cmd('create', self.repository_location + '::test.2', 'input')
self.cmd('extract', '--dry-run', self.repository_location + '::test')
self.cmd('extract', '--dry-run', self.repository_location + '::test.2')
self.cmd('delete', self.repository_location + '::test')
self.cmd('extract', '--dry-run', self.repository_location + '::test.2')
self.cmd('delete', '--stats', self.repository_location + '::test.2')
# Make sure all data except the manifest has been deleted
repository = Repository(self.repository_path)
self.assert_equal(len(repository), 1)
def test_delete_repo(self):
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('dir2/file2', size=1024 * 80)
self.cmd('init', self.repository_location)
self.cmd('create', self.repository_location + '::test', 'input')
self.cmd('create', self.repository_location + '::test.2', 'input')
self.cmd('delete', self.repository_location)
# Make sure the repo is gone
self.assertFalse(os.path.exists(self.repository_path))
def test_corrupted_repository(self):
self.cmd('init', self.repository_location)
self.create_src_archive('test')
self.cmd('extract', '--dry-run', self.repository_location + '::test')
self.cmd('check', self.repository_location)
name = sorted(os.listdir(os.path.join(self.tmpdir, 'repository', 'data', '0')), reverse=True)[0]
with open(os.path.join(self.tmpdir, 'repository', 'data', '0', name), 'r+b') as fd:
fd.seek(100)
fd.write(b'XXXX')
self.cmd('check', self.repository_location, exit_code=1)
def test_readonly_repository(self):
self.cmd('init', self.repository_location)
self.create_src_archive('test')
os.system('chmod -R ugo-w ' + self.repository_path)
try:
self.cmd('extract', '--dry-run', self.repository_location + '::test')
finally:
# Restore permissions so shutil.rmtree is able to delete it
os.system('chmod -R u+w ' + self.repository_path)
def test_umask(self):
self.create_regular_file('file1', size=1024 * 80)
self.cmd('init', self.repository_location)
self.cmd('create', self.repository_location + '::test', 'input')
mode = os.stat(self.repository_path).st_mode
self.assertEqual(stat.S_IMODE(mode), 0o700)
def test_cmdline_compatibility(self):
self.create_regular_file('file1', size=1024 * 80)
self.cmd('init', self.repository_location)
self.cmd('create', self.repository_location + '::test', 'input')
output = self.cmd('verify', '-v', self.repository_location + '::test')
self.assert_in('"borg verify" has been deprecated', output)
output = self.cmd('prune', self.repository_location, '--hourly=1')
self.assert_in('"--hourly" has been deprecated. Use "--keep-hourly" instead', output)
def test_prune_repository(self):
self.cmd('init', self.repository_location)
self.cmd('create', self.repository_location + '::test1', src_dir)
self.cmd('create', self.repository_location + '::test2', src_dir)
output = self.cmd('prune', '-v', '--dry-run', self.repository_location, '--keep-daily=2')
self.assert_in('Keeping archive: test2', output)
self.assert_in('Would prune: test1', output)
output = self.cmd('list', self.repository_location)
self.assert_in('test1', output)
self.assert_in('test2', output)
self.cmd('prune', self.repository_location, '--keep-daily=2')
output = self.cmd('list', self.repository_location)
self.assert_not_in('test1', output)
self.assert_in('test2', output)
def test_prune_repository_prefix(self):
self.cmd('init', self.repository_location)
self.cmd('create', self.repository_location + '::foo-2015-08-12-10:00', src_dir)
self.cmd('create', self.repository_location + '::foo-2015-08-12-20:00', src_dir)
self.cmd('create', self.repository_location + '::bar-2015-08-12-10:00', src_dir)
self.cmd('create', self.repository_location + '::bar-2015-08-12-20:00', src_dir)
output = self.cmd('prune', '-v', '--dry-run', self.repository_location, '--keep-daily=2', '--prefix=foo-')
self.assert_in('Keeping archive: foo-2015-08-12-20:00', output)
self.assert_in('Would prune: foo-2015-08-12-10:00', output)
output = self.cmd('list', self.repository_location)
self.assert_in('foo-2015-08-12-10:00', output)
self.assert_in('foo-2015-08-12-20:00', output)
self.assert_in('bar-2015-08-12-10:00', output)
self.assert_in('bar-2015-08-12-20:00', output)
self.cmd('prune', self.repository_location, '--keep-daily=2', '--prefix=foo-')
output = self.cmd('list', self.repository_location)
self.assert_not_in('foo-2015-08-12-10:00', output)
self.assert_in('foo-2015-08-12-20:00', output)
self.assert_in('bar-2015-08-12-10:00', output)
self.assert_in('bar-2015-08-12-20:00', output)
def test_usage(self):
self.assert_raises(SystemExit, lambda: self.cmd())
self.assert_raises(SystemExit, lambda: self.cmd('-h'))
def test_help(self):
assert 'Borg' in self.cmd('help')
assert 'patterns' in self.cmd('help', 'patterns')
assert 'Initialize' in self.cmd('help', 'init')
assert 'positional arguments' not in self.cmd('help', 'init', '--epilog-only')
assert 'This command initializes' not in self.cmd('help', 'init', '--usage-only')
@unittest.skipUnless(has_llfuse, 'llfuse not installed')
def test_fuse_mount_repository(self):
mountpoint = os.path.join(self.tmpdir, 'mountpoint')
os.mkdir(mountpoint)
self.cmd('init', self.repository_location)
self.create_test_files()
self.cmd('create', self.repository_location + '::archive', 'input')
self.cmd('create', self.repository_location + '::archive2', 'input')
try:
self.cmd('mount', self.repository_location, mountpoint, fork=True)
self.wait_for_mount(mountpoint)
self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive', 'input'))
self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive2', 'input'))
finally:
if sys.platform.startswith('linux'):
os.system('fusermount -u ' + mountpoint)
else:
os.system('umount ' + mountpoint)
os.rmdir(mountpoint)
# Give the daemon some time to exit
time.sleep(.2)
@unittest.skipUnless(has_llfuse, 'llfuse not installed')
def test_fuse_mount_archive(self):
mountpoint = os.path.join(self.tmpdir, 'mountpoint')
os.mkdir(mountpoint)
self.cmd('init', self.repository_location)
self.create_test_files()
self.cmd('create', self.repository_location + '::archive', 'input')
try:
self.cmd('mount', self.repository_location + '::archive', mountpoint, fork=True)
self.wait_for_mount(mountpoint)
self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'input'))
finally:
if sys.platform.startswith('linux'):
os.system('fusermount -u ' + mountpoint)
else:
os.system('umount ' + mountpoint)
os.rmdir(mountpoint)
# Give the daemon some time to exit
time.sleep(.2)
def verify_aes_counter_uniqueness(self, method):
seen = set() # Chunks already seen
used = set() # counter values already used
def verify_uniqueness():
repository = Repository(self.repository_path)
for key, _ in repository.open_index(repository.get_transaction_id()).iteritems():
data = repository.get(key)
hash = sha256(data).digest()
if hash not in seen:
seen.add(hash)
num_blocks = num_aes_blocks(len(data) - 41)
nonce = bytes_to_long(data[33:41])
for counter in range(nonce, nonce + num_blocks):
self.assert_not_in(counter, used)
used.add(counter)
self.create_test_files()
os.environ['BORG_PASSPHRASE'] = 'passphrase'
self.cmd('init', '--encryption=' + method, self.repository_location)
verify_uniqueness()
self.cmd('create', self.repository_location + '::test', 'input')
verify_uniqueness()
self.cmd('create', self.repository_location + '::test.2', 'input')
verify_uniqueness()
self.cmd('delete', self.repository_location + '::test.2')
verify_uniqueness()
self.assert_equal(used, set(range(len(used))))
def test_aes_counter_uniqueness_keyfile(self):
self.verify_aes_counter_uniqueness('keyfile')
def test_aes_counter_uniqueness_passphrase(self):
self.verify_aes_counter_uniqueness('passphrase')
class ArchiverCheckTestCase(ArchiverTestCaseBase):
def setUp(self):
super().setUp()
with patch.object(ChunkBuffer, 'BUFFER_SIZE', 10):
self.cmd('init', self.repository_location)
self.create_src_archive('archive1')
self.create_src_archive('archive2')
def open_archive(self, name):
repository = Repository(self.repository_path)
manifest, key = Manifest.load(repository)
archive = Archive(repository, key, manifest, name)
return archive, repository
def test_check_usage(self):
output = self.cmd('check', self.repository_location, exit_code=0)
self.assert_in('Starting repository check', output)
self.assert_in('Starting archive consistency check', output)
output = self.cmd('check', '--repository-only', self.repository_location, exit_code=0)
self.assert_in('Starting repository check', output)
self.assert_not_in('Starting archive consistency check', output)
output = self.cmd('check', '--archives-only', self.repository_location, exit_code=0)
self.assert_not_in('Starting repository check', output)
self.assert_in('Starting archive consistency check', output)
def test_missing_file_chunk(self):
archive, repository = self.open_archive('archive1')
for item in archive.iter_items():
if item[b'path'].endswith('testsuite/archiver.py'):
repository.delete(item[b'chunks'][-1][0])
break
repository.commit()
self.cmd('check', self.repository_location, exit_code=1)
self.cmd('check', '--repair', self.repository_location, exit_code=0)
self.cmd('check', self.repository_location, exit_code=0)
def test_missing_archive_item_chunk(self):
archive, repository = self.open_archive('archive1')
repository.delete(archive.metadata[b'items'][-5])
repository.commit()
self.cmd('check', self.repository_location, exit_code=1)
self.cmd('check', '--repair', self.repository_location, exit_code=0)
self.cmd('check', self.repository_location, exit_code=0)
def test_missing_archive_metadata(self):
archive, repository = self.open_archive('archive1')
repository.delete(archive.id)
repository.commit()
self.cmd('check', self.repository_location, exit_code=1)
self.cmd('check', '--repair', self.repository_location, exit_code=0)
self.cmd('check', self.repository_location, exit_code=0)
def test_missing_manifest(self):
archive, repository = self.open_archive('archive1')
repository.delete(Manifest.MANIFEST_ID)
repository.commit()
self.cmd('check', self.repository_location, exit_code=1)
output = self.cmd('check', '--repair', self.repository_location, exit_code=0)
self.assert_in('archive1', output)
self.assert_in('archive2', output)
self.cmd('check', self.repository_location, exit_code=0)
def test_extra_chunks(self):
self.cmd('check', self.repository_location, exit_code=0)
repository = Repository(self.repository_location)
repository.put(b'01234567890123456789012345678901', b'xxxx')
repository.commit()
repository.close()
self.cmd('check', self.repository_location, exit_code=1)
self.cmd('check', self.repository_location, exit_code=1)
self.cmd('check', '--repair', self.repository_location, exit_code=0)
self.cmd('check', self.repository_location, exit_code=0)
self.cmd('extract', '--dry-run', self.repository_location + '::archive1', exit_code=0)
class RemoteArchiverTestCase(ArchiverTestCase):
prefix = '__testsuite__:'
def test_remote_repo_restrict_to_path(self):
self.cmd('init', self.repository_location)
path_prefix = os.path.dirname(self.repository_path)
with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-path', '/foo']):
self.assert_raises(PathNotAllowed, lambda: self.cmd('init', self.repository_location + '_1'))
with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-path', path_prefix]):
self.cmd('init', self.repository_location + '_2')
with patch.object(RemoteRepository, 'extra_test_args', ['--restrict-to-path', '/foo', '--restrict-to-path', path_prefix]):
self.cmd('init', self.repository_location + '_3')

31
borg/testsuite/chunker.py Normal file
View file

@ -0,0 +1,31 @@
from io import BytesIO
from ..chunker import Chunker, buzhash, buzhash_update
from ..archive import CHUNK_MAX_EXP
from . import BaseTestCase
class ChunkerTestCase(BaseTestCase):
def test_chunkify(self):
data = b'0' * int(1.5 * (1 << CHUNK_MAX_EXP)) + b'Y'
parts = [bytes(c) for c in Chunker(0, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(data))]
self.assert_equal(len(parts), 2)
self.assert_equal(b''.join(parts), data)
self.assert_equal([bytes(c) for c in Chunker(0, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b''))], [])
self.assert_equal([bytes(c) for c in Chunker(0, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b'foobarboobaz' * 3))], [b'fooba', b'rboobaz', b'fooba', b'rboobaz', b'fooba', b'rboobaz'])
self.assert_equal([bytes(c) for c in Chunker(1, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b'foobarboobaz' * 3))], [b'fo', b'obarb', b'oob', b'azf', b'oobarb', b'oob', b'azf', b'oobarb', b'oobaz'])
self.assert_equal([bytes(c) for c in Chunker(2, 1, CHUNK_MAX_EXP, 2, 2).chunkify(BytesIO(b'foobarboobaz' * 3))], [b'foob', b'ar', b'boobazfoob', b'ar', b'boobazfoob', b'ar', b'boobaz'])
self.assert_equal([bytes(c) for c in Chunker(0, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))], [b'foobarboobaz' * 3])
self.assert_equal([bytes(c) for c in Chunker(1, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))], [b'foobar', b'boobazfo', b'obar', b'boobazfo', b'obar', b'boobaz'])
self.assert_equal([bytes(c) for c in Chunker(2, 2, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))], [b'foob', b'arboobaz', b'foob', b'arboobaz', b'foob', b'arboobaz'])
self.assert_equal([bytes(c) for c in Chunker(0, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))], [b'foobarboobaz' * 3])
self.assert_equal([bytes(c) for c in Chunker(1, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))], [b'foobarbo', b'obazfoobar', b'boobazfo', b'obarboobaz'])
self.assert_equal([bytes(c) for c in Chunker(2, 3, CHUNK_MAX_EXP, 2, 3).chunkify(BytesIO(b'foobarboobaz' * 3))], [b'foobarboobaz', b'foobarboobaz', b'foobarboobaz'])
def test_buzhash(self):
self.assert_equal(buzhash(b'abcdefghijklmnop', 0), 3795437769)
self.assert_equal(buzhash(b'abcdefghijklmnop', 1), 3795400502)
self.assert_equal(buzhash(b'abcdefghijklmnop', 1), buzhash_update(buzhash(b'Xabcdefghijklmno', 1), ord('X'), ord('p'), 16, 1))
# Test with more than 31 bytes to make sure our barrel_shift macro works correctly
self.assert_equal(buzhash(b'abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz', 0), 566521248)

25
attic/testsuite/crypto.py → borg/testsuite/crypto.py
View file

@ -1,9 +1,10 @@
from binascii import hexlify
from attic.testsuite import AtticTestCase
from attic.crypto import AES, bytes_to_long, bytes_to_int, long_to_bytes, pbkdf2_sha256, get_random_bytes
from ..crypto import AES, bytes_to_long, bytes_to_int, long_to_bytes, pbkdf2_sha256, get_random_bytes
from . import BaseTestCase
class CryptoTestCase(AtticTestCase):
class CryptoTestCase(BaseTestCase):
def test_bytes_to_int(self):
self.assert_equal(bytes_to_int(b'\0\0\0\1'), 1)
@ -14,11 +15,11 @@ class CryptoTestCase(AtticTestCase):
def test_pbkdf2_sha256(self):
self.assert_equal(hexlify(pbkdf2_sha256(b'password', b'salt', 1, 32)),
b'120fb6cffcf8b32c43e7225256c4f837a86548c92ccc35480805987cb70be17b')
b'120fb6cffcf8b32c43e7225256c4f837a86548c92ccc35480805987cb70be17b')
self.assert_equal(hexlify(pbkdf2_sha256(b'password', b'salt', 2, 32)),
b'ae4d0c95af6b46d32d0adff928f06dd02a303f8ef3c251dfd6e2d85a95474c43')
b'ae4d0c95af6b46d32d0adff928f06dd02a303f8ef3c251dfd6e2d85a95474c43')
self.assert_equal(hexlify(pbkdf2_sha256(b'password', b'salt', 4096, 32)),
b'c5e478d59288c841aa530db6845c4c8d962893a001ce4e11a4963873aa98134a')
b'c5e478d59288c841aa530db6845c4c8d962893a001ce4e11a4963873aa98134a')
def test_get_random_bytes(self):
bytes = get_random_bytes(10)
@ -30,11 +31,15 @@ class CryptoTestCase(AtticTestCase):
def test_aes(self):
key = b'X' * 32
data = b'foo' * 10
aes = AES(key)
# encrypt
aes = AES(is_encrypt=True, key=key)
self.assert_equal(bytes_to_long(aes.iv, 8), 0)
cdata = aes.encrypt(data)
self.assert_equal(hexlify(cdata), b'c6efb702de12498f34a2c2bbc8149e759996d08bf6dc5c610aefc0c3a466')
self.assert_equal(bytes_to_long(aes.iv, 8), 2)
self.assert_not_equal(data, aes.decrypt(cdata))
aes.reset(iv=b'\0' * 16)
self.assert_equal(data, aes.decrypt(cdata))
# decrypt
aes = AES(is_encrypt=False, key=key)
self.assert_equal(bytes_to_long(aes.iv, 8), 0)
pdata = aes.decrypt(cdata)
self.assert_equal(data, pdata)
self.assert_equal(bytes_to_long(aes.iv, 8), 2)

35
attic/testsuite/hashindex.py → borg/testsuite/hashindex.py
View file

@ -1,11 +1,17 @@
import hashlib
import os
import tempfile
from attic.hashindex import NSIndex, ChunkIndex
from attic.testsuite import AtticTestCase
from ..hashindex import NSIndex, ChunkIndex
from . import BaseTestCase
class HashIndexTestCase(AtticTestCase):
def H(x):
# make some 32byte long thing that depends on x
return bytes('%-0.32d' % x, 'ascii')
class HashIndexTestCase(BaseTestCase):
def _generic_test(self, cls, make_value, sha):
idx = cls()
@ -44,10 +50,12 @@ class HashIndexTestCase(AtticTestCase):
self.assert_equal(len(cls.read(idx_name.name)), 0)
def test_nsindex(self):
self._generic_test(NSIndex, lambda x: (x, x), '369a18ae6a52524eb2884a3c0fdc2824947edd017a2688c5d4d7b3510c245ab9')
self._generic_test(NSIndex, lambda x: (x, x),
'861d6d60069ea45e39d36bed2bdc1d0c07981e0641955f897ac6848be429abac')
def test_chunkindex(self):
self._generic_test(ChunkIndex, lambda x: (x, x, x), 'ed22e8a883400453c0ee79a06c54df72c994a54eeefdc6c0989efdc5ee6d07b7')
self._generic_test(ChunkIndex, lambda x: (x, x, x),
'69464bd0ebbc5866b9f95d838bc48617d21bfe3dcf294682a5c21a2ef6b9dc0b')
def test_resize(self):
n = 2000 # Must be >= MIN_BUCKETS
@ -75,3 +83,20 @@ class HashIndexTestCase(AtticTestCase):
second_half = list(idx.iteritems(marker=all[49][0]))
self.assert_equal(len(second_half), 50)
self.assert_equal(second_half, all[50:])
def test_chunkindex_merge(self):
idx1 = ChunkIndex()
idx1[H(1)] = 1, 100, 100
idx1[H(2)] = 2, 200, 200
idx1[H(3)] = 3, 300, 300
# no H(4) entry
idx2 = ChunkIndex()
idx2[H(1)] = 4, 100, 100
idx2[H(2)] = 5, 200, 200
# no H(3) entry
idx2[H(4)] = 6, 400, 400
idx1.merge(idx2)
assert idx1[H(1)] == (5, 100, 100)
assert idx1[H(2)] == (7, 200, 200)
assert idx1[H(3)] == (3, 300, 300)
assert idx1[H(4)] == (6, 400, 400)

79
attic/testsuite/helpers.py → borg/testsuite/helpers.py
View file

@ -1,16 +1,16 @@
import hashlib
from time import mktime, strptime
from datetime import datetime, timezone, timedelta
import os
import tempfile
import unittest
from attic.helpers import adjust_patterns, exclude_path, Location, format_timedelta, IncludePattern, ExcludePattern, make_path_safe, UpgradableLock, prune_within, prune_split, to_localtime, \
StableDict, int_to_bigint, bigint_to_int, parse_timestamp
from attic.testsuite import AtticTestCase
import msgpack
from ..helpers import adjust_patterns, exclude_path, Location, format_timedelta, ExcludePattern, make_path_safe, \
prune_within, prune_split, \
StableDict, int_to_bigint, bigint_to_int, parse_timestamp
from . import BaseTestCase
class BigIntTestCase(AtticTestCase):
class BigIntTestCase(BaseTestCase):
def test_bigint(self):
self.assert_equal(int_to_bigint(0), 0)
@ -22,7 +22,7 @@ class BigIntTestCase(AtticTestCase):
self.assert_equal(bigint_to_int(int_to_bigint(2**70)), 2**70)
class LocationTestCase(AtticTestCase):
class LocationTestCase(BaseTestCase):
def test(self):
self.assert_equal(
@ -38,8 +38,8 @@ class LocationTestCase(AtticTestCase):
"Location(proto='ssh', user='user', host='host', port=None, path='/some/path', archive='archive')"
)
self.assert_equal(
repr(Location('mybackup.attic::archive')),
"Location(proto='file', user=None, host=None, port=None, path='mybackup.attic', archive='archive')"
repr(Location('path::archive')),
"Location(proto='file', user=None, host=None, port=None, path='path', archive='archive')"
)
self.assert_equal(
repr(Location('/some/absolute/path::archive')),
@ -60,7 +60,7 @@ class LocationTestCase(AtticTestCase):
Location(Location(location).canonical_path()).canonical_path())
class FormatTimedeltaTestCase(AtticTestCase):
class FormatTimedeltaTestCase(BaseTestCase):
def test(self):
t0 = datetime(2001, 1, 1, 10, 20, 3, 0)
@ -71,7 +71,7 @@ class FormatTimedeltaTestCase(AtticTestCase):
)
class PatternTestCase(AtticTestCase):
class PatternTestCase(BaseTestCase):
files = [
'/etc/passwd', '/etc/hosts', '/home',
@ -94,7 +94,7 @@ class PatternTestCase(AtticTestCase):
['/etc/passwd', '/etc/hosts', '/home', '/var/log/messages', '/var/log/dmesg'])
self.assert_equal(self.evaluate(['/home/u'], []), [])
self.assert_equal(self.evaluate(['/', '/home', '/etc/hosts'], ['/']), [])
self.assert_equal(self.evaluate(['/home/'], ['/home/user2']),
self.assert_equal(self.evaluate(['/home/'], ['/home/user2']),
['/home', '/home/user/.profile', '/home/user/.bashrc'])
self.assert_equal(self.evaluate(['/'], ['*.profile', '/var/log']),
['/etc/passwd', '/etc/hosts', '/home', '/home/user/.bashrc', '/home/user2/public_html/index.html'])
@ -104,7 +104,7 @@ class PatternTestCase(AtticTestCase):
['/etc/passwd', '/etc/hosts', '/var/log/messages', '/var/log/dmesg'])
class MakePathSafeTestCase(AtticTestCase):
class MakePathSafeTestCase(BaseTestCase):
def test(self):
self.assert_equal(make_path_safe('/foo/bar'), 'foo/bar')
@ -116,25 +116,8 @@ class MakePathSafeTestCase(AtticTestCase):
self.assert_equal(make_path_safe('/'), '.')
self.assert_equal(make_path_safe('/'), '.')
class UpgradableLockTestCase(AtticTestCase):
def test(self):
file = tempfile.NamedTemporaryFile()
lock = UpgradableLock(file.name)
lock.upgrade()
lock.upgrade()
lock.release()
@unittest.skipIf(os.getuid() == 0, 'Root can always open files for writing')
def test_read_only_lock_file(self):
file = tempfile.NamedTemporaryFile()
os.chmod(file.name, 0o444)
lock = UpgradableLock(file.name)
self.assert_raises(UpgradableLock.WriteLockFailed, lock.upgrade)
lock.release()
class MockArchive(object):
class MockArchive:
def __init__(self, ts):
self.ts = ts
@ -143,12 +126,12 @@ class MockArchive(object):
return repr(self.ts)
class PruneSplitTestCase(AtticTestCase):
class PruneSplitTestCase(BaseTestCase):
def test(self):
def local_to_UTC(month, day):
'Convert noon on the month and day in 2013 to UTC.'
"""Convert noon on the month and day in 2013 to UTC."""
seconds = mktime(strptime('2013-%02d-%02d 12:00' % (month, day), '%Y-%m-%d %H:%M'))
return datetime.fromtimestamp(seconds, tz=timezone.utc)
@ -159,8 +142,8 @@ class PruneSplitTestCase(AtticTestCase):
for ta in test_archives, reversed(test_archives):
self.assert_equal(set(prune_split(ta, '%Y-%m', n, skip)),
subset(test_archives, indices))
test_pairs = [(1,1), (2,1), (2,28), (3,1), (3,2), (3,31), (5,1)]
test_pairs = [(1, 1), (2, 1), (2, 28), (3, 1), (3, 2), (3, 31), (5, 1)]
test_dates = [local_to_UTC(month, day) for month, day in test_pairs]
test_archives = [MockArchive(date) for date in test_dates]
@ -172,7 +155,7 @@ class PruneSplitTestCase(AtticTestCase):
dotest(test_archives, 0, [], [])
class PruneWithinTestCase(AtticTestCase):
class PruneWithinTestCase(BaseTestCase):
def test(self):
@ -183,27 +166,27 @@ class PruneWithinTestCase(AtticTestCase):
for ta in test_archives, reversed(test_archives):
self.assert_equal(set(prune_within(ta, within)),
subset(test_archives, indices))
# 1 minute, 1.5 hours, 2.5 hours, 3.5 hours, 25 hours, 49 hours
test_offsets = [60, 90*60, 150*60, 210*60, 25*60*60, 49*60*60]
now = datetime.now(timezone.utc)
test_dates = [now - timedelta(seconds=s) for s in test_offsets]
test_archives = [MockArchive(date) for date in test_dates]
dotest(test_archives, '1H', [0])
dotest(test_archives, '2H', [0, 1])
dotest(test_archives, '3H', [0, 1, 2])
dotest(test_archives, '1H', [0])
dotest(test_archives, '2H', [0, 1])
dotest(test_archives, '3H', [0, 1, 2])
dotest(test_archives, '24H', [0, 1, 2, 3])
dotest(test_archives, '26H', [0, 1, 2, 3, 4])
dotest(test_archives, '2d', [0, 1, 2, 3, 4])
dotest(test_archives, '2d', [0, 1, 2, 3, 4])
dotest(test_archives, '50H', [0, 1, 2, 3, 4, 5])
dotest(test_archives, '3d', [0, 1, 2, 3, 4, 5])
dotest(test_archives, '1w', [0, 1, 2, 3, 4, 5])
dotest(test_archives, '1m', [0, 1, 2, 3, 4, 5])
dotest(test_archives, '1y', [0, 1, 2, 3, 4, 5])
dotest(test_archives, '3d', [0, 1, 2, 3, 4, 5])
dotest(test_archives, '1w', [0, 1, 2, 3, 4, 5])
dotest(test_archives, '1m', [0, 1, 2, 3, 4, 5])
dotest(test_archives, '1y', [0, 1, 2, 3, 4, 5])
class StableDictTestCase(AtticTestCase):
class StableDictTestCase(BaseTestCase):
def test(self):
d = StableDict(foo=1, bar=2, boo=3, baz=4)
@ -211,7 +194,7 @@ class StableDictTestCase(AtticTestCase):
self.assert_equal(hashlib.md5(msgpack.packb(d)).hexdigest(), 'fc78df42cd60691b3ac3dd2a2b39903f')
class TestParseTimestamp(AtticTestCase):
class TestParseTimestamp(BaseTestCase):
def test(self):
self.assert_equal(parse_timestamp('2015-04-19T20:25:00.226410'), datetime(2015, 4, 19, 20, 25, 0, 226410, timezone.utc))

29
attic/testsuite/key.py → borg/testsuite/key.py
View file

@ -3,19 +3,20 @@ import re
import shutil
import tempfile
from binascii import hexlify
from attic.crypto import bytes_to_long, num_aes_blocks
from attic.testsuite import AtticTestCase
from attic.key import PlaintextKey, PassphraseKey, KeyfileKey
from attic.helpers import Location, unhexlify
from ..crypto import bytes_to_long, num_aes_blocks
from ..key import PlaintextKey, PassphraseKey, KeyfileKey
from ..helpers import Location, unhexlify
from . import BaseTestCase
class KeyTestCase(AtticTestCase):
class KeyTestCase(BaseTestCase):
class MockArgs(object):
class MockArgs:
repository = Location(tempfile.mkstemp()[1])
keyfile2_key_file = """
ATTIC KEY 0000000000000000000000000000000000000000000000000000000000000000
BORG_KEY 0000000000000000000000000000000000000000000000000000000000000000
hqppdGVyYXRpb25zzgABhqCkaGFzaNoAIMyonNI+7Cjv0qHi0AOBM6bLGxACJhfgzVD2oq
bIS9SFqWFsZ29yaXRobaZzaGEyNTakc2FsdNoAINNK5qqJc1JWSUjACwFEWGTdM7Nd0a5l
1uBGPEb+9XM9p3ZlcnNpb24BpGRhdGHaANAYDT5yfPpU099oBJwMomsxouKyx/OG4QIXK2
@ -32,13 +33,13 @@ class KeyTestCase(AtticTestCase):
def setUp(self):
self.tmppath = tempfile.mkdtemp()
os.environ['ATTIC_KEYS_DIR'] = self.tmppath
os.environ['BORG_KEYS_DIR'] = self.tmppath
def tearDown(self):
shutil.rmtree(self.tmppath)
class MockRepository(object):
class _Location(object):
class MockRepository:
class _Location:
orig = '/some/place'
_location = _Location()
@ -51,7 +52,7 @@ class KeyTestCase(AtticTestCase):
self.assert_equal(data, key.decrypt(key.id_hash(data), key.encrypt(data)))
def test_keyfile(self):
os.environ['ATTIC_PASSPHRASE'] = 'test'
os.environ['BORG_PASSPHRASE'] = 'test'
key = KeyfileKey.create(self.MockRepository(), self.MockArgs())
self.assert_equal(bytes_to_long(key.enc_cipher.iv, 8), 0)
manifest = key.encrypt(b'XXX')
@ -70,14 +71,14 @@ class KeyTestCase(AtticTestCase):
self.assert_equal(data, key2.decrypt(key.id_hash(data), key.encrypt(data)))
def test_keyfile2(self):
with open(os.path.join(os.environ['ATTIC_KEYS_DIR'], 'keyfile'), 'w') as fd:
with open(os.path.join(os.environ['BORG_KEYS_DIR'], 'keyfile'), 'w') as fd:
fd.write(self.keyfile2_key_file)
os.environ['ATTIC_PASSPHRASE'] = 'passphrase'
os.environ['BORG_PASSPHRASE'] = 'passphrase'
key = KeyfileKey.detect(self.MockRepository(), self.keyfile2_cdata)
self.assert_equal(key.decrypt(self.keyfile2_id, self.keyfile2_cdata), b'payload')
def test_passphrase(self):
os.environ['ATTIC_PASSPHRASE'] = 'test'
os.environ['BORG_PASSPHRASE'] = 'test'
key = PassphraseKey.create(self.MockRepository(), None)
self.assert_equal(bytes_to_long(key.enc_cipher.iv, 8), 0)
self.assert_equal(hexlify(key.id_key), b'793b0717f9d8fb01c751a487e9b827897ceea62409870600013fbc6b4d8d7ca6')

121
borg/testsuite/locking.py Normal file
View file

@ -0,0 +1,121 @@
import time
import pytest
from ..locking import get_id, TimeoutTimer, ExclusiveLock , UpgradableLock, LockRoster, ADD, REMOVE, SHARED, EXCLUSIVE
ID1 = "foo", 1, 1
ID2 = "bar", 2, 2
def test_id():
hostname, pid, tid = get_id()
assert isinstance(hostname, str)
assert isinstance(pid, int)
assert isinstance(tid, int)
assert len(hostname) > 0
assert pid > 0
class TestTimeoutTimer:
def test_timeout(self):
timeout = 0.5
t = TimeoutTimer(timeout).start()
assert not t.timed_out()
time.sleep(timeout * 1.5)
assert t.timed_out()
def test_notimeout_sleep(self):
timeout, sleep = None, 0.5
t = TimeoutTimer(timeout, sleep).start()
assert not t.timed_out_or_sleep()
assert time.time() >= t.start_time + 1 * sleep
assert not t.timed_out_or_sleep()
assert time.time() >= t.start_time + 2 * sleep
@pytest.fixture()
def lockpath(tmpdir):
return str(tmpdir.join('lock'))
class TestExclusiveLock:
def test_checks(self, lockpath):
with ExclusiveLock(lockpath, timeout=1) as lock:
assert lock.is_locked() and lock.by_me()
def test_acquire_break_reacquire(self, lockpath):
lock = ExclusiveLock(lockpath, id=ID1).acquire()
lock.break_lock()
with ExclusiveLock(lockpath, id=ID2):
pass
def test_timeout(self, lockpath):
with ExclusiveLock(lockpath, id=ID1):
with pytest.raises(ExclusiveLock.LockTimeout):
ExclusiveLock(lockpath, id=ID2, timeout=0.1).acquire()
class TestUpgradableLock:
def test_shared(self, lockpath):
lock1 = UpgradableLock(lockpath, exclusive=False, id=ID1).acquire()
lock2 = UpgradableLock(lockpath, exclusive=False, id=ID2).acquire()
assert len(lock1._roster.get(SHARED)) == 2
assert len(lock1._roster.get(EXCLUSIVE)) == 0
lock1.release()
lock2.release()
def test_exclusive(self, lockpath):
with UpgradableLock(lockpath, exclusive=True, id=ID1) as lock:
assert len(lock._roster.get(SHARED)) == 0
assert len(lock._roster.get(EXCLUSIVE)) == 1
def test_upgrade(self, lockpath):
with UpgradableLock(lockpath, exclusive=False) as lock:
lock.upgrade()
lock.upgrade() # NOP
assert len(lock._roster.get(SHARED)) == 0
assert len(lock._roster.get(EXCLUSIVE)) == 1
def test_downgrade(self, lockpath):
with UpgradableLock(lockpath, exclusive=True) as lock:
lock.downgrade()
lock.downgrade() # NOP
assert len(lock._roster.get(SHARED)) == 1
assert len(lock._roster.get(EXCLUSIVE)) == 0
def test_break(self, lockpath):
lock = UpgradableLock(lockpath, exclusive=True, id=ID1).acquire()
lock.break_lock()
assert len(lock._roster.get(SHARED)) == 0
assert len(lock._roster.get(EXCLUSIVE)) == 0
with UpgradableLock(lockpath, exclusive=True, id=ID2):
pass
@pytest.fixture()
def rosterpath(tmpdir):
return str(tmpdir.join('roster'))
class TestLockRoster:
def test_empty(self, rosterpath):
roster = LockRoster(rosterpath)
empty = roster.load()
roster.save(empty)
assert empty == {}
def test_modify_get(self, rosterpath):
roster1 = LockRoster(rosterpath, id=ID1)
assert roster1.get(SHARED) == set()
roster1.modify(SHARED, ADD)
assert roster1.get(SHARED) == {ID1, }
roster2 = LockRoster(rosterpath, id=ID2)
roster2.modify(SHARED, ADD)
assert roster2.get(SHARED) == {ID1, ID2, }
roster1 = LockRoster(rosterpath, id=ID1)
roster1.modify(SHARED, REMOVE)
assert roster1.get(SHARED) == {ID2, }
roster2 = LockRoster(rosterpath, id=ID2)
roster2.modify(SHARED, REMOVE)
assert roster2.get(SHARED) == set()

6
attic/testsuite/lrucache.py → borg/testsuite/lrucache.py
View file

@ -1,8 +1,8 @@
from attic.lrucache import LRUCache
from attic.testsuite import AtticTestCase
from ..lrucache import LRUCache
from . import BaseTestCase
class LRUCacheTestCase(AtticTestCase):
class LRUCacheTestCase(BaseTestCase):
def test(self):
c = LRUCache(2, dispose=lambda _: None)

10
attic/testsuite/platform.py → borg/testsuite/platform.py
View file

@ -3,8 +3,9 @@ import shutil
import sys
import tempfile
import unittest
from attic.platform import acl_get, acl_set
from attic.testsuite import AtticTestCase
from ..platform import acl_get, acl_set
from . import BaseTestCase
ACCESS_ACL = """
@ -36,7 +37,7 @@ def fakeroot_detected():
@unittest.skipUnless(sys.platform.startswith('linux'), 'linux only test')
@unittest.skipIf(fakeroot_detected(), 'not compatible with fakeroot')
class PlatformLinuxTestCase(AtticTestCase):
class PlatformLinuxTestCase(BaseTestCase):
def setUp(self):
self.tmpdir = tempfile.mkdtemp()
@ -74,7 +75,7 @@ class PlatformLinuxTestCase(AtticTestCase):
@unittest.skipUnless(sys.platform.startswith('darwin'), 'OS X only test')
@unittest.skipIf(fakeroot_detected(), 'not compatible with fakeroot')
class PlatformDarwinTestCase(AtticTestCase):
class PlatformDarwinTestCase(BaseTestCase):
def setUp(self):
self.tmpdir = tempfile.mkdtemp()
@ -101,4 +102,3 @@ class PlatformDarwinTestCase(AtticTestCase):
self.set_acl(file2.name, b'!#acl 1\ngroup:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:staff:0:allow:read\nuser:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read\n', numeric_owner=True)
self.assert_in(b'group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000:wheel:0:allow:read', self.get_acl(file2.name)[b'acl_extended'])
self.assert_in(b'group:ABCDEFAB-CDEF-ABCD-EFAB-CDEF00000000::0:allow:read', self.get_acl(file2.name, numeric_owner=True)[b'acl_extended'])

34
attic/testsuite/repository.py → borg/testsuite/repository.py
View file

@ -1,15 +1,19 @@
import os
import shutil
import tempfile
from attic.testsuite.mock import patch
from attic.hashindex import NSIndex
from attic.helpers import Location, IntegrityError, UpgradableLock
from attic.remote import RemoteRepository
from attic.repository import Repository
from attic.testsuite import AtticTestCase
from mock import patch
from ..hashindex import NSIndex
from ..helpers import Location, IntegrityError
from ..locking import UpgradableLock
from ..remote import RemoteRepository, InvalidRPCMethod
from ..repository import Repository
from . import BaseTestCase
class RepositoryTestCaseBase(AtticTestCase):
class RepositoryTestCaseBase(BaseTestCase):
key_size = 32
def open(self, create=False):
return Repository(os.path.join(self.tmppath, 'repository'), create=create)
@ -154,11 +158,10 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
for name in os.listdir(self.repository.path):
if name.startswith('index.'):
os.unlink(os.path.join(self.repository.path, name))
with patch.object(UpgradableLock, 'upgrade', side_effect=UpgradableLock.WriteLockFailed) as upgrade:
with patch.object(UpgradableLock, 'upgrade', side_effect=UpgradableLock.ExclusiveLockFailed) as upgrade:
self.reopen()
self.assert_raises(UpgradableLock.WriteLockFailed, lambda: len(self.repository))
upgrade.assert_called_once()
self.assert_raises(UpgradableLock.ExclusiveLockFailed, lambda: len(self.repository))
upgrade.assert_called_once_with()
def test_crash_before_write_index(self):
self.add_keys()
@ -308,7 +311,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
# Simulate a crash before compact
with patch.object(Repository, 'compact_segments') as compact:
self.repository.commit()
compact.assert_called_once()
compact.assert_called_once_with()
self.reopen()
self.check(repair=True)
self.assert_equal(self.repository.get(bytes(32)), b'data2')
@ -319,8 +322,15 @@ class RemoteRepositoryTestCase(RepositoryTestCase):
def open(self, create=False):
return RemoteRepository(Location('__testsuite__:' + os.path.join(self.tmppath, 'repository')), create=create)
def test_invalid_rpc(self):
self.assert_raises(InvalidRPCMethod, lambda: self.repository.call('__init__', None))
class RemoteRepositoryCheckTestCase(RepositoryCheckTestCase):
def open(self, create=False):
return RemoteRepository(Location('__testsuite__:' + os.path.join(self.tmppath, 'repository')), create=create)
def test_crash_before_compact(self):
# skip this test, we can't mock-patch a Repository class in another process!
pass

8
attic/testsuite/xattr.py → borg/testsuite/xattr.py
View file

@ -1,11 +1,13 @@
import os
import tempfile
import unittest
from attic.testsuite import AtticTestCase
from attic.xattr import is_enabled, getxattr, setxattr, listxattr
from ..xattr import is_enabled, getxattr, setxattr, listxattr
from . import BaseTestCase
@unittest.skipUnless(is_enabled(), 'xattr not enabled on filesystem')
class XattrTestCase(AtticTestCase):
class XattrTestCase(BaseTestCase):
def setUp(self):
self.tmpfile = tempfile.NamedTemporaryFile()

4
attic/xattr.py → borg/xattr.py
View file

@ -8,10 +8,10 @@ from ctypes import CDLL, create_string_buffer, c_ssize_t, c_size_t, c_char_p, c_
from ctypes.util import find_library
def is_enabled():
def is_enabled(path=None):
"""Determine if xattr is enabled on the filesystem
"""
with tempfile.NamedTemporaryFile() as fd:
with tempfile.NamedTemporaryFile(dir=path, prefix='borg-tmp') as fd:
try:
setxattr(fd.fileno(), 'user.name', b'value')
except OSError:

26
docs/Makefile
View file

@ -73,17 +73,17 @@ qthelp:
@echo
@echo "Build finished; now you can run "qcollectiongenerator" with the" \
".qhcp project file in $(BUILDDIR)/qthelp, like this:"
@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/attic.qhcp"
@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/borg.qhcp"
@echo "To view the help file:"
@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/attic.qhc"
@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/borg.qhc"
devhelp:
$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
@echo
@echo "Build finished."
@echo "To view the help file:"
@echo "# mkdir -p $$HOME/.local/share/devhelp/attic"
@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/attic"
@echo "# mkdir -p $$HOME/.local/share/devhelp/borg"
@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/borg"
@echo "# devhelp"
epub:
@ -130,17 +130,13 @@ doctest:
@echo "Testing of doctests in the sources finished, look at the " \
"results in $(BUILDDIR)/doctest/output.txt."
gh-pages: html
GH_PAGES_CLONE="`mktemp -d`" && \
git clone --branch gh-pages `git rev-parse --show-toplevel` $$GH_PAGES_CLONE && \
(cd $$GH_PAGES_CLONE && git rm -r *) && \
cp -r _build/html/* $$GH_PAGES_CLONE && \
(cd $$GH_PAGES_CLONE && git add -A && git commit -m 'Updated gh-pages' && git push) && \
rm -rf $$GH_PAGES_CLONE
gh-io: html
GH_IO_CLONE="`mktemp -d`" && \
git clone git@github.com:borgbackup/borgbackup.github.io.git $$GH_IO_CLONE && \
(cd $$GH_IO_CLONE && git rm -r *) && \
cp -r _build/html/* $$GH_IO_CLONE && \
(cd $$GH_IO_CLONE && git add -A && git commit -m 'Updated borgbackup.github.io' && git push) && \
rm -rf $$GH_IO_CLONE
inotify: html
while inotifywait -r . --exclude usage.rst --exclude '_build/*' ; do make html ; done
upload: html
rsync -va -e ssh _build/html/ sushi.edgewall.com:/srv/attic/www/

7
docs/_themes/attic/sidebarlogo.html vendored
View file

@ -1,7 +0,0 @@
<link href='//fonts.googleapis.com/css?family=Oswald:700|Inconsolata:400,700'
rel='stylesheet' type='text/css'>
<div class="sidebarlogo">
<a href="{{ pathto('index') }}">
<div class="title">Attic</div>
</a>
</div>

10
docs/_themes/attic/sidebarusefullinks.html vendored
View file

@ -1,10 +0,0 @@
<a href="https://github.com/jborg/attic"><img style="position: fixed; top: 0; right: 0; border: 0;"
src="https://s3.amazonaws.com/github/ribbons/forkme_right_gray_6d6d6d.png" alt="Fork me on GitHub"></a>
<h3>Useful Links</h3>
<ul>
<li><a href="https://pypi.python.org/pypi/Attic">Attic @ PyPI</a></li>
<li><a href="https://github.com/jborg/attic">Attic @ github</a></li>
<li><a href="https://attic-backup.org/">Online Documentation</a></li>
<li><a href="https://github.com/jborg/attic/issues">Issue Tracker</a></li>
</ul>

5
docs/_themes/local/sidebarlogo.html vendored Normal file
View file

@ -0,0 +1,5 @@
<div class="sidebarlogo">
<a href="{{ pathto('index') }}">
<div class="title">Borg</div>
</a>
</div>

20
docs/_themes/local/sidebarusefullinks.html vendored Normal file
View file

@ -0,0 +1,20 @@
<a href="https://github.com/borgbackup/borg"><img style="position: fixed; top: 0; right: 0; border: 0;"
src="https://s3.amazonaws.com/github/ribbons/forkme_right_gray_6d6d6d.png" alt="Fork me on GitHub"></a>
<h3>Useful Links</h3>
<ul>
<li><a href="https://borgbackup.github.io/borgbackup/">Main Web Site</a></li>
<li><a href="https://pypi.python.org/pypi/borgbackup">PyPI packages</a></li>
<li><a href="https://github.com/borgbackup/borg/issues/147">Binary Packages</a></li>
<li><a href="https://github.com/borgbackup/borg/blob/master/CHANGES.rst">Current ChangeLog</a></li>
<li><a href="https://github.com/borgbackup/borg">GitHub</a></li>
<li><a href="https://github.com/borgbackup/borg/issues">Issue Tracker</a></li>
<li><a href="https://www.bountysource.com/teams/borgbackup">Bounties &amp; Fundraisers</a></li>
<li><a href="http://librelist.com/browser/borgbackup/">Mailing List</a></li>
</ul>
<h3>Related Projects</h3>
<ul>
<li><a href="https://borgbackup.github.io/borgweb/">BorgWeb</a></li>
</ul>

96
docs/_themes/attic/static/attic.css_t → docs/_themes/local/static/local.css_t vendored
View file

@ -1,12 +1,14 @@
@import url("basic.css");
@import url(//fonts.googleapis.com/css?family=Black+Ops+One);
body {
font-family: Helvetica;
background-color: white;
font-family: Arial, Helvetica, sans-serif;
background-color: black;
margin: 0;
padding: 0;
position: relative;
}
div.related {
display: none;
background-color: black;
@ -14,10 +16,12 @@ div.related {
width: 800px;
margin: 0 auto;
}
div.related a {
color: white;
text-decoration: none;
}
div.document {
width: 1030px;
margin: 0 auto;
@ -27,114 +31,144 @@ div.documentwrapper {
float: right;
width: 760px;
padding: 0 20px 20px 20px;
background-color: #f3f3f3;
color: #00aa00;
background-color: #000000;
margin-bottom: 2em;
}
div.sphinxsidebar {
margin-left: 0;
parring-right: 20px;
padding-right: 20px;
width: 230px;
background: #e9e9e9;
background: #081008;
position: absolute;
top: 0;
min-height: 100%;
}
h1, h2, h3 {
font-family: "Oswald";
font-weight: normal;
color: #333;
color: #33dd33;
}
h1 {
margin: .8em 0 .5em;
font-size: 200%;
}
h2, h3 {
h2 {
margin: 1.2em 0 .6em;
font-size: 140%;
}
h1 { font-size: 200%;}
h2 { font-size: 140%;}
h3 { font-size: 110%;}
h3 {
margin: 1.2em 0 .6em;
font-size: 110%;
}
ul {
padding-left: 1.2em;
margin-bottom: .3em;
}
ul ul {
font-size: 95%;
}
li {
margin: .1em 0;
}
a:link, a:visited {
color: #00608f;
a:link {
color: #dddd00;
text-decoration: none;
}
a:visited {
color: #990000;
text-decoration: none;
}
a:hover {
color: #00B0E4;
border-bottom: 1px dotted #00B0E4;
color: #dd0000;
border-bottom: 1px dotted #dd0000;
}
div.sphinxsidebar a:link, div.sphinxsidebar a:visited {
color: #555;
border-bottom: 1px dotted #555;
}
div.sphinxsidebar {
color: #00aa00;
background: 0000000;
}
div.sphinxsidebar input {
border: 1px solid #ccc;
color: #00cc00;
background: 0000000;
border: 1px solid #444444;
}
pre {
padding: 10px 20px;
background: white;
color: #222;
background: #101010;
color: #22cc22;
line-height: 1.5em;
border-bottom: 2px solid black;
font-family: "Inconsolata";
}
pre a:link,
pre a:visited {
color: #00B0E4;
color: #00b0e4;
}
div.sidebarlogo .title {
font-family: "Oswald";
font-family: 'Black Ops One', cursive;
font-size: 500%;
}
div.sidebarlogo a {
color: #00dd00;
}
div.sidebarlogo .subtitle {
font-style: italic;
color: #777;
}
tt span.pre {
font-size: 110%;
}
dt {
font-family: "Oswald";
font-size: 95%;
}
div.admonition p.admonition-title + p {
display: inline;
display: inline;
}
div.admonition p {
margin-bottom: 5px;
margin-bottom: 5px;
}
p.admonition-title {
display: inline;
display: inline;
}
p.admonition-title:after {
content: ":";
content: ":";
}
div.note {
background-color: #ff5;
border-bottom: 2px solid #d22;
background-color: #002211;
border-bottom: 2px solid #22dd22;
}
div.seealso {
background-color: #ffe;
border: 1px solid #ff6;
background-color: #0fe;
border: 1px solid #0f6;
border-radius: .4em;
box-shadow: 2px 2px #dd6;
}

2
docs/_themes/attic/theme.conf → docs/_themes/local/theme.conf vendored
View file

@ -1,6 +1,6 @@
[theme]
inherit = basic
stylesheet = attic.css
stylesheet = local.css
pygments_style = tango
[options]

4
docs/changes.rst Normal file
View file

@ -0,0 +1,4 @@
.. include:: global.rst.inc
.. _changelog:
.. include:: ../CHANGES.rst

40
docs/conf.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# -*- coding: utf-8 -*-
#
# Attic documentation build configuration file, created by
# documentation build configuration file, created by
# sphinx-quickstart on Sat Sep 10 18:18:25 2011.
#
# This file is execfile()d with the current directory set to its containing dir.
@ -11,12 +11,13 @@
# All configuration values have a default; values that are commented out
# serve to show the default.
import sys, os, attic
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))
import sys, os
sys.path.insert(0, os.path.abspath('..'))
from borg import __version__ as sw_version
# -- General configuration -----------------------------------------------------
@ -40,15 +41,15 @@ source_suffix = '.rst'
master_doc = 'index'
# General information about the project.
project = 'Attic - Deduplicating Archiver'
copyright = '2010-2014, Jonas Borgström'
project = 'Borg - Deduplicating Archiver'
copyright = '2010-2014, Jonas Borgström, 2015 The Borg Collective (see AUTHORS file)'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = attic.__version__.split('-')[0]
version = sw_version.split('-')[0]
# The full version, including alpha/beta/rc tags.
release = version
@ -91,7 +92,7 @@ pygments_style = 'sphinx'
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = 'attic'
html_theme = 'local'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
@ -120,7 +121,7 @@ html_theme_path = ['_themes']
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
html_static_path = []
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
@ -133,7 +134,7 @@ html_static_path = ['_static']
# Custom sidebar templates, maps document names to template names.
html_sidebars = {
'index': ['sidebarlogo.html', 'sidebarusefullinks.html', 'searchbox.html'],
'**': ['sidebarlogo.html', 'localtoc.html', 'relations.html', 'sidebarusefullinks.html', 'searchbox.html']
'**': ['sidebarlogo.html', 'relations.html', 'searchbox.html', 'localtoc.html', 'sidebarusefullinks.html']
}
# Additional templates that should be rendered to pages, maps page names to
# template names.
@ -166,7 +167,7 @@ html_show_copyright = False
#html_file_suffix = None
# Output file base name for HTML help builder.
htmlhelp_basename = 'atticdoc'
htmlhelp_basename = 'borgdoc'
# -- Options for LaTeX output --------------------------------------------------
@ -180,8 +181,8 @@ htmlhelp_basename = 'atticdoc'
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
('index', 'Attic.tex', 'Attic Documentation',
'Jonas Borgström', 'manual'),
('index', 'Borg.tex', 'Borg Documentation',
'see "AUTHORS" file', 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of
@ -213,14 +214,13 @@ latex_documents = [
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
#man_pages = [
# ('man', 'attic', 'Attic',
# ['Jonas Borgström'], 1)
# ('man', 'borg', 'Borg',
# ['see "AUTHORS" file'], 1)
#]
extensions = ['sphinx.ext.extlinks']
extlinks = {
'issue': ('https://github.com/jborg/attic/issues/%s', '#'),
'targz_url': ('https://pypi.python.org/packages/source/A/Attic/%%s-%s.tar.gz' % version, None),
'artifacts': ('https://attic-backup.org/downloads/releases/%s/%%s' % version, '')
}
'issue': ('https://github.com/borgbackup/borg/issues/%s', '#'),
'targz_url': ('https://pypi.python.org/packages/source/b/borgbackup/%%s-%s.tar.gz' % version, None),
}

67
docs/development.rst Normal file
View file

@ -0,0 +1,67 @@
.. include:: global.rst.inc
.. _development:
Development
===========
This chapter will get you started with |project_name|' development.
|project_name| is written in Python (with a little bit of Cython and C for
the performance critical parts).
Building a development environment
----------------------------------
First, just install borg into a virtual env as described before.
To install some additional packages needed for running the tests, activate your
virtual env and run::
pip install -r requirements.d/development.txt
Running the tests
-----------------
The tests are in the borg/testsuite package.
To run them, you need to have fakeroot, tox and pytest installed.
To run the test suite use the following command::
fakeroot -u tox # run all tests
Some more advanced examples::
# verify a changed tox.ini (run this after any change to tox.ini):
fakeroot -u tox --recreate
fakeroot -u tox -e py32 # run all tests, but only on python 3.2
fakeroot -u tox borg.testsuite.locking # only run 1 test module
fakeroot -u tox borg.testsuite.locking -- -k '"not Timer"' # exclude some tests
fakeroot -u tox borg.testsuite -- -v # verbose py.test
Important notes:
- Without fakeroot -u some tests will fail.
- When using -- to give options to py.test, you MUST also give borg.testsuite[.module].
Building the docs with Sphinx
-----------------------------
The documentation (in reStructuredText format, .rst) is in docs/.
To build the html version of it, you need to have sphinx installed::
pip3 install sphinx
Now run::
cd docs/
make html
Then point a web browser at docs/_build/html/index.html.

108
docs/faq.rst
View file

@ -7,35 +7,127 @@ Frequently asked questions
Which platforms are supported?
Currently Linux, FreeBSD and MacOS X are supported.
You can try your luck on other POSIX-like systems, like Cygwin,
other BSDs, etc. but they are not officially supported.
Can I backup VM disk images?
Yes, the :ref:`deduplication <deduplication_def>` technique used by |project_name|
makes sure only the modified parts of the file are stored.
Also, we have optional simple sparse file support for extract.
Which file attributes are preserved?
The following attributes are preserved:
Can I backup from multiple servers into a single repository?
Yes, but in order for the deduplication used by |project_name| to work, it
needs to keep a local cache containing checksums of all file
chunks already stored in the repository. This cache is stored in
``~/.cache/borg/``. If |project_name| detects that a repository has been
modified since the local cache was updated it will need to rebuild
the cache. This rebuild can be quite time consuming.
So, yes it's possible. But it will be most efficient if a single
repository is only modified from one place. Also keep in mind that
|project_name| will keep an exclusive lock on the repository while creating
or deleting archives, which may make *simultaneous* backups fail.
Which file types, attributes, etc. are preserved?
* Directories
* Regular files
* Hardlinks (considering all files in the same archive)
* Symlinks (stored as symlink, the symlink is not followed)
* Character and block device files
* FIFOs ("named pipes")
* Name
* Contents
* Time of last modification (nanosecond precision with Python >= 3.3)
* User ID of owner
* Group ID of owner
* Unix Permission
* Extended attributes (xattrs)
* Unix Mode/Permissions (u/g/o permissions, suid, sgid, sticky)
* Extended Attributes (xattrs)
* Access Control Lists (ACL_) on Linux, OS X and FreeBSD
* BSD flags on OS X and FreeBSD
Which file types, attributes, etc. are *not* preserved?
* UNIX domain sockets (because it does not make sense - they are meaningless
without the running process that created them and the process needs to
recreate them in any case). So, don't panic if your backup misses a UDS!
* The precise on-disk representation of the holes in a sparse file.
Archive creation has no special support for sparse files, holes are
backed up as (deduplicated and compressed) runs of zero bytes.
Archive extraction has optional support to extract all-zero chunks as
holes in a sparse file.
How can I specify the encryption passphrase programmatically?
The encryption passphrase can be specified programmatically using the
`ATTIC_PASSPHRASE` environment variable. This is convenient when setting up
`BORG_PASSPHRASE` environment variable. This is convenient when setting up
automated encrypted backups. Another option is to use
key file based encryption with a blank passphrase. See
:ref:`encrypted_repos` for more details.
When backing up to remote servers, is data encrypted before leaving the local machine, or do I have to trust that the remote server isn't malicious?
Yes, everything is encrypted before leaving the local machine.
When backing up to remote encrypted repos, is encryption done locally?
Yes, file and directory metadata and data is locally encrypted, before
leaving the local machine. We do not mean the transport layer encryption
by that, but the data/metadata itself. Transport layer encryption (e.g.
when ssh is used as a transport) applies additionally.
If a backup stops mid-way, does the already-backed-up data stay there? I.e. does Attic resume backups?
When backing up to remote servers, do I have to trust the remote server?
Yes and No.
No, as far as data confidentiality is concerned - if you use encryption,
all your files/dirs data and metadata are stored in their encrypted form
into the repository.
Yes, as an attacker with access to the remote server could delete (or
otherwise make unavailable) all your backups.
If a backup stops mid-way, does the already-backed-up data stay there? I.e. does |project_name| resume backups?
Yes, during a backup a special checkpoint archive named ``<archive-name>.checkpoint`` is saved every 5 minutes
containing all the data backed-up until that point. This means that at most 5 minutes worth of data needs to be
retransmitted if a backup needs to be restarted.
If it crashes with a UnicodeError, what can I do?
Check if your encoding is set correctly. For most POSIX-like systems, try::
export LANG=en_US.UTF-8 # or similar, important is correct charset
If I want to run |project_name| on a ARM CPU older than ARM v6?
You need to enable the alignment trap handler to fixup misaligned accesses::
echo "2" > /proc/cpu/alignment
Can |project_name| add redundancy to the backup data to deal with hardware malfunction?
No, it can't. While that at first sounds like a good idea to defend against some
defect HDD sectors or SSD flash blocks, dealing with this in a reliable way needs a lot
of low-level storage layout information and control which we do not have (and also can't
get, even if we wanted).
So, if you need that, consider RAID1 or a filesystems that offers redundant storage.
Can |project_name| verify data integrity of a backup archive?
Yes, if you want to detect accidental data damage (like bit rot), use the ``check``
operation. It will notice corruption using CRCs and hashes.
If you want to be able to detect malicious tampering also, use a encrypted repo.
It will then be able to check using CRCs and HMACs.
Why was Borg forked from Attic?
Borg was created in May 2015 in response to the difficulty of
getting new code or larger changes incorporated into Attic and
establishing a bigger developer community / more open development.
More details can be found in `ticket 217
<https://github.com/jborg/attic/issues/217>`_ that led to the fork.
Borg intends to be:
* simple:
* as simple as possible, but no simpler
* do the right thing by default, but offer options
* open:
* welcome feature requests
* accept pull requests of good quality and coding style
* give feedback on PRs that can't be accepted "as is"
* discuss openly, don't work in the dark
* changing:
* Borg is not compatible with Attic
* do not break compatibility accidentally, without a good reason
or without warning. allow compatibility breaking for other cases.
* if major version number changes, it may have incompatible changes

62
docs/foreword.rst
View file

@ -1,62 +0,0 @@
.. include:: global.rst.inc
.. _foreword:
Foreword
========
|project_name| is a secure backup program for Linux, FreeBSD and Mac OS X.
|project_name| is designed for efficient data storage where only new or
modified data is stored.
Features
--------
Space efficient storage
Variable block size `deduplication`_ is used to reduce the number of bytes
stored by detecting redundant data. Each file is split into a number of
variable length chunks and only chunks that have never been seen before
are compressed and added to the repository.
Optional data encryption
All data can be protected using 256-bit AES_ encryption and data integrity
and authenticity is verified using `HMAC-SHA256`_.
Off-site backups
|project_name| can store data on any remote host accessible over SSH as
long as |project_name| is installed.
Backups mountable as filesystems
Backup archives are :ref:`mountable <attic_mount>` as
`userspace filesystems`_ for easy backup verification and restores.
Glossary
--------
.. _deduplication_def:
Deduplication
Deduplication is a technique for improving storage utilization by
eliminating redundant data.
.. _archive_def:
Archive
An archive is a collection of files along with metadata that include file
permissions, directory structure and various file attributes.
Since each archive in a repository must have a unique name a good naming
convention is ``hostname-YYYY-MM-DD``.
.. _repository_def:
Repository
A repository is a filesystem directory storing data from zero or more
archives. The data in a repository is both deduplicated and
optionally encrypted making it both efficient and safe. Repositories are
created using :ref:`attic_init` and the contents can be listed using
:ref:`attic_list`.
Key file
When a repository is initialized a key file containing a password
protected encryption key is created. It is vital to keep this file safe
since the repository data is totally inaccessible without it.

21
docs/global.rst.inc
View file

@ -1,26 +1,27 @@
.. highlight:: bash
.. |project_name| replace:: ``Attic``
.. |package_dirname| replace:: Attic-|version|
.. |project_name| replace:: ``Borg``
.. |package_dirname| replace:: borgbackup-|version|
.. |package_filename| replace:: |package_dirname|.tar.gz
.. |package_url| replace:: https://pypi.python.org/packages/source/A/Attic/|package_filename|
.. |git_url| replace:: https://github.com/jborg/attic.git
.. |package_url| replace:: https://pypi.python.org/packages/source/b/borgbackup/|package_filename|
.. |git_url| replace:: https://github.com/borgbackup/borg.git
.. _github: https://github.com/borgbackup/borg
.. _issue tracker: https://github.com/borgbackup/borg/issues
.. _deduplication: https://en.wikipedia.org/wiki/Data_deduplication
.. _AES: https://en.wikipedia.org/wiki/Advanced_Encryption_Standard
.. _HMAC-SHA256: http://en.wikipedia.org/wiki/HMAC
.. _SHA256: https://en.wikipedia.org/wiki/SHA-256
.. _PBKDF2: https://en.wikipedia.org/wiki/PBKDF2
.. _ACL: https://en.wikipedia.org/wiki/Access_control_list
.. _github: https://github.com/jborg/attic
.. _libacl: http://savannah.nongnu.org/projects/acl/
.. _OpenSSL: https://www.openssl.org/
.. _Python: http://www.python.org/
.. _Buzhash: https://en.wikipedia.org/wiki/Buzhash
.. _msgpack: http://msgpack.org/
.. _`msgpack-python`: https://pypi.python.org/pypi/msgpack-python/
.. _llfuse: https://pypi.python.org/pypi/llfuse/
.. _homebrew: http://mxcl.github.io/homebrew/
.. _issue tracker: https://github.com/jborg/attic/issues
.. _userspace filesystems: https://en.wikipedia.org/wiki/Filesystem_in_Userspace
.. _librelist: http://librelist.com/
.. _Debian: http://packages.debian.org/attic
.. _Ubuntu: http://packages.ubuntu.com/attic
.. _Arch Linux: https://aur.archlinux.org/packages/attic/
.. _Slackware: http://slackbuilds.org/result/?search=Attic
.. _Cython: http://cython.org/
.. _virtualenv: https://pypi.python.org/pypi/virtualenv/
.. _mailing list discussion about internals: http://librelist.com/browser/attic/2014/5/6/questions-and-suggestions-about-inner-working-of-attic>

66
docs/index.rst
View file

@ -1,70 +1,18 @@
.. include:: global.rst.inc
Welcome to Attic
================
|project_name| is a deduplicating backup program written in Python.
The main goal of |project_name| is to provide an efficient and secure way
to backup data. The data deduplication technique used makes |project_name|
suitable for daily backups since only the changes are stored.
Easy to use
-----------
Initialize a new backup :ref:`repository <repository_def>` and create your
first backup :ref:`archive <archive_def>` in two lines::
$ attic init /somewhere/my-repository.attic
$ attic create /somewhere/my-repository.attic::Monday ~/Documents
$ attic create --stats /somewhere/my-repository.attic::Tuesday ~/Documents
Archive name: Tuesday
Archive fingerprint: 387a5e3f9b0e792e91ce87134b0f4bfe17677d9248cb5337f3fbf3a8e157942a
Start time: Tue Mar 25 12:00:10 2014
End time: Tue Mar 25 12:00:10 2014
Duration: 0.08 seconds
Number of files: 358
Original size Compressed size Deduplicated size
This archive: 57.16 MB 46.78 MB 151.67 kB
All archives: 114.02 MB 93.46 MB 44.81 MB
See the :ref:`quickstart` chapter for a more detailed example.
Easy installation
-----------------
You can use pip to install |project_name| quickly and easily::
$ pip3 install attic
|project_name| is also part of the Debian_, Ubuntu_, `Arch Linux`_ and Slackware_
distributions of GNU/Linux.
Need more help with installing? See :ref:`installation`.
User's Guide
============
Borg Documentation
==================
.. toctree::
:maxdepth: 2
foreword
intro
installation
quickstart
usage
faq
Getting help
============
If you've found a bug or have a concrete feature request, you can add your bug
report or feature request directly to the project's `issue tracker`_. For more
general questions or discussions, a post to the mailing list is preferred.
Mailing list
------------
There is a mailing list for Attic on librelist_ that you can use for feature
requests and general discussions about Attic. A mailing list archive is
available `here <http://librelist.com/browser/attic/>`_.
To subscribe to the list, send an email to attic@librelist.com and reply
to the confirmation mail. Likewise, to unsubscribe, send an email to
attic-unsubscribe@librelist.com and reply to the confirmation mail.
support
changes
internals
development

184
docs/installation.rst
View file

@ -4,58 +4,166 @@
Installation
============
|project_name| requires Python_ 3.2 or above to work. Even though Python 3 is
not the default Python version on most Linux distributions, it is usually
available as an optional install.
|project_name| requires:
Other dependencies:
* `msgpack-python`_ >= 0.1.10
* Python_ >= 3.2
* OpenSSL_ >= 1.0.0
* libacl_
* some python dependencies, see install_requires in setup.py
The OpenSSL version bundled with Mac OS X and FreeBSD is most likey too old.
Newer versions are available from homebrew_ on OS X and from FreeBSD ports.
General notes
-------------
Even though Python 3 is not the default Python version on many systems, it is
usually available as an optional install.
Virtualenv_ can be used to build and install |project_name| without affecting
the system Python or requiring root access.
Important:
if you install into a virtual environment, you need to activate
the virtual env first (``source borg-env/bin/activate``).
Alternatively, directly run ``borg-env/bin/borg`` (or symlink that into some
directory that is in your PATH so you can just run ``borg``).
The llfuse_ python package is also required if you wish to mount an
archive as a FUSE filesystem.
archive as a FUSE filesystem. Only FUSE >= 2.8.0 can support llfuse.
Virtualenv_ can be used to build and install |project_name|
without affecting the system Python or requiring root access.
You only need Cython to compile the .pyx files to the respective .c files
when using |project_name| code from git. For |project_name| releases, the .c
files will be bundled, so you won't need Cython to install a release.
Platform notes
--------------
FreeBSD: You may need to get a recent enough OpenSSL version from FreeBSD ports.
Mac OS X: You may need to get a recent enough OpenSSL version from homebrew_.
Mac OS X: You need OS X FUSE >= 3.0.
Debian / Ubuntu installation (from git)
---------------------------------------
Note: this uses latest, unreleased development code from git.
While we try not to break master, there are no guarantees on anything.
Some of the steps detailled below might be useful also for non-git installs.
.. parsed-literal::
# Python 3.x (>= 3.2) + Headers, Py Package Installer
apt-get install python3 python3-dev python3-pip
# we need OpenSSL + Headers for Crypto
apt-get install libssl-dev openssl
# ACL support Headers + Library
apt-get install libacl1-dev libacl1
# if you do not have gcc / make / etc. yet
apt-get install build-essential
# optional: FUSE support - to mount backup archives
# in case you get complaints about permission denied on /etc/fuse.conf:
# on ubuntu this means your user is not in the "fuse" group. just add
# yourself there, log out and log in again.
apt-get install libfuse-dev fuse
# optional: for unit testing
apt-get install fakeroot
# get |project_name| from github, install it
git clone |git_url|
apt-get install python-virtualenv
virtualenv --python=python3 borg-env
source borg-env/bin/activate # always before using!
# install borg + dependencies into virtualenv
pip install cython # compile .pyx -> .c
pip install tox pytest # optional, for running unit tests
pip install sphinx # optional, to build the docs
pip install llfuse # optional, for FUSE support
cd borg
pip install -e . # in-place editable mode
# optional: run all the tests, on all supported Python versions
fakeroot -u tox
Korora / Fedora 21 installation (from git)
------------------------------------------
Note: this uses latest, unreleased development code from git.
While we try not to break master, there are no guarantees on anything.
Some of the steps detailled below might be useful also for non-git installs.
.. parsed-literal::
# Python 3.x (>= 3.2) + Headers, Py Package Installer
sudo dnf install python3 python3-devel python3-pip
# we need OpenSSL + Headers for Crypto
sudo dnf install openssl-devel openssl
# ACL support Headers + Library
sudo dnf install libacl-devel libacl
# optional: FUSE support - to mount backup archives
sudo dnf install fuse-devel fuse
# optional: for unit testing
sudo dnf install fakeroot
# get |project_name| from github, install it
git clone |git_url|
dnf install python3-virtualenv
virtualenv --python=python3 borg-env
source borg-env/bin/activate # always before using!
# install borg + dependencies into virtualenv
pip install cython # compile .pyx -> .c
pip install tox pytest # optional, for running unit tests
pip install sphinx # optional, to build the docs
pip install llfuse # optional, for FUSE support
cd borg
pip install -e . # in-place editable mode
# optional: run all the tests, on all supported Python versions
fakeroot -u tox
Cygwin (from git)
-----------------
Please note that running under cygwin is rather experimental, stuff has been
tested with CygWin (x86-64) v2.1.0.
You'll need at least (use the cygwin installer to fetch/install these):
Installing from PyPI using pip
------------------------------
::
$ pip3 install Attic
python3
python3-setuptools
python3-cython
binutils
gcc-core
git
libopenssl
make
openssh
openssl-devel
Installing from source tarballs
-------------------------------
.. parsed-literal::
You can then install ``pip`` and ``virtualenv``:
$ curl -O :targz_url:`Attic`
$ tar -xvzf |package_filename|
$ cd |package_dirname|
$ python setup.py install
::
Installing from git
-------------------
.. parsed-literal::
easy_install-3.4 pip
pip install virtualenv
$ git clone |git_url|
$ cd attic
$ python setup.py install
And now continue as for Linux (see above).
Please note that when installing from git, Cython_ is required to generate some files that
are normally bundled with the release tarball.
In case that creation of the virtual env fails, try deleting this file:
Packages
--------
::
|project_name| is also part of the Debian_, Ubuntu_, `Arch Linux`_ and Slackware_
distributions of GNU/Linux.
/usr/lib/python3.4/__pycache__/platform.cpython-34.pyc
Standalone binaries
-------------------
Prebuilt standalone binaries that work on
most Linux systems can be found :artifacts:`here <>`.

391
docs/internals.rst Normal file
View file

@ -0,0 +1,391 @@
.. include:: global.rst.inc
.. _internals:
Internals
=========
This page documents the internal data structures and storage
mechanisms of |project_name|. It is partly based on `mailing list
discussion about internals`_ and also on static code analysis.
It may not be exactly up to date with the current source code.
Repository and Archives
-----------------------
|project_name| stores its data in a `Repository`. Each repository can
hold multiple `Archives`, which represent individual backups that
contain a full archive of the files specified when the backup was
performed. Deduplication is performed across multiple backups, both on
data and metadata, using `Chunks` created by the chunker using the Buzhash_
algorithm.
Each repository has the following file structure:
README
simple text file telling that this is a |project_name| repository
config
repository configuration
data/
directory where the actual data is stored
hints.%d
hints for repository compaction
index.%d
repository index
lock.roster and lock.exclusive/*
used by the locking system to manage shared and exclusive locks
Config file
-----------
Each repository has a ``config`` file which which is a ``INI``-style file
and looks like this::
[repository]
version = 1
segments_per_dir = 10000
max_segment_size = 5242880
id = 57d6c1d52ce76a836b532b0e42e677dec6af9fca3673db511279358828a21ed6
This is where the ``repository.id`` is stored. It is a unique
identifier for repositories. It will not change if you move the
repository around so you can make a local transfer then decide to move
the repository to another (even remote) location at a later time.
Keys
----
The key to address the key/value store is usually computed like this:
key = id = id_hash(unencrypted_data)
The id_hash function is:
* sha256 (no encryption keys available)
* hmac-sha256 (encryption keys available)
Segments and archives
---------------------
A |project_name| repository is a filesystem based transactional key/value
store. It makes extensive use of msgpack_ to store data and, unless
otherwise noted, data is stored in msgpack_ encoded files.
Objects referenced by a key are stored inline in files (`segments`) of approx.
5MB size in numbered subdirectories of ``repo/data``.
They contain:
* header size
* crc
* size
* tag
* key
* data
Segments are built locally, and then uploaded. Those files are
strictly append-only and modified only once.
Tag is either ``PUT``, ``DELETE``, or ``COMMIT``. A segment file is
basically a transaction log where each repository operation is
appended to the file. So if an object is written to the repository a
``PUT`` tag is written to the file followed by the object id and
data. If an object is deleted a ``DELETE`` tag is appended
followed by the object id. A ``COMMIT`` tag is written when a
repository transaction is committed. When a repository is opened any
``PUT`` or ``DELETE`` operations not followed by a ``COMMIT`` tag are
discarded since they are part of a partial/uncommitted transaction.
The manifest
------------
The manifest is an object with an all-zero key that references all the
archives.
It contains:
* version
* list of archive infos
* timestamp
* config
Each archive info contains:
* name
* id
* time
It is the last object stored, in the last segment, and is replaced
each time.
The archive metadata does not contain the file items directly. Only
references to other objects that contain that data. An archive is an
object that contains:
* version
* name
* list of chunks containing item metadata
* cmdline
* hostname
* username
* time
Each item represents a file, directory or other fs item and is stored as an
``item`` dictionary that contains:
* path
* list of data chunks
* user
* group
* uid
* gid
* mode (item type + permissions)
* source (for links)
* rdev (for devices)
* mtime
* xattrs
* acl
* bsdfiles
``ctime`` (change time) is not stored because there is no API to set
it and it is reset every time an inode's metadata is changed.
All items are serialized using msgpack and the resulting byte stream
is fed into the same chunker used for regular file data and turned
into deduplicated chunks. The reference to these chunks is then added
to the archive metadata.
A chunk is stored as an object as well, of course.
Chunks
------
The |project_name| chunker uses a rolling hash computed by the Buzhash_ algorithm.
It triggers (chunks) when the last HASH_MASK_BITS bits of the hash are zero,
producing chunks of 2^HASH_MASK_BITS Bytes on average.
create --chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE
can be used to tune the chunker parameters, the default is:
- CHUNK_MIN_EXP = 10 (minimum chunk size = 2^10 B = 1 kiB)
- CHUNK_MAX_EXP = 23 (maximum chunk size = 2^23 B = 8 MiB)
- HASH_MASK_BITS = 16 (statistical medium chunk size ~= 2^16 B = 64 kiB)
- HASH_WINDOW_SIZE = 4095 [B] (`0xFFF`)
The default parameters are OK for relatively small backup data volumes and
repository sizes and a lot of available memory (RAM) and disk space for the
chunk index. If that does not apply, you are advised to tune these parameters
to keep the chunk count lower than with the defaults.
The buzhash table is altered by XORing it with a seed randomly generated once
for the archive, and stored encrypted in the keyfile. This is to prevent chunk
size based fingerprinting attacks on your encrypted repo contents (to guess
what files you have based on a specific set of chunk sizes).
Indexes / Caches
----------------
The files cache is stored in ``cache/files`` and is indexed on the
``file path hash``. At backup time, it is used to quickly determine whether we
need to chunk a given file (or whether it is unchanged and we already have all
its pieces).
It contains:
* age
* file inode number
* file size
* file mtime_ns
* file content chunk hashes
The inode number is stored to make sure we distinguish between
different files, as a single path may not be unique across different
archives in different setups.
The files cache is stored as a python associative array storing
python objects, which generates a lot of overhead.
The chunks cache is stored in ``cache/chunks`` and is indexed on the
``chunk id_hash``. It is used to determine whether we already have a specific
chunk, to count references to it and also for statistics.
It contains:
* reference count
* size
* encrypted/compressed size
The repository index is stored in ``repo/index.%d`` and is indexed on the
``chunk id_hash``. It is used to determine a chunk's location in the repository.
It contains:
* segment (that contains the chunk)
* offset (where the chunk is located in the segment)
The repository index file is random access.
Hints are stored in a file (``repo/hints.%d``).
It contains:
* version
* list of segments
* compact
hints and index can be recreated if damaged or lost using ``check --repair``.
The chunks cache and the repository index are stored as hash tables, with
only one slot per bucket, but that spreads the collisions to the following
buckets. As a consequence the hash is just a start position for a linear
search, and if the element is not in the table the index is linearly crossed
until an empty bucket is found.
When the hash table is almost full at 90%, its size is doubled. When it's
almost empty at 25%, its size is halved. So operations on it have a variable
complexity between constant and linear with low factor, and memory overhead
varies between 10% and 300%.
Indexes / Caches memory usage
-----------------------------
Here is the estimated memory usage of |project_name|:
chunk_count ~= total_file_size / 2 ^ HASH_MASK_BITS
repo_index_usage = chunk_count * 40
chunks_cache_usage = chunk_count * 44
files_cache_usage = total_file_count * 240 + chunk_count * 80
mem_usage ~= repo_index_usage + chunks_cache_usage + files_cache_usage
= chunk_count * 164 + total_file_count * 240
All units are Bytes.
It is assuming every chunk is referenced exactly once (if you have a lot of
duplicate chunks, you will have less chunks than estimated above).
It is also assuming that typical chunk size is 2^HASH_MASK_BITS (if you have
a lot of files smaller than this statistical medium chunk size, you will have
more chunks than estimated above, because 1 file is at least 1 chunk).
If a remote repository is used the repo index will be allocated on the remote side.
E.g. backing up a total count of 1Mi files with a total size of 1TiB.
a) with create --chunker-params 10,23,16,4095 (default):
mem_usage = 2.8GiB
b) with create --chunker-params 10,23,20,4095 (custom):
mem_usage = 0.4GiB
Note: there is also the --no-files-cache option to switch off the files cache.
You'll save some memory, but it will need to read / chunk all the files then as
it can not skip unmodified files then.
Encryption
----------
AES_ is used in CTR mode (so no need for padding). A 64bit initialization
vector is used, a `HMAC-SHA256`_ is computed on the encrypted chunk with a
random 64bit nonce and both are stored in the chunk.
The header of each chunk is : ``TYPE(1)`` + ``HMAC(32)`` + ``NONCE(8)`` + ``CIPHERTEXT``.
Encryption and HMAC use two different keys.
In AES CTR mode you can think of the IV as the start value for the counter.
The counter itself is incremented by one after each 16 byte block.
The IV/counter is not required to be random but it must NEVER be reused.
So to accomplish this |project_name| initializes the encryption counter to be
higher than any previously used counter value before encrypting new data.
To reduce payload size, only 8 bytes of the 16 bytes nonce is saved in the
payload, the first 8 bytes are always zeros. This does not affect security but
limits the maximum repository capacity to only 295 exabytes (2**64 * 16 bytes).
Encryption keys are either derived from a passphrase or kept in a key file.
The passphrase is passed through the ``BORG_PASSPHRASE`` environment variable
or prompted for interactive usage.
Key files
---------
When initialized with the ``init -e keyfile`` command, |project_name|
needs an associated file in ``$HOME/.borg/keys`` to read and write
the repository. The format is based on msgpack_, base64 encoding and
PBKDF2_ SHA256 hashing, which is then encoded again in a msgpack_.
The internal data structure is as follows:
version
currently always an integer, 1
repository_id
the ``id`` field in the ``config`` ``INI`` file of the repository.
enc_key
the key used to encrypt data with AES (256 bits)
enc_hmac_key
the key used to HMAC the encrypted data (256 bits)
id_key
the key used to HMAC the plaintext chunk data to compute the chunk's id
chunk_seed
the seed for the buzhash chunking table (signed 32 bit integer)
Those fields are processed using msgpack_. The utf-8 encoded passphrase
is processed with PBKDF2_ (SHA256_, 100000 iterations, random 256 bit salt)
to give us a derived key. The derived key is 256 bits long.
A `HMAC-SHA256`_ checksum of the above fields is generated with the derived
key, then the derived key is also used to encrypt the above pack of fields.
Then the result is stored in a another msgpack_ formatted as follows:
version
currently always an integer, 1
salt
random 256 bits salt used to process the passphrase
iterations
number of iterations used to process the passphrase (currently 100000)
algorithm
the hashing algorithm used to process the passphrase and do the HMAC
checksum (currently the string ``sha256``)
hash
the HMAC of the encrypted derived key
data
the derived key, encrypted with AES over a PBKDF2_ SHA256 key
described above
The resulting msgpack_ is then encoded using base64 and written to the
key file, wrapped using the standard ``textwrap`` module with a header.
The header is a single line with a MAGIC string, a space and a hexadecimal
representation of the repository id.
Compression
-----------
|project_name| currently always pipes all data through a zlib compressor which
supports compression levels 0 (no compression, fast) to 9 (high compression, slow).
See ``borg create --help`` about how to specify the compression level and its default.
Note: zlib level 0 creates a little bit more output data than it gets as input,
due to zlib protocol overhead.

7
docs/intro.rst Normal file
View file

@ -0,0 +1,7 @@
.. include:: global.rst.inc
.. _foreword:
Introduction
============
.. include:: ../README.rst

116
docs/misc/create_chunker-params.txt Normal file
View file

@ -0,0 +1,116 @@
About borg create --chunker-params
==================================
--chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE
CHUNK_MIN_EXP and CHUNK_MAX_EXP give the exponent N of the 2^N minimum and
maximum chunk size. Required: CHUNK_MIN_EXP < CHUNK_MAX_EXP.
Defaults: 10 (2^10 == 1KiB) minimum, 23 (2^23 == 8MiB) maximum.
HASH_MASK_BITS is the number of least-significant bits of the rolling hash
that need to be zero to trigger a chunk cut.
Recommended: CHUNK_MIN_EXP + X <= HASH_MASK_BITS <= CHUNK_MAX_EXP - X, X >= 2
(this allows the rolling hash some freedom to make its cut at a place
determined by the windows contents rather than the min/max. chunk size).
Default: 16 (statistically, chunks will be about 2^16 == 64kiB in size)
HASH_WINDOW_SIZE: the size of the window used for the rolling hash computation.
Default: 4095B
Trying it out
=============
I backed up a VM directory to demonstrate how different chunker parameters
influence repo size, index size / chunk count, compression, deduplication.
repo-sm: ~64kiB chunks (16 bits chunk mask), min chunk size 1kiB (2^10B)
(these are attic / borg 0.23 internal defaults)
repo-lg: ~1MiB chunks (20 bits chunk mask), min chunk size 64kiB (2^16B)
repo-xl: 8MiB chunks (2^23B max chunk size), min chunk size 64kiB (2^16B).
The chunk mask bits was set to 31, so it (almost) never triggers.
This degrades the rolling hash based dedup to a fixed-offset dedup
as the cutting point is now (almost) always the end of the buffer
(at 2^23B == 8MiB).
The repo index size is an indicator for the RAM needs of Borg.
In this special case, the total RAM needs are about 2.1x the repo index size.
You see index size of repo-sm is 16x larger than of repo-lg, which corresponds
to the ratio of the different target chunk sizes.
Note: RAM needs were not a problem in this specific case (37GB data size).
But just imagine, you have 37TB of such data and much less than 42GB RAM,
then you'ld definitely want the "lg" chunker params so you only need
2.6GB RAM. Or even bigger chunks than shown for "lg" (see "xl").
You also see compression works better for larger chunks, as expected.
Duplication works worse for larger chunks, also as expected.
small chunks
============
$ borg info /extra/repo-sm::1
Command line: /home/tw/w/borg-env/bin/borg create --chunker-params 10,23,16,4095 /extra/repo-sm::1 /home/tw/win
Number of files: 3
Original size Compressed size Deduplicated size
This archive: 37.12 GB 14.81 GB 12.18 GB
All archives: 37.12 GB 14.81 GB 12.18 GB
Unique chunks Total chunks
Chunk index: 378374 487316
$ ls -l /extra/repo-sm/index*
-rw-rw-r-- 1 tw tw 20971538 Jun 20 23:39 index.2308
$ du -sk /extra/repo-sm
11930840 /extra/repo-sm
large chunks
============
$ borg info /extra/repo-lg::1
Command line: /home/tw/w/borg-env/bin/borg create --chunker-params 16,23,20,4095 /extra/repo-lg::1 /home/tw/win
Number of files: 3
Original size Compressed size Deduplicated size
This archive: 37.10 GB 14.60 GB 13.38 GB
All archives: 37.10 GB 14.60 GB 13.38 GB
Unique chunks Total chunks
Chunk index: 25889 29349
$ ls -l /extra/repo-lg/index*
-rw-rw-r-- 1 tw tw 1310738 Jun 20 23:10 index.2264
$ du -sk /extra/repo-lg
13073928 /extra/repo-lg
xl chunks
=========
(borg-env)tw@tux:~/w/borg$ borg info /extra/repo-xl::1
Command line: /home/tw/w/borg-env/bin/borg create --chunker-params 16,23,31,4095 /extra/repo-xl::1 /home/tw/win
Number of files: 3
Original size Compressed size Deduplicated size
This archive: 37.10 GB 14.59 GB 14.59 GB
All archives: 37.10 GB 14.59 GB 14.59 GB
Unique chunks Total chunks
Chunk index: 4319 4434
$ ls -l /extra/repo-xl/index*
-rw-rw-r-- 1 tw tw 327698 Jun 21 00:52 index.2011
$ du -sk /extra/repo-xl/
14253464 /extra/repo-xl/

130
docs/misc/create_compression.txt Normal file
View file

@ -0,0 +1,130 @@
data compression
================
borg create --compression N repo::archive data
Currently, borg only supports zlib compression. There are plans to expand this
to other, faster or better compression algorithms in the future.
N == 0 -> zlib level 0 == very quick, no compression
N == 1 -> zlib level 1 == quick, low compression
...
N == 9 -> zlib level 9 == slow, high compression
Measurements made on a Haswell Ultrabook, SSD storage, Linux.
Example 1: lots of relatively small text files (linux kernel src)
-----------------------------------------------------------------
N == 1 does a good job here, it saves the additional time needed for
compression because it needs to store less into storage (see N == 0).
N == 6 is also quite ok, a little slower, a little less repo size.
6 was the old default of borg.
High compression levels only give a little more compression, but take a lot
of cpu time.
$ borg create --stats --compression 0
------------------------------------------------------------------------------
Duration: 50.40 seconds
Number of files: 72890
Original size Compressed size Deduplicated size
This archive: 1.17 GB 1.18 GB 1.01 GB
Unique chunks Total chunks
Chunk index: 70263 82309
------------------------------------------------------------------------------
$ borg create --stats --compression 1
------------------------------------------------------------------------------
Duration: 49.29 seconds
Number of files: 72890
Original size Compressed size Deduplicated size
This archive: 1.17 GB 368.62 MB 295.22 MB
Unique chunks Total chunks
Chunk index: 70280 82326
------------------------------------------------------------------------------
$ borg create --stats --compression 5
------------------------------------------------------------------------------
Duration: 59.99 seconds
Number of files: 72890
Original size Compressed size Deduplicated size
This archive: 1.17 GB 331.70 MB 262.20 MB
Unique chunks Total chunks
Chunk index: 70290 82336
------------------------------------------------------------------------------
$ borg create --stats --compression 6
------------------------------------------------------------------------------
Duration: 1 minutes 13.64 seconds
Number of files: 72890
Original size Compressed size Deduplicated size
This archive: 1.17 GB 328.79 MB 259.56 MB
Unique chunks Total chunks
Chunk index: 70279 82325
------------------------------------------------------------------------------
$ borg create --stats --compression 9
------------------------------------------------------------------------------
Duration: 3 minutes 1.58 seconds
Number of files: 72890
Original size Compressed size Deduplicated size
This archive: 1.17 GB 326.57 MB 257.57 MB
Unique chunks Total chunks
Chunk index: 70292 82338
------------------------------------------------------------------------------
Example 2: large VM disk file (sparse file)
-------------------------------------------
The file's directory size is 80GB, but a lot of it is sparse (and reads as
zeros).
$ borg create --stats --compression 0
------------------------------------------------------------------------------
Duration: 13 minutes 48.47 seconds
Number of files: 1
Original size Compressed size Deduplicated size
This archive: 80.54 GB 80.55 GB 10.87 GB
Unique chunks Total chunks
Chunk index: 147307 177109
------------------------------------------------------------------------------
$ borg create --stats --compression 1
------------------------------------------------------------------------------
Duration: 15 minutes 31.34 seconds
Number of files: 1
Original size Compressed size Deduplicated size
This archive: 80.54 GB 6.68 GB 5.67 GB
Unique chunks Total chunks
Chunk index: 147309 177111
------------------------------------------------------------------------------
$ borg create --stats --compression 6
------------------------------------------------------------------------------
Duration: 18 minutes 57.54 seconds
Number of files: 1
Original size Compressed size Deduplicated size
This archive: 80.54 GB 6.19 GB 5.44 GB
Unique chunks Total chunks
Chunk index: 147307 177109
------------------------------------------------------------------------------

62
docs/quickstart.rst
View file

@ -13,16 +13,16 @@ A step by step example
1. Before a backup can be made a repository has to be initialized::
$ attic init /somewhere/my-repository.attic
$ borg init /mnt/backup
2. Backup the ``~/src`` and ``~/Documents`` directories into an archive called
*Monday*::
$ attic create /somewhere/my-repository.attic::Monday ~/src ~/Documents
$ borg create /mnt/backup::Monday ~/src ~/Documents
3. The next day create a new archive called *Tuesday*::
$ attic create --stats /somewhere/my-repository.attic::Tuesday ~/src ~/Documents
$ borg create --stats /mnt/backup::Tuesday ~/src ~/Documents
This backup will be a lot quicker and a lot smaller since only new never
before seen data is stored. The ``--stats`` option causes |project_name| to
@ -42,44 +42,43 @@ A step by step example
4. List all archives in the repository::
$ attic list /somewhere/my-repository.attic
$ borg list /mnt/backup
Monday Mon Mar 24 11:59:35 2014
Tuesday Tue Mar 25 12:00:10 2014
5. List the contents of the *Monday* archive::
$ attic list /somewhere/my-repository.attic::Monday
$ borg list /mnt/backup::Monday
drwxr-xr-x user group 0 Jan 06 15:22 home/user/Documents
-rw-r--r-- user group 7961 Nov 17 2012 home/user/Documents/Important.doc
...
6. Restore the *Monday* archive::
$ attic extract /somwhere/my-repository.attic::Monday
$ borg extract /mnt/backup::Monday
7. Recover disk space by manually deleting the *Monday* archive::
$ attic delete /somwhere/my-backup.attic::Monday
$ borg delete /mnt/backup::Monday
.. Note::
Attic is quiet by default. Add the ``-v`` or ``--verbose`` option to
Borg is quiet by default. Add the ``-v`` or ``--verbose`` option to
get progress reporting during command execution.
Automating backups
------------------
The following example script backs up ``/home`` and
``/var/www`` to a remote server. The script also uses the
:ref:`attic_prune` subcommand to maintain a certain number
of old archives::
The following example script backs up ``/home`` and ``/var/www`` to a remote
server. The script also uses the :ref:`borg_prune` subcommand to maintain a
certain number of old archives::
#!/bin/sh
REPOSITORY=username@remoteserver.com:repository.attic
REPOSITORY=username@remoteserver.com:backup
# Backup all of /home and /var/www except a few
# excluded directories
attic create --stats \
$REPOSITORY::hostname-`date +%Y-%m-%d` \
borg create --stats \
$REPOSITORY::`hostname`-`date +%Y-%m-%d` \
/home \
/var/www \
--exclude /home/*/.cache \
@ -88,7 +87,7 @@ of old archives::
# Use the `prune` subcommand to maintain 7 daily, 4 weekly
# and 6 monthly archives.
attic prune -v $REPOSITORY --keep-daily=7 --keep-weekly=4 --keep-monthly=6
borg prune -v $REPOSITORY --keep-daily=7 --keep-weekly=4 --keep-monthly=6
.. _encrypted_repos:
@ -97,13 +96,13 @@ Repository encryption
Repository encryption is enabled at repository creation time::
$ attic init --encryption=passphrase|keyfile PATH
$ borg init --encryption=passphrase|keyfile PATH
When repository encryption is enabled all data is encrypted using 256-bit AES_
encryption and the integrity and authenticity is verified using `HMAC-SHA256`_.
All data is encrypted before being written to the repository. This means that
an attacker that manages to compromise the host containing an encrypted
an attacker who manages to compromise the host containing an encrypted
archive will not be able to access any of the data.
|project_name| supports two different methods to derive the AES and HMAC keys.
@ -116,11 +115,11 @@ Passphrase based encryption
.. Note::
For automated backups the passphrase can be specified using the
`ATTIC_PASSPHRASE` environment variable.
`BORG_PASSPHRASE` environment variable.
Key file based encryption
This method generates random keys at repository initialization time that
are stored in a password protected file in the ``~/.attic/keys/`` directory.
are stored in a password protected file in the ``~/.borg/keys/`` directory.
The key file is a printable text file. This method is secure and suitable
for automated backups.
@ -138,20 +137,25 @@ Remote repositories
host is accessible using SSH. This is fastest and easiest when |project_name|
is installed on the remote host, in which case the following syntax is used::
$ attic init user@hostname:repository.attic
$ borg init user@hostname:/mnt/backup
or::
$ attic init ssh://user@hostname:port/repository.attic
$ borg init ssh://user@hostname:port//mnt/backup
If it is not possible to install |project_name| on the remote host,
Remote operations over SSH can be automated with SSH keys. You can restrict the
use of the SSH keypair by prepending a forced command to the SSH public key in
the remote server's authorized_keys file. Only the forced command will be run
when the key authenticates a connection. This example will start |project_name| in server
mode, and limit the |project_name| server to a specific filesystem path::
command="borg serve --restrict-to-path /mnt/backup" ssh-rsa AAAAB3[...]
If it is not possible to install |project_name| on the remote host,
it is still possible to use the remote host to store a repository by
mounting the remote filesystem, for example, using sshfs::
$ sshfs user@hostname:/path/to/folder /tmp/mymountpoint
$ attic init /tmp/mymountpoint/repository.attic
$ fusermount -u /tmp/mymountpoint
$ sshfs user@hostname:/mnt /mnt
$ borg init /mnt/backup
$ fusermount -u /mnt
However, be aware that sshfs doesn't fully implement POSIX locks, so
you must be sure to not have two processes trying to access the same
repository at the same time.

34
docs/support.rst Normal file
View file

@ -0,0 +1,34 @@
.. include:: global.rst.inc
.. _support:
Support
=======
Issue Tracker
-------------
If you've found a bug or have a concrete feature request, please create a new
ticket on the project's `issue tracker`_ (after checking whether someone else
already has reported the same thing).
For more general questions or discussions, IRC or mailing list are preferred.
IRC
---
Join us on channel #borgbackup on chat.freenode.net.
As usual on IRC, just ask or tell directly and then patiently wait for replies.
Stay connected.
Mailing list
------------
There is a mailing list for Borg on librelist_ that you can use for feature
requests and general discussions about Borg. A mailing list archive is
available `here <http://librelist.com/browser/borgbackup/>`_.
To subscribe to the list, send an email to borgbackup@librelist.com and reply
to the confirmation mail.
To unsubscribe, send an email to borgbackup-unsubscribe@librelist.com and reply
to the confirmation mail.

12
docs/update_usage.sh
View file

@ -2,12 +2,12 @@
if [ ! -d usage ]; then
mkdir usage
fi
for cmd in change-passphrase check create delete extract info init list mount prune; do
for cmd in change-passphrase check create delete extract info init list mount prune serve; do
FILENAME="usage/$cmd.rst.inc"
LINE=`echo -n attic $cmd | tr 'a-z- ' '-'`
echo -e ".. _attic_$cmd:\n" > $FILENAME
echo -e "attic $cmd\n$LINE\n::\n\n" >> $FILENAME
attic help $cmd --usage-only | sed -e 's/^/ /' >> $FILENAME
LINE=`echo -n borg $cmd | tr 'a-z- ' '-'`
echo -e ".. _borg_$cmd:\n" > $FILENAME
echo -e "borg $cmd\n$LINE\n::\n\n" >> $FILENAME
borg help $cmd --usage-only | sed -e 's/^/ /' >> $FILENAME
echo -e "\nDescription\n~~~~~~~~~~~\n" >> $FILENAME
attic help $cmd --epilog-only >> $FILENAME
borg help $cmd --epilog-only >> $FILENAME
done

199
docs/usage.rst
View file

@ -15,6 +15,90 @@ Like most UNIX commands |project_name| is quiet by default but the ``-v`` or
``--verbose`` option can be used to get the program to output more status
messages as it is processing.
Return codes
------------
|project_name| can exit with the following return codes (rc):
::
0 no error, normal termination
1 some error occurred (this can be a complete or a partial failure)
128+N killed by signal N (e.g. 137 == kill -9)
Note: we are aware that more distinct return codes might be useful, but it is
not clear yet which return codes should be used for which precise conditions.
See issue #61 for a discussion about that. Depending on the outcome of the
discussion there, return codes may change in future (the only thing rather sure
is that 0 will always mean some sort of success and "not 0" will always mean
some sort of warning / error / failure - but the definition of success might
change).
Environment Variables
---------------------
|project_name| uses some environment variables for automation:
::
Specifying a passphrase:
BORG_PASSPHRASE : When set, use the value to answer the passphrase question for encrypted repositories.
Some "yes" sayers (if set, they automatically confirm that you really want to do X even if there is that warning):
BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK : For "Warning: Attempting to access a previously unknown unencrypted repository"
BORG_RELOCATED_REPO_ACCESS_IS_OK : For "Warning: The repository at location ... was previously located at ..."
BORG_CHECK_I_KNOW_WHAT_I_AM_DOING : For "Warning: 'check --repair' is an experimental feature that might result in data loss."
Directories:
BORG_KEYS_DIR : Default to '~/.borg/keys'. This directory contains keys for encrypted repositories.
BORG_CACHE_DIR : Default to '~/.cache/borg'. This directory contains the local cache and might need a lot
of space for dealing with big repositories).
Building:
BORG_OPENSSL_PREFIX : Adds given OpenSSL header file directory to the default locations (setup.py).
General:
TMPDIR : where temporary files are stored (might need a lot of temporary space for some operations)
Please note:
- be very careful when using the "yes" sayers, the warnings with prompt exist for your / your data's security/safety
- also be very careful when putting your passphrase into a script, make sure it has appropriate file permissions
(e.g. mode 600, root:root).
Resource Usage
--------------
|project_name| might use a lot of resources depending on the size of the data set it is dealing with.
CPU: it won't go beyond 100% of 1 core as the code is currently single-threaded.
Memory (RAM): the chunks index and the files index are read into memory for performance reasons.
Temporary files: reading data and metadata from a FUSE mounted repository will consume about the same space as the
deduplicated chunks used to represent them in the repository.
Cache files: chunks index and files index (plus a compressed collection of single-archive chunk indexes).
Chunks index: proportional to the amount of data chunks in your repo. lots of small chunks in your repo implies a big
chunks index. you may need to tweak the chunker params (see create options) if you have a lot of data and
you want to keep the chunks index at some reasonable size.
Files index: proportional to the amount of files in your last backup. can be switched off (see create options), but
next backup will be much slower if you do.
Network: if your repository is remote, all deduplicated (and optionally compressed/encrypted) data of course has to go
over the connection (ssh: repo url). if you use a locally mounted network filesystem, additionally some copy
operations used for transaction support also go over the connection. if you backup multiple sources to one
target repository, additional traffic happens for cache resynchronization.
In case you are interested in more details, please read the internals documentation.
.. include:: usage/init.rst.inc
Examples
@ -22,13 +106,48 @@ Examples
::
# Local repository
$ attic init /data/mybackuprepo.attic
$ borg init /mnt/backup
# Remote repository
$ attic init user@hostname:mybackuprepo.attic
# Remote repository (accesses a remote borg via ssh)
$ borg init user@hostname:backup
# Encrypted remote repository
$ attic init --encryption=passphrase user@hostname:mybackuprepo.attic
# Encrypted remote repository, store the key in the repo
$ borg init --encryption=repokey user@hostname:backup
# Encrypted remote repository, store the key your home dir
$ borg init --encryption=keyfile user@hostname:backup
Important notes about encryption:
Use encryption! Repository encryption protects you e.g. against the case that
an attacker has access to your backup repository.
But be careful with the key / the passphrase:
``--encryption=passphrase`` is DEPRECATED and will be removed in next major release.
This mode has very fundamental, unfixable problems (like you can never change
your passphrase or the pbkdf2 iteration count for an existing repository, because
the encryption / decryption key is directly derived from the passphrase).
If you want "passphrase-only" security, just use the ``repokey`` mode. The key will
be stored inside the repository (in its "config" file). In above mentioned
attack scenario, the attacker will have the key (but not the passphrase).
If you want "passphrase and having-the-key" security, use the ``keyfile`` mode.
The key will be stored in your home directory (in ``.borg/keys``). In the attack
scenario, the attacker who has just access to your repo won't have the key (and
also not the passphrase).
Make a backup copy of the key file (``keyfile`` mode) or repo config file
(``repokey`` mode) and keep it at a safe place, so you still have the key in
case it gets corrupted or lost.
The backup that is encrypted with that key won't help you with that, of course.
Make sure you use a good passphrase. Not too short, not too simple. The real
encryption / decryption key is encrypted with / locked by your passphrase.
If an attacker gets your key, he can't unlock and use it without knowing the
passphrase. In ``repokey`` and ``keyfile`` modes, you can change your passphrase
for existing repos.
.. include:: usage/create.rst.inc
@ -38,17 +157,23 @@ Examples
::
# Backup ~/Documents into an archive named "my-documents"
$ attic create /data/myrepo.attic::my-documents ~/Documents
$ borg create /mnt/backup::my-documents ~/Documents
# Backup ~/Documents and ~/src but exclude pyc files
$ attic create /data/myrepo.attic::my-files \
~/Documents \
~/src \
$ borg create /mnt/backup::my-files \
~/Documents \
~/src \
--exclude '*.pyc'
# Backup the root filesystem into an archive named "root-YYYY-MM-DD"
NAME="root-`date +%Y-%m-%d`"
$ attic create /data/myrepo.attic::$NAME / --do-not-cross-mountpoints
$ borg create /mnt/backup::$NAME / --do-not-cross-mountpoints
# Backup huge files with little chunk management overhead
$ borg create --chunker-params 19,23,21,4095 /mnt/backup::VMs /srv/VMs
# Backup a raw device (must not be active/in use/mounted at that time)
$ dd if=/dev/sda bs=10M | borg create /mnt/backup::my-sda -
.. include:: usage/extract.rst.inc
@ -58,16 +183,19 @@ Examples
::
# Extract entire archive
$ attic extract /data/myrepo::my-files
$ borg extract /mnt/backup::my-files
# Extract entire archive and list files while processing
$ attic extract -v /data/myrepo::my-files
$ borg extract -v /mnt/backup::my-files
# Extract the "src" directory
$ attic extract /data/myrepo::my-files home/USERNAME/src
$ borg extract /mnt/backup::my-files home/USERNAME/src
# Extract the "src" directory but exclude object files
$ attic extract /data/myrepo::my-files home/USERNAME/src --exclude '*.o'
$ borg extract /mnt/backup::my-files home/USERNAME/src --exclude '*.o'
Note: currently, extract always writes into the current working directory ("."),
so make sure you ``cd`` to the right place before calling ``borg extract``.
.. include:: usage/check.rst.inc
@ -79,14 +207,14 @@ Examples
~~~~~~~~
::
$ attic list /data/myrepo
$ borg list /mnt/backup
my-files Thu Aug 1 23:33:22 2013
my-documents Thu Aug 1 23:35:43 2013
root-2013-08-01 Thu Aug 1 23:43:55 2013
root-2013-08-02 Fri Aug 2 15:18:17 2013
...
$ attic list /data/myrepo::root-2013-08-02
$ borg list /mnt/backup::root-2013-08-02
drwxr-xr-x root root 0 Jun 05 12:06 .
lrwxrwxrwx root root 0 May 31 20:40 bin -> usr/bin
drwxr-xr-x root root 0 Aug 01 22:08 etc
@ -102,18 +230,18 @@ Examples
::
# Keep 7 end of day and 4 additional end of week archives:
$ attic prune /data/myrepo --keep-daily=7 --keep-weekly=4
$ borg prune /mnt/backup --keep-daily=7 --keep-weekly=4
# Same as above but only apply to archive names starting with "foo":
$ attic prune /data/myrepo --keep-daily=7 --keep-weekly=4 --prefix=foo
$ borg prune /mnt/backup --keep-daily=7 --keep-weekly=4 --prefix=foo
# Keep 7 end of day, 4 additional end of week archives,
# and an end of month archive for every month:
$ attic prune /data/myrepo --keep-daily=7 --keep-weekly=4 --keep-monthly=-1
$ borg prune /mnt/backup --keep-daily=7 --keep-weekly=4 --keep-monthly=-1
# Keep all backups in the last 10 days, 4 additional end of week archives,
# and an end of month archive for every month:
$ attic prune /data/myrepo --keep-within=10d --keep-weekly=4 --keep-monthly=-1
$ borg prune /mnt/backup --keep-within=10d --keep-weekly=4 --keep-monthly=-1
.. include:: usage/info.rst.inc
@ -122,13 +250,13 @@ Examples
~~~~~~~~
::
$ attic info /data/myrepo::root-2013-08-02
$ borg info /mnt/backup::root-2013-08-02
Name: root-2013-08-02
Fingerprint: bc3902e2c79b6d25f5d769b335c5c49331e6537f324d8d3badcb9a0917536dbb
Hostname: myhostname
Username: root
Time: Fri Aug 2 15:18:17 2013
Command line: /usr/bin/attic create --stats /data/myrepo::root-2013-08-02 / --do-not-cross-mountpoints
Command line: /usr/bin/borg create --stats /mnt/backup::root-2013-08-02 / --do-not-cross-mountpoints
Number of files: 147429
Original size: 5344169493 (4.98 GB)
Compressed size: 1748189642 (1.63 GB)
@ -141,7 +269,7 @@ Examples
~~~~~~~~
::
$ attic mount /data/myrepo::root-2013-08-02 /tmp/mymountpoint
$ borg mount /mnt/backup::root-2013-08-02 /tmp/mymountpoint
$ ls /tmp/mymountpoint
bin boot etc lib lib64 mnt opt root sbin srv usr var
$ fusermount -u /tmp/mymountpoint
@ -154,16 +282,29 @@ Examples
::
# Create a key file protected repository
$ attic init --encryption=keyfile /tmp/encrypted-repo
Initializing repository at "/tmp/encrypted-repo"
$ borg init --encryption=keyfile /mnt/backup
Initializing repository at "/mnt/backup"
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Key file "/home/USER/.attic/keys/tmp_encrypted_repo" created.
Key file "/home/USER/.borg/keys/mnt_backup" created.
Keep this file safe. Your data will be inaccessible without it.
# Change key file passphrase
$ attic change-passphrase /tmp/encrypted-repo
Enter passphrase for key file /home/USER/.attic/keys/tmp_encrypted_repo:
$ borg change-passphrase /mnt/backup
Enter passphrase for key file /home/USER/.borg/keys/mnt_backup:
New passphrase:
Enter same passphrase again:
Key file "/home/USER/.attic/keys/tmp_encrypted_repo" updated
Key file "/home/USER/.borg/keys/mnt_backup" updated
.. include:: usage/serve.rst.inc
Examples
~~~~~~~~
::
# Allow an SSH keypair to only run |project_name|, and only have access to /mnt/backup.
# This will help to secure an automated remote backup system.
$ cat ~/.ssh/authorized_keys
command="borg serve --restrict-to-path /mnt/backup" ssh-rsa AAAAB3[...]

5
requirements.d/development.txt Normal file
View file

@ -0,0 +1,5 @@
tox
mock
pytest
pytest-cov<2.0.0
Cython

4
scripts/attic
View file

@ -1,4 +0,0 @@
#!/usr/bin/env python
from attic.archiver import main
main()

8
setup.cfg Normal file
View file

@ -0,0 +1,8 @@
[pytest]
python_files = testsuite/*.py
[flake8]
ignore = E226,F403
max-line-length = 250
exclude = versioneer.py,docs/conf.py,borg/_version.py,build,dist,.git,.idea,.cache
max-complexity = 100

92
setup.py
View file

@ -4,27 +4,27 @@ import sys
from glob import glob
import versioneer
versioneer.versionfile_source = 'attic/_version.py'
versioneer.versionfile_build = 'attic/_version.py'
versioneer.VCS = 'git'
versioneer.style = 'pep440'
versioneer.versionfile_source = 'borg/_version.py'
versioneer.versionfile_build = 'borg/_version.py'
versioneer.tag_prefix = ''
versioneer.parentdir_prefix = 'Attic-' # dirname like 'myproject-1.2.0'
versioneer.parentdir_prefix = 'borgbackup-' # dirname like 'myproject-1.2.0'
min_python = (3, 2)
if sys.version_info < min_python:
print("Attic requires Python %d.%d or later" % min_python)
print("Borg requires Python %d.%d or later" % min_python)
sys.exit(1)
try:
from setuptools import setup, Extension
except ImportError:
from distutils.core import setup, Extension
crypto_source = 'attic/crypto.pyx'
chunker_source = 'attic/chunker.pyx'
hashindex_source = 'attic/hashindex.pyx'
platform_linux_source = 'attic/platform_linux.pyx'
platform_darwin_source = 'attic/platform_darwin.pyx'
platform_freebsd_source = 'attic/platform_freebsd.pyx'
from setuptools import setup, Extension
crypto_source = 'borg/crypto.pyx'
chunker_source = 'borg/chunker.pyx'
hashindex_source = 'borg/hashindex.pyx'
platform_linux_source = 'borg/platform_linux.pyx'
platform_darwin_source = 'borg/platform_darwin.pyx'
platform_freebsd_source = 'borg/platform_freebsd.pyx'
try:
from Cython.Distutils import build_ext
@ -32,14 +32,20 @@ try:
class Sdist(versioneer.cmd_sdist):
def __init__(self, *args, **kwargs):
for src in glob('attic/*.pyx'):
cython_compiler.compile(glob('attic/*.pyx'),
cython_compiler.default_options)
for src in glob('borg/*.pyx'):
cython_compiler.compile(src, cython_compiler.default_options)
versioneer.cmd_sdist.__init__(self, *args, **kwargs)
def make_distribution(self):
self.filelist.extend(['attic/crypto.c', 'attic/chunker.c', 'attic/_chunker.c', 'attic/hashindex.c', 'attic/_hashindex.c', 'attic/platform_linux.c', 'attic/platform_freebsd.c', 'attic/platform_darwin.c'])
super(Sdist, self).make_distribution()
self.filelist.extend([
'borg/crypto.c',
'borg/chunker.c', 'borg/_chunker.c',
'borg/hashindex.c', 'borg/_hashindex.c',
'borg/platform_linux.c',
'borg/platform_freebsd.c',
'borg/platform_darwin.c',
])
super().make_distribution()
except ImportError:
class Sdist(versioneer.cmd_sdist):
@ -54,7 +60,7 @@ except ImportError:
platform_darwin_source = platform_darwin_source.replace('.pyx', '.c')
from distutils.command.build_ext import build_ext
if not all(os.path.exists(path) for path in [crypto_source, chunker_source, hashindex_source, platform_linux_source, platform_freebsd_source]):
raise ImportError('The GIT version of Attic needs Cython. Install Cython or use a released version')
raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version')
def detect_openssl(prefixes):
@ -66,9 +72,9 @@ def detect_openssl(prefixes):
return prefix
possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', '/usr/local/attic', '/opt/local']
if os.environ.get('ATTIC_OPENSSL_PREFIX'):
possible_openssl_prefixes.insert(0, os.environ.get('ATTIC_OPENSSL_PREFIX'))
possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', '/usr/local/borg', '/opt/local']
if os.environ.get('BORG_OPENSSL_PREFIX'):
possible_openssl_prefixes.insert(0, os.environ.get('BORG_OPENSSL_PREFIX'))
ssl_prefix = detect_openssl(possible_openssl_prefixes)
if not ssl_prefix:
raise Exception('Unable to find OpenSSL >= 1.0 headers. (Looked here: {})'.format(', '.join(possible_openssl_prefixes)))
@ -83,27 +89,27 @@ cmdclass = versioneer.get_cmdclass()
cmdclass.update({'build_ext': build_ext, 'sdist': Sdist})
ext_modules = [
Extension('attic.crypto', [crypto_source], libraries=['crypto'], include_dirs=include_dirs, library_dirs=library_dirs),
Extension('attic.chunker', [chunker_source]),
Extension('attic.hashindex', [hashindex_source])
Extension('borg.crypto', [crypto_source], libraries=['crypto'], include_dirs=include_dirs, library_dirs=library_dirs),
Extension('borg.chunker', [chunker_source]),
Extension('borg.hashindex', [hashindex_source])
]
if sys.platform.startswith('linux'):
ext_modules.append(Extension('attic.platform_linux', [platform_linux_source], libraries=['acl']))
ext_modules.append(Extension('borg.platform_linux', [platform_linux_source], libraries=['acl']))
elif sys.platform.startswith('freebsd'):
ext_modules.append(Extension('attic.platform_freebsd', [platform_freebsd_source]))
ext_modules.append(Extension('borg.platform_freebsd', [platform_freebsd_source]))
elif sys.platform == 'darwin':
ext_modules.append(Extension('attic.platform_darwin', [platform_darwin_source]))
ext_modules.append(Extension('borg.platform_darwin', [platform_darwin_source]))
setup(
name='Attic',
name='borgbackup',
version=versioneer.get_version(),
author='Jonas Borgstrom',
author_email='jonas@borgstrom.se',
url='https://attic-backup.org/',
description='Deduplicated backups',
author='The Borg Collective (see AUTHORS file)',
author_email='borgbackup@librelist.com',
url='https://borgbackup.github.io/',
description='Deduplicated, encrypted, authenticated and compressed backups',
long_description=long_description,
license='BSD',
platforms=['Linux', 'MacOS X'],
platforms=['Linux', 'MacOS X', 'FreeBSD', ],
classifiers=[
'Development Status :: 4 - Beta',
'Environment :: Console',
@ -113,12 +119,22 @@ setup(
'Operating System :: MacOS :: MacOS X',
'Operating System :: POSIX :: Linux',
'Programming Language :: Python',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.2',
'Programming Language :: Python :: 3.3',
'Programming Language :: Python :: 3.4',
'Topic :: Security :: Cryptography',
'Topic :: System :: Archiving :: Backup',
],
packages=['attic', 'attic.testsuite'],
scripts=['scripts/attic'],
packages=['borg', 'borg.testsuite'],
entry_points={
'console_scripts': [
'borg = borg.archiver:main',
]
},
cmdclass=cmdclass,
ext_modules=ext_modules,
install_requires=['msgpack-python']
# msgpack pure python data corruption was fixed in 0.4.6.
# Also, we might use some rather recent API features.
install_requires=['msgpack-python>=0.4.6']
)

16
tox.ini
View file

@ -1,10 +1,14 @@
# tox configuration - if you change anything here, run this to verify:
# fakeroot -u tox --recreate
[tox]
envlist = py32, py33, py34
[testenv]
# Change dir to avoid import problem
changedir = docs
commands = {envpython} -m attic.testsuite.run -bv []
[testenv:py32]
deps = mock
# Change dir to avoid import problem for cython code. The directory does
# not really matter, should be just different from the toplevel dir.
changedir = {toxworkdir}
deps = -rrequirements.d/development.txt
commands = py.test --cov=borg --pyargs {posargs:borg.testsuite}
# fakeroot -u needs some env vars:
passenv = *

1232
versioneer.py
View file

File diff suppressed because it is too large Load diff