diff --git a/.coveragerc b/.coveragerc index 620f29fef..7c4ccf9e1 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,6 +5,7 @@ omit = borg/__init__.py borg/__main__.py borg/_version.py + borg/support/*.py [report] exclude_lines = diff --git a/.gitignore b/.gitignore index 5debd74ed..4f7c67672 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ borg.build/ borg.dist/ borg.exe .coverage +.vagrant diff --git a/CHANGES.rst b/CHANGES.rst index eb7b93667..016a55348 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,14 +1,66 @@ Borg Changelog ============== - -Version 0.26.0 (not released yet) ---------------------------------- +Version 0.27.0 +-------------- New features: +- "borg upgrade" command - attic -> borg one time converter / migration, #21 +- temporary hack to avoid using lots of disk space for chunks.archive.d, #235: + To use it: rm -rf chunks.archive.d ; touch chunks.archive.d +- respect XDG_CACHE_HOME, attic #181 +- add support for arbitrary SSH commands, attic #99 +- borg delete --cache-only REPO (only delete cache, not REPO), attic #123 + + +Bug fixes: + +- use Debian 7 (wheezy) to build pyinstaller borgbackup binaries, fixes slow + down observed when running the Centos6-built binary on Ubuntu, #222 +- do not crash on empty lock.roster, fixes #232 +- fix multiple issues with the cache config version check, #234 +- fix segment entry header size check, attic #352 + plus other error handling improvements / code deduplication there. +- always give segment and offset in repo IntegrityErrors + + +Other changes: + +- stop producing binary wheels, remove docs about it, #147 +- docs: + - add warning about prune + - generate usage include files only as needed + - development docs: add Vagrant section + - update / improve / reformat FAQ + - hint to single-file pyinstaller binaries from README + + +Version 0.26.1 +-------------- + +This is a minor update, just docs and new pyinstaller binaries. + +- docs update about python and binary requirements +- better docs for --read-special, fix #220 +- re-built the binaries, fix #218 and #213 (glibc version issue) +- update web site about single-file pyinstaller binaries + +Note: if you did a python-based installation, there is no need to upgrade. + + +Version 0.26.0 +-------------- + +New features: + +- Faster cache sync (do all in one pass, remove tar/compression stuff), #163 - BORG_REPO env var to specify the default repo, #168 - read special files as if they were regular files, #79 +- implement borg create --dry-run, attic issue #267 +- Normalize paths before pattern matching on OS X, #143 +- support OpenBSD and NetBSD (except xattrs/ACLs) +- support / run tests on Python 3.5 Bug fixes: @@ -16,11 +68,46 @@ Bug fixes: - chunker: use off_t to get 64bit on 32bit platform, #178 - initialize chunker fd to -1, so it's not equal to STDIN_FILENO (0) - fix reaction to "no" answer at delete repo prompt, #182 +- setup.py: detect lz4.h header file location +- to support python < 3.2.4, add less buggy argparse lib from 3.2.6 (#194) +- fix for obtaining 'char *' from temporary Python value (old code causes + a compile error on Mint 17.2) +- llfuse 0.41 install troubles on some platforms, require < 0.41 + (UnicodeDecodeError exception due to non-ascii llfuse setup.py) +- cython code: add some int types to get rid of unspecific python add / + subtract operations (avoid undefined symbol FPE_... error on some platforms) +- fix verbose mode display of stdin backup +- extract: warn if a include pattern never matched, fixes #209, + implement counters for Include/ExcludePatterns +- archive names with slashes are invalid, attic issue #180 +- chunker: add a check whether the POSIX_FADV_DONTNEED constant is defined - + fixes building on OpenBSD. Other changes: - detect inconsistency / corruption / hash collision, #170 - replace versioneer with setuptools_scm, #106 +- docs: + + - pkg-config is needed for llfuse installation + - be more clear about pruning, attic issue #132 +- unit tests: + + - xattr: ignore security.selinux attribute showing up + - ext3 seems to need a bit more space for a sparse file + - do not test lzma level 9 compression (avoid MemoryError) + - work around strange mtime granularity issue on netbsd, fixes #204 + - ignore st_rdev if file is not a block/char device, fixes #203 + - stay away from the setgid and sticky mode bits +- use Vagrant to do easy cross-platform testing (#196), currently: + + - Debian 7 "wheezy" 32bit, Debian 8 "jessie" 64bit + - Ubuntu 12.04 32bit, Ubuntu 14.04 64bit + - Centos 7 64bit + - FreeBSD 10.2 64bit + - OpenBSD 5.7 64bit + - NetBSD 6.1.5 64bit + - Darwin (OS X Yosemite) Version 0.25.0 diff --git a/MANIFEST.in b/MANIFEST.in index d74d9e2c4..309c1f8dc 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,9 @@ -include README.rst AUTHORS LICENSE CHANGES.rst MANIFEST.in versioneer.py +include README.rst AUTHORS LICENSE CHANGES.rst MANIFEST.in recursive-include borg *.pyx recursive-include docs * recursive-exclude docs *.pyc recursive-exclude docs *.pyo prune docs/_build +prune .travis +exclude .coveragerc .gitattributes .gitignore .travis.yml Vagrantfile include borg/_version.py diff --git a/README.rst b/README.rst index 8180fd2ab..310413bfe 100644 --- a/README.rst +++ b/README.rst @@ -63,10 +63,15 @@ Main features Backup archives are mountable as userspace filesystems for easy interactive backup examination and restores (e.g. by using a regular file manager). +**Easy installation** + For Linux, Mac OS X and FreeBSD, we offer a single-file pyinstaller binary + that does not require installing anything - you can just run it. + **Platforms Borg works on** * Linux - * FreeBSD * Mac OS X + * FreeBSD + * OpenBSD and NetBSD (for both: no xattrs/ACLs support yet) * Cygwin (unsupported) **Free and Open Source Software** diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 000000000..72dfdfddd --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,427 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# Automated creation of testing environments / binaries on misc. platforms + +def packages_prepare_wheezy + return <<-EOF + # debian 7 wheezy does not have lz4, but it is available from wheezy-backports: + echo "deb http://http.debian.net/debian wheezy-backports main" > /etc/apt/sources.list.d/wheezy-backports.list + EOF +end + +def packages_prepare_precise + return <<-EOF + # ubuntu 12.04 precise does not have lz4, but it is available from a ppa: + add-apt-repository -y ppa:gezakovacs/lz4 + EOF +end + +def packages_debianoid + return <<-EOF + apt-get update + # for building borgbackup and dependencies: + apt-get install -y libssl-dev libacl1-dev liblz4-dev libfuse-dev fuse pkg-config + apt-get install -y fakeroot build-essential git + apt-get install -y python3-dev python3-setuptools + # for building python: + apt-get install -y zlib1g-dev libbz2-dev libncurses5-dev libreadline-dev liblzma-dev libsqlite3-dev + # this way it works on older dists (like ubuntu 12.04) also: + easy_install3 pip + pip3 install virtualenv + touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile + EOF +end + +def packages_redhatted + return <<-EOF + yum install -y epel-release + yum update -y + # for building borgbackup and dependencies: + yum install -y openssl-devel openssl libacl-devel libacl lz4-devel fuse-devel fuse pkgconfig + usermod -a -G fuse vagrant + yum install -y fakeroot gcc git patch + # for building python: + yum install -y zlib-devel bzip2-devel ncurses-devel readline-devel xz-devel sqlite-devel + #yum install -y python-pip + #pip install virtualenv + touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile + EOF +end + +def packages_darwin + return <<-EOF + # get osxfuse 3.0.x pre-release code from github: + curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.0.5/osxfuse-3.0.5.dmg >osxfuse.dmg + MOUNTDIR=$(echo `hdiutil mount osxfuse.dmg | tail -1 | awk '{$1="" ; print $0}'` | xargs -0 echo) \ + && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for OS X 3.0.5.pkg" -target / + sudo chown -R vagrant /usr/local # brew must be able to create stuff here + ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" + brew update + brew install openssl + brew install lz4 + brew install fakeroot + brew install git + brew install pkgconfig + touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile + EOF +end + +def packages_freebsd + return <<-EOF + # for building borgbackup and dependencies: + pkg install -y openssl liblz4 fusefs-libs pkgconf + pkg install -y fakeroot git bash + # for building python: + pkg install sqlite3 + # make bash default / work: + chsh -s bash vagrant + mount -t fdescfs fdesc /dev/fd + echo 'fdesc /dev/fd fdescfs rw 0 0' >> /etc/fstab + # make FUSE work + echo 'fuse_load="YES"' >> /boot/loader.conf + echo 'vfs.usermount=1' >> /etc/sysctl.conf + kldload fuse + sysctl vfs.usermount=1 + pw groupmod operator -M vagrant + touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile + EOF +end + +def packages_openbsd + return <<-EOF + . ~/.profile + mkdir -p /home/vagrant/borg + rsync -aH /vagrant/borg/ /home/vagrant/borg/ + rm -rf /vagrant/borg + ln -sf /home/vagrant/borg /vagrant/ + pkg_add bash + chsh -s /usr/local/bin/bash vagrant + pkg_add openssl + pkg_add lz4 + # pkg_add fuse # does not install, sdl dependency missing + pkg_add git # no fakeroot + pkg_add python-3.4.2 + pkg_add py3-setuptools + ln -sf /usr/local/bin/python3.4 /usr/local/bin/python3 + ln -sf /usr/local/bin/python3.4 /usr/local/bin/python + easy_install-3.4 pip + pip3 install virtualenv + touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile + EOF +end + +def packages_netbsd + return <<-EOF + hostname netbsd # the box we use has an invalid hostname + PKG_PATH="ftp://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/amd64/6.1.5/All/" + export PKG_PATH + pkg_add mozilla-rootcerts lz4 git bash + chsh -s bash vagrant + mkdir -p /usr/local/opt/lz4/include + mkdir -p /usr/local/opt/lz4/lib + ln -s /usr/pkg/include/lz4*.h /usr/local/opt/lz4/include/ + ln -s /usr/pkg/lib/liblz4* /usr/local/opt/lz4/lib/ + touch /etc/openssl/openssl.cnf # avoids a flood of "can't open ..." + mozilla-rootcerts install + # llfuse does not support netbsd + pkg_add python34 py34-setuptools + ln -s /usr/pkg/bin/python3.4 /usr/pkg/bin/python + ln -s /usr/pkg/bin/python3.4 /usr/pkg/bin/python3 + easy_install-3.4 pip + pip install virtualenv + touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile + EOF +end + +def install_pyenv(boxname) + return <<-EOF + curl -s -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash + echo 'export PATH="$HOME/.pyenv/bin:$PATH"' >> ~/.bash_profile + echo 'eval "$(pyenv init -)"' >> ~/.bash_profile + echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bash_profile + echo 'export PYTHON_CONFIGURE_OPTS="--enable-shared"' >> ~/.bash_profile + EOF +end + +def fix_pyenv_darwin(boxname) + return <<-EOF + echo 'export PYTHON_CONFIGURE_OPTS="--enable-framework"' >> ~/.bash_profile + EOF +end + +def install_pythons(boxname) + return <<-EOF + . ~/.bash_profile + pyenv install 3.2.2 # tests, 3.2(.0) and 3.2.1 deadlock, issue #221 + pyenv install 3.3.0 # tests + pyenv install 3.4.0 # tests + pyenv install 3.5.0 # tests + #pyenv install 3.5.1 # binary build, use latest 3.5.x release + pyenv rehash + EOF +end + +def build_sys_venv(boxname) + return <<-EOF + . ~/.bash_profile + cd /vagrant/borg + virtualenv --python=python3 borg-env + EOF +end + +def build_pyenv_venv(boxname) + return <<-EOF + . ~/.bash_profile + cd /vagrant/borg + # use the latest 3.5 release + pyenv global 3.5.0 + pyenv virtualenv 3.5.0 borg-env + ln -s ~/.pyenv/versions/borg-env . + EOF +end + +def install_borg(boxname) + return <<-EOF + . ~/.bash_profile + cd /vagrant/borg + . borg-env/bin/activate + pip install -U wheel # upgrade wheel, too old for 3.5 + cd borg + # clean up (wrong/outdated) stuff we likely got via rsync: + rm -f borg/*.so borg/*.cpy* + rm -f borg/{chunker,crypto,compress,hashindex,platform_linux}.c + rm -rf borg/__pycache__ borg/support/__pycache__ borg/testsuite/__pycache__ + pip install 'llfuse<0.41' # 0.41 does not install due to UnicodeDecodeError + pip install -r requirements.d/development.txt + pip install -e . + EOF +end + +def install_pyinstaller(boxname) + return <<-EOF + . ~/.bash_profile + cd /vagrant/borg + . borg-env/bin/activate + git clone https://github.com/pyinstaller/pyinstaller.git + cd pyinstaller + git checkout master + pip install -e . + EOF +end + +def install_pyinstaller_bootloader(boxname) + return <<-EOF + . ~/.bash_profile + cd /vagrant/borg + . borg-env/bin/activate + git clone https://github.com/pyinstaller/pyinstaller.git + cd pyinstaller + git checkout master + # build bootloader, if it is not included + cd bootloader + python ./waf all + cd .. + pip install -e . + EOF +end + +def build_binary_with_pyinstaller(boxname) + return <<-EOF + . ~/.bash_profile + cd /vagrant/borg + . borg-env/bin/activate + cd borg + pyinstaller -F -n borg --hidden-import=logging.config borg/__main__.py + EOF +end + +def run_tests(boxname) + return <<-EOF + . ~/.bash_profile + cd /vagrant/borg/borg + . ../borg-env/bin/activate + if which pyenv > /dev/null; then + # for testing, use the earliest point releases of the supported python versions: + pyenv global 3.2.2 3.3.0 3.4.0 3.5.0 + fi + # otherwise: just use the system python + if which fakeroot > /dev/null; then + fakeroot -u tox --skip-missing-interpreters + else + tox --skip-missing-interpreters + fi + EOF +end + +def fix_perms + return <<-EOF + # . ~/.profile + chown -R vagrant /vagrant/borg + EOF +end + +Vagrant.configure(2) do |config| + # use rsync to copy content to the folder + config.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync" + # do not let the VM access . on the host machine via the default shared folder! + config.vm.synced_folder ".", "/vagrant", disabled: true + + # fix permissions on synced folder + config.vm.provision "fix perms", :type => :shell, :inline => fix_perms + + config.vm.provider :virtualbox do |v| + #v.gui = true + v.cpus = 1 + end + + # Linux + config.vm.define "centos7_64" do |b| + b.vm.box = "centos/7" + b.vm.provider :virtualbox do |v| + v.memory = 768 + end + b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted + b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos7_64") + b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos7_64") + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("centos7_64") + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("centos7_64") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos7_64") + end + + config.vm.define "centos6_32" do |b| + b.vm.box = "centos6-32" + b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted + b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_32") + b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos6_32") + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("centos6_32") + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("centos6_32") + b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller("centos6_32") + b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("centos6_32") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos6_32") + end + + config.vm.define "centos6_64" do |b| + b.vm.box = "centos6-64" + b.vm.provider :virtualbox do |v| + v.memory = 768 + end + b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted + b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_64") + b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos6_64") + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("centos6_64") + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("centos6_64") + b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller("centos6_64") + b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("centos6_64") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos6_64") + end + + config.vm.define "trusty64" do |b| + b.vm.box = "ubuntu/trusty64" + b.vm.provider :virtualbox do |v| + v.memory = 768 + end + b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("trusty64") + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("trusty64") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("trusty64") + end + + config.vm.define "precise32" do |b| + b.vm.box = "ubuntu/precise32" + b.vm.provision "packages prepare precise", :type => :shell, :inline => packages_prepare_precise + b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("precise32") + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("precise32") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("precise32") + end + + config.vm.define "jessie64" do |b| + b.vm.box = "debian/jessie64" + b.vm.provider :virtualbox do |v| + v.memory = 768 + end + b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("jessie64") + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("jessie64") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("jessie64") + end + + config.vm.define "wheezy32" do |b| + b.vm.box = "boxcutter/debian79-i386" + b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy + b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid + b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy32") + b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("wheezy32") + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("wheezy32") + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("wheezy32") + b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller("wheezy32") + b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("wheezy32") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("wheezy32") + end + + config.vm.define "wheezy64" do |b| + b.vm.box = "boxcutter/debian79" + b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy + b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid + b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy64") + b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("wheezy64") + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("wheezy64") + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("wheezy64") + b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller("wheezy64") + b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("wheezy64") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("wheezy64") + end + + # OS X + config.vm.define "darwin64" do |b| + b.vm.box = "jhcook/yosemite-clitools" + b.vm.provision "packages darwin", :type => :shell, :privileged => false, :inline => packages_darwin + b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("darwin64") + b.vm.provision "fix pyenv", :type => :shell, :privileged => false, :inline => fix_pyenv_darwin("darwin64") + b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("darwin64") + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("darwin64") + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("darwin64") + b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller("darwin64") + b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("darwin64") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("darwin64") + end + + # BSD + config.vm.define "freebsd64" do |b| + b.vm.box = "geoffgarside/freebsd-10.2" + b.vm.provider :virtualbox do |v| + v.memory = 768 + end + b.vm.provision "install system packages", :type => :shell, :inline => packages_freebsd + b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("freebsd") + b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("freebsd") + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("freebsd") + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("freebsd") + b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller_bootloader("freebsd") + b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("freebsd") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("freebsd") + end + + config.vm.define "openbsd64" do |b| + b.vm.box = "bodgit/openbsd-5.7-amd64" + b.vm.provider :virtualbox do |v| + v.memory = 768 + end + b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd64") + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("openbsd64") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openbsd64") + end + + config.vm.define "netbsd64" do |b| + b.vm.box = "alex-skimlinks/netbsd-6.1.5-amd64" + b.vm.provider :virtualbox do |v| + v.memory = 768 + end + b.vm.provision "packages netbsd", :type => :shell, :inline => packages_netbsd + b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("netbsd64") + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("netbsd64") + b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("netbsd64") + end +end diff --git a/borg/_chunker.c b/borg/_chunker.c index 5fee3b9f4..b75869483 100644 --- a/borg/_chunker.c +++ b/borg/_chunker.c @@ -156,7 +156,7 @@ chunker_fill(Chunker *c, PyThreadState **tstatep) return 0; } length = c->bytes_read - offset; - #if ( _XOPEN_SOURCE >= 600 || _POSIX_C_SOURCE >= 200112L ) + #if ( ( _XOPEN_SOURCE >= 600 || _POSIX_C_SOURCE >= 200112L ) && defined(POSIX_FADV_DONTNEED) ) // We tell the OS that we do not need the data that we just have read any // more (that it maybe has in the cache). This avoids that we spoil the // complete cache with data that we only read once and (due to cache diff --git a/borg/archiver.py b/borg/archiver.py index 8ebbb8344..76cd14605 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -1,4 +1,6 @@ -import argparse +from .support import argparse # see support/__init__.py docstring + # DEPRECATED - remove after requiring py 3.4 + from binascii import hexlify from datetime import datetime from operator import attrgetter @@ -15,11 +17,12 @@ import traceback from . import __version__ from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS from .compress import Compressor, COMPR_BUFFER +from .upgrader import AtticRepositoryUpgrader from .repository import Repository from .cache import Cache from .key import key_creator from .helpers import Error, location_validator, format_time, format_file_size, \ - format_file_mode, ExcludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ + format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec @@ -288,6 +291,9 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") # processing time, archive order is not as traversal order on "create". while dirs: archive.extract_item(dirs.pop(-1)) + for pattern in (patterns or []): + if isinstance(pattern, IncludePattern) and pattern.match_count == 0: + self.print_error("Warning: Include pattern '%s' never matched.", pattern) return self.exit_code def do_rename(self, args): @@ -317,17 +323,19 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") if args.stats: stats.print_('Deleted data:', cache) else: - print("You requested to completely DELETE the repository *including* all archives it contains:") - for archive_info in manifest.list_archive_infos(sort_by='ts'): - print(format_archive(archive_info)) - if not os.environ.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'): - print("""Type "YES" if you understand this and want to continue.\n""") - if input('Do you want to continue? ') != 'YES': - self.exit_code = 1 - return self.exit_code - repository.destroy() + if not args.cache_only: + print("You requested to completely DELETE the repository *including* all archives it contains:") + for archive_info in manifest.list_archive_infos(sort_by='ts'): + print(format_archive(archive_info)) + if not os.environ.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'): + print("""Type "YES" if you understand this and want to continue.\n""") + if input('Do you want to continue? ') != 'YES': + self.exit_code = 1 + return self.exit_code + repository.destroy() + print("Repository deleted.") cache.destroy() - print("Repository and corresponding cache were deleted.") + print("Cache deleted.") return self.exit_code def do_mount(self, args): @@ -461,6 +469,24 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") stats.print_('Deleted data:', cache) return self.exit_code + def do_upgrade(self, args): + """upgrade a repository from a previous version""" + # XXX: currently only upgrades from Attic repositories, but may + # eventually be extended to deal with major upgrades for borg + # itself. + # + # in this case, it should auto-detect the current repository + # format and fire up necessary upgrade mechanism. this remains + # to be implemented. + + # XXX: should auto-detect if it is an attic repository here + repo = AtticRepositoryUpgrader(args.repository.path, create=False) + try: + repo.upgrade(args.dry_run) + except NotImplementedError as e: + print("warning: %s" % e) + return self.exit_code + helptext = {} helptext['patterns'] = ''' Exclude patterns use a variant of shell pattern syntax, with '*' matching any @@ -549,10 +575,10 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") help='verbose output') common_parser.add_argument('--no-files-cache', dest='cache_files', action='store_false', help='do not load/update the file metadata cache used to detect unchanged files') - common_parser.add_argument('--umask', dest='umask', type=lambda s: int(s, 8), default=0o077, metavar='M', - help='set umask to M (local and remote, default: 0o077)') - common_parser.add_argument('--remote-path', dest='remote_path', default='borg', metavar='PATH', - help='set remote path to executable (default: "borg")') + common_parser.add_argument('--umask', dest='umask', type=lambda s: int(s, 8), default=RemoteRepository.umask, metavar='M', + help='set umask to M (local and remote, default: %(default)s)') + common_parser.add_argument('--remote-path', dest='remote_path', default=RemoteRepository.remote_path, metavar='PATH', + help='set remote path to executable (default: "%(default)s")') # We can't use argparse for "serve" since we don't want it to show up in "Available commands" if args: @@ -720,7 +746,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") help='do not create a backup archive') subparser.add_argument('archive', metavar='ARCHIVE', type=location_validator(archive=True), - help='archive to create') + help='name of archive to create (must be also a valid directory name)') subparser.add_argument('paths', metavar='PATH', nargs='+', type=str, help='paths to archive') @@ -791,6 +817,9 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") subparser.add_argument('-s', '--stats', dest='stats', action='store_true', default=False, help='print statistics for the deleted archive') + subparser.add_argument('-c', '--cache-only', dest='cache_only', + action='store_true', default=False, + help='delete only the local cache for the given repository') subparser.add_argument('target', metavar='TARGET', nargs='?', default='', type=location_validator(), help='archive or repository to delete') @@ -864,6 +893,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") If a prefix is set with -p, then only archives that start with the prefix are considered for deletion and only those archives count towards the totals specified by the rules. + Otherwise, *all* archives in the repository are candidates for deletion! """) subparser = subparsers.add_parser('prune', parents=[common_parser], description=self.do_prune.__doc__, @@ -894,6 +924,53 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") type=location_validator(archive=False), help='repository to prune') + upgrade_epilog = textwrap.dedent(""" + upgrade an existing Borg repository in place. this currently + only support converting an Attic repository, but may + eventually be extended to cover major Borg upgrades as well. + + it will change the magic strings in the repository's segments + to match the new Borg magic strings. the keyfiles found in + $ATTIC_KEYS_DIR or ~/.attic/keys/ will also be converted and + copied to $BORG_KEYS_DIR or ~/.borg/keys. + + the cache files are converted, from $ATTIC_CACHE_DIR or + ~/.cache/attic to $BORG_CACHE_DIR or ~/.cache/borg, but the + cache layout between Borg and Attic changed, so it is possible + the first backup after the conversion takes longer than expected + due to the cache resync. + + it is recommended you run this on a copy of the Attic + repository, in case something goes wrong, for example: + + cp -a attic borg + borg upgrade -n borg + borg upgrade borg + + upgrade should be able to resume if interrupted, although it + will still iterate over all segments. if you want to start + from scratch, use `borg delete` over the copied repository to + make sure the cache files are also removed: + + borg delete borg + + the conversion can PERMANENTLY DAMAGE YOUR REPOSITORY! Attic + will also NOT BE ABLE TO READ THE BORG REPOSITORY ANYMORE, as + the magic strings will have changed. + + you have been warned.""") + subparser = subparsers.add_parser('upgrade', parents=[common_parser], + description=self.do_upgrade.__doc__, + epilog=upgrade_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparser.set_defaults(func=self.do_upgrade) + subparser.add_argument('-n', '--dry-run', dest='dry_run', + default=False, action='store_true', + help='do not change repository') + subparser.add_argument('repository', metavar='REPOSITORY', nargs='?', default='', + type=location_validator(archive=False), + help='path to the repository to be upgraded') + subparser = subparsers.add_parser('help', parents=[common_parser], description='Extra help') subparser.add_argument('--epilog-only', dest='epilog_only', diff --git a/borg/cache.py b/borg/cache.py index 76329ec5f..13819a212 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -1,4 +1,4 @@ -from configparser import RawConfigParser +import configparser from .remote import cache_if_remote import errno import msgpack @@ -93,7 +93,7 @@ class Cache: os.makedirs(self.path) with open(os.path.join(self.path, 'README'), 'w') as fd: fd.write('This is a Borg cache') - config = RawConfigParser() + config = configparser.RawConfigParser() config.add_section('cache') config.set('cache', 'version', '1') config.set('cache', 'repository', hexlify(self.repository.id).decode('ascii')) @@ -101,8 +101,7 @@ class Cache: with open(os.path.join(self.path, 'config'), 'w') as fd: config.write(fd) ChunkIndex().write(os.path.join(self.path, 'chunks').encode('utf-8')) - with open(os.path.join(self.path, 'chunks.archive'), 'wb') as fd: - pass # empty file + os.makedirs(os.path.join(self.path, 'chunks.archive.d')) with open(os.path.join(self.path, 'files'), 'wb') as fd: pass # empty file @@ -114,10 +113,17 @@ class Cache: shutil.rmtree(self.path) def _do_open(self): - self.config = RawConfigParser() - self.config.read(os.path.join(self.path, 'config')) - if self.config.getint('cache', 'version') != 1: - raise Exception('%s Does not look like a Borg cache') + self.config = configparser.RawConfigParser() + config_path = os.path.join(self.path, 'config') + self.config.read(config_path) + try: + cache_version = self.config.getint('cache', 'version') + wanted_version = 1 + if cache_version != wanted_version: + raise Exception('%s has unexpected cache version %d (wanted: %d).' % ( + config_path, cache_version, wanted_version)) + except configparser.NoSectionError as e: + raise Exception('%s does not look like a Borg cache.' % config_path) self.id = self.config.get('cache', 'repository') self.manifest_id = unhexlify(self.config.get('cache', 'manifest')) self.timestamp = self.config.get('cache', 'timestamp', fallback=None) @@ -158,7 +164,6 @@ class Cache: os.mkdir(txn_dir) shutil.copy(os.path.join(self.path, 'config'), txn_dir) shutil.copy(os.path.join(self.path, 'chunks'), txn_dir) - shutil.copy(os.path.join(self.path, 'chunks.archive'), txn_dir) shutil.copy(os.path.join(self.path, 'files'), txn_dir) os.rename(os.path.join(self.path, 'txn.tmp'), os.path.join(self.path, 'txn.active')) @@ -200,7 +205,6 @@ class Cache: if os.path.exists(txn_dir): shutil.copy(os.path.join(txn_dir, 'config'), self.path) shutil.copy(os.path.join(txn_dir, 'chunks'), self.path) - shutil.copy(os.path.join(txn_dir, 'chunks.archive'), self.path) shutil.copy(os.path.join(txn_dir, 'files'), self.path) os.rename(txn_dir, os.path.join(self.path, 'txn.tmp')) if os.path.exists(os.path.join(self.path, 'txn.tmp')): @@ -211,54 +215,34 @@ class Cache: def sync(self): """Re-synchronize chunks cache with repository. - If present, uses a compressed tar archive of known backup archive - indices, so it only needs to fetch infos from repo and build a chunk - index once per backup archive. - If out of sync, the tar gets rebuilt from known + fetched chunk infos, - so it has complete and current information about all backup archives. - Finally, it builds the master chunks index by merging all indices from - the tar. - - Note: compression (esp. xz) is very effective in keeping the tar - relatively small compared to the files it contains. + Maintains a directory with known backup archive indexes, so it only + needs to fetch infos from repo and build a chunk index once per backup + archive. + If out of sync, missing archive indexes get added, outdated indexes + get removed and a new master chunks index is built by merging all + archive indexes. """ - in_archive_path = os.path.join(self.path, 'chunks.archive') - out_archive_path = os.path.join(self.path, 'chunks.archive.tmp') + archive_path = os.path.join(self.path, 'chunks.archive.d') - def open_in_archive(): - try: - tf = tarfile.open(in_archive_path, 'r') - except OSError as e: - if e.errno != errno.ENOENT: - raise - # file not found - tf = None - except tarfile.ReadError: - # empty file? - tf = None - return tf + def mkpath(id, suffix=''): + id_hex = hexlify(id).decode('ascii') + path = os.path.join(archive_path, id_hex + suffix) + return path.encode('utf-8') - def open_out_archive(): - for compression in ('xz', 'bz2', 'gz'): - # xz needs py 3.3, bz2 and gz also work on 3.2 - try: - tf = tarfile.open(out_archive_path, 'w:'+compression, format=tarfile.PAX_FORMAT) - break - except tarfile.CompressionError: - continue - else: # shouldn't happen - tf = None - return tf + def cached_archives(): + if self.do_cache: + fns = os.listdir(archive_path) + # filenames with 64 hex digits == 256bit + return set(unhexlify(fn) for fn in fns if len(fn) == 64) + else: + return set() - def close_archive(tf): - if tf: - tf.close() + def repo_archives(): + return set(info[b'id'] for info in self.manifest.archives.values()) - def delete_in_archive(): - os.unlink(in_archive_path) - - def rename_out_archive(): - os.rename(out_archive_path, in_archive_path) + def cleanup_outdated(ids): + for id in ids: + os.unlink(mkpath(id)) def add(chunk_idx, id, size, csize, incr=1): try: @@ -267,16 +251,7 @@ class Cache: except KeyError: chunk_idx[id] = incr, size, csize - def transfer_known_idx(archive_id, tf_in, tf_out): - archive_id_hex = hexlify(archive_id).decode('ascii') - tarinfo = tf_in.getmember(archive_id_hex) - archive_name = tarinfo.pax_headers['archive_name'] - print('Already known archive:', archive_name) - f_in = tf_in.extractfile(archive_id_hex) - tf_out.addfile(tarinfo, f_in) - return archive_name - - def fetch_and_build_idx(archive_id, repository, key, tmp_dir, tf_out): + def fetch_and_build_idx(archive_id, repository, key): chunk_idx = ChunkIndex() cdata = repository.get(archive_id) data = key.decrypt(archive_id, cdata) @@ -285,7 +260,6 @@ class Cache: if archive[b'version'] != 1: raise Exception('Unknown archive metadata version') decode_dict(archive, (b'name',)) - print('Analyzing new archive:', archive[b'name']) unpacker = msgpack.Unpacker() for item_id, chunk in zip(archive[b'items'], repository.get_many(archive[b'items'])): data = key.decrypt(item_id, chunk) @@ -298,56 +272,76 @@ class Cache: if b'chunks' in item: for chunk_id, size, csize in item[b'chunks']: add(chunk_idx, chunk_id, size, csize) - archive_id_hex = hexlify(archive_id).decode('ascii') - file_tmp = os.path.join(tmp_dir, archive_id_hex).encode('utf-8') - chunk_idx.write(file_tmp) - tarinfo = tf_out.gettarinfo(file_tmp, archive_id_hex) - tarinfo.pax_headers['archive_name'] = archive[b'name'] - with open(file_tmp, 'rb') as f: - tf_out.addfile(tarinfo, f) - os.unlink(file_tmp) + if self.do_cache: + fn = mkpath(archive_id) + fn_tmp = mkpath(archive_id, suffix='.tmp') + try: + chunk_idx.write(fn_tmp) + except Exception: + os.unlink(fn_tmp) + else: + os.rename(fn_tmp, fn) + return chunk_idx - def create_master_idx(chunk_idx, tf_in, tmp_dir): + def lookup_name(archive_id): + for name, info in self.manifest.archives.items(): + if info[b'id'] == archive_id: + return name + + def create_master_idx(chunk_idx): + print('Synchronizing chunks cache...') + cached_ids = cached_archives() + archive_ids = repo_archives() + print('Archives: %d, w/ cached Idx: %d, w/ outdated Idx: %d, w/o cached Idx: %d.' % ( + len(archive_ids), len(cached_ids), + len(cached_ids - archive_ids), len(archive_ids - cached_ids), )) + # deallocates old hashindex, creates empty hashindex: chunk_idx.clear() - for tarinfo in tf_in: - archive_id_hex = tarinfo.name - archive_name = tarinfo.pax_headers['archive_name'] - print("- extracting archive %s ..." % archive_name) - tf_in.extract(archive_id_hex, tmp_dir) - chunk_idx_path = os.path.join(tmp_dir, archive_id_hex).encode('utf-8') - print("- reading archive ...") - archive_chunk_idx = ChunkIndex.read(chunk_idx_path) - print("- merging archive ...") - chunk_idx.merge(archive_chunk_idx) - os.unlink(chunk_idx_path) + cleanup_outdated(cached_ids - archive_ids) + if archive_ids: + chunk_idx = None + for archive_id in archive_ids: + archive_name = lookup_name(archive_id) + if archive_id in cached_ids: + archive_chunk_idx_path = mkpath(archive_id) + print("Reading cached archive chunk index for %s ..." % archive_name) + archive_chunk_idx = ChunkIndex.read(archive_chunk_idx_path) + else: + print('Fetching and building archive index for %s ...' % archive_name) + archive_chunk_idx = fetch_and_build_idx(archive_id, repository, self.key) + print("Merging into master chunks index ...") + if chunk_idx is None: + # we just use the first archive's idx as starting point, + # to avoid growing the hash table from 0 size and also + # to save 1 merge call. + chunk_idx = archive_chunk_idx + else: + chunk_idx.merge(archive_chunk_idx) + print('Done.') + return chunk_idx + + def legacy_cleanup(): + """bring old cache dirs into the desired state (cleanup and adapt)""" + try: + os.unlink(os.path.join(self.path, 'chunks.archive')) + except: + pass + try: + os.unlink(os.path.join(self.path, 'chunks.archive.tmp')) + except: + pass + try: + os.mkdir(archive_path) + except: + pass self.begin_txn() - print('Synchronizing chunks cache...') - # XXX we have to do stuff on disk due to lacking ChunkIndex api - with tempfile.TemporaryDirectory(prefix='borg-tmp') as tmp_dir: - repository = cache_if_remote(self.repository) - out_archive = open_out_archive() - in_archive = open_in_archive() - if in_archive: - known_ids = set(unhexlify(hexid) for hexid in in_archive.getnames()) - else: - known_ids = set() - archive_ids = set(info[b'id'] for info in self.manifest.archives.values()) - print('Rebuilding archive collection. Known: %d Repo: %d Unknown: %d' % ( - len(known_ids), len(archive_ids), len(archive_ids - known_ids), )) - for archive_id in archive_ids & known_ids: - transfer_known_idx(archive_id, in_archive, out_archive) - close_archive(in_archive) - delete_in_archive() # free disk space - for archive_id in archive_ids - known_ids: - fetch_and_build_idx(archive_id, repository, self.key, tmp_dir, out_archive) - close_archive(out_archive) - rename_out_archive() - print('Merging collection into master chunks cache...') - in_archive = open_in_archive() - create_master_idx(self.chunks, in_archive, tmp_dir) - close_archive(in_archive) - print('Done.') + repository = cache_if_remote(self.repository) + legacy_cleanup() + # TEMPORARY HACK: to avoid archive index caching, create a FILE named ~/.cache/borg/REPOID/chunks.archive.d - + # this is only recommended if you have a fast, low latency connection to your repo (e.g. if repo is local disk) + self.do_cache = os.path.isdir(archive_path) + self.chunks = create_master_idx(self.chunks) def add_chunk(self, id, data, stats): if not self.txn_active: diff --git a/borg/chunker.pyx b/borg/chunker.pyx index 1d4897db1..0faa06f38 100644 --- a/borg/chunker.pyx +++ b/borg/chunker.pyx @@ -20,7 +20,7 @@ cdef extern from "_chunker.c": cdef class Chunker: cdef _Chunker *chunker - def __cinit__(self, seed, chunk_min_exp, chunk_max_exp, hash_mask_bits, hash_window_size): + def __cinit__(self, int seed, int chunk_min_exp, int chunk_max_exp, int hash_mask_bits, int hash_window_size): min_size = 1 << chunk_min_exp max_size = 1 << chunk_max_exp hash_mask = (1 << hash_mask_bits) - 1 diff --git a/borg/crypto.pyx b/borg/crypto.pyx index 61dbc42d5..d8143bdbc 100644 --- a/borg/crypto.pyx +++ b/borg/crypto.pyx @@ -52,7 +52,7 @@ bytes_to_long = lambda x, offset=0: _long.unpack_from(x, offset)[0] long_to_bytes = lambda x: _long.pack(x) -def num_aes_blocks(length): +def num_aes_blocks(int length): """Return the number of AES blocks required to encrypt/decrypt *length* bytes of data. Note: this is only correct for modes without padding, like AES-CTR. """ diff --git a/borg/hashindex.pyx b/borg/hashindex.pyx index 6652e057f..0b4dc2605 100644 --- a/borg/hashindex.pyx +++ b/borg/hashindex.pyx @@ -37,7 +37,8 @@ cdef class IndexBase: def __cinit__(self, capacity=0, path=None, key_size=32): self.key_size = key_size if path: - self.index = hashindex_read(os.fsencode(path)) + path = os.fsencode(path) + self.index = hashindex_read(path) if not self.index: raise Exception('hashindex_read failed') else: @@ -54,7 +55,8 @@ cdef class IndexBase: return cls(path=path) def write(self, path): - if not hashindex_write(self.index, os.fsencode(path)): + path = os.fsencode(path) + if not hashindex_write(self.index, path): raise Exception('hashindex_write failed') def clear(self): diff --git a/borg/helpers.py b/borg/helpers.py index 79017c376..82a23654f 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -1,6 +1,9 @@ -import argparse +from .support import argparse # see support/__init__.py docstring + # DEPRECATED - remove after requiring py 3.4 + import binascii from collections import namedtuple +from functools import wraps import grp import os import pwd @@ -8,6 +11,8 @@ import queue import re import sys import time +import unicodedata + from datetime import datetime, timezone, timedelta from fnmatch import translate from operator import attrgetter @@ -170,8 +175,8 @@ def get_keys_dir(): def get_cache_dir(): """Determine where to repository keys and cache""" - return os.environ.get('BORG_CACHE_DIR', - os.path.join(os.path.expanduser('~'), '.cache', 'borg')) + xdg_cache = os.environ.get('XDG_CACHE_HOME', os.path.join(os.path.expanduser('~'), '.cache')) + return os.environ.get('BORG_CACHE_DIR', os.path.join(xdg_cache, 'borg')) def to_localtime(ts): @@ -223,6 +228,24 @@ def exclude_path(path, patterns): # unify the two cases, we add a path separator to the end of # the path before matching. +def normalized(func): + """ Decorator for the Pattern match methods, returning a wrapper that + normalizes OSX paths to match the normalized pattern on OSX, and + returning the original method on other platforms""" + @wraps(func) + def normalize_wrapper(self, path): + return func(self, unicodedata.normalize("NFD", path)) + + if sys.platform in ('darwin',): + # HFS+ converts paths to a canonical form, so users shouldn't be + # required to enter an exact match + return normalize_wrapper + else: + # Windows and Unix filesystems allow different forms, so users + # always have to enter an exact match + return func + + class IncludePattern: """Literal files or directories listed on the command line for some operations (e.g. extract, but not create). @@ -230,34 +253,61 @@ class IncludePattern: path match as well. A trailing slash makes no difference. """ def __init__(self, pattern): + self.pattern_orig = pattern + self.match_count = 0 + + if sys.platform in ('darwin',): + pattern = unicodedata.normalize("NFD", pattern) + self.pattern = os.path.normpath(pattern).rstrip(os.path.sep)+os.path.sep + @normalized def match(self, path): - return (path+os.path.sep).startswith(self.pattern) + matches = (path+os.path.sep).startswith(self.pattern) + if matches: + self.match_count += 1 + return matches def __repr__(self): return '%s(%s)' % (type(self), self.pattern) + def __str__(self): + return self.pattern_orig + class ExcludePattern(IncludePattern): """Shell glob patterns to exclude. A trailing slash means to exclude the contents of a directory, but not the directory itself. """ def __init__(self, pattern): + self.pattern_orig = pattern + self.match_count = 0 + if pattern.endswith(os.path.sep): self.pattern = os.path.normpath(pattern).rstrip(os.path.sep)+os.path.sep+'*'+os.path.sep else: self.pattern = os.path.normpath(pattern)+os.path.sep+'*' + + if sys.platform in ('darwin',): + self.pattern = unicodedata.normalize("NFD", self.pattern) + # fnmatch and re.match both cache compiled regular expressions. # Nevertheless, this is about 10 times faster. self.regex = re.compile(translate(self.pattern)) + @normalized def match(self, path): - return self.regex.match(path+os.path.sep) is not None + matches = self.regex.match(path+os.path.sep) is not None + if matches: + self.match_count += 1 + return matches def __repr__(self): return '%s(%s)' % (type(self), self.pattern) + def __str__(self): + return self.pattern_orig + def timestamp(s): """Convert a --timestamp=s argument to a datetime object""" @@ -462,18 +512,20 @@ class Location: """Object representing a repository / archive location """ proto = user = host = port = path = archive = None + # borg mount's FUSE filesystem creates one level of directories from + # the archive names. Thus, we must not accept "/" in archive names. ssh_re = re.compile(r'(?Pssh)://(?:(?P[^@]+)@)?' r'(?P[^:/#]+)(?::(?P\d+))?' - r'(?P[^:]+)(?:::(?P.+))?$') + r'(?P[^:]+)(?:::(?P[^/]+))?$') file_re = re.compile(r'(?Pfile)://' - r'(?P[^:]+)(?:::(?P.+))?$') + r'(?P[^:]+)(?:::(?P[^/]+))?$') scp_re = re.compile(r'((?:(?P[^@]+)@)?(?P[^:/]+):)?' - r'(?P[^:]+)(?:::(?P.+))?$') + r'(?P[^:]+)(?:::(?P[^/]+))?$') # get the repo from BORG_RE env and the optional archive from param. # if the syntax requires giving REPOSITORY (see "borg mount"), # use "::" to let it use the env var. # if REPOSITORY argument is optional, it'll automatically use the env. - env_re = re.compile(r'(?:::(?P.+)?)?$') + env_re = re.compile(r'(?:::(?P[^/]+)?)?$') def __init__(self, text=''): self.orig = text diff --git a/borg/locking.py b/borg/locking.py index 8e4f1a41f..b2beac345 100644 --- a/borg/locking.py +++ b/borg/locking.py @@ -169,6 +169,9 @@ class LockRoster: if err.errno != errno.ENOENT: raise data = {} + except ValueError: + # corrupt/empty roster file? + data = {} return data def save(self, data): diff --git a/borg/remote.py b/borg/remote.py index 3a274b214..b9847c7e4 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -3,6 +3,7 @@ import fcntl import msgpack import os import select +import shlex from subprocess import Popen, PIPE import sys import tempfile @@ -108,8 +109,9 @@ class RepositoryServer: # pragma: no cover class RemoteRepository: extra_test_args = [] - remote_path = None - umask = None + remote_path = 'borg' + # default umask, overriden by --umask, defaults to read/write only for owner + umask = 0o077 class RPCError(Exception): def __init__(self, name): @@ -125,19 +127,14 @@ class RemoteRepository: self.responses = {} self.unpacker = msgpack.Unpacker(use_list=False) self.p = None - # use local umask also for the remote process - umask = ['--umask', '%03o' % self.umask] + # XXX: ideally, the testsuite would subclass Repository and + # override ssh_cmd() instead of this crude hack, although + # __testsuite__ is not a valid domain name so this is pretty + # safe. if location.host == '__testsuite__': - args = [sys.executable, '-m', 'borg.archiver', 'serve'] + umask + self.extra_test_args + args = [sys.executable, '-m', 'borg.archiver', 'serve' ] + self.extra_test_args else: # pragma: no cover - args = ['ssh'] - if location.port: - args += ['-p', str(location.port)] - if location.user: - args.append('%s@%s' % (location.user, location.host)) - else: - args.append('%s' % location.host) - args += [self.remote_path, 'serve'] + umask + args = self.ssh_cmd(location) self.p = Popen(args, bufsize=0, stdin=PIPE, stdout=PIPE) self.stdin_fd = self.p.stdin.fileno() self.stdout_fd = self.p.stdout.fileno() @@ -160,6 +157,21 @@ class RemoteRepository: def __repr__(self): return '<%s %s>' % (self.__class__.__name__, self.location.canonical_path()) + def umask_flag(self): + return ['--umask', '%03o' % self.umask] + + def ssh_cmd(self, location): + args = shlex.split(os.environ.get('BORG_RSH', 'ssh')) + if location.port: + args += ['-p', str(location.port)] + if location.user: + args.append('%s@%s' % (location.user, location.host)) + else: + args.append('%s' % location.host) + # use local umask also for the remote process + args += [self.remote_path, 'serve'] + self.umask_flag() + return args + def call(self, cmd, *args, **kw): for resp in self.call_many(cmd, [args], **kw): return resp diff --git a/borg/repository.py b/borg/repository.py index f43161fb6..932e4fef3 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -301,7 +301,7 @@ class Repository: try: objects = list(self.io.iter_objects(segment)) except IntegrityError as err: - report_error('Error reading segment {}: {}'.format(segment, err)) + report_error(str(err)) objects = [] if repair: self.io.recover_segment(segment, filename) @@ -530,30 +530,14 @@ class LoggedIO: fd = self.get_fd(segment) fd.seek(0) if fd.read(MAGIC_LEN) != MAGIC: - raise IntegrityError('Invalid segment magic') + raise IntegrityError('Invalid segment magic [segment {}, offset {}]'.format(segment, 0)) offset = MAGIC_LEN header = fd.read(self.header_fmt.size) while header: - try: - crc, size, tag = self.header_fmt.unpack(header) - except struct.error as err: - raise IntegrityError('Invalid segment entry header [offset {}]: {}'.format(offset, err)) - if size > MAX_OBJECT_SIZE: - raise IntegrityError('Invalid segment entry size [offset {}]'.format(offset)) - length = size - self.header_fmt.size - rest = fd.read(length) - if len(rest) != length: - raise IntegrityError('Segment entry data short read [offset {}]: expected: {}, got {} bytes'.format( - offset, length, len(rest))) - if crc32(rest, crc32(memoryview(header)[4:])) & 0xffffffff != crc: - raise IntegrityError('Segment entry checksum mismatch [offset {}]'.format(offset)) - if tag not in (TAG_PUT, TAG_DELETE, TAG_COMMIT): - raise IntegrityError('Invalid segment entry tag [offset {}]'.format(offset)) - key = None - if tag in (TAG_PUT, TAG_DELETE): - key = rest[:32] + size, tag, key, data = self._read(fd, self.header_fmt, header, segment, offset, + (TAG_PUT, TAG_DELETE, TAG_COMMIT)) if include_data: - yield tag, key, offset, rest[32:] + yield tag, key, offset, data else: yield tag, key, offset offset += size @@ -586,16 +570,44 @@ class LoggedIO: fd = self.get_fd(segment) fd.seek(offset) header = fd.read(self.put_header_fmt.size) - crc, size, tag, key = self.put_header_fmt.unpack(header) - if size > MAX_OBJECT_SIZE: - raise IntegrityError('Invalid segment object size') - data = fd.read(size - self.put_header_fmt.size) - if crc32(data, crc32(memoryview(header)[4:])) & 0xffffffff != crc: - raise IntegrityError('Segment checksum mismatch') - if tag != TAG_PUT or id != key: - raise IntegrityError('Invalid segment entry header') + size, tag, key, data = self._read(fd, self.put_header_fmt, header, segment, offset, (TAG_PUT, )) + if id != key: + raise IntegrityError('Invalid segment entry header, is not for wanted id [segment {}, offset {}]'.format( + segment, offset)) return data + def _read(self, fd, fmt, header, segment, offset, acceptable_tags): + # some code shared by read() and iter_objects() + try: + hdr_tuple = fmt.unpack(header) + except struct.error as err: + raise IntegrityError('Invalid segment entry header [segment {}, offset {}]: {}'.format( + segment, offset, err)) + if fmt is self.put_header_fmt: + crc, size, tag, key = hdr_tuple + elif fmt is self.header_fmt: + crc, size, tag = hdr_tuple + key = None + else: + raise TypeError("_read called with unsupported format") + if size > MAX_OBJECT_SIZE or size < fmt.size: + raise IntegrityError('Invalid segment entry size [segment {}, offset {}]'.format( + segment, offset)) + length = size - fmt.size + data = fd.read(length) + if len(data) != length: + raise IntegrityError('Segment entry data short read [segment {}, offset {}]: expected {}, got {} bytes'.format( + segment, offset, length, len(data))) + if crc32(data, crc32(memoryview(header)[4:])) & 0xffffffff != crc: + raise IntegrityError('Segment entry checksum mismatch [segment {}, offset {}]'.format( + segment, offset)) + if tag not in acceptable_tags: + raise IntegrityError('Invalid segment entry header, did not get acceptable tag [segment {}, offset {}]'.format( + segment, offset)) + if key is None and tag in (TAG_PUT, TAG_DELETE): + key, data = data[:32], data[32:] + return size, tag, key, data + def write_put(self, id, data): size = len(data) + self.put_header_fmt.size fd = self.get_write_fd() diff --git a/borg/support/__init__.py b/borg/support/__init__.py new file mode 100644 index 000000000..449fcebfc --- /dev/null +++ b/borg/support/__init__.py @@ -0,0 +1,16 @@ +""" +3rd party stuff that needed fixing + +Note: linux package maintainers feel free to remove any of these hacks + IF your python version is not affected. + +argparse is broken with default args (double conversion): +affects: 3.2.0 <= python < 3.2.4 +affects: 3.3.0 <= python < 3.3.1 + +as we still support 3.2 and 3.3 there is no other way than to bundle +a fixed version (I just took argparse.py from 3.2.6) and import it from +here (see import in archiver.py). +DEPRECATED - remove support.argparse after requiring python 3.4. +""" + diff --git a/borg/support/argparse.py b/borg/support/argparse.py new file mode 100644 index 000000000..da73bc5fc --- /dev/null +++ b/borg/support/argparse.py @@ -0,0 +1,2383 @@ +# Author: Steven J. Bethard . + +"""Command-line parsing library + +This module is an optparse-inspired command-line parsing library that: + + - handles both optional and positional arguments + - produces highly informative usage messages + - supports parsers that dispatch to sub-parsers + +The following is a simple usage example that sums integers from the +command-line and writes the result to a file:: + + parser = argparse.ArgumentParser( + description='sum the integers at the command line') + parser.add_argument( + 'integers', metavar='int', nargs='+', type=int, + help='an integer to be summed') + parser.add_argument( + '--log', default=sys.stdout, type=argparse.FileType('w'), + help='the file where the sum should be written') + args = parser.parse_args() + args.log.write('%s' % sum(args.integers)) + args.log.close() + +The module contains the following public classes: + + - ArgumentParser -- The main entry point for command-line parsing. As the + example above shows, the add_argument() method is used to populate + the parser with actions for optional and positional arguments. Then + the parse_args() method is invoked to convert the args at the + command-line into an object with attributes. + + - ArgumentError -- The exception raised by ArgumentParser objects when + there are errors with the parser's actions. Errors raised while + parsing the command-line are caught by ArgumentParser and emitted + as command-line messages. + + - FileType -- A factory for defining types of files to be created. As the + example above shows, instances of FileType are typically passed as + the type= argument of add_argument() calls. + + - Action -- The base class for parser actions. Typically actions are + selected by passing strings like 'store_true' or 'append_const' to + the action= argument of add_argument(). However, for greater + customization of ArgumentParser actions, subclasses of Action may + be defined and passed as the action= argument. + + - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter, + ArgumentDefaultsHelpFormatter -- Formatter classes which + may be passed as the formatter_class= argument to the + ArgumentParser constructor. HelpFormatter is the default, + RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser + not to change the formatting for help text, and + ArgumentDefaultsHelpFormatter adds information about argument defaults + to the help. + +All other classes in this module are considered implementation details. +(Also note that HelpFormatter and RawDescriptionHelpFormatter are only +considered public as object names -- the API of the formatter objects is +still considered an implementation detail.) +""" + +__version__ = '1.1' +__all__ = [ + 'ArgumentParser', + 'ArgumentError', + 'ArgumentTypeError', + 'FileType', + 'HelpFormatter', + 'ArgumentDefaultsHelpFormatter', + 'RawDescriptionHelpFormatter', + 'RawTextHelpFormatter', + 'Namespace', + 'Action', + 'ONE_OR_MORE', + 'OPTIONAL', + 'PARSER', + 'REMAINDER', + 'SUPPRESS', + 'ZERO_OR_MORE', +] + + +import collections as _collections +import copy as _copy +import os as _os +import re as _re +import sys as _sys +import textwrap as _textwrap + +try: + from gettext import gettext, ngettext +except ImportError: + def gettext(message): + return message + def ngettext(msg1, msg2, n): + return msg1 if n == 1 else msg2 +_ = gettext + + +SUPPRESS = '==SUPPRESS==' + +OPTIONAL = '?' +ZERO_OR_MORE = '*' +ONE_OR_MORE = '+' +PARSER = 'A...' +REMAINDER = '...' +_UNRECOGNIZED_ARGS_ATTR = '_unrecognized_args' + +# ============================= +# Utility functions and classes +# ============================= + +class _AttributeHolder(object): + """Abstract base class that provides __repr__. + + The __repr__ method returns a string in the format:: + ClassName(attr=name, attr=name, ...) + The attributes are determined either by a class-level attribute, + '_kwarg_names', or by inspecting the instance __dict__. + """ + + def __repr__(self): + type_name = type(self).__name__ + arg_strings = [] + for arg in self._get_args(): + arg_strings.append(repr(arg)) + for name, value in self._get_kwargs(): + arg_strings.append('%s=%r' % (name, value)) + return '%s(%s)' % (type_name, ', '.join(arg_strings)) + + def _get_kwargs(self): + return sorted(self.__dict__.items()) + + def _get_args(self): + return [] + + +def _ensure_value(namespace, name, value): + if getattr(namespace, name, None) is None: + setattr(namespace, name, value) + return getattr(namespace, name) + + +# =============== +# Formatting Help +# =============== + +class HelpFormatter(object): + """Formatter for generating usage messages and argument help strings. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def __init__(self, + prog, + indent_increment=2, + max_help_position=24, + width=None): + + # default setting for width + if width is None: + try: + width = int(_os.environ['COLUMNS']) + except (KeyError, ValueError): + width = 80 + width -= 2 + + self._prog = prog + self._indent_increment = indent_increment + self._max_help_position = max_help_position + self._width = width + + self._current_indent = 0 + self._level = 0 + self._action_max_length = 0 + + self._root_section = self._Section(self, None) + self._current_section = self._root_section + + self._whitespace_matcher = _re.compile(r'\s+') + self._long_break_matcher = _re.compile(r'\n\n\n+') + + # =============================== + # Section and indentation methods + # =============================== + def _indent(self): + self._current_indent += self._indent_increment + self._level += 1 + + def _dedent(self): + self._current_indent -= self._indent_increment + assert self._current_indent >= 0, 'Indent decreased below 0.' + self._level -= 1 + + class _Section(object): + + def __init__(self, formatter, parent, heading=None): + self.formatter = formatter + self.parent = parent + self.heading = heading + self.items = [] + + def format_help(self): + # format the indented section + if self.parent is not None: + self.formatter._indent() + join = self.formatter._join_parts + for func, args in self.items: + func(*args) + item_help = join([func(*args) for func, args in self.items]) + if self.parent is not None: + self.formatter._dedent() + + # return nothing if the section was empty + if not item_help: + return '' + + # add the heading if the section was non-empty + if self.heading is not SUPPRESS and self.heading is not None: + current_indent = self.formatter._current_indent + heading = '%*s%s:\n' % (current_indent, '', self.heading) + else: + heading = '' + + # join the section-initial newline, the heading and the help + return join(['\n', heading, item_help, '\n']) + + def _add_item(self, func, args): + self._current_section.items.append((func, args)) + + # ======================== + # Message building methods + # ======================== + def start_section(self, heading): + self._indent() + section = self._Section(self, self._current_section, heading) + self._add_item(section.format_help, []) + self._current_section = section + + def end_section(self): + self._current_section = self._current_section.parent + self._dedent() + + def add_text(self, text): + if text is not SUPPRESS and text is not None: + self._add_item(self._format_text, [text]) + + def add_usage(self, usage, actions, groups, prefix=None): + if usage is not SUPPRESS: + args = usage, actions, groups, prefix + self._add_item(self._format_usage, args) + + def add_argument(self, action): + if action.help is not SUPPRESS: + + # find all invocations + get_invocation = self._format_action_invocation + invocations = [get_invocation(action)] + for subaction in self._iter_indented_subactions(action): + invocations.append(get_invocation(subaction)) + + # update the maximum item length + invocation_length = max([len(s) for s in invocations]) + action_length = invocation_length + self._current_indent + self._action_max_length = max(self._action_max_length, + action_length) + + # add the item to the list + self._add_item(self._format_action, [action]) + + def add_arguments(self, actions): + for action in actions: + self.add_argument(action) + + # ======================= + # Help-formatting methods + # ======================= + def format_help(self): + help = self._root_section.format_help() + if help: + help = self._long_break_matcher.sub('\n\n', help) + help = help.strip('\n') + '\n' + return help + + def _join_parts(self, part_strings): + return ''.join([part + for part in part_strings + if part and part is not SUPPRESS]) + + def _format_usage(self, usage, actions, groups, prefix): + if prefix is None: + prefix = _('usage: ') + + # if usage is specified, use that + if usage is not None: + usage = usage % dict(prog=self._prog) + + # if no optionals or positionals are available, usage is just prog + elif usage is None and not actions: + usage = '%(prog)s' % dict(prog=self._prog) + + # if optionals and positionals are available, calculate usage + elif usage is None: + prog = '%(prog)s' % dict(prog=self._prog) + + # split optionals from positionals + optionals = [] + positionals = [] + for action in actions: + if action.option_strings: + optionals.append(action) + else: + positionals.append(action) + + # build full usage string + format = self._format_actions_usage + action_usage = format(optionals + positionals, groups) + usage = ' '.join([s for s in [prog, action_usage] if s]) + + # wrap the usage parts if it's too long + text_width = self._width - self._current_indent + if len(prefix) + len(usage) > text_width: + + # break usage into wrappable parts + part_regexp = r'\(.*?\)+|\[.*?\]+|\S+' + opt_usage = format(optionals, groups) + pos_usage = format(positionals, groups) + opt_parts = _re.findall(part_regexp, opt_usage) + pos_parts = _re.findall(part_regexp, pos_usage) + assert ' '.join(opt_parts) == opt_usage + assert ' '.join(pos_parts) == pos_usage + + # helper for wrapping lines + def get_lines(parts, indent, prefix=None): + lines = [] + line = [] + if prefix is not None: + line_len = len(prefix) - 1 + else: + line_len = len(indent) - 1 + for part in parts: + if line_len + 1 + len(part) > text_width: + lines.append(indent + ' '.join(line)) + line = [] + line_len = len(indent) - 1 + line.append(part) + line_len += len(part) + 1 + if line: + lines.append(indent + ' '.join(line)) + if prefix is not None: + lines[0] = lines[0][len(indent):] + return lines + + # if prog is short, follow it with optionals or positionals + if len(prefix) + len(prog) <= 0.75 * text_width: + indent = ' ' * (len(prefix) + len(prog) + 1) + if opt_parts: + lines = get_lines([prog] + opt_parts, indent, prefix) + lines.extend(get_lines(pos_parts, indent)) + elif pos_parts: + lines = get_lines([prog] + pos_parts, indent, prefix) + else: + lines = [prog] + + # if prog is long, put it on its own line + else: + indent = ' ' * len(prefix) + parts = opt_parts + pos_parts + lines = get_lines(parts, indent) + if len(lines) > 1: + lines = [] + lines.extend(get_lines(opt_parts, indent)) + lines.extend(get_lines(pos_parts, indent)) + lines = [prog] + lines + + # join lines into usage + usage = '\n'.join(lines) + + # prefix with 'usage:' + return '%s%s\n\n' % (prefix, usage) + + def _format_actions_usage(self, actions, groups): + # find group indices and identify actions in groups + group_actions = set() + inserts = {} + for group in groups: + try: + start = actions.index(group._group_actions[0]) + except ValueError: + continue + else: + end = start + len(group._group_actions) + if actions[start:end] == group._group_actions: + for action in group._group_actions: + group_actions.add(action) + if not group.required: + if start in inserts: + inserts[start] += ' [' + else: + inserts[start] = '[' + inserts[end] = ']' + else: + if start in inserts: + inserts[start] += ' (' + else: + inserts[start] = '(' + inserts[end] = ')' + for i in range(start + 1, end): + inserts[i] = '|' + + # collect all actions format strings + parts = [] + for i, action in enumerate(actions): + + # suppressed arguments are marked with None + # remove | separators for suppressed arguments + if action.help is SUPPRESS: + parts.append(None) + if inserts.get(i) == '|': + inserts.pop(i) + elif inserts.get(i + 1) == '|': + inserts.pop(i + 1) + + # produce all arg strings + elif not action.option_strings: + part = self._format_args(action, action.dest) + + # if it's in a group, strip the outer [] + if action in group_actions: + if part[0] == '[' and part[-1] == ']': + part = part[1:-1] + + # add the action string to the list + parts.append(part) + + # produce the first way to invoke the option in brackets + else: + option_string = action.option_strings[0] + + # if the Optional doesn't take a value, format is: + # -s or --long + if action.nargs == 0: + part = '%s' % option_string + + # if the Optional takes a value, format is: + # -s ARGS or --long ARGS + else: + default = action.dest.upper() + args_string = self._format_args(action, default) + part = '%s %s' % (option_string, args_string) + + # make it look optional if it's not required or in a group + if not action.required and action not in group_actions: + part = '[%s]' % part + + # add the action string to the list + parts.append(part) + + # insert things at the necessary indices + for i in sorted(inserts, reverse=True): + parts[i:i] = [inserts[i]] + + # join all the action items with spaces + text = ' '.join([item for item in parts if item is not None]) + + # clean up separators for mutually exclusive groups + open = r'[\[(]' + close = r'[\])]' + text = _re.sub(r'(%s) ' % open, r'\1', text) + text = _re.sub(r' (%s)' % close, r'\1', text) + text = _re.sub(r'%s *%s' % (open, close), r'', text) + text = _re.sub(r'\(([^|]*)\)', r'\1', text) + text = text.strip() + + # return the text + return text + + def _format_text(self, text): + if '%(prog)' in text: + text = text % dict(prog=self._prog) + text_width = self._width - self._current_indent + indent = ' ' * self._current_indent + return self._fill_text(text, text_width, indent) + '\n\n' + + def _format_action(self, action): + # determine the required width and the entry label + help_position = min(self._action_max_length + 2, + self._max_help_position) + help_width = self._width - help_position + action_width = help_position - self._current_indent - 2 + action_header = self._format_action_invocation(action) + + # ho nelp; start on same line and add a final newline + if not action.help: + tup = self._current_indent, '', action_header + action_header = '%*s%s\n' % tup + + # short action name; start on the same line and pad two spaces + elif len(action_header) <= action_width: + tup = self._current_indent, '', action_width, action_header + action_header = '%*s%-*s ' % tup + indent_first = 0 + + # long action name; start on the next line + else: + tup = self._current_indent, '', action_header + action_header = '%*s%s\n' % tup + indent_first = help_position + + # collect the pieces of the action help + parts = [action_header] + + # if there was help for the action, add lines of help text + if action.help: + help_text = self._expand_help(action) + help_lines = self._split_lines(help_text, help_width) + parts.append('%*s%s\n' % (indent_first, '', help_lines[0])) + for line in help_lines[1:]: + parts.append('%*s%s\n' % (help_position, '', line)) + + # or add a newline if the description doesn't end with one + elif not action_header.endswith('\n'): + parts.append('\n') + + # if there are any sub-actions, add their help as well + for subaction in self._iter_indented_subactions(action): + parts.append(self._format_action(subaction)) + + # return a single string + return self._join_parts(parts) + + def _format_action_invocation(self, action): + if not action.option_strings: + metavar, = self._metavar_formatter(action, action.dest)(1) + return metavar + + else: + parts = [] + + # if the Optional doesn't take a value, format is: + # -s, --long + if action.nargs == 0: + parts.extend(action.option_strings) + + # if the Optional takes a value, format is: + # -s ARGS, --long ARGS + else: + default = action.dest.upper() + args_string = self._format_args(action, default) + for option_string in action.option_strings: + parts.append('%s %s' % (option_string, args_string)) + + return ', '.join(parts) + + def _metavar_formatter(self, action, default_metavar): + if action.metavar is not None: + result = action.metavar + elif action.choices is not None: + choice_strs = [str(choice) for choice in action.choices] + result = '{%s}' % ','.join(choice_strs) + else: + result = default_metavar + + def format(tuple_size): + if isinstance(result, tuple): + return result + else: + return (result, ) * tuple_size + return format + + def _format_args(self, action, default_metavar): + get_metavar = self._metavar_formatter(action, default_metavar) + if action.nargs is None: + result = '%s' % get_metavar(1) + elif action.nargs == OPTIONAL: + result = '[%s]' % get_metavar(1) + elif action.nargs == ZERO_OR_MORE: + result = '[%s [%s ...]]' % get_metavar(2) + elif action.nargs == ONE_OR_MORE: + result = '%s [%s ...]' % get_metavar(2) + elif action.nargs == REMAINDER: + result = '...' + elif action.nargs == PARSER: + result = '%s ...' % get_metavar(1) + else: + formats = ['%s' for _ in range(action.nargs)] + result = ' '.join(formats) % get_metavar(action.nargs) + return result + + def _expand_help(self, action): + params = dict(vars(action), prog=self._prog) + for name in list(params): + if params[name] is SUPPRESS: + del params[name] + for name in list(params): + if hasattr(params[name], '__name__'): + params[name] = params[name].__name__ + if params.get('choices') is not None: + choices_str = ', '.join([str(c) for c in params['choices']]) + params['choices'] = choices_str + return self._get_help_string(action) % params + + def _iter_indented_subactions(self, action): + try: + get_subactions = action._get_subactions + except AttributeError: + pass + else: + self._indent() + for subaction in get_subactions(): + yield subaction + self._dedent() + + def _split_lines(self, text, width): + text = self._whitespace_matcher.sub(' ', text).strip() + return _textwrap.wrap(text, width) + + def _fill_text(self, text, width, indent): + text = self._whitespace_matcher.sub(' ', text).strip() + return _textwrap.fill(text, width, initial_indent=indent, + subsequent_indent=indent) + + def _get_help_string(self, action): + return action.help + + +class RawDescriptionHelpFormatter(HelpFormatter): + """Help message formatter which retains any formatting in descriptions. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _fill_text(self, text, width, indent): + return ''.join([indent + line for line in text.splitlines(True)]) + + +class RawTextHelpFormatter(RawDescriptionHelpFormatter): + """Help message formatter which retains formatting of all help text. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _split_lines(self, text, width): + return text.splitlines() + + +class ArgumentDefaultsHelpFormatter(HelpFormatter): + """Help message formatter which adds default values to argument help. + + Only the name of this class is considered a public API. All the methods + provided by the class are considered an implementation detail. + """ + + def _get_help_string(self, action): + help = action.help + if '%(default)' not in action.help: + if action.default is not SUPPRESS: + defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] + if action.option_strings or action.nargs in defaulting_nargs: + help += ' (default: %(default)s)' + return help + + +# ===================== +# Options and Arguments +# ===================== + +def _get_action_name(argument): + if argument is None: + return None + elif argument.option_strings: + return '/'.join(argument.option_strings) + elif argument.metavar not in (None, SUPPRESS): + return argument.metavar + elif argument.dest not in (None, SUPPRESS): + return argument.dest + else: + return None + + +class ArgumentError(Exception): + """An error from creating or using an argument (optional or positional). + + The string value of this exception is the message, augmented with + information about the argument that caused it. + """ + + def __init__(self, argument, message): + self.argument_name = _get_action_name(argument) + self.message = message + + def __str__(self): + if self.argument_name is None: + format = '%(message)s' + else: + format = 'argument %(argument_name)s: %(message)s' + return format % dict(message=self.message, + argument_name=self.argument_name) + + +class ArgumentTypeError(Exception): + """An error from trying to convert a command line string to a type.""" + pass + + +# ============== +# Action classes +# ============== + +class Action(_AttributeHolder): + """Information about how to convert command line strings to Python objects. + + Action objects are used by an ArgumentParser to represent the information + needed to parse a single argument from one or more strings from the + command line. The keyword arguments to the Action constructor are also + all attributes of Action instances. + + Keyword Arguments: + + - option_strings -- A list of command-line option strings which + should be associated with this action. + + - dest -- The name of the attribute to hold the created object(s) + + - nargs -- The number of command-line arguments that should be + consumed. By default, one argument will be consumed and a single + value will be produced. Other values include: + - N (an integer) consumes N arguments (and produces a list) + - '?' consumes zero or one arguments + - '*' consumes zero or more arguments (and produces a list) + - '+' consumes one or more arguments (and produces a list) + Note that the difference between the default and nargs=1 is that + with the default, a single value will be produced, while with + nargs=1, a list containing a single value will be produced. + + - const -- The value to be produced if the option is specified and the + option uses an action that takes no values. + + - default -- The value to be produced if the option is not specified. + + - type -- A callable that accepts a single string argument, and + returns the converted value. The standard Python types str, int, + float, and complex are useful examples of such callables. If None, + str is used. + + - choices -- A container of values that should be allowed. If not None, + after a command-line argument has been converted to the appropriate + type, an exception will be raised if it is not a member of this + collection. + + - required -- True if the action must always be specified at the + command line. This is only meaningful for optional command-line + arguments. + + - help -- The help string describing the argument. + + - metavar -- The name to be used for the option's argument with the + help string. If None, the 'dest' value will be used as the name. + """ + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + self.option_strings = option_strings + self.dest = dest + self.nargs = nargs + self.const = const + self.default = default + self.type = type + self.choices = choices + self.required = required + self.help = help + self.metavar = metavar + + def _get_kwargs(self): + names = [ + 'option_strings', + 'dest', + 'nargs', + 'const', + 'default', + 'type', + 'choices', + 'help', + 'metavar', + ] + return [(name, getattr(self, name)) for name in names] + + def __call__(self, parser, namespace, values, option_string=None): + raise NotImplementedError(_('.__call__() not defined')) + + +class _StoreAction(Action): + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + if nargs == 0: + raise ValueError('nargs for store actions must be > 0; if you ' + 'have nothing to store, actions such as store ' + 'true or store const may be more appropriate') + if const is not None and nargs != OPTIONAL: + raise ValueError('nargs must be %r to supply const' % OPTIONAL) + super(_StoreAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=nargs, + const=const, + default=default, + type=type, + choices=choices, + required=required, + help=help, + metavar=metavar) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, values) + + +class _StoreConstAction(Action): + + def __init__(self, + option_strings, + dest, + const, + default=None, + required=False, + help=None, + metavar=None): + super(_StoreConstAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + const=const, + default=default, + required=required, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, self.const) + + +class _StoreTrueAction(_StoreConstAction): + + def __init__(self, + option_strings, + dest, + default=False, + required=False, + help=None): + super(_StoreTrueAction, self).__init__( + option_strings=option_strings, + dest=dest, + const=True, + default=default, + required=required, + help=help) + + +class _StoreFalseAction(_StoreConstAction): + + def __init__(self, + option_strings, + dest, + default=True, + required=False, + help=None): + super(_StoreFalseAction, self).__init__( + option_strings=option_strings, + dest=dest, + const=False, + default=default, + required=required, + help=help) + + +class _AppendAction(Action): + + def __init__(self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + type=None, + choices=None, + required=False, + help=None, + metavar=None): + if nargs == 0: + raise ValueError('nargs for append actions must be > 0; if arg ' + 'strings are not supplying the value to append, ' + 'the append const action may be more appropriate') + if const is not None and nargs != OPTIONAL: + raise ValueError('nargs must be %r to supply const' % OPTIONAL) + super(_AppendAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=nargs, + const=const, + default=default, + type=type, + choices=choices, + required=required, + help=help, + metavar=metavar) + + def __call__(self, parser, namespace, values, option_string=None): + items = _copy.copy(_ensure_value(namespace, self.dest, [])) + items.append(values) + setattr(namespace, self.dest, items) + + +class _AppendConstAction(Action): + + def __init__(self, + option_strings, + dest, + const, + default=None, + required=False, + help=None, + metavar=None): + super(_AppendConstAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + const=const, + default=default, + required=required, + help=help, + metavar=metavar) + + def __call__(self, parser, namespace, values, option_string=None): + items = _copy.copy(_ensure_value(namespace, self.dest, [])) + items.append(self.const) + setattr(namespace, self.dest, items) + + +class _CountAction(Action): + + def __init__(self, + option_strings, + dest, + default=None, + required=False, + help=None): + super(_CountAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + default=default, + required=required, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + new_count = _ensure_value(namespace, self.dest, 0) + 1 + setattr(namespace, self.dest, new_count) + + +class _HelpAction(Action): + + def __init__(self, + option_strings, + dest=SUPPRESS, + default=SUPPRESS, + help=None): + super(_HelpAction, self).__init__( + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help) + + def __call__(self, parser, namespace, values, option_string=None): + parser.print_help() + parser.exit() + + +class _VersionAction(Action): + + def __init__(self, + option_strings, + version=None, + dest=SUPPRESS, + default=SUPPRESS, + help="show program's version number and exit"): + super(_VersionAction, self).__init__( + option_strings=option_strings, + dest=dest, + default=default, + nargs=0, + help=help) + self.version = version + + def __call__(self, parser, namespace, values, option_string=None): + version = self.version + if version is None: + version = parser.version + formatter = parser._get_formatter() + formatter.add_text(version) + parser.exit(message=formatter.format_help()) + + +class _SubParsersAction(Action): + + class _ChoicesPseudoAction(Action): + + def __init__(self, name, aliases, help): + metavar = dest = name + if aliases: + metavar += ' (%s)' % ', '.join(aliases) + sup = super(_SubParsersAction._ChoicesPseudoAction, self) + sup.__init__(option_strings=[], dest=dest, help=help, + metavar=metavar) + + def __init__(self, + option_strings, + prog, + parser_class, + dest=SUPPRESS, + help=None, + metavar=None): + + self._prog_prefix = prog + self._parser_class = parser_class + self._name_parser_map = _collections.OrderedDict() + self._choices_actions = [] + + super(_SubParsersAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=PARSER, + choices=self._name_parser_map, + help=help, + metavar=metavar) + + def add_parser(self, name, **kwargs): + # set prog from the existing prefix + if kwargs.get('prog') is None: + kwargs['prog'] = '%s %s' % (self._prog_prefix, name) + + aliases = kwargs.pop('aliases', ()) + + # create a pseudo-action to hold the choice help + if 'help' in kwargs: + help = kwargs.pop('help') + choice_action = self._ChoicesPseudoAction(name, aliases, help) + self._choices_actions.append(choice_action) + + # create the parser and add it to the map + parser = self._parser_class(**kwargs) + self._name_parser_map[name] = parser + + # make parser available under aliases also + for alias in aliases: + self._name_parser_map[alias] = parser + + return parser + + def _get_subactions(self): + return self._choices_actions + + def __call__(self, parser, namespace, values, option_string=None): + parser_name = values[0] + arg_strings = values[1:] + + # set the parser name if requested + if self.dest is not SUPPRESS: + setattr(namespace, self.dest, parser_name) + + # select the parser + try: + parser = self._name_parser_map[parser_name] + except KeyError: + args = {'parser_name': parser_name, + 'choices': ', '.join(self._name_parser_map)} + msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args + raise ArgumentError(self, msg) + + # parse all the remaining options into the namespace + # store any unrecognized options on the object, so that the top + # level parser can decide what to do with them + namespace, arg_strings = parser.parse_known_args(arg_strings, namespace) + if arg_strings: + vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) + getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) + + +# ============== +# Type classes +# ============== + +class FileType(object): + """Factory for creating file object types + + Instances of FileType are typically passed as type= arguments to the + ArgumentParser add_argument() method. + + Keyword Arguments: + - mode -- A string indicating how the file is to be opened. Accepts the + same values as the builtin open() function. + - bufsize -- The file's desired buffer size. Accepts the same values as + the builtin open() function. + """ + + def __init__(self, mode='r', bufsize=-1): + self._mode = mode + self._bufsize = bufsize + + def __call__(self, string): + # the special argument "-" means sys.std{in,out} + if string == '-': + if 'r' in self._mode: + return _sys.stdin + elif 'w' in self._mode: + return _sys.stdout + else: + msg = _('argument "-" with mode %r') % self._mode + raise ValueError(msg) + + # all other arguments are used as file names + try: + return open(string, self._mode, self._bufsize) + except IOError as e: + message = _("can't open '%s': %s") + raise ArgumentTypeError(message % (string, e)) + + def __repr__(self): + args = self._mode, self._bufsize + args_str = ', '.join(repr(arg) for arg in args if arg != -1) + return '%s(%s)' % (type(self).__name__, args_str) + +# =========================== +# Optional and Positional Parsing +# =========================== + +class Namespace(_AttributeHolder): + """Simple object for storing attributes. + + Implements equality by attribute names and values, and provides a simple + string representation. + """ + + def __init__(self, **kwargs): + for name in kwargs: + setattr(self, name, kwargs[name]) + + def __eq__(self, other): + return vars(self) == vars(other) + + def __ne__(self, other): + return not (self == other) + + def __contains__(self, key): + return key in self.__dict__ + + +class _ActionsContainer(object): + + def __init__(self, + description, + prefix_chars, + argument_default, + conflict_handler): + super(_ActionsContainer, self).__init__() + + self.description = description + self.argument_default = argument_default + self.prefix_chars = prefix_chars + self.conflict_handler = conflict_handler + + # set up registries + self._registries = {} + + # register actions + self.register('action', None, _StoreAction) + self.register('action', 'store', _StoreAction) + self.register('action', 'store_const', _StoreConstAction) + self.register('action', 'store_true', _StoreTrueAction) + self.register('action', 'store_false', _StoreFalseAction) + self.register('action', 'append', _AppendAction) + self.register('action', 'append_const', _AppendConstAction) + self.register('action', 'count', _CountAction) + self.register('action', 'help', _HelpAction) + self.register('action', 'version', _VersionAction) + self.register('action', 'parsers', _SubParsersAction) + + # raise an exception if the conflict handler is invalid + self._get_handler() + + # action storage + self._actions = [] + self._option_string_actions = {} + + # groups + self._action_groups = [] + self._mutually_exclusive_groups = [] + + # defaults storage + self._defaults = {} + + # determines whether an "option" looks like a negative number + self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$') + + # whether or not there are any optionals that look like negative + # numbers -- uses a list so it can be shared and edited + self._has_negative_number_optionals = [] + + # ==================== + # Registration methods + # ==================== + def register(self, registry_name, value, object): + registry = self._registries.setdefault(registry_name, {}) + registry[value] = object + + def _registry_get(self, registry_name, value, default=None): + return self._registries[registry_name].get(value, default) + + # ================================== + # Namespace default accessor methods + # ================================== + def set_defaults(self, **kwargs): + self._defaults.update(kwargs) + + # if these defaults match any existing arguments, replace + # the previous default on the object with the new one + for action in self._actions: + if action.dest in kwargs: + action.default = kwargs[action.dest] + + def get_default(self, dest): + for action in self._actions: + if action.dest == dest and action.default is not None: + return action.default + return self._defaults.get(dest, None) + + + # ======================= + # Adding argument actions + # ======================= + def add_argument(self, *args, **kwargs): + """ + add_argument(dest, ..., name=value, ...) + add_argument(option_string, option_string, ..., name=value, ...) + """ + + # if no positional args are supplied or only one is supplied and + # it doesn't look like an option string, parse a positional + # argument + chars = self.prefix_chars + if not args or len(args) == 1 and args[0][0] not in chars: + if args and 'dest' in kwargs: + raise ValueError('dest supplied twice for positional argument') + kwargs = self._get_positional_kwargs(*args, **kwargs) + + # otherwise, we're adding an optional argument + else: + kwargs = self._get_optional_kwargs(*args, **kwargs) + + # if no default was supplied, use the parser-level default + if 'default' not in kwargs: + dest = kwargs['dest'] + if dest in self._defaults: + kwargs['default'] = self._defaults[dest] + elif self.argument_default is not None: + kwargs['default'] = self.argument_default + + # create the action object, and add it to the parser + action_class = self._pop_action_class(kwargs) + if not callable(action_class): + raise ValueError('unknown action "%s"' % (action_class,)) + action = action_class(**kwargs) + + # raise an error if the action type is not callable + type_func = self._registry_get('type', action.type, action.type) + if not callable(type_func): + raise ValueError('%r is not callable' % (type_func,)) + + # raise an error if the metavar does not match the type + if hasattr(self, "_get_formatter"): + try: + self._get_formatter()._format_args(action, None) + except TypeError: + raise ValueError("length of metavar tuple does not match nargs") + + return self._add_action(action) + + def add_argument_group(self, *args, **kwargs): + group = _ArgumentGroup(self, *args, **kwargs) + self._action_groups.append(group) + return group + + def add_mutually_exclusive_group(self, **kwargs): + group = _MutuallyExclusiveGroup(self, **kwargs) + self._mutually_exclusive_groups.append(group) + return group + + def _add_action(self, action): + # resolve any conflicts + self._check_conflict(action) + + # add to actions list + self._actions.append(action) + action.container = self + + # index the action by any option strings it has + for option_string in action.option_strings: + self._option_string_actions[option_string] = action + + # set the flag if any option strings look like negative numbers + for option_string in action.option_strings: + if self._negative_number_matcher.match(option_string): + if not self._has_negative_number_optionals: + self._has_negative_number_optionals.append(True) + + # return the created action + return action + + def _remove_action(self, action): + self._actions.remove(action) + + def _add_container_actions(self, container): + # collect groups by titles + title_group_map = {} + for group in self._action_groups: + if group.title in title_group_map: + msg = _('cannot merge actions - two groups are named %r') + raise ValueError(msg % (group.title)) + title_group_map[group.title] = group + + # map each action to its group + group_map = {} + for group in container._action_groups: + + # if a group with the title exists, use that, otherwise + # create a new group matching the container's group + if group.title not in title_group_map: + title_group_map[group.title] = self.add_argument_group( + title=group.title, + description=group.description, + conflict_handler=group.conflict_handler) + + # map the actions to their new group + for action in group._group_actions: + group_map[action] = title_group_map[group.title] + + # add container's mutually exclusive groups + # NOTE: if add_mutually_exclusive_group ever gains title= and + # description= then this code will need to be expanded as above + for group in container._mutually_exclusive_groups: + mutex_group = self.add_mutually_exclusive_group( + required=group.required) + + # map the actions to their new mutex group + for action in group._group_actions: + group_map[action] = mutex_group + + # add all actions to this container or their group + for action in container._actions: + group_map.get(action, self)._add_action(action) + + def _get_positional_kwargs(self, dest, **kwargs): + # make sure required is not specified + if 'required' in kwargs: + msg = _("'required' is an invalid argument for positionals") + raise TypeError(msg) + + # mark positional arguments as required if at least one is + # always required + if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]: + kwargs['required'] = True + if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs: + kwargs['required'] = True + + # return the keyword arguments with no option strings + return dict(kwargs, dest=dest, option_strings=[]) + + def _get_optional_kwargs(self, *args, **kwargs): + # determine short and long option strings + option_strings = [] + long_option_strings = [] + for option_string in args: + # error on strings that don't start with an appropriate prefix + if not option_string[0] in self.prefix_chars: + args = {'option': option_string, + 'prefix_chars': self.prefix_chars} + msg = _('invalid option string %(option)r: ' + 'must start with a character %(prefix_chars)r') + raise ValueError(msg % args) + + # strings starting with two prefix characters are long options + option_strings.append(option_string) + if option_string[0] in self.prefix_chars: + if len(option_string) > 1: + if option_string[1] in self.prefix_chars: + long_option_strings.append(option_string) + + # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x' + dest = kwargs.pop('dest', None) + if dest is None: + if long_option_strings: + dest_option_string = long_option_strings[0] + else: + dest_option_string = option_strings[0] + dest = dest_option_string.lstrip(self.prefix_chars) + if not dest: + msg = _('dest= is required for options like %r') + raise ValueError(msg % option_string) + dest = dest.replace('-', '_') + + # return the updated keyword arguments + return dict(kwargs, dest=dest, option_strings=option_strings) + + def _pop_action_class(self, kwargs, default=None): + action = kwargs.pop('action', default) + return self._registry_get('action', action, action) + + def _get_handler(self): + # determine function from conflict handler string + handler_func_name = '_handle_conflict_%s' % self.conflict_handler + try: + return getattr(self, handler_func_name) + except AttributeError: + msg = _('invalid conflict_resolution value: %r') + raise ValueError(msg % self.conflict_handler) + + def _check_conflict(self, action): + + # find all options that conflict with this option + confl_optionals = [] + for option_string in action.option_strings: + if option_string in self._option_string_actions: + confl_optional = self._option_string_actions[option_string] + confl_optionals.append((option_string, confl_optional)) + + # resolve any conflicts + if confl_optionals: + conflict_handler = self._get_handler() + conflict_handler(action, confl_optionals) + + def _handle_conflict_error(self, action, conflicting_actions): + message = ngettext('conflicting option string: %s', + 'conflicting option strings: %s', + len(conflicting_actions)) + conflict_string = ', '.join([option_string + for option_string, action + in conflicting_actions]) + raise ArgumentError(action, message % conflict_string) + + def _handle_conflict_resolve(self, action, conflicting_actions): + + # remove all conflicting options + for option_string, action in conflicting_actions: + + # remove the conflicting option + action.option_strings.remove(option_string) + self._option_string_actions.pop(option_string, None) + + # if the option now has no option string, remove it from the + # container holding it + if not action.option_strings: + action.container._remove_action(action) + + +class _ArgumentGroup(_ActionsContainer): + + def __init__(self, container, title=None, description=None, **kwargs): + # add any missing keyword arguments by checking the container + update = kwargs.setdefault + update('conflict_handler', container.conflict_handler) + update('prefix_chars', container.prefix_chars) + update('argument_default', container.argument_default) + super_init = super(_ArgumentGroup, self).__init__ + super_init(description=description, **kwargs) + + # group attributes + self.title = title + self._group_actions = [] + + # share most attributes with the container + self._registries = container._registries + self._actions = container._actions + self._option_string_actions = container._option_string_actions + self._defaults = container._defaults + self._has_negative_number_optionals = \ + container._has_negative_number_optionals + self._mutually_exclusive_groups = container._mutually_exclusive_groups + + def _add_action(self, action): + action = super(_ArgumentGroup, self)._add_action(action) + self._group_actions.append(action) + return action + + def _remove_action(self, action): + super(_ArgumentGroup, self)._remove_action(action) + self._group_actions.remove(action) + + +class _MutuallyExclusiveGroup(_ArgumentGroup): + + def __init__(self, container, required=False): + super(_MutuallyExclusiveGroup, self).__init__(container) + self.required = required + self._container = container + + def _add_action(self, action): + if action.required: + msg = _('mutually exclusive arguments must be optional') + raise ValueError(msg) + action = self._container._add_action(action) + self._group_actions.append(action) + return action + + def _remove_action(self, action): + self._container._remove_action(action) + self._group_actions.remove(action) + + +class ArgumentParser(_AttributeHolder, _ActionsContainer): + """Object for parsing command line strings into Python objects. + + Keyword Arguments: + - prog -- The name of the program (default: sys.argv[0]) + - usage -- A usage message (default: auto-generated from arguments) + - description -- A description of what the program does + - epilog -- Text following the argument descriptions + - parents -- Parsers whose arguments should be copied into this one + - formatter_class -- HelpFormatter class for printing help messages + - prefix_chars -- Characters that prefix optional arguments + - fromfile_prefix_chars -- Characters that prefix files containing + additional arguments + - argument_default -- The default value for all arguments + - conflict_handler -- String indicating how to handle conflicts + - add_help -- Add a -h/-help option + """ + + def __init__(self, + prog=None, + usage=None, + description=None, + epilog=None, + version=None, + parents=[], + formatter_class=HelpFormatter, + prefix_chars='-', + fromfile_prefix_chars=None, + argument_default=None, + conflict_handler='error', + add_help=True): + + if version is not None: + import warnings + warnings.warn( + """The "version" argument to ArgumentParser is deprecated. """ + """Please use """ + """"add_argument(..., action='version', version="N", ...)" """ + """instead""", DeprecationWarning) + + superinit = super(ArgumentParser, self).__init__ + superinit(description=description, + prefix_chars=prefix_chars, + argument_default=argument_default, + conflict_handler=conflict_handler) + + # default setting for prog + if prog is None: + prog = _os.path.basename(_sys.argv[0]) + + self.prog = prog + self.usage = usage + self.epilog = epilog + self.version = version + self.formatter_class = formatter_class + self.fromfile_prefix_chars = fromfile_prefix_chars + self.add_help = add_help + + add_group = self.add_argument_group + self._positionals = add_group(_('positional arguments')) + self._optionals = add_group(_('optional arguments')) + self._subparsers = None + + # register types + def identity(string): + return string + self.register('type', None, identity) + + # add help and version arguments if necessary + # (using explicit default to override global argument_default) + default_prefix = '-' if '-' in prefix_chars else prefix_chars[0] + if self.add_help: + self.add_argument( + default_prefix+'h', default_prefix*2+'help', + action='help', default=SUPPRESS, + help=_('show this help message and exit')) + if self.version: + self.add_argument( + default_prefix+'v', default_prefix*2+'version', + action='version', default=SUPPRESS, + version=self.version, + help=_("show program's version number and exit")) + + # add parent arguments and defaults + for parent in parents: + self._add_container_actions(parent) + try: + defaults = parent._defaults + except AttributeError: + pass + else: + self._defaults.update(defaults) + + # ======================= + # Pretty __repr__ methods + # ======================= + def _get_kwargs(self): + names = [ + 'prog', + 'usage', + 'description', + 'version', + 'formatter_class', + 'conflict_handler', + 'add_help', + ] + return [(name, getattr(self, name)) for name in names] + + # ================================== + # Optional/Positional adding methods + # ================================== + def add_subparsers(self, **kwargs): + if self._subparsers is not None: + self.error(_('cannot have multiple subparser arguments')) + + # add the parser class to the arguments if it's not present + kwargs.setdefault('parser_class', type(self)) + + if 'title' in kwargs or 'description' in kwargs: + title = _(kwargs.pop('title', 'subcommands')) + description = _(kwargs.pop('description', None)) + self._subparsers = self.add_argument_group(title, description) + else: + self._subparsers = self._positionals + + # prog defaults to the usage message of this parser, skipping + # optional arguments and with no "usage:" prefix + if kwargs.get('prog') is None: + formatter = self._get_formatter() + positionals = self._get_positional_actions() + groups = self._mutually_exclusive_groups + formatter.add_usage(self.usage, positionals, groups, '') + kwargs['prog'] = formatter.format_help().strip() + + # create the parsers action and add it to the positionals list + parsers_class = self._pop_action_class(kwargs, 'parsers') + action = parsers_class(option_strings=[], **kwargs) + self._subparsers._add_action(action) + + # return the created parsers action + return action + + def _add_action(self, action): + if action.option_strings: + self._optionals._add_action(action) + else: + self._positionals._add_action(action) + return action + + def _get_optional_actions(self): + return [action + for action in self._actions + if action.option_strings] + + def _get_positional_actions(self): + return [action + for action in self._actions + if not action.option_strings] + + # ===================================== + # Command line argument parsing methods + # ===================================== + def parse_args(self, args=None, namespace=None): + args, argv = self.parse_known_args(args, namespace) + if argv: + msg = _('unrecognized arguments: %s') + self.error(msg % ' '.join(argv)) + return args + + def parse_known_args(self, args=None, namespace=None): + if args is None: + # args default to the system args + args = _sys.argv[1:] + else: + # make sure that args are mutable + args = list(args) + + # default Namespace built from parser defaults + if namespace is None: + namespace = Namespace() + + # add any action defaults that aren't present + for action in self._actions: + if action.dest is not SUPPRESS: + if not hasattr(namespace, action.dest): + if action.default is not SUPPRESS: + setattr(namespace, action.dest, action.default) + + # add any parser defaults that aren't present + for dest in self._defaults: + if not hasattr(namespace, dest): + setattr(namespace, dest, self._defaults[dest]) + + # parse the arguments and exit if there are any errors + try: + namespace, args = self._parse_known_args(args, namespace) + if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): + args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR)) + delattr(namespace, _UNRECOGNIZED_ARGS_ATTR) + return namespace, args + except ArgumentError: + err = _sys.exc_info()[1] + self.error(str(err)) + + def _parse_known_args(self, arg_strings, namespace): + # replace arg strings that are file references + if self.fromfile_prefix_chars is not None: + arg_strings = self._read_args_from_files(arg_strings) + + # map all mutually exclusive arguments to the other arguments + # they can't occur with + action_conflicts = {} + for mutex_group in self._mutually_exclusive_groups: + group_actions = mutex_group._group_actions + for i, mutex_action in enumerate(mutex_group._group_actions): + conflicts = action_conflicts.setdefault(mutex_action, []) + conflicts.extend(group_actions[:i]) + conflicts.extend(group_actions[i + 1:]) + + # find all option indices, and determine the arg_string_pattern + # which has an 'O' if there is an option at an index, + # an 'A' if there is an argument, or a '-' if there is a '--' + option_string_indices = {} + arg_string_pattern_parts = [] + arg_strings_iter = iter(arg_strings) + for i, arg_string in enumerate(arg_strings_iter): + + # all args after -- are non-options + if arg_string == '--': + arg_string_pattern_parts.append('-') + for arg_string in arg_strings_iter: + arg_string_pattern_parts.append('A') + + # otherwise, add the arg to the arg strings + # and note the index if it was an option + else: + option_tuple = self._parse_optional(arg_string) + if option_tuple is None: + pattern = 'A' + else: + option_string_indices[i] = option_tuple + pattern = 'O' + arg_string_pattern_parts.append(pattern) + + # join the pieces together to form the pattern + arg_strings_pattern = ''.join(arg_string_pattern_parts) + + # converts arg strings to the appropriate and then takes the action + seen_actions = set() + seen_non_default_actions = set() + + def take_action(action, argument_strings, option_string=None): + seen_actions.add(action) + argument_values = self._get_values(action, argument_strings) + + # error if this argument is not allowed with other previously + # seen arguments, assuming that actions that use the default + # value don't really count as "present" + if argument_values is not action.default: + seen_non_default_actions.add(action) + for conflict_action in action_conflicts.get(action, []): + if conflict_action in seen_non_default_actions: + msg = _('not allowed with argument %s') + action_name = _get_action_name(conflict_action) + raise ArgumentError(action, msg % action_name) + + # take the action if we didn't receive a SUPPRESS value + # (e.g. from a default) + if argument_values is not SUPPRESS: + action(self, namespace, argument_values, option_string) + + # function to convert arg_strings into an optional action + def consume_optional(start_index): + + # get the optional identified at this index + option_tuple = option_string_indices[start_index] + action, option_string, explicit_arg = option_tuple + + # identify additional optionals in the same arg string + # (e.g. -xyz is the same as -x -y -z if no args are required) + match_argument = self._match_argument + action_tuples = [] + while True: + + # if we found no optional action, skip it + if action is None: + extras.append(arg_strings[start_index]) + return start_index + 1 + + # if there is an explicit argument, try to match the + # optional's string arguments to only this + if explicit_arg is not None: + arg_count = match_argument(action, 'A') + + # if the action is a single-dash option and takes no + # arguments, try to parse more single-dash options out + # of the tail of the option string + chars = self.prefix_chars + if arg_count == 0 and option_string[1] not in chars: + action_tuples.append((action, [], option_string)) + char = option_string[0] + option_string = char + explicit_arg[0] + new_explicit_arg = explicit_arg[1:] or None + optionals_map = self._option_string_actions + if option_string in optionals_map: + action = optionals_map[option_string] + explicit_arg = new_explicit_arg + else: + msg = _('ignored explicit argument %r') + raise ArgumentError(action, msg % explicit_arg) + + # if the action expect exactly one argument, we've + # successfully matched the option; exit the loop + elif arg_count == 1: + stop = start_index + 1 + args = [explicit_arg] + action_tuples.append((action, args, option_string)) + break + + # error if a double-dash option did not use the + # explicit argument + else: + msg = _('ignored explicit argument %r') + raise ArgumentError(action, msg % explicit_arg) + + # if there is no explicit argument, try to match the + # optional's string arguments with the following strings + # if successful, exit the loop + else: + start = start_index + 1 + selected_patterns = arg_strings_pattern[start:] + arg_count = match_argument(action, selected_patterns) + stop = start + arg_count + args = arg_strings[start:stop] + action_tuples.append((action, args, option_string)) + break + + # add the Optional to the list and return the index at which + # the Optional's string args stopped + assert action_tuples + for action, args, option_string in action_tuples: + take_action(action, args, option_string) + return stop + + # the list of Positionals left to be parsed; this is modified + # by consume_positionals() + positionals = self._get_positional_actions() + + # function to convert arg_strings into positional actions + def consume_positionals(start_index): + # match as many Positionals as possible + match_partial = self._match_arguments_partial + selected_pattern = arg_strings_pattern[start_index:] + arg_counts = match_partial(positionals, selected_pattern) + + # slice off the appropriate arg strings for each Positional + # and add the Positional and its args to the list + for action, arg_count in zip(positionals, arg_counts): + args = arg_strings[start_index: start_index + arg_count] + start_index += arg_count + take_action(action, args) + + # slice off the Positionals that we just parsed and return the + # index at which the Positionals' string args stopped + positionals[:] = positionals[len(arg_counts):] + return start_index + + # consume Positionals and Optionals alternately, until we have + # passed the last option string + extras = [] + start_index = 0 + if option_string_indices: + max_option_string_index = max(option_string_indices) + else: + max_option_string_index = -1 + while start_index <= max_option_string_index: + + # consume any Positionals preceding the next option + next_option_string_index = min([ + index + for index in option_string_indices + if index >= start_index]) + if start_index != next_option_string_index: + positionals_end_index = consume_positionals(start_index) + + # only try to parse the next optional if we didn't consume + # the option string during the positionals parsing + if positionals_end_index > start_index: + start_index = positionals_end_index + continue + else: + start_index = positionals_end_index + + # if we consumed all the positionals we could and we're not + # at the index of an option string, there were extra arguments + if start_index not in option_string_indices: + strings = arg_strings[start_index:next_option_string_index] + extras.extend(strings) + start_index = next_option_string_index + + # consume the next optional and any arguments for it + start_index = consume_optional(start_index) + + # consume any positionals following the last Optional + stop_index = consume_positionals(start_index) + + # if we didn't consume all the argument strings, there were extras + extras.extend(arg_strings[stop_index:]) + + # if we didn't use all the Positional objects, there were too few + # arg strings supplied. + if positionals: + self.error(_('too few arguments')) + + # make sure all required actions were present, and convert defaults. + for action in self._actions: + if action not in seen_actions: + if action.required: + name = _get_action_name(action) + self.error(_('argument %s is required') % name) + else: + # Convert action default now instead of doing it before + # parsing arguments to avoid calling convert functions + # twice (which may fail) if the argument was given, but + # only if it was defined already in the namespace + if (action.default is not None and + isinstance(action.default, str) and + hasattr(namespace, action.dest) and + action.default is getattr(namespace, action.dest)): + setattr(namespace, action.dest, + self._get_value(action, action.default)) + + # make sure all required groups had one option present + for group in self._mutually_exclusive_groups: + if group.required: + for action in group._group_actions: + if action in seen_non_default_actions: + break + + # if no actions were used, report the error + else: + names = [_get_action_name(action) + for action in group._group_actions + if action.help is not SUPPRESS] + msg = _('one of the arguments %s is required') + self.error(msg % ' '.join(names)) + + # return the updated namespace and the extra arguments + return namespace, extras + + def _read_args_from_files(self, arg_strings): + # expand arguments referencing files + new_arg_strings = [] + for arg_string in arg_strings: + + # for regular arguments, just add them back into the list + if not arg_string or arg_string[0] not in self.fromfile_prefix_chars: + new_arg_strings.append(arg_string) + + # replace arguments referencing files with the file content + else: + try: + args_file = open(arg_string[1:]) + try: + arg_strings = [] + for arg_line in args_file.read().splitlines(): + for arg in self.convert_arg_line_to_args(arg_line): + arg_strings.append(arg) + arg_strings = self._read_args_from_files(arg_strings) + new_arg_strings.extend(arg_strings) + finally: + args_file.close() + except IOError: + err = _sys.exc_info()[1] + self.error(str(err)) + + # return the modified argument list + return new_arg_strings + + def convert_arg_line_to_args(self, arg_line): + return [arg_line] + + def _match_argument(self, action, arg_strings_pattern): + # match the pattern for this action to the arg strings + nargs_pattern = self._get_nargs_pattern(action) + match = _re.match(nargs_pattern, arg_strings_pattern) + + # raise an exception if we weren't able to find a match + if match is None: + nargs_errors = { + None: _('expected one argument'), + OPTIONAL: _('expected at most one argument'), + ONE_OR_MORE: _('expected at least one argument'), + } + default = ngettext('expected %s argument', + 'expected %s arguments', + action.nargs) % action.nargs + msg = nargs_errors.get(action.nargs, default) + raise ArgumentError(action, msg) + + # return the number of arguments matched + return len(match.group(1)) + + def _match_arguments_partial(self, actions, arg_strings_pattern): + # progressively shorten the actions list by slicing off the + # final actions until we find a match + result = [] + for i in range(len(actions), 0, -1): + actions_slice = actions[:i] + pattern = ''.join([self._get_nargs_pattern(action) + for action in actions_slice]) + match = _re.match(pattern, arg_strings_pattern) + if match is not None: + result.extend([len(string) for string in match.groups()]) + break + + # return the list of arg string counts + return result + + def _parse_optional(self, arg_string): + # if it's an empty string, it was meant to be a positional + if not arg_string: + return None + + # if it doesn't start with a prefix, it was meant to be positional + if not arg_string[0] in self.prefix_chars: + return None + + # if the option string is present in the parser, return the action + if arg_string in self._option_string_actions: + action = self._option_string_actions[arg_string] + return action, arg_string, None + + # if it's just a single character, it was meant to be positional + if len(arg_string) == 1: + return None + + # if the option string before the "=" is present, return the action + if '=' in arg_string: + option_string, explicit_arg = arg_string.split('=', 1) + if option_string in self._option_string_actions: + action = self._option_string_actions[option_string] + return action, option_string, explicit_arg + + # search through all possible prefixes of the option string + # and all actions in the parser for possible interpretations + option_tuples = self._get_option_tuples(arg_string) + + # if multiple actions match, the option string was ambiguous + if len(option_tuples) > 1: + options = ', '.join([option_string + for action, option_string, explicit_arg in option_tuples]) + args = {'option': arg_string, 'matches': options} + msg = _('ambiguous option: %(option)s could match %(matches)s') + self.error(msg % args) + + # if exactly one action matched, this segmentation is good, + # so return the parsed action + elif len(option_tuples) == 1: + option_tuple, = option_tuples + return option_tuple + + # if it was not found as an option, but it looks like a negative + # number, it was meant to be positional + # unless there are negative-number-like options + if self._negative_number_matcher.match(arg_string): + if not self._has_negative_number_optionals: + return None + + # if it contains a space, it was meant to be a positional + if ' ' in arg_string: + return None + + # it was meant to be an optional but there is no such option + # in this parser (though it might be a valid option in a subparser) + return None, arg_string, None + + def _get_option_tuples(self, option_string): + result = [] + + # option strings starting with two prefix characters are only + # split at the '=' + chars = self.prefix_chars + if option_string[0] in chars and option_string[1] in chars: + if '=' in option_string: + option_prefix, explicit_arg = option_string.split('=', 1) + else: + option_prefix = option_string + explicit_arg = None + for option_string in self._option_string_actions: + if option_string.startswith(option_prefix): + action = self._option_string_actions[option_string] + tup = action, option_string, explicit_arg + result.append(tup) + + # single character options can be concatenated with their arguments + # but multiple character options always have to have their argument + # separate + elif option_string[0] in chars and option_string[1] not in chars: + option_prefix = option_string + explicit_arg = None + short_option_prefix = option_string[:2] + short_explicit_arg = option_string[2:] + + for option_string in self._option_string_actions: + if option_string == short_option_prefix: + action = self._option_string_actions[option_string] + tup = action, option_string, short_explicit_arg + result.append(tup) + elif option_string.startswith(option_prefix): + action = self._option_string_actions[option_string] + tup = action, option_string, explicit_arg + result.append(tup) + + # shouldn't ever get here + else: + self.error(_('unexpected option string: %s') % option_string) + + # return the collected option tuples + return result + + def _get_nargs_pattern(self, action): + # in all examples below, we have to allow for '--' args + # which are represented as '-' in the pattern + nargs = action.nargs + + # the default (None) is assumed to be a single argument + if nargs is None: + nargs_pattern = '(-*A-*)' + + # allow zero or one arguments + elif nargs == OPTIONAL: + nargs_pattern = '(-*A?-*)' + + # allow zero or more arguments + elif nargs == ZERO_OR_MORE: + nargs_pattern = '(-*[A-]*)' + + # allow one or more arguments + elif nargs == ONE_OR_MORE: + nargs_pattern = '(-*A[A-]*)' + + # allow any number of options or arguments + elif nargs == REMAINDER: + nargs_pattern = '([-AO]*)' + + # allow one argument followed by any number of options or arguments + elif nargs == PARSER: + nargs_pattern = '(-*A[-AO]*)' + + # all others should be integers + else: + nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs) + + # if this is an optional action, -- is not allowed + if action.option_strings: + nargs_pattern = nargs_pattern.replace('-*', '') + nargs_pattern = nargs_pattern.replace('-', '') + + # return the pattern + return nargs_pattern + + # ======================== + # Value conversion methods + # ======================== + def _get_values(self, action, arg_strings): + # for everything but PARSER, REMAINDER args, strip out first '--' + if action.nargs not in [PARSER, REMAINDER]: + try: + arg_strings.remove('--') + except ValueError: + pass + + # optional argument produces a default when not present + if not arg_strings and action.nargs == OPTIONAL: + if action.option_strings: + value = action.const + else: + value = action.default + if isinstance(value, str): + value = self._get_value(action, value) + self._check_value(action, value) + + # when nargs='*' on a positional, if there were no command-line + # args, use the default if it is anything other than None + elif (not arg_strings and action.nargs == ZERO_OR_MORE and + not action.option_strings): + if action.default is not None: + value = action.default + else: + value = arg_strings + self._check_value(action, value) + + # single argument or optional argument produces a single value + elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: + arg_string, = arg_strings + value = self._get_value(action, arg_string) + self._check_value(action, value) + + # REMAINDER arguments convert all values, checking none + elif action.nargs == REMAINDER: + value = [self._get_value(action, v) for v in arg_strings] + + # PARSER arguments convert all values, but check only the first + elif action.nargs == PARSER: + value = [self._get_value(action, v) for v in arg_strings] + self._check_value(action, value[0]) + + # all other types of nargs produce a list + else: + value = [self._get_value(action, v) for v in arg_strings] + for v in value: + self._check_value(action, v) + + # return the converted value + return value + + def _get_value(self, action, arg_string): + type_func = self._registry_get('type', action.type, action.type) + if not callable(type_func): + msg = _('%r is not callable') + raise ArgumentError(action, msg % type_func) + + # convert the value to the appropriate type + try: + result = type_func(arg_string) + + # ArgumentTypeErrors indicate errors + except ArgumentTypeError: + name = getattr(action.type, '__name__', repr(action.type)) + msg = str(_sys.exc_info()[1]) + raise ArgumentError(action, msg) + + # TypeErrors or ValueErrors also indicate errors + except (TypeError, ValueError): + name = getattr(action.type, '__name__', repr(action.type)) + args = {'type': name, 'value': arg_string} + msg = _('invalid %(type)s value: %(value)r') + raise ArgumentError(action, msg % args) + + # return the converted value + return result + + def _check_value(self, action, value): + # converted value must be one of the choices (if specified) + if action.choices is not None and value not in action.choices: + args = {'value': value, + 'choices': ', '.join(map(repr, action.choices))} + msg = _('invalid choice: %(value)r (choose from %(choices)s)') + raise ArgumentError(action, msg % args) + + # ======================= + # Help-formatting methods + # ======================= + def format_usage(self): + formatter = self._get_formatter() + formatter.add_usage(self.usage, self._actions, + self._mutually_exclusive_groups) + return formatter.format_help() + + def format_help(self): + formatter = self._get_formatter() + + # usage + formatter.add_usage(self.usage, self._actions, + self._mutually_exclusive_groups) + + # description + formatter.add_text(self.description) + + # positionals, optionals and user-defined groups + for action_group in self._action_groups: + formatter.start_section(action_group.title) + formatter.add_text(action_group.description) + formatter.add_arguments(action_group._group_actions) + formatter.end_section() + + # epilog + formatter.add_text(self.epilog) + + # determine help from format above + return formatter.format_help() + + def format_version(self): + import warnings + warnings.warn( + 'The format_version method is deprecated -- the "version" ' + 'argument to ArgumentParser is no longer supported.', + DeprecationWarning) + formatter = self._get_formatter() + formatter.add_text(self.version) + return formatter.format_help() + + def _get_formatter(self): + return self.formatter_class(prog=self.prog) + + # ===================== + # Help-printing methods + # ===================== + def print_usage(self, file=None): + if file is None: + file = _sys.stdout + self._print_message(self.format_usage(), file) + + def print_help(self, file=None): + if file is None: + file = _sys.stdout + self._print_message(self.format_help(), file) + + def print_version(self, file=None): + import warnings + warnings.warn( + 'The print_version method is deprecated -- the "version" ' + 'argument to ArgumentParser is no longer supported.', + DeprecationWarning) + self._print_message(self.format_version(), file) + + def _print_message(self, message, file=None): + if message: + if file is None: + file = _sys.stderr + file.write(message) + + # =============== + # Exiting methods + # =============== + def exit(self, status=0, message=None): + if message: + self._print_message(message, _sys.stderr) + _sys.exit(status) + + def error(self, message): + """error(message: string) + + Prints a usage message incorporating the message to stderr and + exits. + + If you override this in a subclass, it should not return -- it + should either exit or raise an exception. + """ + self.print_usage(_sys.stderr) + args = {'prog': self.prog, 'message': message} + self.exit(2, _('%(prog)s: error: %(message)s\n') % args) diff --git a/borg/testsuite/__init__.py b/borg/testsuite/__init__.py index 9872edeb6..cd790b571 100644 --- a/borg/testsuite/__init__.py +++ b/borg/testsuite/__init__.py @@ -2,6 +2,7 @@ from contextlib import contextmanager import filecmp import os import posix +import stat import sys import sysconfig import time @@ -27,6 +28,8 @@ elif 'HAVE_UTIMES' in sysconfig.get_config_vars(): else: st_mtime_ns_round = -9 +if sys.platform.startswith('netbsd'): + st_mtime_ns_round = -4 # only >1 microsecond resolution here? has_mtime_ns = sys.version >= '3.3' utime_supports_fd = os.utime in getattr(os, 'supports_fd', {}) @@ -72,6 +75,11 @@ class BaseTestCase(unittest.TestCase): attrs.append('st_nlink') d1 = [filename] + [getattr(s1, a) for a in attrs] d2 = [filename] + [getattr(s2, a) for a in attrs] + # ignore st_rdev if file is not a block/char device, fixes #203 + if not stat.S_ISCHR(d1[1]) and not stat.S_ISBLK(d1[1]): + d1[4] = None + if not stat.S_ISCHR(d2[1]) and not stat.S_ISBLK(d2[1]): + d2[4] = None if not os.path.islink(path1) or utime_supports_fd: # Older versions of llfuse do not support ns precision properly if fuse and not have_fuse_mtime_ns: diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 0263588e2..754af8dd6 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -162,7 +162,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): # Directory self.create_regular_file('dir2/file2', size=1024 * 80) # File mode - os.chmod('input/file1', 0o7755) + os.chmod('input/file1', 0o4755) # Hard link os.link(os.path.join(self.input_path, 'file1'), os.path.join(self.input_path, 'hardlink')) @@ -264,7 +264,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): st = os.stat(filename) self.assert_equal(st.st_size, total_len) if sparse_support and hasattr(st, 'st_blocks'): - self.assert_true(st.st_blocks * 512 < total_len / 10) # is input sparse? + self.assert_true(st.st_blocks * 512 < total_len / 9) # is input sparse? self.cmd('init', self.repository_location) self.cmd('create', self.repository_location + '::test', 'input') with changedir('output'): @@ -279,7 +279,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): st = os.stat(filename) self.assert_equal(st.st_size, total_len) if sparse_support and hasattr(st, 'st_blocks'): - self.assert_true(st.st_blocks * 512 < total_len / 10) # is output sparse? + self.assert_true(st.st_blocks * 512 < total_len / 9) # is output sparse? def test_unusual_filenames(self): filenames = ['normal', 'with some blanks', '(with_parens)', ] diff --git a/borg/testsuite/compress.py b/borg/testsuite/compress.py index 8019925b2..ce46c9d30 100644 --- a/borg/testsuite/compress.py +++ b/borg/testsuite/compress.py @@ -93,7 +93,7 @@ def test_compressor(): params_list += [ dict(name='lzma', level=0, buffer=buffer), dict(name='lzma', level=6, buffer=buffer), - dict(name='lzma', level=9, buffer=buffer), + # we do not test lzma on level 9 because of the huge memory needs ] for params in params_list: c = Compressor(**params) diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 95531df83..620a77c14 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -1,12 +1,14 @@ import hashlib from time import mktime, strptime from datetime import datetime, timezone, timedelta +import os import pytest +import sys import msgpack -from ..helpers import adjust_patterns, exclude_path, Location, format_timedelta, ExcludePattern, make_path_safe, \ - prune_within, prune_split, \ +from ..helpers import adjust_patterns, exclude_path, Location, format_timedelta, IncludePattern, ExcludePattern, make_path_safe, \ + prune_within, prune_split, get_cache_dir, \ StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams from . import BaseTestCase @@ -80,6 +82,11 @@ class TestLocationWithoutEnv: with pytest.raises(ValueError): Location('ssh://localhost:22/path:archive') + def test_no_slashes(self, monkeypatch): + monkeypatch.delenv('BORG_REPO', raising=False) + with pytest.raises(ValueError): + Location('/some/path/to/repo::archive_name_with/slashes/is_invalid') + def test_canonical_path(self, monkeypatch): monkeypatch.delenv('BORG_REPO', raising=False) locations = ['some/path::archive', 'file://some/path::archive', 'host:some/path::archive', @@ -133,6 +140,11 @@ class TestLocationWithEnv: assert repr(Location()) == \ "Location(proto='file', user=None, host=None, port=None, path='some/relative/path', archive=None)" + def test_no_slashes(self, monkeypatch): + monkeypatch.setenv('BORG_REPO', '/some/absolute/path') + with pytest.raises(ValueError): + Location('::archive_name_with/slashes/is_invalid') + class FormatTimedeltaTestCase(BaseTestCase): @@ -178,6 +190,72 @@ class PatternTestCase(BaseTestCase): ['/etc/passwd', '/etc/hosts', '/var/log/messages', '/var/log/dmesg']) +@pytest.mark.skipif(sys.platform in ('darwin',), reason='all but OS X test') +class PatternNonAsciiTestCase(BaseTestCase): + def testComposedUnicode(self): + pattern = 'b\N{LATIN SMALL LETTER A WITH ACUTE}' + i = IncludePattern(pattern) + e = ExcludePattern(pattern) + + assert i.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") + assert not i.match("ba\N{COMBINING ACUTE ACCENT}/foo") + assert e.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") + assert not e.match("ba\N{COMBINING ACUTE ACCENT}/foo") + + def testDecomposedUnicode(self): + pattern = 'ba\N{COMBINING ACUTE ACCENT}' + i = IncludePattern(pattern) + e = ExcludePattern(pattern) + + assert not i.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") + assert i.match("ba\N{COMBINING ACUTE ACCENT}/foo") + assert not e.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") + assert e.match("ba\N{COMBINING ACUTE ACCENT}/foo") + + def testInvalidUnicode(self): + pattern = str(b'ba\x80', 'latin1') + i = IncludePattern(pattern) + e = ExcludePattern(pattern) + + assert not i.match("ba/foo") + assert i.match(str(b"ba\x80/foo", 'latin1')) + assert not e.match("ba/foo") + assert e.match(str(b"ba\x80/foo", 'latin1')) + + +@pytest.mark.skipif(sys.platform not in ('darwin',), reason='OS X test') +class OSXPatternNormalizationTestCase(BaseTestCase): + def testComposedUnicode(self): + pattern = 'b\N{LATIN SMALL LETTER A WITH ACUTE}' + i = IncludePattern(pattern) + e = ExcludePattern(pattern) + + assert i.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") + assert i.match("ba\N{COMBINING ACUTE ACCENT}/foo") + assert e.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") + assert e.match("ba\N{COMBINING ACUTE ACCENT}/foo") + + def testDecomposedUnicode(self): + pattern = 'ba\N{COMBINING ACUTE ACCENT}' + i = IncludePattern(pattern) + e = ExcludePattern(pattern) + + assert i.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") + assert i.match("ba\N{COMBINING ACUTE ACCENT}/foo") + assert e.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") + assert e.match("ba\N{COMBINING ACUTE ACCENT}/foo") + + def testInvalidUnicode(self): + pattern = str(b'ba\x80', 'latin1') + i = IncludePattern(pattern) + e = ExcludePattern(pattern) + + assert not i.match("ba/foo") + assert i.match(str(b"ba\x80/foo", 'latin1')) + assert not e.match("ba/foo") + assert e.match(str(b"ba\x80/foo", 'latin1')) + + def test_compression_specs(): with pytest.raises(ValueError): CompressionSpec('') @@ -304,3 +382,20 @@ class TestParseTimestamp(BaseTestCase): def test(self): self.assert_equal(parse_timestamp('2015-04-19T20:25:00.226410'), datetime(2015, 4, 19, 20, 25, 0, 226410, timezone.utc)) self.assert_equal(parse_timestamp('2015-04-19T20:25:00'), datetime(2015, 4, 19, 20, 25, 0, 0, timezone.utc)) + + +def test_get_cache_dir(): + """test that get_cache_dir respects environement""" + # reset BORG_CACHE_DIR in order to test default + old_env = None + if os.environ.get('BORG_CACHE_DIR'): + old_env = os.environ['BORG_CACHE_DIR'] + del(os.environ['BORG_CACHE_DIR']) + assert get_cache_dir() == os.path.join(os.path.expanduser('~'), '.cache', 'borg') + os.environ['XDG_CACHE_HOME'] = '/var/tmp/.cache' + assert get_cache_dir() == os.path.join('/var/tmp/.cache', 'borg') + os.environ['BORG_CACHE_DIR'] = '/var/tmp' + assert get_cache_dir() == '/var/tmp' + # reset old env + if old_env is not None: + os.environ['BORG_CACHE_DIR'] = old_env diff --git a/borg/testsuite/repository.py b/borg/testsuite/repository.py index 74996b717..2b99b83d6 100644 --- a/borg/testsuite/repository.py +++ b/borg/testsuite/repository.py @@ -325,6 +325,15 @@ class RemoteRepositoryTestCase(RepositoryTestCase): def test_invalid_rpc(self): self.assert_raises(InvalidRPCMethod, lambda: self.repository.call('__init__', None)) + def test_ssh_cmd(self): + assert self.repository.umask is not None + assert self.repository.ssh_cmd(Location('example.com:foo')) == ['ssh', 'example.com', 'borg', 'serve'] + self.repository.umask_flag() + assert self.repository.ssh_cmd(Location('ssh://example.com/foo')) == ['ssh', 'example.com', 'borg', 'serve'] + self.repository.umask_flag() + assert self.repository.ssh_cmd(Location('ssh://user@example.com/foo')) == ['ssh', 'user@example.com', 'borg', 'serve'] + self.repository.umask_flag() + assert self.repository.ssh_cmd(Location('ssh://user@example.com:1234/foo')) == ['ssh', '-p', '1234', 'user@example.com', 'borg', 'serve'] + self.repository.umask_flag() + os.environ['BORG_RSH'] = 'ssh --foo' + assert self.repository.ssh_cmd(Location('example.com:foo')) == ['ssh', '--foo', 'example.com', 'borg', 'serve'] + self.repository.umask_flag() + class RemoteRepositoryCheckTestCase(RepositoryCheckTestCase): diff --git a/borg/testsuite/upgrader.py b/borg/testsuite/upgrader.py new file mode 100644 index 000000000..22278f9ac --- /dev/null +++ b/borg/testsuite/upgrader.py @@ -0,0 +1,163 @@ +import os +import shutil +import tempfile + +import pytest + +try: + import attic.repository + import attic.key + import attic.helpers +except ImportError: + attic = None + +from ..upgrader import AtticRepositoryUpgrader, AtticKeyfileKey +from ..helpers import get_keys_dir +from ..key import KeyfileKey +from ..repository import Repository, MAGIC + +pytestmark = pytest.mark.skipif(attic is None, + reason='cannot find an attic install') + + +def repo_valid(path): + """ + utility function to check if borg can open a repository + + :param path: the path to the repository + :returns: if borg can check the repository + """ + repository = Repository(str(path), create=False) + # can't check raises() because check() handles the error + state = repository.check() + repository.close() + return state + + +def key_valid(path): + """ + check that the new keyfile is alright + + :param path: the path to the key file + :returns: if the file starts with the borg magic string + """ + keyfile = os.path.join(get_keys_dir(), + os.path.basename(path)) + with open(keyfile, 'r') as f: + return f.read().startswith(KeyfileKey.FILE_ID) + + +@pytest.fixture() +def attic_repo(tmpdir): + """ + create an attic repo with some stuff in it + + :param tmpdir: path to the repository to be created + :returns: a attic.repository.Repository object + """ + attic_repo = attic.repository.Repository(str(tmpdir), create=True) + # throw some stuff in that repo, copied from `RepositoryTestCase.test1` + for x in range(100): + attic_repo.put(('%-32d' % x).encode('ascii'), b'SOMEDATA') + attic_repo.commit() + attic_repo.close() + return attic_repo + + +def test_convert_segments(tmpdir, attic_repo): + """test segment conversion + + this will load the given attic repository, list all the segments + then convert them one at a time. we need to close the repo before + conversion otherwise we have errors from borg + + :param tmpdir: a temporary directory to run the test in (builtin + fixture) + :param attic_repo: a populated attic repository (fixture) + """ + # check should fail because of magic number + assert not repo_valid(tmpdir) + print("opening attic repository with borg and converting") + repo = AtticRepositoryUpgrader(str(tmpdir), create=False) + segments = [filename for i, filename in repo.io.segment_iterator()] + repo.close() + repo.convert_segments(segments, dryrun=False) + repo.convert_cache(dryrun=False) + assert repo_valid(tmpdir) + + +class MockArgs: + """ + mock attic location + + this is used to simulate a key location with a properly loaded + repository object to create a key file + """ + def __init__(self, path): + self.repository = attic.helpers.Location(path) + + +@pytest.fixture() +def attic_key_file(attic_repo, tmpdir): + """ + create an attic key file from the given repo, in the keys + subdirectory of the given tmpdir + + :param attic_repo: an attic.repository.Repository object (fixture + define above) + :param tmpdir: a temporary directory (a builtin fixture) + :returns: the KeyfileKey object as returned by + attic.key.KeyfileKey.create() + """ + keys_dir = str(tmpdir.mkdir('keys')) + + # we use the repo dir for the created keyfile, because we do + # not want to clutter existing keyfiles + os.environ['ATTIC_KEYS_DIR'] = keys_dir + + # we use the same directory for the converted files, which + # will clutter the previously created one, which we don't care + # about anyways. in real runs, the original key will be retained. + os.environ['BORG_KEYS_DIR'] = keys_dir + os.environ['ATTIC_PASSPHRASE'] = 'test' + return attic.key.KeyfileKey.create(attic_repo, + MockArgs(keys_dir)) + + +def test_keys(tmpdir, attic_repo, attic_key_file): + """test key conversion + + test that we can convert the given key to a properly formatted + borg key. assumes that the ATTIC_KEYS_DIR and BORG_KEYS_DIR have + been properly populated by the attic_key_file fixture. + + :param tmpdir: a temporary directory (a builtin fixture) + :param attic_repo: an attic.repository.Repository object (fixture + define above) + :param attic_key_file: an attic.key.KeyfileKey (fixture created above) + """ + repository = AtticRepositoryUpgrader(str(tmpdir), create=False) + keyfile = AtticKeyfileKey.find_key_file(repository) + AtticRepositoryUpgrader.convert_keyfiles(keyfile, dryrun=False) + assert key_valid(attic_key_file.path) + + +def test_convert_all(tmpdir, attic_repo, attic_key_file): + """test all conversion steps + + this runs everything. mostly redundant test, since everything is + done above. yet we expect a NotImplementedError because we do not + convert caches yet. + + :param tmpdir: a temporary directory (a builtin fixture) + :param attic_repo: an attic.repository.Repository object (fixture + define above) + :param attic_key_file: an attic.key.KeyfileKey (fixture created above) + """ + # check should fail because of magic number + assert not repo_valid(tmpdir) + print("opening attic repository with borg and converting") + repo = AtticRepositoryUpgrader(str(tmpdir), create=False) + repo.upgrade(dryrun=False) + assert key_valid(attic_key_file.path) + assert repo_valid(tmpdir) diff --git a/borg/testsuite/xattr.py b/borg/testsuite/xattr.py index d73856953..df0130c90 100644 --- a/borg/testsuite/xattr.py +++ b/borg/testsuite/xattr.py @@ -17,17 +17,23 @@ class XattrTestCase(BaseTestCase): def tearDown(self): os.unlink(self.symlink) + def assert_equal_se(self, is_x, want_x): + # check 2 xattr lists for equality, but ignore security.selinux attr + is_x = set(is_x) - {'security.selinux'} + want_x = set(want_x) + self.assert_equal(is_x, want_x) + def test(self): - self.assert_equal(listxattr(self.tmpfile.name), []) - self.assert_equal(listxattr(self.tmpfile.fileno()), []) - self.assert_equal(listxattr(self.symlink), []) + self.assert_equal_se(listxattr(self.tmpfile.name), []) + self.assert_equal_se(listxattr(self.tmpfile.fileno()), []) + self.assert_equal_se(listxattr(self.symlink), []) setxattr(self.tmpfile.name, 'user.foo', b'bar') setxattr(self.tmpfile.fileno(), 'user.bar', b'foo') setxattr(self.tmpfile.name, 'user.empty', None) - self.assert_equal(set(listxattr(self.tmpfile.name)), set(['user.foo', 'user.bar', 'user.empty'])) - self.assert_equal(set(listxattr(self.tmpfile.fileno())), set(['user.foo', 'user.bar', 'user.empty'])) - self.assert_equal(set(listxattr(self.symlink)), set(['user.foo', 'user.bar', 'user.empty'])) - self.assert_equal(listxattr(self.symlink, follow_symlinks=False), []) + self.assert_equal_se(listxattr(self.tmpfile.name), ['user.foo', 'user.bar', 'user.empty']) + self.assert_equal_se(listxattr(self.tmpfile.fileno()), ['user.foo', 'user.bar', 'user.empty']) + self.assert_equal_se(listxattr(self.symlink), ['user.foo', 'user.bar', 'user.empty']) + self.assert_equal_se(listxattr(self.symlink, follow_symlinks=False), []) self.assert_equal(getxattr(self.tmpfile.name, 'user.foo'), b'bar') self.assert_equal(getxattr(self.tmpfile.fileno(), 'user.foo'), b'bar') self.assert_equal(getxattr(self.symlink, 'user.foo'), b'bar') diff --git a/borg/upgrader.py b/borg/upgrader.py new file mode 100644 index 000000000..33ef2d388 --- /dev/null +++ b/borg/upgrader.py @@ -0,0 +1,233 @@ +from binascii import hexlify +import os +import shutil +import time + +from .helpers import get_keys_dir, get_cache_dir +from .locking import UpgradableLock +from .repository import Repository, MAGIC +from .key import KeyfileKey, KeyfileNotFoundError + +ATTIC_MAGIC = b'ATTICSEG' + + +class AtticRepositoryUpgrader(Repository): + def upgrade(self, dryrun=True): + """convert an attic repository to a borg repository + + those are the files that need to be upgraded here, from most + important to least important: segments, key files, and various + caches, the latter being optional, as they will be rebuilt if + missing. + + we nevertheless do the order in reverse, as we prefer to do + the fast stuff first, to improve interactivity. + """ + print("reading segments from attic repository using borg") + # we need to open it to load the configuration and other fields + self.open(self.path, exclusive=False) + segments = [filename for i, filename in self.io.segment_iterator()] + try: + keyfile = self.find_attic_keyfile() + except KeyfileNotFoundError: + print("no key file found for repository") + else: + self.convert_keyfiles(keyfile, dryrun) + self.close() + # partial open: just hold on to the lock + self.lock = UpgradableLock(os.path.join(self.path, 'lock'), + exclusive=True).acquire() + try: + self.convert_cache(dryrun) + self.convert_segments(segments, dryrun) + finally: + self.lock.release() + self.lock = None + + @staticmethod + def convert_segments(segments, dryrun): + """convert repository segments from attic to borg + + replacement pattern is `s/ATTICSEG/BORG_SEG/` in files in + `$ATTIC_REPO/data/**`. + + luckily the magic string length didn't change so we can just + replace the 8 first bytes of all regular files in there.""" + print("converting %d segments..." % len(segments)) + i = 0 + for filename in segments: + i += 1 + print("\rconverting segment %d/%d in place, %.2f%% done (%s)" + % (i, len(segments), 100*float(i)/len(segments), filename), end='') + if dryrun: + time.sleep(0.001) + else: + AtticRepositoryUpgrader.header_replace(filename, ATTIC_MAGIC, MAGIC) + print() + + @staticmethod + def header_replace(filename, old_magic, new_magic): + with open(filename, 'r+b') as segment: + segment.seek(0) + # only write if necessary + if segment.read(len(old_magic)) == old_magic: + segment.seek(0) + segment.write(new_magic) + + def find_attic_keyfile(self): + """find the attic keyfiles + + the keyfiles are loaded by `KeyfileKey.find_key_file()`. that + finds the keys with the right identifier for the repo. + + this is expected to look into $HOME/.attic/keys or + $ATTIC_KEYS_DIR for key files matching the given Borg + repository. + + it is expected to raise an exception (KeyfileNotFoundError) if + no key is found. whether that exception is from Borg or Attic + is unclear. + + this is split in a separate function in case we want to use + the attic code here directly, instead of our local + implementation.""" + return AtticKeyfileKey.find_key_file(self) + + @staticmethod + def convert_keyfiles(keyfile, dryrun): + + """convert key files from attic to borg + + replacement pattern is `s/ATTIC KEY/BORG_KEY/` in + `get_keys_dir()`, that is `$ATTIC_KEYS_DIR` or + `$HOME/.attic/keys`, and moved to `$BORG_KEYS_DIR` or + `$HOME/.borg/keys`. + + no need to decrypt to convert. we need to rewrite the whole + key file because magic string length changed, but that's not a + problem because the keyfiles are small (compared to, say, + all the segments).""" + print("converting keyfile %s" % keyfile) + with open(keyfile, 'r') as f: + data = f.read() + data = data.replace(AtticKeyfileKey.FILE_ID, KeyfileKey.FILE_ID, 1) + keyfile = os.path.join(get_keys_dir(), os.path.basename(keyfile)) + print("writing borg keyfile to %s" % keyfile) + if not dryrun: + with open(keyfile, 'w') as f: + f.write(data) + + def convert_cache(self, dryrun): + """convert caches from attic to borg + + those are all hash indexes, so we need to + `s/ATTICIDX/BORG_IDX/` in a few locations: + + * the repository index (in `$ATTIC_REPO/index.%d`, where `%d` + is the `Repository.get_index_transaction_id()`), which we + should probably update, with a lock, see + `Repository.open()`, which i'm not sure we should use + because it may write data on `Repository.close()`... + + * the `files` and `chunks` cache (in `$ATTIC_CACHE_DIR` or + `$HOME/.cache/attic//`), which we could just drop, + but if we'd want to convert, we could open it with the + `Cache.open()`, edit in place and then `Cache.close()` to + make sure we have locking right + """ + caches = [] + transaction_id = self.get_index_transaction_id() + if transaction_id is None: + print('no index file found for repository %s' % self.path) + else: + caches += [os.path.join(self.path, 'index.%d' % transaction_id).encode('utf-8')] + + # copy of attic's get_cache_dir() + attic_cache_dir = os.environ.get('ATTIC_CACHE_DIR', + os.path.join(os.path.expanduser('~'), + '.cache', 'attic')) + attic_cache_dir = os.path.join(attic_cache_dir, hexlify(self.id).decode('ascii')) + borg_cache_dir = os.path.join(get_cache_dir(), hexlify(self.id).decode('ascii')) + + def copy_cache_file(path): + """copy the given attic cache path into the borg directory + + does nothing if dryrun is True. also expects + attic_cache_dir and borg_cache_dir to be set in the parent + scope, to the directories path including the repository + identifier. + + :params path: the basename of the cache file to copy + (example: "files" or "chunks") as a string + + :returns: the borg file that was created or None if non + was created. + + """ + attic_file = os.path.join(attic_cache_dir, path) + if os.path.exists(attic_file): + borg_file = os.path.join(borg_cache_dir, path) + if os.path.exists(borg_file): + print("borg cache file already exists in %s, skipping conversion of %s" % (borg_file, attic_file)) + else: + print("copying attic cache file from %s to %s" % (attic_file, borg_file)) + if not dryrun: + shutil.copyfile(attic_file, borg_file) + return borg_file + else: + print("no %s cache file found in %s" % (path, attic_file)) + return None + + # XXX: untested, because generating cache files is a PITA, see + # Archiver.do_create() for proof + if os.path.exists(attic_cache_dir): + if not os.path.exists(borg_cache_dir): + os.makedirs(borg_cache_dir) + + # file that we don't have a header to convert, just copy + for cache in ['config', 'files']: + copy_cache_file(cache) + + # we need to convert the headers of those files, copy first + for cache in ['chunks']: + copied = copy_cache_file(cache) + if copied: + print("converting cache %s" % cache) + if not dryrun: + AtticRepositoryUpgrader.header_replace(cache, b'ATTICIDX', b'BORG_IDX') + + +class AtticKeyfileKey(KeyfileKey): + """backwards compatible Attic key file parser""" + FILE_ID = 'ATTIC KEY' + + # verbatim copy from attic + @staticmethod + def get_keys_dir(): + """Determine where to repository keys and cache""" + return os.environ.get('ATTIC_KEYS_DIR', + os.path.join(os.path.expanduser('~'), '.attic', 'keys')) + + @classmethod + def find_key_file(cls, repository): + """copy of attic's `find_key_file`_ + + this has two small modifications: + + 1. it uses the above `get_keys_dir`_ instead of the global one, + assumed to be borg's + + 2. it uses `repository.path`_ instead of + `repository._location.canonical_path`_ because we can't + assume the repository has been opened by the archiver yet + """ + get_keys_dir = cls.get_keys_dir + id = hexlify(repository.id).decode('ascii') + keys_dir = get_keys_dir() + for name in os.listdir(keys_dir): + filename = os.path.join(keys_dir, name) + with open(filename, 'r') as fd: + line = fd.readline().strip() + if line and line.startswith(cls.FILE_ID) and line[10:] == id: + return filename + raise KeyfileNotFoundError(repository.path, get_keys_dir()) diff --git a/docs/Makefile b/docs/Makefile index 387195a2a..133080cdb 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -36,8 +36,7 @@ help: clean: -rm -rf $(BUILDDIR)/* -html: - ./update_usage.sh +html: usage api.rst $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." @@ -140,3 +139,32 @@ gh-io: html inotify: html while inotifywait -r . --exclude usage.rst --exclude '_build/*' ; do make html ; done + +# generate list of targets +usage: $(shell borg help | grep -A1 "Available commands:" | tail -1 | sed 's/[{} ]//g;s/,\|^/.rst.inc usage\//g;s/^.rst.inc//;s/usage\/help//') + +# generate help file based on usage +usage/%.rst.inc: ../borg/archiver.py + @echo generating usage for $* + @printf ".. _borg_$*:\n\n" > $@ + @printf "borg $*\n" >> $@ + @echo -n borg $* | tr 'a-z- ' '-' >> $@ + @printf "\n::\n\n" >> $@ + @borg help $* --usage-only | sed -e 's/^/ /' >> $@ + @printf "\nDescription\n~~~~~~~~~~~\n" >> $@ + @borg help $* --epilog-only >> $@ + +api.rst: Makefile + @echo "auto-generating API documentation" + @echo "Borg Backup API documentation" > $@ + @echo "=============================" >> $@ + @echo "" >> $@ + @for mod in ../borg/*.pyx ../borg/*.py; do \ + if echo "$$mod" | grep -q "/_"; then \ + continue ; \ + fi ; \ + printf ".. automodule:: "; \ + echo "$$mod" | sed "s!\.\./!!;s/\.pyx\?//;s!/!.!"; \ + echo " :members:"; \ + echo " :undoc-members:"; \ + done >> $@ diff --git a/docs/_themes/local/sidebarusefullinks.html b/docs/_themes/local/sidebarusefullinks.html index 368dee25f..47de85364 100644 --- a/docs/_themes/local/sidebarusefullinks.html +++ b/docs/_themes/local/sidebarusefullinks.html @@ -5,7 +5,7 @@
  • Main Web Site
  • PyPI packages
    • -
  • Binary Packages
    • +
  • Binaries
  • Current ChangeLog
  • GitHub
  • Issue Tracker
    • diff --git a/docs/conf.py b/docs/conf.py index 772d88498..eba5c841e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -218,7 +218,7 @@ latex_documents = [ # ['see "AUTHORS" file'], 1) #] -extensions = ['sphinx.ext.extlinks'] +extensions = ['sphinx.ext.extlinks', 'sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode'] extlinks = { 'issue': ('https://github.com/borgbackup/borg/issues/%s', '#'), diff --git a/docs/development.rst b/docs/development.rst index be8405c18..9b4c0d893 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -51,6 +51,7 @@ Important notes: - When using -- to give options to py.test, you MUST also give borg.testsuite[.module]. + Building the docs with Sphinx ----------------------------- @@ -58,7 +59,7 @@ The documentation (in reStructuredText format, .rst) is in docs/. To build the html version of it, you need to have sphinx installed:: - pip3 install sphinx + pip3 install sphinx # important: this will install sphinx with Python 3 Now run:: @@ -66,3 +67,73 @@ Now run:: make html Then point a web browser at docs/_build/html/index.html. + +Using Vagrant +------------- + +We use Vagrant for the automated creation of testing environment and borgbackup +standalone binaries for various platforms. + +For better security, there is no automatic sync in the VM to host direction. +The plugin `vagrant-scp` is useful to copy stuff from the VMs to the host. + +Usage:: + + To create and provision the VM: + vagrant up OS + To create an ssh session to the VM: + vagrant ssh OS command + To shut down the VM: + vagrant halt OS + To shut down and destroy the VM: + vagrant destroy OS + To copy files from the VM (in this case, the generated binary): + vagrant scp OS:/vagrant/borg/borg/dist/borg . + + +Creating a new release +---------------------- + +Checklist:: + +- all issues for this milestone closed? +- any low hanging fruit left on the issue tracker? +- run tox on all supported platforms via vagrant, check for test fails. +- is Travis CI happy also? +- update CHANGES.rst (compare to git log). check version number of upcoming release. +- check MANIFEST.in and setup.py - are they complete? +- tag the release:: + + git tag -s -m "tagged release" 0.26.0 + +- cd docs ; make html # to update the usage include files +- update website with the html +- create a release on PyPi:: + + python setup.py register sdist upload --identity="Thomas Waldmann" --sign + +- close release milestone. +- announce on:: + + - mailing list + - Twitter + - IRC channel (topic) + +- create standalone binaries and link them from issue tracker: https://github.com/borgbackup/borg/issues/214 + + +Creating standalone binaries +---------------------------- + +Make sure you have everything built and installed (including llfuse and fuse). + +With virtual env activated:: + + pip install pyinstaller>=3.0 # or git checkout master + pyinstaller -F -n borg-PLATFORM --hidden-import=logging.config borg/__main__.py + ls -l dist/* + +If you encounter issues, see also our `Vagrantfile` for details. + +Note: Standalone binaries built with pyinstaller are supposed to work on same OS, + same architecture (x86 32bit, amd64 64bit) without external dependencies. diff --git a/docs/faq.rst b/docs/faq.rst index d13fe67f1..c43936ce2 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -4,15 +4,9 @@ Frequently asked questions ========================== -Which platforms are supported? - Currently Linux, FreeBSD and MacOS X are supported. - - You can try your luck on other POSIX-like systems, like Cygwin, - other BSDs, etc. but they are not officially supported. - Can I backup VM disk images? - Yes, the :ref:`deduplication ` technique used by |project_name| - makes sure only the modified parts of the file are stored. + Yes, the :ref:`deduplication ` technique used by + |project_name| makes sure only the modified parts of the file are stored. Also, we have optional simple sparse file support for extract. Can I backup from multiple servers into a single repository? @@ -41,14 +35,15 @@ Which file types, attributes, etc. are preserved? * User ID of owner * Group ID of owner * Unix Mode/Permissions (u/g/o permissions, suid, sgid, sticky) - * Extended Attributes (xattrs) + * Extended Attributes (xattrs) on Linux, OS X and FreeBSD * Access Control Lists (ACL_) on Linux, OS X and FreeBSD * BSD flags on OS X and FreeBSD Which file types, attributes, etc. are *not* preserved? - * UNIX domain sockets (because it does not make sense - they are meaningless - without the running process that created them and the process needs to - recreate them in any case). So, don't panic if your backup misses a UDS! + * UNIX domain sockets (because it does not make sense - they are + meaningless without the running process that created them and the process + needs to recreate them in any case). So, don't panic if your backup + misses a UDS! * The precise on-disk representation of the holes in a sparse file. Archive creation has no special support for sparse files, holes are backed up as (deduplicated and compressed) runs of zero bytes. @@ -76,52 +71,51 @@ When backing up to remote servers, do I have to trust the remote server? Yes, as an attacker with access to the remote server could delete (or otherwise make unavailable) all your backups. -If a backup stops mid-way, does the already-backed-up data stay there? I.e. does |project_name| resume backups? - Yes, during a backup a special checkpoint archive named ``.checkpoint`` is saved every 5 minutes - containing all the data backed-up until that point. This means that at most 5 minutes worth of data needs to be - retransmitted if a backup needs to be restarted. +If a backup stops mid-way, does the already-backed-up data stay there? + Yes, |project_name| supports resuming backups. + During a backup a special checkpoint archive named ``.checkpoint`` + is saved every checkpoint interval (the default value for this is 5 + minutes) containing all the data backed-up until that point. This means + that at most worth of data needs to be retransmitted + if a backup needs to be restarted. + Once your backup has finished successfully, you can delete all ``*.checkpoint`` + archives. If it crashes with a UnicodeError, what can I do? Check if your encoding is set correctly. For most POSIX-like systems, try:: export LANG=en_US.UTF-8 # or similar, important is correct charset -I can't extract non-ascii filenames by giving them on the commandline on OS X!? - This is due to different ways to represent some characters in unicode. - HFS+ likes the decomposed form while the commandline seems to be the composed - form usually. If you run into that, for now maybe just try: +I can't extract non-ascii filenames by giving them on the commandline!? + This might be due to different ways to represent some characters in unicode + or due to other non-ascii encoding issues. + If you run into that, try this: - - avoiding the non-ascii characters on the commandline by e.g. extracting + - avoid the non-ascii characters on the commandline by e.g. extracting the parent directory (or even everything) - - try to enter the composed form on the commandline - mount the repo using FUSE and use some file manager - See issue #143 on the issue tracker for more about this. - -If I want to run |project_name| on a ARM CPU older than ARM v6? - You need to enable the alignment trap handler to fixup misaligned accesses:: - - echo "2" > /proc/cpu/alignment - Can |project_name| add redundancy to the backup data to deal with hardware malfunction? - No, it can't. While that at first sounds like a good idea to defend against some - defect HDD sectors or SSD flash blocks, dealing with this in a reliable way needs a lot - of low-level storage layout information and control which we do not have (and also can't - get, even if we wanted). + No, it can't. While that at first sounds like a good idea to defend against + some defect HDD sectors or SSD flash blocks, dealing with this in a + reliable way needs a lot of low-level storage layout information and + control which we do not have (and also can't get, even if we wanted). - So, if you need that, consider RAID1 or a filesystem that offers redundant storage - or just make 2 backups to different locations / different hardware. + So, if you need that, consider RAID or a filesystem that offers redundant + storage or just make backups to different locations / different hardware. + + See also `ticket 225 `_. Can |project_name| verify data integrity of a backup archive? - Yes, if you want to detect accidental data damage (like bit rot), use the ``check`` - operation. It will notice corruption using CRCs and hashes. - If you want to be able to detect malicious tampering also, use a encrypted repo. - It will then be able to check using CRCs and HMACs. + Yes, if you want to detect accidental data damage (like bit rot), use the + ``check`` operation. It will notice corruption using CRCs and hashes. + If you want to be able to detect malicious tampering also, use a encrypted + repo. It will then be able to check using CRCs and HMACs. Why was Borg forked from Attic? - Borg was created in May 2015 in response to the difficulty of - getting new code or larger changes incorporated into Attic and - establishing a bigger developer community / more open development. + Borg was created in May 2015 in response to the difficulty of getting new + code or larger changes incorporated into Attic and establishing a bigger + developer community / more open development. More details can be found in `ticket 217 `_ that led to the fork. diff --git a/docs/global.rst.inc b/docs/global.rst.inc index c8c490498..265ad2658 100644 --- a/docs/global.rst.inc +++ b/docs/global.rst.inc @@ -13,6 +13,7 @@ .. _PBKDF2: https://en.wikipedia.org/wiki/PBKDF2 .. _ACL: https://en.wikipedia.org/wiki/Access_control_list .. _libacl: http://savannah.nongnu.org/projects/acl/ +.. _libattr: http://savannah.nongnu.org/projects/attr/ .. _liblz4: https://github.com/Cyan4973/lz4 .. _OpenSSL: https://www.openssl.org/ .. _Python: http://www.python.org/ diff --git a/docs/index.rst b/docs/index.rst index a871ef353..6a42dce0f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,3 +16,4 @@ Borg Documentation changes internals development + api diff --git a/docs/installation.rst b/docs/installation.rst index 6bc38a0aa..50957d17a 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -4,11 +4,17 @@ Installation ============ -|project_name| requires: +|project_name| pyinstaller binary installation requires: -* Python_ >= 3.2 +* Linux: glibc >= 2.12 (ok for most supported Linux releases) +* MacOS X: 10.10 (unknown whether it works for older releases) +* FreeBSD: 10.2 (unknown whether it works for older releases) + +|project_name| non-binary installation requires: + +* Python_ >= 3.2.2 * OpenSSL_ >= 1.0.0 -* libacl_ +* libacl_ (that pulls in libattr_ also) * liblz4_ * some python dependencies, see install_requires in setup.py @@ -21,11 +27,10 @@ Below, we describe different ways to install |project_name|. - **dist package** - easy and fast, needs a distribution and platform specific binary package (for your Linux/*BSD/OS X/... distribution). -- **wheel** - easy and fast, needs a platform specific borgbackup binary wheel, - which matches your platform [OS and CPU]). +- **pyinstaller binary** - easy and fast, we provide a ready-to-use binary file + that just works on the supported platforms - **pypi** - installing a source package from pypi needs more installation steps - and will compile stuff - try this if there is no binary wheel that works for - you. + and will need a compiler, development headers, etc.. - **git** - for developers and power users who want to have the latest code or use revision control (each release is tagged). @@ -74,33 +79,13 @@ and compare that to our latest release and review the change log (see links on our web site). -Debian Jessie / Ubuntu 14.04 preparations (wheel) -------------------------------------------------- +Installation (pyinstaller binary) +--------------------------------- +For some platforms we offer a ready-to-use standalone borg binary. -.. parsed-literal:: +It is supposed to work without requiring installation or preparations. - # Python stuff we need - apt-get install python3 python3-pip - - # Libraries we need (fuse is optional) - apt-get install openssl libacl1 liblz4-1 fuse - - -Installation (wheel) --------------------- - -This uses the latest binary wheel release. - -.. parsed-literal:: - - # Check https://github.com/borgbackup/borg/issues/147 for the correct - # platform-specific binary wheel, download and install it: - - # system-wide installation, needs sudo/root permissions: - sudo pip install borgbackup.whl - - # home directory installation, no sudo/root needed: - pip install --user borgbackup.whl +Check https://github.com/borgbackup/borg/issues/214 for available binaries. Debian Jessie / Ubuntu 14.04 preparations (git/pypi) @@ -127,7 +112,7 @@ Debian Jessie / Ubuntu 14.04 preparations (git/pypi) # in case you get complaints about permission denied on /etc/fuse.conf: # on ubuntu this means your user is not in the "fuse" group. just add # yourself there, log out and log in again. - apt-get install libfuse-dev fuse + apt-get install libfuse-dev fuse pkg-config # optional: for unit testing apt-get install fakeroot @@ -151,7 +136,7 @@ Korora / Fedora 21 preparations (git/pypi) sudo dnf install lz4-devel # optional: FUSE support - to mount backup archives - sudo dnf install fuse-devel fuse + sudo dnf install fuse-devel fuse pkgconfig # optional: for unit testing sudo dnf install fakeroot @@ -201,7 +186,8 @@ This uses the latest (source package) release from PyPi. source borg-env/bin/activate # always before using! # install borg + dependencies into virtualenv - pip install llfuse # optional, for FUSE support + pip install 'llfuse<0.41' # optional, for FUSE support + # 0.41 and 0.41.1 have unicode issues at install time pip install borgbackup Note: we install into a virtual environment here, but this is not a requirement. @@ -223,7 +209,8 @@ While we try not to break master, there are no guarantees on anything. # install borg + dependencies into virtualenv pip install sphinx # optional, to build the docs - pip install llfuse # optional, for FUSE support + pip install 'llfuse<0.41' # optional, for FUSE support + # 0.41 and 0.41.1 have unicode issues at install time cd borg pip install -r requirements.d/development.txt pip install -e . # in-place editable mode diff --git a/docs/quickstart.rst b/docs/quickstart.rst index b6c4c42df..32218fc67 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -85,9 +85,12 @@ certain number of old archives:: --exclude /home/Ben/Music/Justin\ Bieber \ --exclude '*.pyc' - # Use the `prune` subcommand to maintain 7 daily, 4 weekly - # and 6 monthly archives. - borg prune -v $REPOSITORY --keep-daily=7 --keep-weekly=4 --keep-monthly=6 + # Use the `prune` subcommand to maintain 7 daily, 4 weekly and 6 monthly + # archives of THIS machine. --prefix `hostname`- is very important to + # limit prune's operation to this machine's archives and not apply to + # other machine's archives also. + borg prune -v $REPOSITORY --prefix `hostname`- \ + --keep-daily=7 --keep-weekly=4 --keep-monthly=6 .. backup_compression: diff --git a/docs/update_usage.sh b/docs/update_usage.sh deleted file mode 100755 index 9e79f4e88..000000000 --- a/docs/update_usage.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -if [ ! -d usage ]; then - mkdir usage -fi -for cmd in change-passphrase check create delete extract info init list mount prune serve; do - FILENAME="usage/$cmd.rst.inc" - LINE=`echo -n borg $cmd | tr 'a-z- ' '-'` - echo -e ".. _borg_$cmd:\n" > $FILENAME - echo -e "borg $cmd\n$LINE\n::\n\n" >> $FILENAME - borg help $cmd --usage-only | sed -e 's/^/ /' >> $FILENAME - echo -e "\nDescription\n~~~~~~~~~~~\n" >> $FILENAME - borg help $cmd --epilog-only >> $FILENAME -done diff --git a/docs/usage.rst b/docs/usage.rst index 0ce547b93..6bd292e14 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -48,6 +48,8 @@ General: can either leave it away or abbreviate as `::`, if a positional parameter is required. BORG_PASSPHRASE When set, use the value to answer the passphrase question for encrypted repositories. + BORG_RSH + When set, use this command instead of ``ssh``. TMPDIR where temporary files are stored (might need a lot of temporary space for some operations) @@ -69,6 +71,8 @@ Directories: Building: BORG_OPENSSL_PREFIX Adds given OpenSSL header file directory to the default locations (setup.py). + BORG_LZ4_PREFIX + Adds given LZ4 header file directory to the default locations (setup.py). Please note: @@ -212,12 +216,6 @@ Examples # Even slower, even higher compression (N = 0..9) $ borg create --compression lzma,N /mnt/backup::repo ~ - # Backup some LV snapshots (you have to create the snapshots before this - # and remove them afterwards). We also backup the output of lvdisplay so - # we can see the LV sizes at restore time. See also "borg extract" examples. - $ lvdisplay > lvdisplay.txt - $ borg create --read-special /mnt/backup::repo lvdisplay.txt /dev/vg0/*-snapshot - .. include:: usage/extract.rst.inc Examples @@ -236,11 +234,6 @@ Examples # Extract the "src" directory but exclude object files $ borg extract /mnt/backup::my-files home/USERNAME/src --exclude '*.o' - # Restore LV snapshots (the target LVs /dev/vg0/* of correct size have - # to be already available and will be overwritten by this command!) - $ borg extract --stdout /mnt/backup::repo dev/vg0/root-snapshot > /dev/vg0/root - $ borg extract --stdout /mnt/backup::repo dev/vg0/home-snapshot > /dev/vg0/home - Note: currently, extract always writes into the current working directory ("."), so make sure you ``cd`` to the right place before calling ``borg extract``. @@ -274,10 +267,23 @@ Examples Examples ~~~~~~~~ + +Be careful, prune is potentially dangerous command, it will remove backup +archives. + +The default of prune is to apply to **all archives in the repository** unless +you restrict its operation to a subset of the archives using `--prefix`. +When using --prefix, be careful to choose a good prefix - e.g. do not use a +prefix "foo" if you do not also want to match "foobar". + +It is strongly recommended to always run `prune --dry-run ...` first so you +will see what it would do without it actually doing anything. + :: - # Keep 7 end of day and 4 additional end of week archives: - $ borg prune /mnt/backup --keep-daily=7 --keep-weekly=4 + # Keep 7 end of day and 4 additional end of week archives. + # Do a dry-run without actually deleting anything. + $ borg prune /mnt/backup --dry-run --keep-daily=7 --keep-weekly=4 # Same as above but only apply to archive names starting with "foo": $ borg prune /mnt/backup --keep-daily=7 --keep-weekly=4 --prefix=foo @@ -355,3 +361,70 @@ Examples $ cat ~/.ssh/authorized_keys command="borg serve --restrict-to-path /mnt/backup" ssh-rsa AAAAB3[...] + +Additional Notes +================ + +Here are misc. notes about topics that are maybe not covered in enough detail in the usage section. + +--read-special +-------------- + +The option --read-special is not intended for normal, filesystem-level (full or +partly-recursive) backups. You only give this option if you want to do something +rather ... special - and if you have hand-picked some files that you want to treat +that way. + +`borg create --read-special` will open all files without doing any special treatment +according to the file type (the only exception here are directories: they will be +recursed into). Just imagine what happens if you do `cat filename` - the content +you will see there is what borg will backup for that filename. + +So, for example, symlinks will be followed, block device content will be read, +named pipes / UNIX domain sockets will be read. + +You need to be careful with what you give as filename when using --read-special, +e.g. if you give /dev/zero, your backup will never terminate. + +The given files' metadata is saved as it would be saved without --read-special +(e.g. its name, its size [might be 0], its mode, etc.) - but additionally, also +the content read from it will be saved for it. + +Restoring such files' content is currently only supported one at a time via --stdout +option (and you have to redirect stdout to where ever it shall go, maybe directly +into an existing device file of your choice or indirectly via dd). + +Example +~~~~~~~ + +Imagine you have made some snapshots of logical volumes (LVs) you want to backup. + +Note: For some scenarios, this is a good method to get "crash-like" consistency +(I call it crash-like because it is the same as you would get if you just hit the +reset button or your machine would abrubtly and completely crash). +This is better than no consistency at all and a good method for some use cases, +but likely not good enough if you have databases running. + +Then you create a backup archive of all these snapshots. The backup process will +see a "frozen" state of the logical volumes, while the processes working in the +original volumes continue changing the data stored there. + +You also add the output of `lvdisplay` to your backup, so you can see the LV sizes +in case you ever need to recreate and restore them. + +After the backup has completed, you remove the snapshots again. + +:: + $ # create snapshots here + $ lvdisplay > lvdisplay.txt + $ borg create --read-special /mnt/backup::repo lvdisplay.txt /dev/vg0/*-snapshot + $ # remove snapshots here + +Now, let's see how to restore some LVs from such a backup. + + $ borg extract /mnt/backup::repo lvdisplay.txt + $ # create empty LVs with correct sizes here (look into lvdisplay.txt). + $ # we assume that you created an empty root and home LV and overwrite it now: + $ borg extract --stdout /mnt/backup::repo dev/vg0/root-snapshot > /dev/vg0/root + $ borg extract --stdout /mnt/backup::repo dev/vg0/home-snapshot > /dev/vg0/home + diff --git a/setup.cfg b/setup.cfg index 19a49eea6..8a128d6e1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,5 +4,5 @@ python_files = testsuite/*.py [flake8] ignore = E226,F403 max-line-length = 250 -exclude = versioneer.py,docs/conf.py,borg/_version.py,build,dist,.git,.idea,.cache +exclude = docs/conf.py,borg/_version.py,build,dist,.git,.idea,.cache max-complexity = 100 diff --git a/setup.py b/setup.py index 667ba4ee2..68a3db8d3 100644 --- a/setup.py +++ b/setup.py @@ -4,10 +4,16 @@ import sys from glob import glob min_python = (3, 2) -if sys.version_info < min_python: +my_python = sys.version_info + +if my_python < min_python: print("Borg requires Python %d.%d or later" % min_python) sys.exit(1) +# msgpack pure python data corruption was fixed in 0.4.6. +# Also, we might use some rather recent API features. +install_requires=['msgpack-python>=0.4.6', ] + from setuptools import setup, Extension from setuptools.command.sdist import sdist @@ -59,7 +65,7 @@ except ImportError: if not all(os.path.exists(path) for path in [ compress_source, crypto_source, chunker_source, hashindex_source, platform_linux_source, platform_freebsd_source]): - raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version') + raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.') def detect_openssl(prefixes): @@ -71,14 +77,36 @@ def detect_openssl(prefixes): return prefix +def detect_lz4(prefixes): + for prefix in prefixes: + filename = os.path.join(prefix, 'include', 'lz4.h') + if os.path.exists(filename): + with open(filename, 'r') as fd: + if 'LZ4_decompress_safe' in fd.read(): + return prefix + + +include_dirs = [] +library_dirs = [] + possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', '/usr/local/borg', '/opt/local'] if os.environ.get('BORG_OPENSSL_PREFIX'): possible_openssl_prefixes.insert(0, os.environ.get('BORG_OPENSSL_PREFIX')) ssl_prefix = detect_openssl(possible_openssl_prefixes) if not ssl_prefix: raise Exception('Unable to find OpenSSL >= 1.0 headers. (Looked here: {})'.format(', '.join(possible_openssl_prefixes))) -include_dirs = [os.path.join(ssl_prefix, 'include')] -library_dirs = [os.path.join(ssl_prefix, 'lib')] +include_dirs.append(os.path.join(ssl_prefix, 'include')) +library_dirs.append(os.path.join(ssl_prefix, 'lib')) + + +possible_lz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4', '/usr/local/borg', '/opt/local'] +if os.environ.get('BORG_LZ4_PREFIX'): + possible_openssl_prefixes.insert(0, os.environ.get('BORG_LZ4_PREFIX')) +lz4_prefix = detect_lz4(possible_lz4_prefixes) +if not lz4_prefix: + raise Exception('Unable to find LZ4 headers. (Looked here: {})'.format(', '.join(possible_lz4_prefixes))) +include_dirs.append(os.path.join(lz4_prefix, 'include')) +library_dirs.append(os.path.join(lz4_prefix, 'lib')) with open('README.rst', 'r') as fd: @@ -87,7 +115,7 @@ with open('README.rst', 'r') as fd: cmdclass = {'build_ext': build_ext, 'sdist': Sdist} ext_modules = [ - Extension('borg.compress', [compress_source], libraries=['lz4']), + Extension('borg.compress', [compress_source], libraries=['lz4'], include_dirs=include_dirs, library_dirs=library_dirs), Extension('borg.crypto', [crypto_source], libraries=['crypto'], include_dirs=include_dirs, library_dirs=library_dirs), Extension('borg.chunker', [chunker_source]), Extension('borg.hashindex', [hashindex_source]) @@ -110,13 +138,15 @@ setup( description='Deduplicated, encrypted, authenticated and compressed backups', long_description=long_description, license='BSD', - platforms=['Linux', 'MacOS X', 'FreeBSD', ], + platforms=['Linux', 'MacOS X', 'FreeBSD', 'OpenBSD', 'NetBSD', ], classifiers=[ 'Development Status :: 4 - Beta', 'Environment :: Console', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: BSD License', 'Operating System :: POSIX :: BSD :: FreeBSD', + 'Operating System :: POSIX :: BSD :: OpenBSD', + 'Operating System :: POSIX :: BSD :: NetBSD', 'Operating System :: MacOS :: MacOS X', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', @@ -124,10 +154,11 @@ setup( 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', 'Topic :: Security :: Cryptography', 'Topic :: System :: Archiving :: Backup', ], - packages=['borg', 'borg.testsuite'], + packages=['borg', 'borg.testsuite', 'borg.support', ], entry_points={ 'console_scripts': [ 'borg = borg.archiver:main', @@ -136,7 +167,5 @@ setup( cmdclass=cmdclass, ext_modules=ext_modules, setup_requires=['setuptools_scm>=1.7'], - # msgpack pure python data corruption was fixed in 0.4.6. - # Also, we might use some rather recent API features. - install_requires=['msgpack-python>=0.4.6'], + install_requires=install_requires, ) diff --git a/tox.ini b/tox.ini index a120a237a..a9ccb5e04 100644 --- a/tox.ini +++ b/tox.ini @@ -2,13 +2,15 @@ # fakeroot -u tox --recreate [tox] -envlist = py32, py33, py34 +envlist = py{32,33,34,35} [testenv] # Change dir to avoid import problem for cython code. The directory does # not really matter, should be just different from the toplevel dir. changedir = {toxworkdir} -deps = -rrequirements.d/development.txt +deps = + -rrequirements.d/development.txt + attic commands = py.test --cov=borg --pyargs {posargs:borg.testsuite} # fakeroot -u needs some env vars: passenv = *