diff --git a/.gitignore b/.gitignore index 26af5221f..7fd06517e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,6 @@ src/borg/platform/posix.c src/borg/_version.py *.egg-info *.pyc -*.pyo *.so .idea/ .cache/ diff --git a/.travis.yml b/.travis.yml index e12266b9d..572cc9a08 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,14 +6,9 @@ cache: directories: - $HOME/.cache/pip -# note: use py 3.5.2, it has lzma support. 3.5(.0) on travis.org/trusty does not. matrix: include: - - python: 3.4 - os: linux - dist: trusty - env: TOXENV=py34 - - python: 3.5.2 + - python: 3.5 os: linux dist: trusty env: TOXENV=py35 @@ -21,7 +16,7 @@ matrix: os: linux dist: trusty env: TOXENV=py36 - - python: 3.4 + - python: 3.5 os: linux dist: trusty env: TOXENV=flake8 @@ -29,10 +24,6 @@ matrix: os: linux dist: trusty env: TOXENV=py36 - - language: generic - os: osx - osx_image: xcode6.4 - env: TOXENV=py34 - language: generic os: osx osx_image: xcode6.4 diff --git a/.travis/install.sh b/.travis/install.sh index 708b8c81b..4ffc99359 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -21,10 +21,6 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then brew install Caskroom/cask/osxfuse case "${TOXENV}" in - py34) - pyenv install 3.4.5 - pyenv global 3.4.5 - ;; py35) pyenv install 3.5.2 pyenv global 3.5.2 diff --git a/MANIFEST.in b/MANIFEST.in index 2ec72dc24..bdabc64d0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,4 +5,4 @@ prune .github graft src graft docs prune docs/_build -global-exclude *.py[co] *.orig *.so *.dll +global-exclude *.pyc *.orig *.so *.dll diff --git a/Vagrantfile b/Vagrantfile index 87897c023..669c0975a 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -7,13 +7,6 @@ $cpus = Integer(ENV.fetch('VMCPUS', '4')) # create VMs with that many cpus $xdistn = Integer(ENV.fetch('XDISTN', '4')) # dispatch tests to that many pytest workers $wmem = $xdistn * 256 # give the VM additional memory for workers [MB] -def packages_prepare_wheezy - return <<-EOF - # debian 7 wheezy does not have lz4, but it is available from wheezy-backports: - echo "deb http://http.debian.net/debian wheezy-backports main" > /etc/apt/sources.list.d/wheezy-backports.list - EOF -end - def packages_debianoid return <<-EOF if id "vagrant" >/dev/null 2>&1; then @@ -35,199 +28,16 @@ def packages_debianoid apt-get install -y python3-dev python3-setuptools # for building python: apt-get install -y zlib1g-dev libbz2-dev libncurses5-dev libreadline-dev liblzma-dev libsqlite3-dev - # this way it works on older dists (like ubuntu 12.04) also: - # for python 3.2 on ubuntu 12.04 we need pip<8 and virtualenv<14 as - # newer versions are not compatible with py 3.2 any more. - easy_install3 'pip<8.0' - pip3 install 'virtualenv<14.0' + easy_install3 'pip' + pip3 install 'virtualenv' touch $home_dir/.bash_profile ; chown $username $home_dir/.bash_profile EOF end -def packages_redhatted - return <<-EOF - yum install -y epel-release - yum update -y - # for building borgbackup and dependencies: - yum install -y openssl-devel openssl libacl-devel libacl lz4-devel fuse-devel fuse pkgconfig - usermod -a -G fuse vagrant - chgrp fuse /dev/fuse - chmod 666 /dev/fuse - yum install -y fakeroot gcc git patch - # needed to compile msgpack-python (otherwise it will use slow fallback code): - yum install -y gcc-c++ - # for building python: - yum install -y zlib-devel bzip2-devel ncurses-devel readline-devel xz xz-devel sqlite-devel - #yum install -y python-pip - #pip install virtualenv - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile - EOF -end - -def packages_darwin - return <<-EOF - # install all the (security and other) updates - sudo softwareupdate --ignore iTunesX - sudo softwareupdate --ignore iTunes - sudo softwareupdate --install --all - # get osxfuse 3.x release code from github: - curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.6.3/osxfuse-3.6.3.dmg >osxfuse.dmg - MOUNTDIR=$(echo `hdiutil mount osxfuse.dmg | tail -1 | awk '{$1="" ; print $0}'` | xargs -0 echo) \ - && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for macOS 3.6.3.pkg" -target / - sudo chown -R vagrant /usr/local # brew must be able to create stuff here - ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - brew update - brew install openssl - brew install lz4 - brew install xz # required for python lzma module - brew install fakeroot - brew install git - brew install pkg-config - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile - EOF -end - -def packages_freebsd - return <<-EOF - # VM has no hostname set - hostname freebsd - # install all the (security and other) updates, base system - freebsd-update --not-running-from-cron fetch install - # for building borgbackup and dependencies: - pkg install -y openssl liblz4 fusefs-libs pkgconf - pkg install -y git bash - # for building python: - pkg install -y sqlite3 - # make bash default / work: - chsh -s bash vagrant - mount -t fdescfs fdesc /dev/fd - echo 'fdesc /dev/fd fdescfs rw 0 0' >> /etc/fstab - # make FUSE work - echo 'fuse_load="YES"' >> /boot/loader.conf - echo 'vfs.usermount=1' >> /etc/sysctl.conf - kldload fuse - sysctl vfs.usermount=1 - pw groupmod operator -M vagrant - # /dev/fuse has group operator - chmod 666 /dev/fuse - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile - # install all the (security and other) updates, packages - pkg update - yes | pkg upgrade - EOF -end - -def packages_openbsd - return <<-EOF - echo 'installpath = http://ftp.hostserver.de/pub/OpenBSD/6.0/packages/amd64/' > /etc/pkg.conf - echo 'export PKG_PATH=http://ftp.hostserver.de/pub/OpenBSD/6.0/packages/amd64/' >> ~/.profile - . ~/.profile - pkg_add bash - chsh -s /usr/local/bin/bash vagrant - pkg_add openssl - pkg_add lz4 - pkg_add git # no fakeroot - pkg_add py3-setuptools - ln -sf /usr/local/bin/python3.4 /usr/local/bin/python3 - ln -sf /usr/local/bin/python3.4 /usr/local/bin/python - easy_install-3.4 pip - pip3 install virtualenv - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile - EOF -end - -def packages_netbsd - return <<-EOF - hostname netbsd # the box we use has an invalid hostname - PKG_PATH="ftp://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/amd64/6.1.5/All/" - export PKG_PATH - pkg_add mozilla-rootcerts lz4 git bash - chsh -s bash vagrant - mkdir -p /usr/local/opt/lz4/include - mkdir -p /usr/local/opt/lz4/lib - ln -s /usr/pkg/include/lz4*.h /usr/local/opt/lz4/include/ - ln -s /usr/pkg/lib/liblz4* /usr/local/opt/lz4/lib/ - touch /etc/openssl/openssl.cnf # avoids a flood of "can't open ..." - mozilla-rootcerts install - pkg_add pkg-config # avoids some "pkg-config missing" error msg, even without fuse pkg - # pkg_add fuse # llfuse supports netbsd, but is still buggy. - # https://bitbucket.org/nikratio/python-llfuse/issues/70/perfuse_open-setsockopt-no-buffer-space - pkg_add python34 py34-setuptools - ln -s /usr/pkg/bin/python3.4 /usr/pkg/bin/python - ln -s /usr/pkg/bin/python3.4 /usr/pkg/bin/python3 - easy_install-3.4 pip - pip install virtualenv - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile - EOF -end - -def packages_openindiana - return <<-EOF - #pkg update # XXX needs separate provisioning step + reboot - pkg install python-34 clang-3.4 lz4 git - python3 -m ensurepip - pip3 install -U setuptools pip wheel virtualenv - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile - EOF -end - -# Install required cygwin packages and configure environment -# -# Microsoft/EdgeOnWindows10 image has MLS-OpenSSH installed by default, -# which is based on cygwin x86_64 but should not be used together with cygwin. -# In order to have have cygwin compatible bash 'ImagePath' is replaced with -# cygrunsrv of newly installed cygwin -# -# supported cygwin versions: -# x86_64 -# x86 -def packages_cygwin(version) - setup_exe = "setup-#{version}.exe" - - return <<-EOF - mkdir -p /cygdrive/c/cygwin - powershell -Command '$client = new-object System.Net.WebClient; $client.DownloadFile("https://www.cygwin.com/#{setup_exe}","C:\\cygwin\\#{setup_exe}")' - echo ' - REM --- Change to use different CygWin platform and final install path - set CYGSETUP=#{setup_exe} - REM --- Install build version of CygWin in a subfolder - set OURPATH=%cd% - set CYGBUILD="C:\\cygwin\\CygWin" - set CYGMIRROR=http://mirrors.kernel.org/sourceware/cygwin/ - set BASEPKGS=openssh,rsync - set BUILDPKGS=python3,python3-setuptools,python-devel,binutils,gcc-g++,libopenssl,openssl-devel,git,make,liblz4-devel,liblz4_1,curl - %CYGSETUP% -q -B -o -n -R %CYGBUILD% -L -D -s %CYGMIRROR% -P %BASEPKGS%,%BUILDPKGS% - cd /d C:\\cygwin\\CygWin\\bin - regtool set /HKLM/SYSTEM/CurrentControlSet/Services/OpenSSHd/ImagePath "C:\\cygwin\\CygWin\\bin\\cygrunsrv.exe" - bash -c "ssh-host-config --no" - bash -c "chown sshd_server /cygdrive/c/cygwin/CygWin/var/empty" - ' > /cygdrive/c/cygwin/install.bat - - echo "alias mkdir='mkdir -p'" > ~/.profile - echo "export CYGWIN_ROOT=/cygdrive/c/cygwin/CygWin" >> ~/.profile - echo 'export PATH=$CYGWIN_ROOT/bin:$PATH' >> ~/.profile - - echo '' > ~/.bash_profile - - cmd.exe /c 'setx /m PATH "C:\\cygwin\\CygWin\\bin;%PATH%"' - source ~/.profile - cd /cygdrive/c/cygwin && cmd.exe /c install.bat - - echo 'db_home: windows' > $CYGWIN_ROOT/etc/nsswitch.conf - EOF -end - -def install_cygwin_venv - return <<-EOF - python3 -m ensurepip -U --default-pip - pip install virtualenv - EOF -end - def install_pyenv(boxname) script = <<-EOF curl -s -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash - echo 'export PATH="$HOME/.pyenv/bin:/vagrant/borg:$PATH"' >> ~/.bash_profile + echo 'export PATH="$HOME/.pyenv/bin:$PATH"' >> ~/.bash_profile echo 'eval "$(pyenv init -)"' >> ~/.bash_profile echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bash_profile echo 'export PYTHON_CONFIGURE_OPTS="--enable-shared"' >> ~/.bash_profile @@ -246,7 +56,6 @@ end def install_pythons(boxname) return <<-EOF . ~/.bash_profile - pyenv install 3.4.0 # tests pyenv install 3.5.0 # tests pyenv install 3.6.0 # tests pyenv install 3.5.3 # binary build, use latest 3.5.x release @@ -320,6 +129,7 @@ def build_binary_with_pyinstaller(boxname) . borg-env/bin/activate cd borg pyinstaller --clean --distpath=/vagrant/borg scripts/borg.exe.spec + echo 'export PATH="/vagrant/borg:$PATH"' >> ~/.bash_profile EOF end @@ -330,8 +140,8 @@ def run_tests(boxname) . ../borg-env/bin/activate if which pyenv 2> /dev/null; then # for testing, use the earliest point releases of the supported python versions: - pyenv global 3.4.0 3.5.0 3.6.0 - pyenv local 3.4.0 3.5.0 3.6.0 + pyenv global 3.5.0 3.6.0 + pyenv local 3.5.0 3.6.0 fi # otherwise: just use the system python if which fakeroot 2> /dev/null; then @@ -369,46 +179,6 @@ Vagrant.configure(2) do |config| v.cpus = $cpus end - # Linux - config.vm.define "centos7_64" do |b| - b.vm.box = "centos/7" - b.vm.provider :virtualbox do |v| - v.memory = 1024 + $wmem - end - b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos7_64") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos7_64") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("centos7_64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos7_64") - end - - config.vm.define "centos6_32" do |b| - b.vm.box = "centos6-32" - b.vm.provider :virtualbox do |v| - v.memory = 768 + $wmem - end - b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_32") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos6_32") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("centos6_32") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false) - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos6_32") - end - - config.vm.define "centos6_64" do |b| - b.vm.box = "centos6-64" - b.vm.provider :virtualbox do |v| - v.memory = 1024 + $wmem - end - b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_64") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos6_64") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("centos6_64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false) - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos6_64") - end - config.vm.define "xenial64" do |b| b.vm.box = "ubuntu/xenial64" b.vm.provider :virtualbox do |v| @@ -420,17 +190,6 @@ Vagrant.configure(2) do |config| b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("xenial64") end - config.vm.define "trusty64" do |b| - b.vm.box = "ubuntu/trusty64" - b.vm.provider :virtualbox do |v| - v.memory = 1024 + $wmem - end - b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("trusty64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("trusty64") - end - config.vm.define "stretch64" do |b| b.vm.box = "debian/stretch64" b.vm.provider :virtualbox do |v| @@ -439,155 +198,11 @@ Vagrant.configure(2) do |config| b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("stretch64") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) + b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() + b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("stretch64") b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("stretch64") end - config.vm.define "jessie64" do |b| - b.vm.box = "debian/jessie64" - b.vm.provider :virtualbox do |v| - v.memory = 1024 + $wmem - end - b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("jessie64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("jessie64") - end - - config.vm.define "wheezy32" do |b| - b.vm.box = "boxcutter/debian7-i386" - b.vm.provider :virtualbox do |v| - v.memory = 768 + $wmem - end - b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy - b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy32") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("wheezy32") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("wheezy32") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) - b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() - b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("wheezy32") - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("wheezy32") - end - - config.vm.define "wheezy64" do |b| - b.vm.box = "boxcutter/debian7" - b.vm.provider :virtualbox do |v| - v.memory = 1024 + $wmem - end - b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy - b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy64") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("wheezy64") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("wheezy64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) - b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() - b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("wheezy64") - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("wheezy64") - end - - # OS X - config.vm.define "darwin64" do |b| - b.vm.box = "jhcook/yosemite-clitools" - b.vm.provider :virtualbox do |v| - v.memory = 1536 + $wmem - v.customize ['modifyvm', :id, '--ostype', 'MacOS1010_64'] - v.customize ['modifyvm', :id, '--paravirtprovider', 'default'] - # Adjust CPU settings according to - # https://github.com/geerlingguy/macos-virtualbox-vm - v.customize ['modifyvm', :id, '--cpuidset', - '00000001', '000306a9', '00020800', '80000201', '178bfbff'] - # Disable USB variant requiring Virtualbox proprietary extension pack - v.customize ["modifyvm", :id, '--usbehci', 'off', '--usbxhci', 'off'] - end - b.vm.provision "packages darwin", :type => :shell, :privileged => false, :inline => packages_darwin - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("darwin64") - b.vm.provision "fix pyenv", :type => :shell, :privileged => false, :inline => fix_pyenv_darwin("darwin64") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("darwin64") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("darwin64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) - b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() - b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("darwin64") - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("darwin64") - end - - # BSD - # note: the FreeBSD-10.3-RELEASE box needs "vagrant up" twice to start. - config.vm.define "freebsd64" do |b| - b.vm.box = "freebsd/FreeBSD-10.3-RELEASE" - b.vm.provider :virtualbox do |v| - v.memory = 1024 + $wmem - end - b.ssh.shell = "sh" - b.vm.provision "install system packages", :type => :shell, :inline => packages_freebsd - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("freebsd") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("freebsd") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("freebsd") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) - b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() - b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("freebsd") - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("freebsd") - end - - config.vm.define "openbsd64" do |b| - b.vm.box = "openbsd60-64" # note: basic openbsd install for vagrant WITH sudo and rsync pre-installed - b.vm.provider :virtualbox do |v| - v.memory = 1024 + $wmem - end - b.ssh.shell = "sh" - b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false) - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openbsd64") - end - - config.vm.define "netbsd64" do |b| - b.vm.box = "netbsd70-64" - b.vm.provider :virtualbox do |v| - v.memory = 1024 + $wmem - end - b.vm.provision "packages netbsd", :type => :shell, :inline => packages_netbsd - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("netbsd64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false) - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("netbsd64") - end - - # rsync on openindiana has troubles, does not set correct owner for /vagrant/borg and thus gives lots of - # permission errors. can be manually fixed in the VM by: sudo chown -R vagrant /vagrant/borg ; then rsync again. - config.vm.define "openindiana64" do |b| - b.vm.box = "openindiana/hipster" - b.vm.provider :virtualbox do |v| - v.memory = 1536 + $wmem - end - b.vm.provision "packages openindiana", :type => :shell, :inline => packages_openindiana - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openindiana64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false) - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openindiana64") - end - - config.vm.define "windows10" do |b| - b.vm.box = "Microsoft/EdgeOnWindows10" - b.vm.guest = :windows - b.vm.boot_timeout = 180 - b.vm.graceful_halt_timeout = 120 - - b.ssh.shell = "sh -l" - b.ssh.username = "IEUser" - b.ssh.password = "Passw0rd!" - b.ssh.insert_key = false - - b.vm.provider :virtualbox do |v| - v.memory = 1536 + $wmem - #v.gui = true - end - - # fix permissions placeholder - b.vm.provision "fix perms", :type => :shell, :privileged => false, :inline => "echo 'fix permission placeholder'" - - b.vm.provision "packages cygwin", :type => :shell, :privileged => false, :inline => packages_cygwin("x86_64") - b.vm.provision :reload - b.vm.provision "cygwin install pip", :type => :shell, :privileged => false, :inline => install_cygwin_venv - b.vm.provision "cygwin build env", :type => :shell, :privileged => false, :inline => build_sys_venv("windows10") - b.vm.provision "cygwin install borg", :type => :shell, :privileged => false, :inline => install_borg(false) - b.vm.provision "cygwin run tests", :type => :shell, :privileged => false, :inline => run_tests("windows10") - end + # TODO: create more VMs with python 3.5+ and openssl 1.1. + # See branch 1.1-maint for a better equipped Vagrantfile (but still on py34 and openssl 1.0). end diff --git a/docs/changes.rst b/docs/changes.rst index 25909cf30..ec436b7e2 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -131,6 +131,16 @@ The best check that everything is ok is to run a dry-run extraction:: Changelog ========= +Version 1.2.0dev0 (not released yet) +------------------------------------ + +Compatibility notes: + +- dropped support and testing for Python 3.4, minimum requirement is 3.5.0. + In case your OS does not provide Python >= 3.5, consider using our binary, + which does not need an external Python interpreter. + + Version 1.1.0rc1 (2017-07-24) ----------------------------- diff --git a/docs/development.rst b/docs/development.rst index 37cc9ad68..e9868a1b9 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -182,7 +182,7 @@ Some more advanced examples:: # verify a changed tox.ini (run this after any change to tox.ini): fakeroot -u tox --recreate - fakeroot -u tox -e py34 # run all tests, but only on python 3.4 + fakeroot -u tox -e py35 # run all tests, but only on python 3.5 fakeroot -u tox borg.testsuite.locking # only run 1 test module @@ -315,7 +315,7 @@ Checklist: - check version number of upcoming release in ``CHANGES.rst`` - verify that ``MANIFEST.in`` and ``setup.py`` are complete - ``python setup.py build_usage ; python setup.py build_man`` and - commit (be sure to build with Python 3.4 or 3.5 as Python 3.6 added `more + commit (be sure to build with Python 3.5 as Python 3.6 added `more guaranteed hashing algorithms `_) - tag the release:: diff --git a/docs/installation.rst b/docs/installation.rst index 353254c38..683560fc7 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -136,7 +136,7 @@ Dependencies To install |project_name| from a source package (including pip), you have to install the following dependencies first: -* `Python 3`_ >= 3.4.0, plus development headers. Even though Python 3 is not +* `Python 3`_ >= 3.5.0, plus development headers. Even though Python 3 is not the default Python version on most systems, it is usually available as an optional install. * OpenSSL_ >= 1.0.0, plus development headers. @@ -181,7 +181,7 @@ group, log out and log in again. Fedora / Korora +++++++++++++++ -.. todo:: Add zeromq +.. todo:: Add zeromq, use python 3.5 or 3.6 Install the dependencies with development headers:: @@ -196,7 +196,7 @@ Install the dependencies with development headers:: openSUSE Tumbleweed / Leap ++++++++++++++++++++++++++ -.. todo:: Add zeromq +.. todo:: Add zeromq, use python 3.5 or 3.6 Install the dependencies automatically using zypper:: @@ -213,7 +213,7 @@ Alternatively, you can enumerate all build dependencies in the command line:: Mac OS X ++++++++ -.. todo:: Add zeromq +.. todo:: Add zeromq, use python 3.5 or 3.6 Assuming you have installed homebrew_, the following steps will install all the dependencies:: @@ -231,7 +231,7 @@ FUSE for OS X, which is available as a pre-release_. FreeBSD ++++++++ -.. todo:: Add zeromq +.. todo:: Add zeromq, use python 3.5 or 3.6 Listed below are packages you will need to install Borg, its dependencies, and commands to make FUSE work for using the mount command. @@ -264,7 +264,7 @@ Cygwin Running under Cygwin is experimental and has only been tested with Cygwin (x86-64) v2.5.2. Remote repositories are known broken, local repositories should work. -.. todo:: Add zeromq +.. todo:: Add zeromq, use python 3.5 or 3.6 Use the Cygwin installer to install the dependencies:: diff --git a/docs/usage_general.rst.inc b/docs/usage_general.rst.inc index 3c6191438..8a20275c7 100644 --- a/docs/usage_general.rst.inc +++ b/docs/usage_general.rst.inc @@ -215,7 +215,7 @@ Please note: (e.g. mode 600, root:root). -.. _INI: https://docs.python.org/3.4/library/logging.config.html#configuration-file-format +.. _INI: https://docs.python.org/3.5/library/logging.config.html#configuration-file-format .. _file-systems: diff --git a/setup.py b/setup.py index 43936623e..d73fa089f 100644 --- a/setup.py +++ b/setup.py @@ -764,7 +764,7 @@ setup( license='BSD', platforms=['Linux', 'MacOS X', 'FreeBSD', 'OpenBSD', 'NetBSD', ], classifiers=[ - 'Development Status :: 4 - Beta', + 'Development Status :: 2 - Pre-Alpha', 'Environment :: Console', 'Intended Audience :: System Administrators', 'License :: OSI Approved :: BSD License', @@ -775,7 +775,6 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Topic :: Security :: Cryptography', diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 4dc5dacce..7d27e8c64 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2022,11 +2022,11 @@ class Archiver: {now} The current local date and time, by default in ISO-8601 format. - You can also supply your own `format string `_, e.g. {now:%Y-%m-%d_%H:%M:%S} + You can also supply your own `format string `_, e.g. {now:%Y-%m-%d_%H:%M:%S} {utcnow} The current UTC date and time, by default in ISO-8601 format. - You can also supply your own `format string `_, e.g. {utcnow:%Y-%m-%d_%H:%M:%S} + You can also supply your own `format string `_, e.g. {utcnow:%Y-%m-%d_%H:%M:%S} {user} The user name (or UID, if no name is available) of the user running borg. diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 2094c8652..d5a803db5 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -1967,71 +1967,9 @@ def consume(iterator, n=None): # advance to the empty slice starting at position n next(islice(iterator, n, n), None) -# GenericDirEntry, scandir_generic (c) 2012 Ben Hoyt -# from the python-scandir package (3-clause BSD license, just like us, so no troubles here) -# note: simplified version - - -class GenericDirEntry: - __slots__ = ('name', '_scandir_path', '_path') - - def __init__(self, scandir_path, name): - self._scandir_path = scandir_path - self.name = name - self._path = None - - @property - def path(self): - if self._path is None: - self._path = os.path.join(self._scandir_path, self.name) - return self._path - - def stat(self, follow_symlinks=True): - assert not follow_symlinks - return os.stat(self.path, follow_symlinks=follow_symlinks) - - def _check_type(self, type): - st = self.stat(False) - return stat.S_IFMT(st.st_mode) == type - - def is_dir(self, follow_symlinks=True): - assert not follow_symlinks - return self._check_type(stat.S_IFDIR) - - def is_file(self, follow_symlinks=True): - assert not follow_symlinks - return self._check_type(stat.S_IFREG) - - def is_symlink(self): - return self._check_type(stat.S_IFLNK) - - def inode(self): - st = self.stat(False) - return st.st_ino - - def __repr__(self): - return '<{0}: {1!r}>'.format(self.__class__.__name__, self.path) - - -def scandir_generic(path='.'): - """Like os.listdir(), but yield DirEntry objects instead of returning a list of names.""" - for name in sorted(os.listdir(path)): - yield GenericDirEntry(path, name) - - -try: - from os import scandir -except ImportError: - try: - # Try python-scandir on Python 3.4 - from scandir import scandir - except ImportError: - # If python-scandir is not installed, then use a version that is just as slow as listdir. - scandir = scandir_generic - def scandir_inorder(path='.'): - return sorted(scandir(path), key=lambda dirent: dirent.inode()) + return sorted(os.scandir(path), key=lambda dirent: dirent.inode()) def clean_lines(lines, lstrip=None, rstrip=None, remove_empty=True, remove_comments=True): diff --git a/src/borg/remote.py b/src/borg/remote.py index d131a8266..1cc455c2e 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -1,5 +1,4 @@ import errno -import fcntl import functools import inspect import json @@ -190,15 +189,9 @@ class RepositoryServer: # pragma: no cover stdin_fd = sys.stdin.fileno() stdout_fd = sys.stdout.fileno() stderr_fd = sys.stdout.fileno() - # Make stdin non-blocking - fl = fcntl.fcntl(stdin_fd, fcntl.F_GETFL) - fcntl.fcntl(stdin_fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) - # Make stdout blocking - fl = fcntl.fcntl(stdout_fd, fcntl.F_GETFL) - fcntl.fcntl(stdout_fd, fcntl.F_SETFL, fl & ~os.O_NONBLOCK) - # Make stderr blocking - fl = fcntl.fcntl(stderr_fd, fcntl.F_GETFL) - fcntl.fcntl(stderr_fd, fcntl.F_SETFL, fl & ~os.O_NONBLOCK) + os.set_blocking(stdin_fd, False) + os.set_blocking(stdout_fd, True) + os.set_blocking(stderr_fd, True) unpacker = get_limited_unpacker('server') while True: r, w, es = select.select([stdin_fd], [], [], 10) @@ -557,9 +550,9 @@ class RemoteRepository: self.stdin_fd = self.p.stdin.fileno() self.stdout_fd = self.p.stdout.fileno() self.stderr_fd = self.p.stderr.fileno() - fcntl.fcntl(self.stdin_fd, fcntl.F_SETFL, fcntl.fcntl(self.stdin_fd, fcntl.F_GETFL) | os.O_NONBLOCK) - fcntl.fcntl(self.stdout_fd, fcntl.F_SETFL, fcntl.fcntl(self.stdout_fd, fcntl.F_GETFL) | os.O_NONBLOCK) - fcntl.fcntl(self.stderr_fd, fcntl.F_SETFL, fcntl.fcntl(self.stderr_fd, fcntl.F_GETFL) | os.O_NONBLOCK) + os.set_blocking(self.stdin_fd, False) + os.set_blocking(self.stdout_fd, False) + os.set_blocking(self.stderr_fd, False) self.r_fds = [self.stdout_fd, self.stderr_fd] self.x_fds = [self.stdin_fd, self.stdout_fd, self.stderr_fd] diff --git a/tox.ini b/tox.ini index 2e04bf2e7..8fb3f31bd 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ # fakeroot -u tox --recreate [tox] -envlist = py{34,35,36},flake8 +envlist = py{35,36},flake8 [testenv] deps =