From 126921da48aaa6cbb4434e4b69dd68b1e820f754 Mon Sep 17 00:00:00 2001 From: Antti Aalto Date: Mon, 18 Apr 2016 04:14:10 +0300 Subject: [PATCH] Add initial native Windows support. --- .gitignore | 1 + borg/_hashindex.c | 24 ++++++-- borg/archive.py | 39 +++++++----- borg/archiver.py | 8 ++- borg/helpers.py | 115 ++++++++++++++++++++++++++--------- borg/remote.py | 12 ++-- borg/testsuite/__init__.py | 7 ++- buildwin32.py | 120 +++++++++++++++++++++++++++++++++++++ docs/development.rst | 20 +++++++ docs/installation.rst | 14 +++++ setup.py | 35 ++++++++++- 11 files changed, 336 insertions(+), 59 deletions(-) create mode 100644 buildwin32.py diff --git a/.gitignore b/.gitignore index 2d77951bd..0e13ddabf 100644 --- a/.gitignore +++ b/.gitignore @@ -21,5 +21,6 @@ borg/_version.py borg.build/ borg.dist/ borg.exe +*.dll .coverage .vagrant diff --git a/borg/_hashindex.c b/borg/_hashindex.c index 4599fe0fa..f79d10b3d 100644 --- a/borg/_hashindex.c +++ b/borg/_hashindex.c @@ -11,13 +11,27 @@ #if defined (__SVR4) && defined (__sun) #include #endif - -#if (defined(BYTE_ORDER)&&(BYTE_ORDER == BIG_ENDIAN)) || \ - (defined(_BIG_ENDIAN)&&defined(__SVR4)&&defined(__sun)) +#if (defined(_BIG_ENDIAN)&&defined(__SVR4)&&defined(__sun)) +#define BIG_ENDIAN_DETECTED +#endif + +#if (defined(__MINGW32__) && defined(_WIN32)) || \ + (defined(_LITTLE_ENDIAN)&&defined(__SVR4)&&defined(__sun)) +#define LITTLE_ENDIAN_DETECTED +#endif // __MINGW32__ + +#if !defined(BIG_ENDIAN_DETECTED) && !defined(LITTLE_ENDIAN_DETECTED) +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define LITTLE_ENDIAN_DETECTED +#else +#define BIG_ENDIAN_DETECTED +#endif +#endif + +#ifdef BIG_ENDIAN_DETECTED #define _le32toh(x) __builtin_bswap32(x) #define _htole32(x) __builtin_bswap32(x) -#elif (defined(BYTE_ORDER)&&(BYTE_ORDER == LITTLE_ENDIAN)) || \ - (defined(_LITTLE_ENDIAN)&&defined(__SVR4)&&defined(__sun)) +#elif defined(LITTLE_ENDIAN_DETECTED) #define _le32toh(x) (x) #define _htole32(x) (x) #else diff --git a/borg/archive.py b/borg/archive.py index d37fb09eb..0b5bc773d 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -323,7 +323,7 @@ Number of files: {0.stats.nfiles}'''.format( original_path = original_path or item[b'path'] dest = self.cwd - if item[b'path'].startswith('/') or item[b'path'].startswith('..'): + if item[b'path'].startswith('/') or item[b'path'].startswith('..') or (sys.platform == 'win32' and item[b'path'][1] == ':'): raise Exception('Path should be relative and local') path = os.path.join(dest, item[b'path']) # Attempt to remove existing files, ignore errors on failure @@ -367,7 +367,12 @@ Number of files: {0.stats.nfiles}'''.format( pos = fd.tell() fd.truncate(pos) fd.flush() - self.restore_attrs(path, item, fd=fd.fileno()) + if sys.platform != 'win32': + self.restore_attrs(path, item, fd=fd.fileno()) + else: + # File needs to be closed or timestamps are rewritten at close + fd.close() + self.restore_attrs(path, item) if hardlink_masters: # Update master entry with extracted file path, so that following hardlinks don't extract twice. hardlink_masters[item.get(b'source') or original_path] = (None, path) @@ -406,26 +411,30 @@ Number of files: {0.stats.nfiles}'''.format( uid = item[b'uid'] if uid is None else uid gid = item[b'gid'] if gid is None else gid # This code is a bit of a mess due to os specific differences - try: + if sys.platform != 'win32': + try: + if fd: + os.fchown(fd, uid, gid) + else: + os.lchown(path, uid, gid) + except OSError: + pass + if sys.platform != 'win32': if fd: - os.fchown(fd, uid, gid) - else: - os.lchown(path, uid, gid) - except OSError: - pass - if fd: - os.fchmod(fd, item[b'mode']) - elif not symlink: - os.chmod(path, item[b'mode']) - elif has_lchmod: # Not available on Linux - os.lchmod(path, item[b'mode']) + os.fchmod(fd, item[b'mode']) + elif not symlink: + os.chmod(path, item[b'mode']) + elif has_lchmod: # Not available on Linux + os.lchmod(path, item[b'mode']) mtime = bigint_to_int(item[b'mtime']) if b'atime' in item: atime = bigint_to_int(item[b'atime']) else: # old archives only had mtime in item metadata atime = mtime - if fd: + if sys.platform == 'win32': + os.utime(path, ns=(atime, mtime)) + elif fd: os.utime(fd, None, ns=(atime, mtime)) else: os.utime(path, None, ns=(atime, mtime), follow_symlinks=False) diff --git a/borg/archiver.py b/borg/archiver.py index 6a68eaa4f..a6a0e3241 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -38,6 +38,9 @@ from .hashindex import ChunkIndexEntry has_lchflags = hasattr(os, 'lchflags') +if sys.platform == 'win32': + import posixpath + def argument(args, str_or_bool): """If bool is passed, return it. If str is passed, retrieve named attribute from args.""" @@ -247,7 +250,10 @@ class Archiver: status = '-' self.print_file_status(status, path) continue - path = os.path.normpath(path) + if sys.platform == 'win32': + path = posixpath.normpath(path.replace('\\', '/')) + else: + path = os.path.normpath(path) try: st = os.lstat(path) except OSError as e: diff --git a/borg/helpers.py b/borg/helpers.py index 994ad9a3e..ae33f0077 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -2,17 +2,22 @@ import argparse from binascii import hexlify from collections import namedtuple, deque from functools import wraps, partial -import grp +import sys +if sys.platform != 'win32': + import grp + import pwd +else: + import posixpath import hashlib from itertools import islice import os import os.path import stat import textwrap -import pwd + import re from shutil import get_terminal_size -import sys + from string import Formatter import platform import time @@ -37,7 +42,6 @@ import msgpack.fallback import socket - # meta dict, data bytes _Chunk = namedtuple('_Chunk', 'meta data') @@ -387,10 +391,16 @@ class PathPrefixPattern(PatternBase): PREFIX = "pp" def _prepare(self, pattern): - self.pattern = os.path.normpath(pattern).rstrip(os.path.sep) + os.path.sep + if sys.platform != 'win32': + self.pattern = os.path.normpath(pattern).rstrip(os.path.sep) + os.path.sep + else: + self.pattern = posixpath.normpath(pattern).rstrip(posixpath.sep) + posixpath.sep def _match(self, path): - return (path + os.path.sep).startswith(self.pattern) + if sys.platform != 'win32': + return (path + os.path.sep).startswith(self.pattern) + else: + return (path + posixpath.sep).startswith(self.pattern) class FnmatchPattern(PatternBase): @@ -682,7 +692,10 @@ def memoize(function): @memoize def uid2user(uid, default=None): try: - return pwd.getpwuid(uid).pw_name + if sys.platform != 'win32': + return pwd.getpwuid(uid).pw_name + else: + return os.getlogin() except KeyError: return default @@ -690,7 +703,10 @@ def uid2user(uid, default=None): @memoize def user2uid(user, default=None): try: - return user and pwd.getpwnam(user).pw_uid + if sys.platform != 'win32': + return user and pwd.getpwnam(user).pw_uid + else: + return user and 0 except KeyError: return default @@ -698,17 +714,32 @@ def user2uid(user, default=None): @memoize def gid2group(gid, default=None): try: - return grp.getgrgid(gid).gr_name + if sys.platform != 'win32': + return grp.getgrgid(gid).gr_name + else: + return '' except KeyError: return default @memoize def group2gid(group, default=None): - try: - return group and grp.getgrnam(group).gr_gid - except KeyError: - return default + if sys.platform != 'win32': + if group == '': + return 0 # From windows + try: + return group and grp.getgrnam(group).gr_gid + except KeyError: + return default + else: + return 0 + + +def getuid(): + if sys.platform != 'win32': + return os.getuid() + else: + return 0 def posix_acl_use_stored_uid_gid(acl): @@ -748,8 +779,13 @@ class Location: ssh_re = re.compile(r'(?Pssh)://(?:(?P[^@]+)@)?' r'(?P[^:/#]+)(?::(?P\d+))?' r'(?P[^:]+)(?:::(?P[^/]+))?$') - file_re = re.compile(r'(?Pfile)://' - r'(?P[^:]+)(?:::(?P[^/]+))?$') + file_re = None + if sys.platform != 'win32': + file_re = re.compile(r'(?Pfile)://' + r'(?P[^:]+)(?:::(?P[^/]+))?$') + else: + file_re = re.compile(r'((?Pfile)://)?' + r'(?P[a-zA-Z])?:[\\/](?P[^:]+)(?:::(?P[^/]+))?$') scp_re = re.compile(r'((?:(?P[^@]+)@)?(?P[^:/]+):)?' r'(?P[^:]+)(?:::(?P[^/]+))?$') # get the repo from BORG_RE env and the optional archive from param. @@ -772,7 +808,7 @@ class Location: 'hostname': socket.gethostname(), 'now': current_time.now(), 'utcnow': current_time.utcnow(), - 'user': uid2user(os.getuid(), os.getuid()) + 'user': uid2user(getuid(), getuid()) } return format_line(text, data) @@ -794,26 +830,41 @@ class Location: return True def _parse(self, text): + if sys.platform == 'win32': + m = self.file_re.match(text) + if m: + self.proto = m.group('proto') + self.path = posixpath.normpath(m.group('drive') + ":\\" + m.group('path')) + self.archive = m.group('archive') + return True + m = self.ssh_re.match(text) if m: self.proto = m.group('proto') self.user = m.group('user') self.host = m.group('host') self.port = m.group('port') and int(m.group('port')) or None - self.path = os.path.normpath(m.group('path')) - self.archive = m.group('archive') - return True - m = self.file_re.match(text) - if m: - self.proto = m.group('proto') - self.path = os.path.normpath(m.group('path')) + if sys.platform != 'win32': + self.path = os.path.normpath(m.group('path')) + else: + self.path = posixpath.normpath(m.group('path')) self.archive = m.group('archive') return True + if sys.platform != 'win32': + m = self.file_re.match(text) + if m: + self.proto = m.group('proto') + self.path = os.path.normpath(m.group('path')) + self.archive = m.group('archive') + return True m = self.scp_re.match(text) if m: self.user = m.group('user') self.host = m.group('host') - self.path = os.path.normpath(m.group('path')) + if sys.platform != 'win32': + self.path = os.path.normpath(m.group('path')) + else: + self.path = posixpath.normpath(m.group('path')) self.archive = m.group('archive') self.proto = self.host and 'ssh' or 'file' return True @@ -889,14 +940,24 @@ def remove_surrogates(s, errors='replace'): """ return s.encode('utf-8', errors).decode('utf-8') - -_safe_re = re.compile(r'^((\.\.)?/+)+') +_safe_re = None +if sys.platform != 'win32': + _safe_re = re.compile(r'^((\.\.)?/+)+') +else: + _safe_re = re.compile(r'^((\.\.)?[/\\]+)+') def make_path_safe(path): """Make path safe by making it relative and local """ - return _safe_re.sub('', path) or '.' + if sys.platform != 'win32': + return _safe_re.sub('', path) or '.' + else: + tail = path + if len(path) > 2 and (path[0:2] == '//' or path[0:2] == '\\\\' or path[1] == ':'): + drive, tail = os.path.splitdrive(path) + tail = tail.replace('\\', '/') + return posixpath.normpath(_safe_re.sub('', tail) or '.') def daemonize(): diff --git a/borg/remote.py b/borg/remote.py index 5444f05bf..37a425ed6 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -1,11 +1,12 @@ import errno -import fcntl +import sys +if sys.platform != 'win32': + import fcntl import logging import os import select import shlex from subprocess import Popen, PIPE -import sys import tempfile from . import __version__ @@ -157,9 +158,10 @@ class RemoteRepository: self.stdin_fd = self.p.stdin.fileno() self.stdout_fd = self.p.stdout.fileno() self.stderr_fd = self.p.stderr.fileno() - fcntl.fcntl(self.stdin_fd, fcntl.F_SETFL, fcntl.fcntl(self.stdin_fd, fcntl.F_GETFL) | os.O_NONBLOCK) - fcntl.fcntl(self.stdout_fd, fcntl.F_SETFL, fcntl.fcntl(self.stdout_fd, fcntl.F_GETFL) | os.O_NONBLOCK) - fcntl.fcntl(self.stderr_fd, fcntl.F_SETFL, fcntl.fcntl(self.stderr_fd, fcntl.F_GETFL) | os.O_NONBLOCK) + if sys.platform != 'win32': + fcntl.fcntl(self.stdin_fd, fcntl.F_SETFL, fcntl.fcntl(self.stdin_fd, fcntl.F_GETFL) | os.O_NONBLOCK) + fcntl.fcntl(self.stdout_fd, fcntl.F_SETFL, fcntl.fcntl(self.stdout_fd, fcntl.F_GETFL) | os.O_NONBLOCK) + fcntl.fcntl(self.stderr_fd, fcntl.F_SETFL, fcntl.fcntl(self.stderr_fd, fcntl.F_GETFL) | os.O_NONBLOCK) self.r_fds = [self.stdout_fd, self.stderr_fd] self.x_fds = [self.stdin_fd, self.stdout_fd, self.stderr_fd] diff --git a/borg/testsuite/__init__.py b/borg/testsuite/__init__.py index 5c1a0a6fd..be676cd21 100644 --- a/borg/testsuite/__init__.py +++ b/borg/testsuite/__init__.py @@ -1,9 +1,10 @@ from contextlib import contextmanager import filecmp import os -import posix -import stat import sys +if sys.platform != 'win32': + import posix +import stat import sysconfig import time import unittest @@ -21,7 +22,7 @@ has_lchflags = hasattr(os, 'lchflags') # The mtime get/set precision varies on different OS and Python versions -if 'HAVE_FUTIMENS' in getattr(posix, '_have_functions', []): +if sys.platform != 'win32' and 'HAVE_FUTIMENS' in getattr(posix, '_have_functions', []): st_mtime_ns_round = 0 elif 'HAVE_UTIMES' in sysconfig.get_config_vars(): st_mtime_ns_round = -6 diff --git a/buildwin32.py b/buildwin32.py new file mode 100644 index 000000000..07b7d22ea --- /dev/null +++ b/buildwin32.py @@ -0,0 +1,120 @@ +import shutil +import os +import subprocess +from modulefinder import ModuleFinder + +# Creates standalone Windows executable +# First build by following instructions from installation.rst + +builddir = 'win32exe' + +if os.path.exists(builddir): + shutil.rmtree(builddir) +os.mkdir(builddir) +os.mkdir(builddir + '/bin') +os.mkdir(builddir + '/lib') + +print('Compiling wrapper') + +gccpath = '' # check for compiler, path needed later +for p in os.environ['PATH'].split(';'): + if os.path.exists(os.path.join(p, 'gcc.exe')): + gccpath = p + break +if gccpath == '': + print('gcc not found.') + exit(1) + +source = open('wrapper.c', 'w') +source.write( +""" +#include +#include +#include +#include "Shlwapi.h" + +int wmain(int argc , wchar_t *argv[] ) +{ + + wchar_t *program = argv[0]; + Py_SetProgramName(program); + Py_Initialize(); + + PySys_SetArgv(argc, argv); + + wchar_t path[MAX_PATH]; + GetModuleFileNameW(NULL, path, MAX_PATH); + PathRemoveFileSpecW(path); + + FILE* file_1 = _wfopen(wcsncat(path, L"/borg/__main__.py", 17), L"r"); + PyRun_AnyFile(file_1, "borg/__main__.py"); + + Py_Finalize(); + PyMem_RawFree(program); + return 0; +} +""") +source.close() +subprocess.run('gcc wrapper.c -lpython3.5m -lshlwapi -municode -o ' + builddir + '/bin/borg.exe') +os.remove('wrapper.c') + +print('Searching modules') + +modulepath = os.path.abspath(os.path.join(gccpath, '../lib/python3.5/')) + +shutil.copytree(os.path.join(modulepath, 'encodings'), os.path.join(builddir, 'lib/python3.5/encodings')) + +finder = ModuleFinder() +finder.run_script('borg/__main__.py') +extramodules = [os.path.join(modulepath, 'site.py')] + +for module in extramodules: + finder.run_script(module) + +print('Copying files') + + +def finddlls(exe): + re = [] + output = subprocess.check_output(['ntldd', '-R', exe]) + for line in output.decode('utf-8').split('\n'): + if 'not found' in line: + continue + if 'windows' in line.lower(): + continue + words = line.split() + if len(words) < 3: + if len(words) == 2: + re.append(words[0]) + continue + dll = words[2] + re.append(dll) + return re + +items = finder.modules.items() +for name, mod in items: + file = mod.__file__ + if file is None: + continue + lib = file.find('lib') + if lib == -1: + relpath = os.path.relpath(file) + os.makedirs(os.path.join(builddir, 'bin', os.path.split(relpath)[0]), exist_ok=True) + shutil.copyfile(file, os.path.join(builddir, 'bin', relpath)) + continue + relativepath = file[file.find('lib')+4:] + os.makedirs(os.path.join(builddir, 'lib', os.path.split(relativepath)[0]), exist_ok=True) + shutil.copyfile(file, os.path.join(builddir, 'lib', relativepath)) + if file[-4:] == '.dll' or file[-4:] == '.DLL': + for dll in finddlls(file): + if builddir not in dll: + shutil.copyfile(dll, os.path.join(builddir, 'bin', os.path.split(dll)[1])) +for dll in finddlls(os.path.join(builddir, "bin/borg.exe")): + if builddir not in dll: + shutil.copyfile(dll, os.path.join(builddir, 'bin', os.path.split(dll)[1])) +shutil.copyfile('borg/__main__.py', os.path.join(builddir, 'bin/borg/__main__.py')) + +for extmodule in ['borg/chunker-cpython-35m.dll', 'borg/compress-cpython-35m.dll', 'borg/crypto-cpython-35m.dll', 'borg/hashindex-cpython-35m.dll']: + for dll in finddlls(extmodule): + if builddir not in dll: + shutil.copyfile(dll, os.path.join(builddir, 'bin', os.path.split(dll)[1])) diff --git a/docs/development.rst b/docs/development.rst index 524957e01..6a930e0c5 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -40,6 +40,20 @@ virtual env and run:: pip install -r requirements.d/development.txt +Building on Windows ++++++++++++++++++++ + +Download and install MSYS from https://msys2.github.io/ + +Use `Mingw64-w64 64bit Shell`:: + + pacman -S mingw-w64-x86_64-python3 git mingw-w64-x86_64-lz4 mingw-w64-x86_64-python3-pip \ + mingw-w64-x86_64-cython mingw-w64-x86_64-gcc mingw-w64-x86_64-ntldd-git + +Use git to get the source and checkout `windows` branch then:: + + pip3 install -r requirements.d/development.txt + pip3 install -e . Running the tests ----------------- @@ -71,6 +85,9 @@ Important notes: - When using ``--`` to give options to py.test, you MUST also give ``borg.testsuite[.module]``. +As tox doesn't run on Windows you have to manually run command:: + + py.test --cov=borg --cov-config=.coveragerc --benchmark-skip --pyargs borg/testsuite Regenerate usage files ---------------------- @@ -149,6 +166,9 @@ If you encounter issues, see also our `Vagrantfile` for details. work on same OS, same architecture (x86 32bit, amd64 64bit) without external dependencies. +On Windows use `python buildwin32.py` to build standalone executable in `win32exe` directory +with all necessary files to run. + Creating a new release ---------------------- diff --git a/docs/installation.rst b/docs/installation.rst index fae4517b3..d3863a27b 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -102,6 +102,12 @@ You can change the temporary directory by setting the ``TEMP`` environment varia If a new version is released, you will have to manually download it and replace the old version using the same steps as shown above. +Windows zip ++++++++++++ +Tested on Windows10. (Should work on Vista and up) + +To install on Windows just extract the zip anywhere and add the bin directory to your ``PATH`` environment variable. + .. _pyinstaller: http://www.pyinstaller.org .. _releases: https://github.com/borgbackup/borg/releases @@ -200,6 +206,14 @@ and commands to make fuse work for using the mount command. sysctl vfs.usermount=1 +Windows ++++++++ + +See development_ on how to build on windows. +run `python3 buildwin32.py` to create standalone windows executable in `win32exe`. +You can rename or move that folder. Add the bin folder to your ``PATH`` and you can run ``borg``. + + Cygwin ++++++ diff --git a/setup.py b/setup.py index 30e8466ed..4a85bb0a2 100644 --- a/setup.py +++ b/setup.py @@ -2,6 +2,7 @@ import os import re import sys +import subprocess from glob import glob from distutils.command.build import build @@ -106,7 +107,22 @@ def detect_lz4(prefixes): include_dirs = [] library_dirs = [] -possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', '/usr/local/borg', '/opt/local'] +windowsIncludeDirs = [] +if sys.platform == 'win32': + gccpath = "" + for p in os.environ["PATH"].split(";"): + if os.path.exists(os.path.join(p, "gcc.exe")): + gccpath = p + break + windowsIncludeDirs.append(os.path.abspath(os.path.join(gccpath, ".."))) + windowsIncludeDirs.append(os.path.abspath(os.path.join(gccpath, "..", ".."))) + + +possible_openssl_prefixes = None +if sys.platform == 'win32': + possible_openssl_prefixes = windowsIncludeDirs +else: + possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', '/usr/local/borg', '/opt/local'] if os.environ.get('BORG_OPENSSL_PREFIX'): possible_openssl_prefixes.insert(0, os.environ.get('BORG_OPENSSL_PREFIX')) ssl_prefix = detect_openssl(possible_openssl_prefixes) @@ -115,8 +131,11 @@ if not ssl_prefix: include_dirs.append(os.path.join(ssl_prefix, 'include')) library_dirs.append(os.path.join(ssl_prefix, 'lib')) - -possible_lz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4', '/usr/local/borg', '/opt/local'] +possible_lz4_prefixes = None +if sys.platform == 'win32': + possible_lz4_prefixes = windowsIncludeDirs +else: + possible_lz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4', '/usr/local/borg', '/opt/local'] if os.environ.get('BORG_LZ4_PREFIX'): possible_lz4_prefixes.insert(0, os.environ.get('BORG_LZ4_PREFIX')) lz4_prefix = detect_lz4(possible_lz4_prefixes) @@ -291,10 +310,20 @@ if not on_rtd: elif sys.platform == 'darwin': ext_modules.append(Extension('borg.platform_darwin', [platform_darwin_source])) + +def parse(root, describe_command=None): + file = open('borg/_version.py', 'w') + output = subprocess.check_output("git describe --tags --long").decode().strip() + file.write('version = "' + output + '"\n') + return output + +parse_function = parse if sys.platform == 'win32' else None + setup( name='borgbackup', use_scm_version={ 'write_to': 'borg/_version.py', + 'parse': parse_function, }, author='The Borg Collective (see AUTHORS file)', author_email='borgbackup@python.org',