Add initial native Windows support.

This commit is contained in:
Antti Aalto 2016-04-18 04:14:10 +03:00
parent ddc7687d9e
commit 126921da48
11 changed files with 336 additions and 59 deletions

1
.gitignore vendored
View file

@ -21,5 +21,6 @@ borg/_version.py
borg.build/
borg.dist/
borg.exe
*.dll
.coverage
.vagrant

View file

@ -11,13 +11,27 @@
#if defined (__SVR4) && defined (__sun)
#include <sys/isa_defs.h>
#endif
#if (defined(BYTE_ORDER)&&(BYTE_ORDER == BIG_ENDIAN)) || \
(defined(_BIG_ENDIAN)&&defined(__SVR4)&&defined(__sun))
#if (defined(_BIG_ENDIAN)&&defined(__SVR4)&&defined(__sun))
#define BIG_ENDIAN_DETECTED
#endif
#if (defined(__MINGW32__) && defined(_WIN32)) || \
(defined(_LITTLE_ENDIAN)&&defined(__SVR4)&&defined(__sun))
#define LITTLE_ENDIAN_DETECTED
#endif // __MINGW32__
#if !defined(BIG_ENDIAN_DETECTED) && !defined(LITTLE_ENDIAN_DETECTED)
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
#define LITTLE_ENDIAN_DETECTED
#else
#define BIG_ENDIAN_DETECTED
#endif
#endif
#ifdef BIG_ENDIAN_DETECTED
#define _le32toh(x) __builtin_bswap32(x)
#define _htole32(x) __builtin_bswap32(x)
#elif (defined(BYTE_ORDER)&&(BYTE_ORDER == LITTLE_ENDIAN)) || \
(defined(_LITTLE_ENDIAN)&&defined(__SVR4)&&defined(__sun))
#elif defined(LITTLE_ENDIAN_DETECTED)
#define _le32toh(x) (x)
#define _htole32(x) (x)
#else

View file

@ -323,7 +323,7 @@ Number of files: {0.stats.nfiles}'''.format(
original_path = original_path or item[b'path']
dest = self.cwd
if item[b'path'].startswith('/') or item[b'path'].startswith('..'):
if item[b'path'].startswith('/') or item[b'path'].startswith('..') or (sys.platform == 'win32' and item[b'path'][1] == ':'):
raise Exception('Path should be relative and local')
path = os.path.join(dest, item[b'path'])
# Attempt to remove existing files, ignore errors on failure
@ -367,7 +367,12 @@ Number of files: {0.stats.nfiles}'''.format(
pos = fd.tell()
fd.truncate(pos)
fd.flush()
self.restore_attrs(path, item, fd=fd.fileno())
if sys.platform != 'win32':
self.restore_attrs(path, item, fd=fd.fileno())
else:
# File needs to be closed or timestamps are rewritten at close
fd.close()
self.restore_attrs(path, item)
if hardlink_masters:
# Update master entry with extracted file path, so that following hardlinks don't extract twice.
hardlink_masters[item.get(b'source') or original_path] = (None, path)
@ -406,26 +411,30 @@ Number of files: {0.stats.nfiles}'''.format(
uid = item[b'uid'] if uid is None else uid
gid = item[b'gid'] if gid is None else gid
# This code is a bit of a mess due to os specific differences
try:
if sys.platform != 'win32':
try:
if fd:
os.fchown(fd, uid, gid)
else:
os.lchown(path, uid, gid)
except OSError:
pass
if sys.platform != 'win32':
if fd:
os.fchown(fd, uid, gid)
else:
os.lchown(path, uid, gid)
except OSError:
pass
if fd:
os.fchmod(fd, item[b'mode'])
elif not symlink:
os.chmod(path, item[b'mode'])
elif has_lchmod: # Not available on Linux
os.lchmod(path, item[b'mode'])
os.fchmod(fd, item[b'mode'])
elif not symlink:
os.chmod(path, item[b'mode'])
elif has_lchmod: # Not available on Linux
os.lchmod(path, item[b'mode'])
mtime = bigint_to_int(item[b'mtime'])
if b'atime' in item:
atime = bigint_to_int(item[b'atime'])
else:
# old archives only had mtime in item metadata
atime = mtime
if fd:
if sys.platform == 'win32':
os.utime(path, ns=(atime, mtime))
elif fd:
os.utime(fd, None, ns=(atime, mtime))
else:
os.utime(path, None, ns=(atime, mtime), follow_symlinks=False)

View file

@ -38,6 +38,9 @@ from .hashindex import ChunkIndexEntry
has_lchflags = hasattr(os, 'lchflags')
if sys.platform == 'win32':
import posixpath
def argument(args, str_or_bool):
"""If bool is passed, return it. If str is passed, retrieve named attribute from args."""
@ -247,7 +250,10 @@ class Archiver:
status = '-'
self.print_file_status(status, path)
continue
path = os.path.normpath(path)
if sys.platform == 'win32':
path = posixpath.normpath(path.replace('\\', '/'))
else:
path = os.path.normpath(path)
try:
st = os.lstat(path)
except OSError as e:

View file

@ -2,17 +2,22 @@ import argparse
from binascii import hexlify
from collections import namedtuple, deque
from functools import wraps, partial
import grp
import sys
if sys.platform != 'win32':
import grp
import pwd
else:
import posixpath
import hashlib
from itertools import islice
import os
import os.path
import stat
import textwrap
import pwd
import re
from shutil import get_terminal_size
import sys
from string import Formatter
import platform
import time
@ -37,7 +42,6 @@ import msgpack.fallback
import socket
# meta dict, data bytes
_Chunk = namedtuple('_Chunk', 'meta data')
@ -387,10 +391,16 @@ class PathPrefixPattern(PatternBase):
PREFIX = "pp"
def _prepare(self, pattern):
self.pattern = os.path.normpath(pattern).rstrip(os.path.sep) + os.path.sep
if sys.platform != 'win32':
self.pattern = os.path.normpath(pattern).rstrip(os.path.sep) + os.path.sep
else:
self.pattern = posixpath.normpath(pattern).rstrip(posixpath.sep) + posixpath.sep
def _match(self, path):
return (path + os.path.sep).startswith(self.pattern)
if sys.platform != 'win32':
return (path + os.path.sep).startswith(self.pattern)
else:
return (path + posixpath.sep).startswith(self.pattern)
class FnmatchPattern(PatternBase):
@ -682,7 +692,10 @@ def memoize(function):
@memoize
def uid2user(uid, default=None):
try:
return pwd.getpwuid(uid).pw_name
if sys.platform != 'win32':
return pwd.getpwuid(uid).pw_name
else:
return os.getlogin()
except KeyError:
return default
@ -690,7 +703,10 @@ def uid2user(uid, default=None):
@memoize
def user2uid(user, default=None):
try:
return user and pwd.getpwnam(user).pw_uid
if sys.platform != 'win32':
return user and pwd.getpwnam(user).pw_uid
else:
return user and 0
except KeyError:
return default
@ -698,17 +714,32 @@ def user2uid(user, default=None):
@memoize
def gid2group(gid, default=None):
try:
return grp.getgrgid(gid).gr_name
if sys.platform != 'win32':
return grp.getgrgid(gid).gr_name
else:
return ''
except KeyError:
return default
@memoize
def group2gid(group, default=None):
try:
return group and grp.getgrnam(group).gr_gid
except KeyError:
return default
if sys.platform != 'win32':
if group == '':
return 0 # From windows
try:
return group and grp.getgrnam(group).gr_gid
except KeyError:
return default
else:
return 0
def getuid():
if sys.platform != 'win32':
return os.getuid()
else:
return 0
def posix_acl_use_stored_uid_gid(acl):
@ -748,8 +779,13 @@ class Location:
ssh_re = re.compile(r'(?P<proto>ssh)://(?:(?P<user>[^@]+)@)?'
r'(?P<host>[^:/#]+)(?::(?P<port>\d+))?'
r'(?P<path>[^:]+)(?:::(?P<archive>[^/]+))?$')
file_re = re.compile(r'(?P<proto>file)://'
r'(?P<path>[^:]+)(?:::(?P<archive>[^/]+))?$')
file_re = None
if sys.platform != 'win32':
file_re = re.compile(r'(?P<proto>file)://'
r'(?P<path>[^:]+)(?:::(?P<archive>[^/]+))?$')
else:
file_re = re.compile(r'((?P<proto>file)://)?'
r'(?P<drive>[a-zA-Z])?:[\\/](?P<path>[^:]+)(?:::(?P<archive>[^/]+))?$')
scp_re = re.compile(r'((?:(?P<user>[^@]+)@)?(?P<host>[^:/]+):)?'
r'(?P<path>[^:]+)(?:::(?P<archive>[^/]+))?$')
# get the repo from BORG_RE env and the optional archive from param.
@ -772,7 +808,7 @@ class Location:
'hostname': socket.gethostname(),
'now': current_time.now(),
'utcnow': current_time.utcnow(),
'user': uid2user(os.getuid(), os.getuid())
'user': uid2user(getuid(), getuid())
}
return format_line(text, data)
@ -794,26 +830,41 @@ class Location:
return True
def _parse(self, text):
if sys.platform == 'win32':
m = self.file_re.match(text)
if m:
self.proto = m.group('proto')
self.path = posixpath.normpath(m.group('drive') + ":\\" + m.group('path'))
self.archive = m.group('archive')
return True
m = self.ssh_re.match(text)
if m:
self.proto = m.group('proto')
self.user = m.group('user')
self.host = m.group('host')
self.port = m.group('port') and int(m.group('port')) or None
self.path = os.path.normpath(m.group('path'))
self.archive = m.group('archive')
return True
m = self.file_re.match(text)
if m:
self.proto = m.group('proto')
self.path = os.path.normpath(m.group('path'))
if sys.platform != 'win32':
self.path = os.path.normpath(m.group('path'))
else:
self.path = posixpath.normpath(m.group('path'))
self.archive = m.group('archive')
return True
if sys.platform != 'win32':
m = self.file_re.match(text)
if m:
self.proto = m.group('proto')
self.path = os.path.normpath(m.group('path'))
self.archive = m.group('archive')
return True
m = self.scp_re.match(text)
if m:
self.user = m.group('user')
self.host = m.group('host')
self.path = os.path.normpath(m.group('path'))
if sys.platform != 'win32':
self.path = os.path.normpath(m.group('path'))
else:
self.path = posixpath.normpath(m.group('path'))
self.archive = m.group('archive')
self.proto = self.host and 'ssh' or 'file'
return True
@ -889,14 +940,24 @@ def remove_surrogates(s, errors='replace'):
"""
return s.encode('utf-8', errors).decode('utf-8')
_safe_re = re.compile(r'^((\.\.)?/+)+')
_safe_re = None
if sys.platform != 'win32':
_safe_re = re.compile(r'^((\.\.)?/+)+')
else:
_safe_re = re.compile(r'^((\.\.)?[/\\]+)+')
def make_path_safe(path):
"""Make path safe by making it relative and local
"""
return _safe_re.sub('', path) or '.'
if sys.platform != 'win32':
return _safe_re.sub('', path) or '.'
else:
tail = path
if len(path) > 2 and (path[0:2] == '//' or path[0:2] == '\\\\' or path[1] == ':'):
drive, tail = os.path.splitdrive(path)
tail = tail.replace('\\', '/')
return posixpath.normpath(_safe_re.sub('', tail) or '.')
def daemonize():

View file

@ -1,11 +1,12 @@
import errno
import fcntl
import sys
if sys.platform != 'win32':
import fcntl
import logging
import os
import select
import shlex
from subprocess import Popen, PIPE
import sys
import tempfile
from . import __version__
@ -157,9 +158,10 @@ class RemoteRepository:
self.stdin_fd = self.p.stdin.fileno()
self.stdout_fd = self.p.stdout.fileno()
self.stderr_fd = self.p.stderr.fileno()
fcntl.fcntl(self.stdin_fd, fcntl.F_SETFL, fcntl.fcntl(self.stdin_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
fcntl.fcntl(self.stdout_fd, fcntl.F_SETFL, fcntl.fcntl(self.stdout_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
fcntl.fcntl(self.stderr_fd, fcntl.F_SETFL, fcntl.fcntl(self.stderr_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
if sys.platform != 'win32':
fcntl.fcntl(self.stdin_fd, fcntl.F_SETFL, fcntl.fcntl(self.stdin_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
fcntl.fcntl(self.stdout_fd, fcntl.F_SETFL, fcntl.fcntl(self.stdout_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
fcntl.fcntl(self.stderr_fd, fcntl.F_SETFL, fcntl.fcntl(self.stderr_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
self.r_fds = [self.stdout_fd, self.stderr_fd]
self.x_fds = [self.stdin_fd, self.stdout_fd, self.stderr_fd]

View file

@ -1,9 +1,10 @@
from contextlib import contextmanager
import filecmp
import os
import posix
import stat
import sys
if sys.platform != 'win32':
import posix
import stat
import sysconfig
import time
import unittest
@ -21,7 +22,7 @@ has_lchflags = hasattr(os, 'lchflags')
# The mtime get/set precision varies on different OS and Python versions
if 'HAVE_FUTIMENS' in getattr(posix, '_have_functions', []):
if sys.platform != 'win32' and 'HAVE_FUTIMENS' in getattr(posix, '_have_functions', []):
st_mtime_ns_round = 0
elif 'HAVE_UTIMES' in sysconfig.get_config_vars():
st_mtime_ns_round = -6

120
buildwin32.py Normal file
View file

@ -0,0 +1,120 @@
import shutil
import os
import subprocess
from modulefinder import ModuleFinder
# Creates standalone Windows executable
# First build by following instructions from installation.rst
builddir = 'win32exe'
if os.path.exists(builddir):
shutil.rmtree(builddir)
os.mkdir(builddir)
os.mkdir(builddir + '/bin')
os.mkdir(builddir + '/lib')
print('Compiling wrapper')
gccpath = '' # check for compiler, path needed later
for p in os.environ['PATH'].split(';'):
if os.path.exists(os.path.join(p, 'gcc.exe')):
gccpath = p
break
if gccpath == '':
print('gcc not found.')
exit(1)
source = open('wrapper.c', 'w')
source.write(
"""
#include <python3.5m/python.h>
#include <windows.h>
#include <wchar.h>
#include "Shlwapi.h"
int wmain(int argc , wchar_t *argv[] )
{
wchar_t *program = argv[0];
Py_SetProgramName(program);
Py_Initialize();
PySys_SetArgv(argc, argv);
wchar_t path[MAX_PATH];
GetModuleFileNameW(NULL, path, MAX_PATH);
PathRemoveFileSpecW(path);
FILE* file_1 = _wfopen(wcsncat(path, L"/borg/__main__.py", 17), L"r");
PyRun_AnyFile(file_1, "borg/__main__.py");
Py_Finalize();
PyMem_RawFree(program);
return 0;
}
""")
source.close()
subprocess.run('gcc wrapper.c -lpython3.5m -lshlwapi -municode -o ' + builddir + '/bin/borg.exe')
os.remove('wrapper.c')
print('Searching modules')
modulepath = os.path.abspath(os.path.join(gccpath, '../lib/python3.5/'))
shutil.copytree(os.path.join(modulepath, 'encodings'), os.path.join(builddir, 'lib/python3.5/encodings'))
finder = ModuleFinder()
finder.run_script('borg/__main__.py')
extramodules = [os.path.join(modulepath, 'site.py')]
for module in extramodules:
finder.run_script(module)
print('Copying files')
def finddlls(exe):
re = []
output = subprocess.check_output(['ntldd', '-R', exe])
for line in output.decode('utf-8').split('\n'):
if 'not found' in line:
continue
if 'windows' in line.lower():
continue
words = line.split()
if len(words) < 3:
if len(words) == 2:
re.append(words[0])
continue
dll = words[2]
re.append(dll)
return re
items = finder.modules.items()
for name, mod in items:
file = mod.__file__
if file is None:
continue
lib = file.find('lib')
if lib == -1:
relpath = os.path.relpath(file)
os.makedirs(os.path.join(builddir, 'bin', os.path.split(relpath)[0]), exist_ok=True)
shutil.copyfile(file, os.path.join(builddir, 'bin', relpath))
continue
relativepath = file[file.find('lib')+4:]
os.makedirs(os.path.join(builddir, 'lib', os.path.split(relativepath)[0]), exist_ok=True)
shutil.copyfile(file, os.path.join(builddir, 'lib', relativepath))
if file[-4:] == '.dll' or file[-4:] == '.DLL':
for dll in finddlls(file):
if builddir not in dll:
shutil.copyfile(dll, os.path.join(builddir, 'bin', os.path.split(dll)[1]))
for dll in finddlls(os.path.join(builddir, "bin/borg.exe")):
if builddir not in dll:
shutil.copyfile(dll, os.path.join(builddir, 'bin', os.path.split(dll)[1]))
shutil.copyfile('borg/__main__.py', os.path.join(builddir, 'bin/borg/__main__.py'))
for extmodule in ['borg/chunker-cpython-35m.dll', 'borg/compress-cpython-35m.dll', 'borg/crypto-cpython-35m.dll', 'borg/hashindex-cpython-35m.dll']:
for dll in finddlls(extmodule):
if builddir not in dll:
shutil.copyfile(dll, os.path.join(builddir, 'bin', os.path.split(dll)[1]))

View file

@ -40,6 +40,20 @@ virtual env and run::
pip install -r requirements.d/development.txt
Building on Windows
+++++++++++++++++++
Download and install MSYS from https://msys2.github.io/
Use `Mingw64-w64 64bit Shell`::
pacman -S mingw-w64-x86_64-python3 git mingw-w64-x86_64-lz4 mingw-w64-x86_64-python3-pip \
mingw-w64-x86_64-cython mingw-w64-x86_64-gcc mingw-w64-x86_64-ntldd-git
Use git to get the source and checkout `windows` branch then::
pip3 install -r requirements.d/development.txt
pip3 install -e .
Running the tests
-----------------
@ -71,6 +85,9 @@ Important notes:
- When using ``--`` to give options to py.test, you MUST also give ``borg.testsuite[.module]``.
As tox doesn't run on Windows you have to manually run command::
py.test --cov=borg --cov-config=.coveragerc --benchmark-skip --pyargs borg/testsuite
Regenerate usage files
----------------------
@ -149,6 +166,9 @@ If you encounter issues, see also our `Vagrantfile` for details.
work on same OS, same architecture (x86 32bit, amd64 64bit)
without external dependencies.
On Windows use `python buildwin32.py` to build standalone executable in `win32exe` directory
with all necessary files to run.
Creating a new release
----------------------

View file

@ -102,6 +102,12 @@ You can change the temporary directory by setting the ``TEMP`` environment varia
If a new version is released, you will have to manually download it and replace
the old version using the same steps as shown above.
Windows zip
+++++++++++
Tested on Windows10. (Should work on Vista and up)
To install on Windows just extract the zip anywhere and add the bin directory to your ``PATH`` environment variable.
.. _pyinstaller: http://www.pyinstaller.org
.. _releases: https://github.com/borgbackup/borg/releases
@ -200,6 +206,14 @@ and commands to make fuse work for using the mount command.
sysctl vfs.usermount=1
Windows
+++++++
See development_ on how to build on windows.
run `python3 buildwin32.py` to create standalone windows executable in `win32exe`.
You can rename or move that folder. Add the bin folder to your ``PATH`` and you can run ``borg``.
Cygwin
++++++

View file

@ -2,6 +2,7 @@
import os
import re
import sys
import subprocess
from glob import glob
from distutils.command.build import build
@ -106,7 +107,22 @@ def detect_lz4(prefixes):
include_dirs = []
library_dirs = []
possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', '/usr/local/borg', '/opt/local']
windowsIncludeDirs = []
if sys.platform == 'win32':
gccpath = ""
for p in os.environ["PATH"].split(";"):
if os.path.exists(os.path.join(p, "gcc.exe")):
gccpath = p
break
windowsIncludeDirs.append(os.path.abspath(os.path.join(gccpath, "..")))
windowsIncludeDirs.append(os.path.abspath(os.path.join(gccpath, "..", "..")))
possible_openssl_prefixes = None
if sys.platform == 'win32':
possible_openssl_prefixes = windowsIncludeDirs
else:
possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', '/usr/local/borg', '/opt/local']
if os.environ.get('BORG_OPENSSL_PREFIX'):
possible_openssl_prefixes.insert(0, os.environ.get('BORG_OPENSSL_PREFIX'))
ssl_prefix = detect_openssl(possible_openssl_prefixes)
@ -115,8 +131,11 @@ if not ssl_prefix:
include_dirs.append(os.path.join(ssl_prefix, 'include'))
library_dirs.append(os.path.join(ssl_prefix, 'lib'))
possible_lz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4', '/usr/local/borg', '/opt/local']
possible_lz4_prefixes = None
if sys.platform == 'win32':
possible_lz4_prefixes = windowsIncludeDirs
else:
possible_lz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4', '/usr/local/borg', '/opt/local']
if os.environ.get('BORG_LZ4_PREFIX'):
possible_lz4_prefixes.insert(0, os.environ.get('BORG_LZ4_PREFIX'))
lz4_prefix = detect_lz4(possible_lz4_prefixes)
@ -291,10 +310,20 @@ if not on_rtd:
elif sys.platform == 'darwin':
ext_modules.append(Extension('borg.platform_darwin', [platform_darwin_source]))
def parse(root, describe_command=None):
file = open('borg/_version.py', 'w')
output = subprocess.check_output("git describe --tags --long").decode().strip()
file.write('version = "' + output + '"\n')
return output
parse_function = parse if sys.platform == 'win32' else None
setup(
name='borgbackup',
use_scm_version={
'write_to': 'borg/_version.py',
'parse': parse_function,
},
author='The Borg Collective (see AUTHORS file)',
author_email='borgbackup@python.org',