From 6b426d08d76d47037ec1e01a105a7b17bad79551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Rast?= Date: Fri, 9 Aug 2019 00:01:38 +0200 Subject: [PATCH 1/3] Initial work to build and run borg under windows - Created a batch file to build borg on windows - Adjusted setup.py to be runnable on windows and build the windows extension - Extracted the free space check to a function in the platform module - Created the minimal needed (dummy) functions for the windows platform module --- .gitignore | 2 ++ MANIFEST.in | 2 +- README_WINDOWS.rst | 19 +++++++++++ scripts/buildwin.bat | 20 ++++++++++++ setup.py | 18 ++++++++--- src/borg/platform/__init__.py | 4 +++ src/borg/platform/base.py | 5 +++ src/borg/platform/windows.pyx | 60 +++++++++++++++++++++++++++++++++++ src/borg/repository.py | 9 ++---- 9 files changed, 127 insertions(+), 12 deletions(-) create mode 100644 README_WINDOWS.rst create mode 100644 scripts/buildwin.bat create mode 100644 src/borg/platform/windows.pyx diff --git a/.gitignore b/.gitignore index 7fd06517e..91ee405fd 100644 --- a/.gitignore +++ b/.gitignore @@ -14,9 +14,11 @@ src/borg/platform/darwin.c src/borg/platform/freebsd.c src/borg/platform/linux.c src/borg/platform/posix.c +src/borg/platform/windows.c src/borg/_version.py *.egg-info *.pyc +*.pyd *.so .idea/ .cache/ diff --git a/MANIFEST.in b/MANIFEST.in index a439280d5..d6f230182 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,4 +4,4 @@ exclude .coafile .editorconfig .gitattributes .gitignore .mailmap .travis.yml Vagrantfile prune .travis prune .github -include src/borg/platform/darwin.c src/borg/platform/freebsd.c src/borg/platform/linux.c src/borg/platform/posix.c +include src/borg/platform/darwin.c src/borg/platform/freebsd.c src/borg/platform/linux.c src/borg/platform/posix.c src/borg/platform/windows.c diff --git a/README_WINDOWS.rst b/README_WINDOWS.rst new file mode 100644 index 000000000..8ad610976 --- /dev/null +++ b/README_WINDOWS.rst @@ -0,0 +1,19 @@ +Borg Native on Windows +====================== + +Build Requirements +------------------ + +- VC 14.0 Compiler +- OpenSSL Library (https://slproweb.com/products/Win32OpenSSL.html) +- Patience and a lot of coffee / beer + +What's working +-------------- + +- Borg does not crash if called with ``borg`` +- ``borg init --encryption none ./demoRepo`` runs without an error/warning. + Note that only relative paths work at the moment. + + + diff --git a/scripts/buildwin.bat b/scripts/buildwin.bat new file mode 100644 index 000000000..33d745671 --- /dev/null +++ b/scripts/buildwin.bat @@ -0,0 +1,20 @@ + +REM Use the downloaded OpenSSL, for all other libraries the bundled version is used. +REM On Appveyor different OpenSSL versions are available, therefore the directory contains the version information. +set BORG_OPENSSL_PREFIX=C:\OpenSSL-v111-Win64 +set BORG_USE_BUNDLED_B2=YES +set BORG_USE_BUNDLED_LZ4=YES +set BORG_USE_BUNDLED_ZSTD=YES +set BORG_USE_BUNDLED_XXHASH=YES + +REM Somehow on my machine rc.exe was not found. Adding the Windows Kit to the path worked. +set PATH=%PATH%;C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64 + +REM Run the build in the project directory. +SET WORKPATH=%~dp0\.. +pushd %WORKPATH% + +python setup.py clean +pip install -v -e . + +popd diff --git a/setup.py b/setup.py index 71ec53686..ba257e96a 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,8 @@ import setup_compress import setup_crypto import setup_docs +is_win32 = sys.platform.startswith('win32') + # How the build process finds the system libs / uses the bundled code: # # 1. it will try to use (system) libs (see 1.1. and 1.2.), @@ -60,6 +62,7 @@ system_prefix_libzstd = os.environ.get('BORG_LIBZSTD_PREFIX') prefer_system_libxxhash = not bool(os.environ.get('BORG_USE_BUNDLED_XXHASH')) system_prefix_libxxhash = os.environ.get('BORG_LIBXXHASH_PREFIX') +# Number of threads to use for cythonize, not used on windows cpu_threads = multiprocessing.cpu_count() if multiprocessing else 1 # Are we building on ReadTheDocs? @@ -97,6 +100,7 @@ platform_posix_source = 'src/borg/platform/posix.pyx' platform_linux_source = 'src/borg/platform/linux.pyx' platform_darwin_source = 'src/borg/platform/darwin.pyx' platform_freebsd_source = 'src/borg/platform/freebsd.pyx' +platform_windows_source = 'src/borg/platform/windows.pyx' cython_sources = [ compress_source, @@ -110,6 +114,7 @@ cython_sources = [ platform_linux_source, platform_freebsd_source, platform_darwin_source, + platform_windows_source, ] if cythonize: @@ -199,9 +204,12 @@ if not on_rtd: linux_ext = Extension('borg.platform.linux', [platform_linux_source], libraries=['acl']) freebsd_ext = Extension('borg.platform.freebsd', [platform_freebsd_source]) darwin_ext = Extension('borg.platform.darwin', [platform_darwin_source]) + windows_ext = Extension('borg.platform.windows', [platform_windows_source]) - if not sys.platform.startswith(('win32', )): + if not is_win32: ext_modules.append(posix_ext) + else: + ext_modules.append(windows_ext) if sys.platform == 'linux': ext_modules.append(linux_ext) elif sys.platform.startswith('freebsd'): @@ -216,13 +224,15 @@ if not on_rtd: if cythonize and cythonizing: cython_opts = dict( - # compile .pyx extensions to .c in parallel - nthreads=cpu_threads + 1, # default language_level will be '3str' starting from Cython 3.0.0, # but old cython versions (< 0.29) do not know that, thus we use 3 for now. compiler_directives={'language_level': 3}, ) - cythonize([posix_ext, linux_ext, freebsd_ext, darwin_ext], **cython_opts) + if not is_win32: + # compile .pyx extensions to .c in parallel, does not work on windows + cython_opts['nthreads'] = cpu_threads + 1 + + cythonize([posix_ext, linux_ext, freebsd_ext, darwin_ext, windows_ext], **cython_opts) ext_modules = cythonize(ext_modules, **cython_opts) diff --git a/src/borg/platform/__init__.py b/src/borg/platform/__init__.py index 80a7d7618..98f926230 100644 --- a/src/borg/platform/__init__.py +++ b/src/borg/platform/__init__.py @@ -22,6 +22,10 @@ if not is_win32: from .posix import get_errno from .posix import uid2user, user2uid, gid2group, group2gid, getosusername +else: + from .windows import process_alive, local_pid_alive + from .windows import uid2user, user2uid, gid2group, group2gid, getosusername + if is_linux: # pragma: linux only from .linux import API_VERSION as OS_API_VERSION from .linux import listxattr, getxattr, setxattr diff --git a/src/borg/platform/base.py b/src/borg/platform/base.py index a3575eac8..5e5e2eb56 100644 --- a/src/borg/platform/base.py +++ b/src/borg/platform/base.py @@ -4,6 +4,7 @@ import socket import uuid from borg.helpers import truncate_and_unlink +from borg.platformflags import is_win32 """ platform base module @@ -94,6 +95,10 @@ def get_flags(path, st, fd=None): def sync_dir(path): + if is_win32: + # Opening directories is not supported on windows. + # TODO: do we need to handle this in some other way? + return fd = os.open(path, os.O_RDONLY) try: os.fsync(fd) diff --git a/src/borg/platform/windows.pyx b/src/borg/platform/windows.pyx new file mode 100644 index 000000000..503353526 --- /dev/null +++ b/src/borg/platform/windows.pyx @@ -0,0 +1,60 @@ +import os +import platform +from functools import lru_cache + + +cdef extern from 'windows.h': + ctypedef void* HANDLE + ctypedef int BOOL + ctypedef unsigned long DWORD + + BOOL CloseHandle(HANDLE hObject) + HANDLE OpenProcess(DWORD dwDesiredAccess, BOOL bInheritHandle, DWORD dbProcessId) + + cdef extern int PROCESS_QUERY_INFORMATION + + +@lru_cache(maxsize=None) +def uid2user(uid, default=None): + return default + + +@lru_cache(maxsize=None) +def user2uid(user, default=None): + return default + + +@lru_cache(maxsize=None) +def gid2group(gid, default=None): + return default + + +@lru_cache(maxsize=None) +def group2gid(group, default=None): + return default + + +def getosusername(): + """Return the os user name.""" + return os.getlogin() + + +def process_alive(host, pid, thread): + """ + Check if the (host, pid, thread_id) combination corresponds to a potentially alive process. + """ + if host.split('@')[0].lower() != platform.node().lower(): + # Not running on the same node, assume running. + return True + + # If the process can be opened, the process is alive. + handle = OpenProcess(PROCESS_QUERY_INFORMATION, False, pid) + if handle != NULL: + CloseHandle(handle) + return True + return False + + +def local_pid_alive(pid): + """Return whether *pid* is alive.""" + raise NotImplementedError diff --git a/src/borg/repository.py b/src/borg/repository.py index e53cec6d0..8db65dcb9 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -662,17 +662,12 @@ class Repository: else: # Keep one full worst-case segment free in non-append-only mode required_free_space += full_segment_size + try: - st_vfs = os.statvfs(self.path) + free_space = shutil.disk_usage(self.path).free except OSError as os_error: logger.warning('Failed to check free space before committing: ' + str(os_error)) return - except AttributeError: - # TODO move the call to statvfs to platform - logger.warning('Failed to check free space before committing: no statvfs method available') - return - # f_bavail: even as root - don't touch the Federal Block Reserve! - free_space = st_vfs.f_bavail * st_vfs.f_frsize logger.debug('check_free_space: required bytes {}, free bytes {}'.format(required_free_space, free_space)) if free_space < required_free_space: if self.created: From bff97a99e16f992203c8af0df5550b4def02218b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Rast?= Date: Thu, 8 Aug 2019 23:48:23 +0200 Subject: [PATCH 2/3] Windows specific directory handling On windows os.open does not work for directories. If borg tries to open an directory on windows, None is returned as file descriptor. The archive and archiver where adjusted to handle the case if a file descriptor is None. --- src/borg/archive.py | 4 +++- src/borg/archiver.py | 6 ++++-- src/borg/helpers/fs.py | 4 ++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 212fd3bfa..86e8a62e0 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -239,7 +239,9 @@ def OsOpen(*, flags, path=None, parent_fd=None, name=None, noatime=False, op='op try: yield fd finally: - os.close(fd) + # On windows fd is None for directories. + if fd is not None: + os.close(fd) class DownloadPipeline: diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 8fb6439d5..55ef22cde 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -625,8 +625,10 @@ class Archiver: elif stat.S_ISDIR(st.st_mode): with OsOpen(path=path, parent_fd=parent_fd, name=name, flags=flags_dir, noatime=True, op='dir_open') as child_fd: - with backup_io('fstat'): - st = stat_update_check(st, os.fstat(child_fd)) + # child_fd is None for directories on windows, in that case a race condition check is not possible. + if child_fd is not None: + with backup_io('fstat'): + st = stat_update_check(st, os.fstat(child_fd)) if recurse: tag_names = dir_is_tagged(path, exclude_caches, exclude_if_present) if tag_names: diff --git a/src/borg/helpers/fs.py b/src/borg/helpers/fs.py index cb3f0ba8e..eecff277a 100644 --- a/src/borg/helpers/fs.py +++ b/src/borg/helpers/fs.py @@ -8,6 +8,7 @@ import sys import textwrap from .process import prepare_subprocess_env +from ..platformflags import is_win32 from ..constants import * # NOQA @@ -230,6 +231,9 @@ def os_open(*, flags, path=None, parent_fd=None, name=None, noatime=False): fname = name # use name relative to parent_fd else: fname, parent_fd = path, None # just use the path + if is_win32 and os.path.isdir(fname): + # Directories can not be opened on Windows. + return None _flags_normal = flags if noatime: _flags_noatime = _flags_normal | O_('NOATIME') From a81dffa54e9f2ced0a2345d9df9cc80ea0206f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Rast?= Date: Thu, 8 Aug 2019 23:51:21 +0200 Subject: [PATCH 3/3] Updated windows readme with current state --- README_WINDOWS.rst | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/README_WINDOWS.rst b/README_WINDOWS.rst index 8ad610976..07ba50523 100644 --- a/README_WINDOWS.rst +++ b/README_WINDOWS.rst @@ -1,19 +1,34 @@ Borg Native on Windows ====================== +Running borg natively on windows is in a early alpha stage. Expect many things to fail. +Do not use the native windows build on any data which you do not want to lose! + Build Requirements ------------------ - VC 14.0 Compiler -- OpenSSL Library (https://slproweb.com/products/Win32OpenSSL.html) +- OpenSSL Library v1.1.1c, 64bit (available at https://slproweb.com/products/Win32OpenSSL.html) - Patience and a lot of coffee / beer What's working -------------- +.. note:: + The following examples assume that the `BORG_REPO` and `BORG_PASSPHRASE` environment variables are set + if the repo or passphrase is not explicitly given. + - Borg does not crash if called with ``borg`` -- ``borg init --encryption none ./demoRepo`` runs without an error/warning. - Note that only relative paths work at the moment. - +- ``borg init --encryption repokey-blake2 ./demoRepo`` runs without an error/warning. + Note that absolute paths only work if the protocol is explicitly set to file:// +- ``borg create ::backup-{now} D:\DemoData`` works as expected. +- ``borg list`` works as expected. +- ``borg extract --strip-components 1 ::backup-XXXX`` works. + If absolute paths are extracted, it's important to pass ``--strip-components 1`` as + otherwise the data is resotred to the original location! +What's NOT working +------------------ +- Extracting a backup which was created on windows machine on a non windows machine will fail. +- And many things more.