diff --git a/.gitignore b/.gitignore index 7fd06517e..91ee405fd 100644 --- a/.gitignore +++ b/.gitignore @@ -14,9 +14,11 @@ src/borg/platform/darwin.c src/borg/platform/freebsd.c src/borg/platform/linux.c src/borg/platform/posix.c +src/borg/platform/windows.c src/borg/_version.py *.egg-info *.pyc +*.pyd *.so .idea/ .cache/ diff --git a/MANIFEST.in b/MANIFEST.in index a439280d5..d6f230182 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,4 +4,4 @@ exclude .coafile .editorconfig .gitattributes .gitignore .mailmap .travis.yml Vagrantfile prune .travis prune .github -include src/borg/platform/darwin.c src/borg/platform/freebsd.c src/borg/platform/linux.c src/borg/platform/posix.c +include src/borg/platform/darwin.c src/borg/platform/freebsd.c src/borg/platform/linux.c src/borg/platform/posix.c src/borg/platform/windows.c diff --git a/README_WINDOWS.rst b/README_WINDOWS.rst new file mode 100644 index 000000000..07ba50523 --- /dev/null +++ b/README_WINDOWS.rst @@ -0,0 +1,34 @@ +Borg Native on Windows +====================== + +Running borg natively on windows is in a early alpha stage. Expect many things to fail. +Do not use the native windows build on any data which you do not want to lose! + +Build Requirements +------------------ + +- VC 14.0 Compiler +- OpenSSL Library v1.1.1c, 64bit (available at https://slproweb.com/products/Win32OpenSSL.html) +- Patience and a lot of coffee / beer + +What's working +-------------- + +.. note:: + The following examples assume that the `BORG_REPO` and `BORG_PASSPHRASE` environment variables are set + if the repo or passphrase is not explicitly given. + +- Borg does not crash if called with ``borg`` +- ``borg init --encryption repokey-blake2 ./demoRepo`` runs without an error/warning. + Note that absolute paths only work if the protocol is explicitly set to file:// +- ``borg create ::backup-{now} D:\DemoData`` works as expected. +- ``borg list`` works as expected. +- ``borg extract --strip-components 1 ::backup-XXXX`` works. + If absolute paths are extracted, it's important to pass ``--strip-components 1`` as + otherwise the data is resotred to the original location! + +What's NOT working +------------------ + +- Extracting a backup which was created on windows machine on a non windows machine will fail. +- And many things more. diff --git a/scripts/buildwin.bat b/scripts/buildwin.bat new file mode 100644 index 000000000..33d745671 --- /dev/null +++ b/scripts/buildwin.bat @@ -0,0 +1,20 @@ + +REM Use the downloaded OpenSSL, for all other libraries the bundled version is used. +REM On Appveyor different OpenSSL versions are available, therefore the directory contains the version information. +set BORG_OPENSSL_PREFIX=C:\OpenSSL-v111-Win64 +set BORG_USE_BUNDLED_B2=YES +set BORG_USE_BUNDLED_LZ4=YES +set BORG_USE_BUNDLED_ZSTD=YES +set BORG_USE_BUNDLED_XXHASH=YES + +REM Somehow on my machine rc.exe was not found. Adding the Windows Kit to the path worked. +set PATH=%PATH%;C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64 + +REM Run the build in the project directory. +SET WORKPATH=%~dp0\.. +pushd %WORKPATH% + +python setup.py clean +pip install -v -e . + +popd diff --git a/setup.py b/setup.py index 71ec53686..ba257e96a 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,8 @@ import setup_compress import setup_crypto import setup_docs +is_win32 = sys.platform.startswith('win32') + # How the build process finds the system libs / uses the bundled code: # # 1. it will try to use (system) libs (see 1.1. and 1.2.), @@ -60,6 +62,7 @@ system_prefix_libzstd = os.environ.get('BORG_LIBZSTD_PREFIX') prefer_system_libxxhash = not bool(os.environ.get('BORG_USE_BUNDLED_XXHASH')) system_prefix_libxxhash = os.environ.get('BORG_LIBXXHASH_PREFIX') +# Number of threads to use for cythonize, not used on windows cpu_threads = multiprocessing.cpu_count() if multiprocessing else 1 # Are we building on ReadTheDocs? @@ -97,6 +100,7 @@ platform_posix_source = 'src/borg/platform/posix.pyx' platform_linux_source = 'src/borg/platform/linux.pyx' platform_darwin_source = 'src/borg/platform/darwin.pyx' platform_freebsd_source = 'src/borg/platform/freebsd.pyx' +platform_windows_source = 'src/borg/platform/windows.pyx' cython_sources = [ compress_source, @@ -110,6 +114,7 @@ cython_sources = [ platform_linux_source, platform_freebsd_source, platform_darwin_source, + platform_windows_source, ] if cythonize: @@ -199,9 +204,12 @@ if not on_rtd: linux_ext = Extension('borg.platform.linux', [platform_linux_source], libraries=['acl']) freebsd_ext = Extension('borg.platform.freebsd', [platform_freebsd_source]) darwin_ext = Extension('borg.platform.darwin', [platform_darwin_source]) + windows_ext = Extension('borg.platform.windows', [platform_windows_source]) - if not sys.platform.startswith(('win32', )): + if not is_win32: ext_modules.append(posix_ext) + else: + ext_modules.append(windows_ext) if sys.platform == 'linux': ext_modules.append(linux_ext) elif sys.platform.startswith('freebsd'): @@ -216,13 +224,15 @@ if not on_rtd: if cythonize and cythonizing: cython_opts = dict( - # compile .pyx extensions to .c in parallel - nthreads=cpu_threads + 1, # default language_level will be '3str' starting from Cython 3.0.0, # but old cython versions (< 0.29) do not know that, thus we use 3 for now. compiler_directives={'language_level': 3}, ) - cythonize([posix_ext, linux_ext, freebsd_ext, darwin_ext], **cython_opts) + if not is_win32: + # compile .pyx extensions to .c in parallel, does not work on windows + cython_opts['nthreads'] = cpu_threads + 1 + + cythonize([posix_ext, linux_ext, freebsd_ext, darwin_ext, windows_ext], **cython_opts) ext_modules = cythonize(ext_modules, **cython_opts) diff --git a/src/borg/archive.py b/src/borg/archive.py index 212fd3bfa..86e8a62e0 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -239,7 +239,9 @@ def OsOpen(*, flags, path=None, parent_fd=None, name=None, noatime=False, op='op try: yield fd finally: - os.close(fd) + # On windows fd is None for directories. + if fd is not None: + os.close(fd) class DownloadPipeline: diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 5b9e07ab6..f1f406f52 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -625,8 +625,10 @@ class Archiver: elif stat.S_ISDIR(st.st_mode): with OsOpen(path=path, parent_fd=parent_fd, name=name, flags=flags_dir, noatime=True, op='dir_open') as child_fd: - with backup_io('fstat'): - st = stat_update_check(st, os.fstat(child_fd)) + # child_fd is None for directories on windows, in that case a race condition check is not possible. + if child_fd is not None: + with backup_io('fstat'): + st = stat_update_check(st, os.fstat(child_fd)) if recurse: tag_names = dir_is_tagged(path, exclude_caches, exclude_if_present) if tag_names: diff --git a/src/borg/helpers/fs.py b/src/borg/helpers/fs.py index cb3f0ba8e..eecff277a 100644 --- a/src/borg/helpers/fs.py +++ b/src/borg/helpers/fs.py @@ -8,6 +8,7 @@ import sys import textwrap from .process import prepare_subprocess_env +from ..platformflags import is_win32 from ..constants import * # NOQA @@ -230,6 +231,9 @@ def os_open(*, flags, path=None, parent_fd=None, name=None, noatime=False): fname = name # use name relative to parent_fd else: fname, parent_fd = path, None # just use the path + if is_win32 and os.path.isdir(fname): + # Directories can not be opened on Windows. + return None _flags_normal = flags if noatime: _flags_noatime = _flags_normal | O_('NOATIME') diff --git a/src/borg/platform/__init__.py b/src/borg/platform/__init__.py index 80a7d7618..98f926230 100644 --- a/src/borg/platform/__init__.py +++ b/src/borg/platform/__init__.py @@ -22,6 +22,10 @@ if not is_win32: from .posix import get_errno from .posix import uid2user, user2uid, gid2group, group2gid, getosusername +else: + from .windows import process_alive, local_pid_alive + from .windows import uid2user, user2uid, gid2group, group2gid, getosusername + if is_linux: # pragma: linux only from .linux import API_VERSION as OS_API_VERSION from .linux import listxattr, getxattr, setxattr diff --git a/src/borg/platform/base.py b/src/borg/platform/base.py index a3575eac8..5e5e2eb56 100644 --- a/src/borg/platform/base.py +++ b/src/borg/platform/base.py @@ -4,6 +4,7 @@ import socket import uuid from borg.helpers import truncate_and_unlink +from borg.platformflags import is_win32 """ platform base module @@ -94,6 +95,10 @@ def get_flags(path, st, fd=None): def sync_dir(path): + if is_win32: + # Opening directories is not supported on windows. + # TODO: do we need to handle this in some other way? + return fd = os.open(path, os.O_RDONLY) try: os.fsync(fd) diff --git a/src/borg/platform/windows.pyx b/src/borg/platform/windows.pyx new file mode 100644 index 000000000..503353526 --- /dev/null +++ b/src/borg/platform/windows.pyx @@ -0,0 +1,60 @@ +import os +import platform +from functools import lru_cache + + +cdef extern from 'windows.h': + ctypedef void* HANDLE + ctypedef int BOOL + ctypedef unsigned long DWORD + + BOOL CloseHandle(HANDLE hObject) + HANDLE OpenProcess(DWORD dwDesiredAccess, BOOL bInheritHandle, DWORD dbProcessId) + + cdef extern int PROCESS_QUERY_INFORMATION + + +@lru_cache(maxsize=None) +def uid2user(uid, default=None): + return default + + +@lru_cache(maxsize=None) +def user2uid(user, default=None): + return default + + +@lru_cache(maxsize=None) +def gid2group(gid, default=None): + return default + + +@lru_cache(maxsize=None) +def group2gid(group, default=None): + return default + + +def getosusername(): + """Return the os user name.""" + return os.getlogin() + + +def process_alive(host, pid, thread): + """ + Check if the (host, pid, thread_id) combination corresponds to a potentially alive process. + """ + if host.split('@')[0].lower() != platform.node().lower(): + # Not running on the same node, assume running. + return True + + # If the process can be opened, the process is alive. + handle = OpenProcess(PROCESS_QUERY_INFORMATION, False, pid) + if handle != NULL: + CloseHandle(handle) + return True + return False + + +def local_pid_alive(pid): + """Return whether *pid* is alive.""" + raise NotImplementedError diff --git a/src/borg/repository.py b/src/borg/repository.py index 53b512781..e59e3acd8 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -673,17 +673,12 @@ class Repository: else: # Keep one full worst-case segment free in non-append-only mode required_free_space += full_segment_size + try: - st_vfs = os.statvfs(self.path) + free_space = shutil.disk_usage(self.path).free except OSError as os_error: logger.warning('Failed to check free space before committing: ' + str(os_error)) return - except AttributeError: - # TODO move the call to statvfs to platform - logger.warning('Failed to check free space before committing: no statvfs method available') - return - # f_bavail: even as root - don't touch the Federal Block Reserve! - free_space = st_vfs.f_bavail * st_vfs.f_frsize logger.debug('check_free_space: required bytes {}, free bytes {}'.format(required_free_space, free_space)) if free_space < required_free_space: if self.created: