Merge pull request #4696 from jrast/win10

WIP jrast/borg:win10, PR for better review and testing

.gitignore

@@ -14,9 +14,11 @@ src/borg/platform/darwin.c
 src/borg/platform/freebsd.c
 src/borg/platform/linux.c
 src/borg/platform/posix.c
+src/borg/platform/windows.c
 src/borg/_version.py
 *.egg-info
 *.pyc
+*.pyd
 *.so
 .idea/
 .cache/

MANIFEST.in

@@ -4,4 +4,4 @@
 exclude .coafile .editorconfig .gitattributes .gitignore .mailmap .travis.yml Vagrantfile
 prune .travis
 prune .github
-include src/borg/platform/darwin.c src/borg/platform/freebsd.c src/borg/platform/linux.c src/borg/platform/posix.c
+include src/borg/platform/darwin.c src/borg/platform/freebsd.c src/borg/platform/linux.c src/borg/platform/posix.c src/borg/platform/windows.c

README_WINDOWS.rst

@@ -0,0 +1,34 @@
+Borg Native on Windows
+======================
+
+Running borg natively on windows is in a early alpha stage. Expect many things to fail.
+Do not use the native windows build on any data which you do not want to lose!
+
+Build Requirements
+------------------
+
+- VC 14.0 Compiler
+- OpenSSL Library v1.1.1c, 64bit (available at https://slproweb.com/products/Win32OpenSSL.html)
+- Patience and a lot of coffee / beer
+
+What's working
+--------------
+
+.. note::
+   The following examples assume that the `BORG_REPO` and `BORG_PASSPHRASE` environment variables are set
+   if the repo or passphrase is not explicitly given.
+
+- Borg does not crash if called with ``borg``
+- ``borg init --encryption repokey-blake2 ./demoRepo`` runs without an error/warning.
+  Note that absolute paths only work if the protocol is explicitly set to file://
+- ``borg create ::backup-{now} D:\DemoData`` works as expected.
+- ``borg list`` works as expected.
+- ``borg extract --strip-components 1 ::backup-XXXX`` works. 
+  If absolute paths are extracted, it's important to pass ``--strip-components 1`` as
+  otherwise the data is resotred to the original location!
+
+What's NOT working
+------------------
+
+- Extracting a backup which was created on windows machine on a non windows machine will fail.
+- And many things more.

scripts/buildwin.bat

@@ -0,0 +1,20 @@
+
+REM Use the downloaded OpenSSL, for all other libraries the bundled version is used.
+REM On Appveyor different OpenSSL versions are available, therefore the directory contains the version information.
+set BORG_OPENSSL_PREFIX=C:\OpenSSL-v111-Win64
+set BORG_USE_BUNDLED_B2=YES
+set BORG_USE_BUNDLED_LZ4=YES
+set BORG_USE_BUNDLED_ZSTD=YES
+set BORG_USE_BUNDLED_XXHASH=YES
+
+REM Somehow on my machine rc.exe was not found. Adding the Windows Kit to the path worked.
+set PATH=%PATH%;C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64
+
+REM Run the build in the project directory.
+SET WORKPATH=%~dp0\..
+pushd %WORKPATH%
+
+python setup.py clean
+pip install -v -e .
+
+popd

setup.py

@@ -25,6 +25,8 @@ import setup_compress
 import setup_crypto
 import setup_docs
 
+is_win32 = sys.platform.startswith('win32')
+
 # How the build process finds the system libs / uses the bundled code:
 #
 # 1. it will try to use (system) libs (see 1.1. and 1.2.),
@@ -60,6 +62,7 @@ system_prefix_libzstd = os.environ.get('BORG_LIBZSTD_PREFIX')
 prefer_system_libxxhash = not bool(os.environ.get('BORG_USE_BUNDLED_XXHASH'))
 system_prefix_libxxhash = os.environ.get('BORG_LIBXXHASH_PREFIX')
 
+# Number of threads to use for cythonize, not used on windows
 cpu_threads = multiprocessing.cpu_count() if multiprocessing else 1
 
 # Are we building on ReadTheDocs?
@@ -97,6 +100,7 @@ platform_posix_source = 'src/borg/platform/posix.pyx'
 platform_linux_source = 'src/borg/platform/linux.pyx'
 platform_darwin_source = 'src/borg/platform/darwin.pyx'
 platform_freebsd_source = 'src/borg/platform/freebsd.pyx'
+platform_windows_source = 'src/borg/platform/windows.pyx'
 
 cython_sources = [
     compress_source,
@@ -110,6 +114,7 @@ cython_sources = [
     platform_linux_source,
     platform_freebsd_source,
     platform_darwin_source,
+    platform_windows_source,
 ]
 
 if cythonize:
@@ -199,9 +204,12 @@ if not on_rtd:
     linux_ext = Extension('borg.platform.linux', [platform_linux_source], libraries=['acl'])
     freebsd_ext = Extension('borg.platform.freebsd', [platform_freebsd_source])
     darwin_ext = Extension('borg.platform.darwin', [platform_darwin_source])
+    windows_ext = Extension('borg.platform.windows', [platform_windows_source])
 
-    if not sys.platform.startswith(('win32', )):
+    if not is_win32:
         ext_modules.append(posix_ext)
+    else:
+        ext_modules.append(windows_ext)
     if sys.platform == 'linux':
         ext_modules.append(linux_ext)
     elif sys.platform.startswith('freebsd'):
@@ -216,13 +224,15 @@ if not on_rtd:
 
     if cythonize and cythonizing:
         cython_opts = dict(
-            # compile .pyx extensions to .c in parallel
-            nthreads=cpu_threads + 1,
             # default language_level will be '3str' starting from Cython 3.0.0,
             # but old cython versions (< 0.29) do not know that, thus we use 3 for now.
             compiler_directives={'language_level': 3},
         )
-        cythonize([posix_ext, linux_ext, freebsd_ext, darwin_ext], **cython_opts)
+        if not is_win32:
+            # compile .pyx extensions to .c in parallel, does not work on windows
+            cython_opts['nthreads'] = cpu_threads + 1
+
+        cythonize([posix_ext, linux_ext, freebsd_ext, darwin_ext, windows_ext], **cython_opts)
         ext_modules = cythonize(ext_modules, **cython_opts)
 
 

src/borg/archive.py

@@ -239,7 +239,9 @@ def OsOpen(*, flags, path=None, parent_fd=None, name=None, noatime=False, op='op
     try:
         yield fd
     finally:
-        os.close(fd)
+        # On windows fd is None for directories.
+        if fd is not None:
+            os.close(fd)
 
 
 class DownloadPipeline:

src/borg/archiver.py

@@ -625,8 +625,10 @@ class Archiver:
             elif stat.S_ISDIR(st.st_mode):
                 with OsOpen(path=path, parent_fd=parent_fd, name=name, flags=flags_dir,
                             noatime=True, op='dir_open') as child_fd:
-                    with backup_io('fstat'):
-                        st = stat_update_check(st, os.fstat(child_fd))
+                    # child_fd is None for directories on windows, in that case a race condition check is not possible.
+                    if child_fd is not None:
+                        with backup_io('fstat'):
+                            st = stat_update_check(st, os.fstat(child_fd))
                     if recurse:
                         tag_names = dir_is_tagged(path, exclude_caches, exclude_if_present)
                         if tag_names:

src/borg/helpers/fs.py

@@ -8,6 +8,7 @@ import sys
 import textwrap
 
 from .process import prepare_subprocess_env
+from ..platformflags import is_win32
 
 from ..constants import *  # NOQA
 
@@ -230,6 +231,9 @@ def os_open(*, flags, path=None, parent_fd=None, name=None, noatime=False):
         fname = name  # use name relative to parent_fd
     else:
         fname, parent_fd = path, None  # just use the path
+    if is_win32 and os.path.isdir(fname):
+        # Directories can not be opened on Windows.
+        return None
     _flags_normal = flags
     if noatime:
         _flags_noatime = _flags_normal | O_('NOATIME')

src/borg/platform/__init__.py

@@ -22,6 +22,10 @@ if not is_win32:
     from .posix import get_errno
     from .posix import uid2user, user2uid, gid2group, group2gid, getosusername
 
+else:
+    from .windows import process_alive, local_pid_alive
+    from .windows import uid2user, user2uid, gid2group, group2gid, getosusername
+
 if is_linux:  # pragma: linux only
     from .linux import API_VERSION as OS_API_VERSION
     from .linux import listxattr, getxattr, setxattr

src/borg/platform/base.py

@@ -4,6 +4,7 @@ import socket
 import uuid
 
 from borg.helpers import truncate_and_unlink
+from borg.platformflags import is_win32
 
 """
 platform base module
@@ -94,6 +95,10 @@ def get_flags(path, st, fd=None):
 
 
 def sync_dir(path):
+    if is_win32:
+        # Opening directories is not supported on windows.
+        # TODO: do we need to handle this in some other way?
+        return
     fd = os.open(path, os.O_RDONLY)
     try:
         os.fsync(fd)

src/borg/platform/windows.pyx

@@ -0,0 +1,60 @@
+import os
+import platform
+from functools import lru_cache
+
+
+cdef extern from 'windows.h':
+    ctypedef void* HANDLE
+    ctypedef int BOOL
+    ctypedef unsigned long DWORD
+
+    BOOL CloseHandle(HANDLE hObject)
+    HANDLE OpenProcess(DWORD dwDesiredAccess, BOOL bInheritHandle, DWORD dbProcessId)
+
+    cdef extern int PROCESS_QUERY_INFORMATION
+
+
+@lru_cache(maxsize=None)
+def uid2user(uid, default=None):
+    return default
+
+
+@lru_cache(maxsize=None)
+def user2uid(user, default=None):
+    return default
+
+
+@lru_cache(maxsize=None)
+def gid2group(gid, default=None):
+    return default
+    
+
+@lru_cache(maxsize=None)
+def group2gid(group, default=None):
+    return default
+    
+
+def getosusername():
+    """Return the os user name."""
+    return os.getlogin()
+
+
+def process_alive(host, pid, thread):
+    """
+    Check if the (host, pid, thread_id) combination corresponds to a potentially alive process.
+    """
+    if host.split('@')[0].lower() != platform.node().lower():
+        # Not running on the same node, assume running.
+        return True
+
+    # If the process can be opened, the process is alive.
+    handle = OpenProcess(PROCESS_QUERY_INFORMATION, False, pid)
+    if handle != NULL:
+        CloseHandle(handle)
+        return True
+    return False
+
+
+def local_pid_alive(pid):
+    """Return whether *pid* is alive."""
+    raise NotImplementedError

src/borg/repository.py

@@ -673,17 +673,12 @@ class Repository:
             else:
                 # Keep one full worst-case segment free in non-append-only mode
                 required_free_space += full_segment_size
+
         try:
-            st_vfs = os.statvfs(self.path)
+            free_space = shutil.disk_usage(self.path).free
         except OSError as os_error:
             logger.warning('Failed to check free space before committing: ' + str(os_error))
             return
-        except AttributeError:
-            # TODO move the call to statvfs to platform
-            logger.warning('Failed to check free space before committing: no statvfs method available')
-            return
-        # f_bavail: even as root - don't touch the Federal Block Reserve!
-        free_space = st_vfs.f_bavail * st_vfs.f_frsize
         logger.debug('check_free_space: required bytes {}, free bytes {}'.format(required_free_space, free_space))
         if free_space < required_free_space:
             if self.created: