From 064d2a96d1f515d3a87facdbefc341f3d2961e58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Caron?= Date: Fri, 28 Sep 2018 10:18:48 -0400 Subject: [PATCH] Add support for running tests on Windows (#90) * Add batch file clones of the unix shell scripts * Update go-winio * Document procedure for running tests on Windows --- README.md | 13 + scripts/runAccTests.bat | 104 +++++++ scripts/testing/setup_private_registry.bat | 87 ++++++ vendor/github.com/Microsoft/go-winio/ea.go | 274 +++++++++--------- .../github.com/Microsoft/go-winio/fileinfo.go | 3 +- vendor/github.com/Microsoft/go-winio/pipe.go | 57 ++-- vendor/vendor.json | 6 +- 7 files changed, 373 insertions(+), 171 deletions(-) create mode 100644 scripts/runAccTests.bat create mode 100644 scripts/testing/setup_private_registry.bat diff --git a/README.md b/README.md index c75123ed..bf0f858f 100644 --- a/README.md +++ b/README.md @@ -80,3 +80,16 @@ provider "docker" { Don't forget to run `terraform init` each time you rebuild the provider. Check [here](https://www.youtube.com/watch?v=TMmovxyo5sY&t=30m14s) for a more detailed explanation. You can check the latest released version of a provider at https://releases.hashicorp.com/terraform-provider-docker/. + +Developing on Windows +--------------------- + +You can build and test on Widows without `make`. Run `go install` to +build and `Scripts\runAccTests.bat` to run the test suite. + +Continuous integration for Windows is not available at the moment due +to lack of a CI provider that is free for open source projects *and* +supports running Linux containers in Docker for Windows. For example, +AppVeyor is free for open source projects and provides Docker on its +Windows builds, but only offers Linux containers on Windows as a paid +upgrade. diff --git a/scripts/runAccTests.bat b/scripts/runAccTests.bat new file mode 100644 index 00000000..9c3e8683 --- /dev/null +++ b/scripts/runAccTests.bat @@ -0,0 +1,104 @@ +@echo off +setlocal + +:: As of `go-dockerclient` v1.2.0, the default endpoint to the Docker daemon +:: is a UNIX socket. We need to force it to use the Windows named pipe when +:: running against Docker for Windows. +set DOCKER_HOST=npipe:////.//pipe//docker_engine + +:: Note: quoting these values breaks the tests! +set DOCKER_REGISTRY_ADDRESS=127.0.0.1:15000 +set DOCKER_REGISTRY_USER=testuser +set DOCKER_REGISTRY_PASS=testpwd +set DOCKER_PRIVATE_IMAGE=127.0.0.1:15000/tftest-service:v1 +set TF_ACC=1 + +call:setup +if %ErrorLevel% neq 0 ( + call:print "Failed to set up acceptance test fixtures." + exit /b %ErrorLevel% +) + +call:run +if %ErrorLevel% neq 0 ( + call:print "Acceptance tests failed." + set outcome=1 +) else ( + set outcome=0 +) + +call:cleanup +if %ErrorLevel% neq 0 ( + call:print "Failed to clean up acceptance test fixtures." + exit /b %ErrorLevel% +) + +exit /b %outcome% + + +:print + if "%~1" == "" ( + echo. + ) else ( + echo %~1 + ) + exit /b 0 + + +:log + call:print "" + call:print "##################################" + call:print "-------- %~1" + call:print "##################################" + exit /b 0 + + +:setup + call:log "setup" + call %~dp0testing\setup_private_registry.bat + exit /b %ErrorLevel% + + +:run + call:log "run" + call go test ./docker -v -timeout 120m + exit /b %ErrorLevel% + + +:cleanup + call:log "cleanup" + call:print "### unsetted env ###" + for /F %%p in ('docker container ls -f "name=private_registry" -q') do ( + call docker stop %%p + call docker rm -f -v %%p + ) + call:print "### stopped private registry ###" + rmdir /q /s %~dp0testing\auth + rmdir /q /s %~dp0testing\certs + call:print "### removed auth and certs ###" + for %%r in ("container" "volume") do ( + call docker %%r ls -f "name=tftest-" -q + for /F %%i in ('docker %%r ls -f "name=tf-test" -q') do ( + echo Deleting %%r %%i + call docker %%r rm -f -v %%i + ) + for /F %%i in ('docker %%r ls -f "name=tftest-" -q') do ( + echo Deleting %%r %%i + call docker %%r rm -f -v %%i + ) + call:print "### removed %%r ###" + ) + for %%r in ("config" "secret" "network") do ( + call docker %%r ls -f "name=tftest-" -q + for /F %%i in ('docker %%r ls -f "name=tftest-" -q') do ( + echo Deleting %%r %%i + call docker %%r rm %%i + ) + call:print "### removed %%r ###" + ) + for /F %%i in ('docker images -aq 127.0.0.1:5000/tftest-service') do ( + echo Deleting imag %%i + docker rmi -f %%i + ) + call:print "### removed service images ###" + exit /b %ErrorLevel% diff --git a/scripts/testing/setup_private_registry.bat b/scripts/testing/setup_private_registry.bat new file mode 100644 index 00000000..653ff030 --- /dev/null +++ b/scripts/testing/setup_private_registry.bat @@ -0,0 +1,87 @@ +@echo off +setlocal + +:: Create self-signed certificate. +call:mkdirp %~dp0certs +call openssl req ^ + -newkey rsa:2048 ^ + -nodes ^ + -x509 ^ + -days 365 ^ + -subj "/C=US/ST=Denial/L=Springfield/O=Dis/CN=127.0.0.1" ^ + -keyout %~dp0certs\registry_auth.key ^ + -out %~dp0certs\registry_auth.crt +if %ErrorLevel% neq 0 ( + call:print "Failed to generate self-signed certificate." + exit /b %ErrorLevel% +) + +:: Generate random credentials. +call:mkdirp %~dp0auth +call docker run ^ + --rm ^ + --entrypoint htpasswd ^ + registry:2 ^ + -Bbn testuser testpwd ^ + > %~dp0auth\htpasswd +if %ErrorLevel% neq 0 ( + call:print "Failed to generate random credentials." + exit /b %ErrorLevel% +) + +:: Start an ephemeral Docker registry in a container. +:: --rm ^ +@echo on +call docker run ^ + -d ^ + --name private_registry ^ + -p 15000:5000 ^ + -v %~dp0auth:/auth ^ + -e "REGISTRY_AUTH=htpasswd" ^ + -e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" ^ + -e "REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd" ^ + -v %~dp0certs:/certs ^ + -e "REGISTRY_HTTP_TLS_CERTIFICATE=/certs/registry_auth.crt" ^ + -e "REGISTRY_HTTP_TLS_KEY=/certs/registry_auth.key" ^ + registry:2 +if %ErrorLevel% neq 0 ( + call:print "Failed to create ephemeral Docker registry." + exit /b %ErrorLevel% +) + +:: Wait until the container is responsive (*crosses fingers*). +timeout /t 5 + +:: Point our Docker Daemon to this ephemeral registry. +call docker login -u testuser -p testpwd 127.0.0.1:15000 +if %ErrorLevel% neq 0 ( + call:print "Failed to log in to ephemeral Docker registry." + exit /b %ErrorLevel% +) + +:: Build a few private images. +for /L %%i in (1,1,3) do ( + call docker build ^ + -t tftest-service ^ + %~dp0 ^ + -f %~dp0Dockerfile_v%%i + call docker tag ^ + tftest-service ^ + 127.0.0.1:15000/tftest-service:v%%i + call docker push ^ + 127.0.0.1:15000/tftest-service:v%%i +) + +exit /b %ErrorLevel% + + +:print + echo %~1 + exit /b 0 + + +:mkdirp + if not exist %~1\nul ( + mkdir %~1 + ) + exit /b %ErrorLevel% diff --git a/vendor/github.com/Microsoft/go-winio/ea.go b/vendor/github.com/Microsoft/go-winio/ea.go index b37e930d..4051c1b3 100644 --- a/vendor/github.com/Microsoft/go-winio/ea.go +++ b/vendor/github.com/Microsoft/go-winio/ea.go @@ -1,137 +1,137 @@ -package winio - -import ( - "bytes" - "encoding/binary" - "errors" -) - -type fileFullEaInformation struct { - NextEntryOffset uint32 - Flags uint8 - NameLength uint8 - ValueLength uint16 -} - -var ( - fileFullEaInformationSize = binary.Size(&fileFullEaInformation{}) - - errInvalidEaBuffer = errors.New("invalid extended attribute buffer") - errEaNameTooLarge = errors.New("extended attribute name too large") - errEaValueTooLarge = errors.New("extended attribute value too large") -) - -// ExtendedAttribute represents a single Windows EA. -type ExtendedAttribute struct { - Name string - Value []byte - Flags uint8 -} - -func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) { - var info fileFullEaInformation - err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info) - if err != nil { - err = errInvalidEaBuffer - return - } - - nameOffset := fileFullEaInformationSize - nameLen := int(info.NameLength) - valueOffset := nameOffset + int(info.NameLength) + 1 - valueLen := int(info.ValueLength) - nextOffset := int(info.NextEntryOffset) - if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) { - err = errInvalidEaBuffer - return - } - - ea.Name = string(b[nameOffset : nameOffset+nameLen]) - ea.Value = b[valueOffset : valueOffset+valueLen] - ea.Flags = info.Flags - if info.NextEntryOffset != 0 { - nb = b[info.NextEntryOffset:] - } - return -} - -// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION -// buffer retrieved from BackupRead, ZwQueryEaFile, etc. -func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) { - for len(b) != 0 { - ea, nb, err := parseEa(b) - if err != nil { - return nil, err - } - - eas = append(eas, ea) - b = nb - } - return -} - -func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error { - if int(uint8(len(ea.Name))) != len(ea.Name) { - return errEaNameTooLarge - } - if int(uint16(len(ea.Value))) != len(ea.Value) { - return errEaValueTooLarge - } - entrySize := uint32(fileFullEaInformationSize + len(ea.Name) + 1 + len(ea.Value)) - withPadding := (entrySize + 3) &^ 3 - nextOffset := uint32(0) - if !last { - nextOffset = withPadding - } - info := fileFullEaInformation{ - NextEntryOffset: nextOffset, - Flags: ea.Flags, - NameLength: uint8(len(ea.Name)), - ValueLength: uint16(len(ea.Value)), - } - - err := binary.Write(buf, binary.LittleEndian, &info) - if err != nil { - return err - } - - _, err = buf.Write([]byte(ea.Name)) - if err != nil { - return err - } - - err = buf.WriteByte(0) - if err != nil { - return err - } - - _, err = buf.Write(ea.Value) - if err != nil { - return err - } - - _, err = buf.Write([]byte{0, 0, 0}[0 : withPadding-entrySize]) - if err != nil { - return err - } - - return nil -} - -// EncodeExtendedAttributes encodes a list of EAs into a FILE_FULL_EA_INFORMATION -// buffer for use with BackupWrite, ZwSetEaFile, etc. -func EncodeExtendedAttributes(eas []ExtendedAttribute) ([]byte, error) { - var buf bytes.Buffer - for i := range eas { - last := false - if i == len(eas)-1 { - last = true - } - - err := writeEa(&buf, &eas[i], last) - if err != nil { - return nil, err - } - } - return buf.Bytes(), nil -} +package winio + +import ( + "bytes" + "encoding/binary" + "errors" +) + +type fileFullEaInformation struct { + NextEntryOffset uint32 + Flags uint8 + NameLength uint8 + ValueLength uint16 +} + +var ( + fileFullEaInformationSize = binary.Size(&fileFullEaInformation{}) + + errInvalidEaBuffer = errors.New("invalid extended attribute buffer") + errEaNameTooLarge = errors.New("extended attribute name too large") + errEaValueTooLarge = errors.New("extended attribute value too large") +) + +// ExtendedAttribute represents a single Windows EA. +type ExtendedAttribute struct { + Name string + Value []byte + Flags uint8 +} + +func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) { + var info fileFullEaInformation + err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info) + if err != nil { + err = errInvalidEaBuffer + return + } + + nameOffset := fileFullEaInformationSize + nameLen := int(info.NameLength) + valueOffset := nameOffset + int(info.NameLength) + 1 + valueLen := int(info.ValueLength) + nextOffset := int(info.NextEntryOffset) + if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) { + err = errInvalidEaBuffer + return + } + + ea.Name = string(b[nameOffset : nameOffset+nameLen]) + ea.Value = b[valueOffset : valueOffset+valueLen] + ea.Flags = info.Flags + if info.NextEntryOffset != 0 { + nb = b[info.NextEntryOffset:] + } + return +} + +// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION +// buffer retrieved from BackupRead, ZwQueryEaFile, etc. +func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) { + for len(b) != 0 { + ea, nb, err := parseEa(b) + if err != nil { + return nil, err + } + + eas = append(eas, ea) + b = nb + } + return +} + +func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error { + if int(uint8(len(ea.Name))) != len(ea.Name) { + return errEaNameTooLarge + } + if int(uint16(len(ea.Value))) != len(ea.Value) { + return errEaValueTooLarge + } + entrySize := uint32(fileFullEaInformationSize + len(ea.Name) + 1 + len(ea.Value)) + withPadding := (entrySize + 3) &^ 3 + nextOffset := uint32(0) + if !last { + nextOffset = withPadding + } + info := fileFullEaInformation{ + NextEntryOffset: nextOffset, + Flags: ea.Flags, + NameLength: uint8(len(ea.Name)), + ValueLength: uint16(len(ea.Value)), + } + + err := binary.Write(buf, binary.LittleEndian, &info) + if err != nil { + return err + } + + _, err = buf.Write([]byte(ea.Name)) + if err != nil { + return err + } + + err = buf.WriteByte(0) + if err != nil { + return err + } + + _, err = buf.Write(ea.Value) + if err != nil { + return err + } + + _, err = buf.Write([]byte{0, 0, 0}[0 : withPadding-entrySize]) + if err != nil { + return err + } + + return nil +} + +// EncodeExtendedAttributes encodes a list of EAs into a FILE_FULL_EA_INFORMATION +// buffer for use with BackupWrite, ZwSetEaFile, etc. +func EncodeExtendedAttributes(eas []ExtendedAttribute) ([]byte, error) { + var buf bytes.Buffer + for i := range eas { + last := false + if i == len(eas)-1 { + last = true + } + + err := writeEa(&buf, &eas[i], last) + if err != nil { + return nil, err + } + } + return buf.Bytes(), nil +} diff --git a/vendor/github.com/Microsoft/go-winio/fileinfo.go b/vendor/github.com/Microsoft/go-winio/fileinfo.go index b1d60abb..ada2fbab 100644 --- a/vendor/github.com/Microsoft/go-winio/fileinfo.go +++ b/vendor/github.com/Microsoft/go-winio/fileinfo.go @@ -20,7 +20,8 @@ const ( // FileBasicInfo contains file access time and file attributes information. type FileBasicInfo struct { CreationTime, LastAccessTime, LastWriteTime, ChangeTime syscall.Filetime - FileAttributes uintptr // includes padding + FileAttributes uint32 + pad uint32 // padding } // GetFileBasicInfo retrieves times and attributes for a file. diff --git a/vendor/github.com/Microsoft/go-winio/pipe.go b/vendor/github.com/Microsoft/go-winio/pipe.go index 82cbe7af..d99eedb6 100644 --- a/vendor/github.com/Microsoft/go-winio/pipe.go +++ b/vendor/github.com/Microsoft/go-winio/pipe.go @@ -15,7 +15,6 @@ import ( //sys connectNamedPipe(pipe syscall.Handle, o *syscall.Overlapped) (err error) = ConnectNamedPipe //sys createNamedPipe(name string, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error) [failretval==syscall.InvalidHandle] = CreateNamedPipeW //sys createFile(name string, access uint32, mode uint32, sa *syscall.SecurityAttributes, createmode uint32, attrs uint32, templatefile syscall.Handle) (handle syscall.Handle, err error) [failretval==syscall.InvalidHandle] = CreateFileW -//sys waitNamedPipe(name string, timeout uint32) (err error) = WaitNamedPipeW //sys getNamedPipeInfo(pipe syscall.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) = GetNamedPipeInfo //sys getNamedPipeHandleState(pipe syscall.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) = GetNamedPipeHandleStateW //sys localAlloc(uFlags uint32, length uint32) (ptr uintptr) = LocalAlloc @@ -121,6 +120,11 @@ func (f *win32MessageBytePipe) Read(b []byte) (int, error) { // zero-byte message, ensure that all future Read() calls // also return EOF. f.readEOF = true + } else if err == syscall.ERROR_MORE_DATA { + // ERROR_MORE_DATA indicates that the pipe's read mode is message mode + // and the message still has more bytes. Treat this as a success, since + // this package presents all named pipes as byte streams. + err = nil } return n, err } @@ -134,12 +138,14 @@ func (s pipeAddress) String() string { } // DialPipe connects to a named pipe by path, timing out if the connection -// takes longer than the specified duration. If timeout is nil, then the timeout -// is the default timeout established by the pipe server. +// takes longer than the specified duration. If timeout is nil, then we use +// a default timeout of 5 seconds. (We do not use WaitNamedPipe.) func DialPipe(path string, timeout *time.Duration) (net.Conn, error) { var absTimeout time.Time if timeout != nil { absTimeout = time.Now().Add(*timeout) + } else { + absTimeout = time.Now().Add(time.Second * 2) } var err error var h syscall.Handle @@ -148,22 +154,13 @@ func DialPipe(path string, timeout *time.Duration) (net.Conn, error) { if err != cERROR_PIPE_BUSY { break } - now := time.Now() - var ms uint32 - if absTimeout.IsZero() { - ms = cNMPWAIT_USE_DEFAULT_WAIT - } else if now.After(absTimeout) { - ms = cNMPWAIT_NOWAIT - } else { - ms = uint32(absTimeout.Sub(now).Nanoseconds() / 1000 / 1000) - } - err = waitNamedPipe(path, ms) - if err != nil { - if err == cERROR_SEM_TIMEOUT { - return nil, ErrTimeout - } - break + if time.Now().After(absTimeout) { + return nil, ErrTimeout } + + // Wait 10 msec and try again. This is a rather simplistic + // view, as we always try each 10 milliseconds. + time.Sleep(time.Millisecond * 10) } if err != nil { return nil, &os.PathError{Op: "open", Path: path, Err: err} @@ -175,16 +172,6 @@ func DialPipe(path string, timeout *time.Duration) (net.Conn, error) { return nil, err } - var state uint32 - err = getNamedPipeHandleState(h, &state, nil, nil, nil, nil, 0) - if err != nil { - return nil, err - } - - if state&cPIPE_READMODE_MESSAGE != 0 { - return nil, &os.PathError{Op: "open", Path: path, Err: errors.New("message readmode pipes not supported")} - } - f, err := makeWin32File(h) if err != nil { syscall.Close(h) @@ -354,13 +341,23 @@ func ListenPipe(path string, c *PipeConfig) (net.Listener, error) { if err != nil { return nil, err } - // Immediately open and then close a client handle so that the named pipe is - // created but not currently accepting connections. + // Create a client handle and connect it. This results in the pipe + // instance always existing, so that clients see ERROR_PIPE_BUSY + // rather than ERROR_FILE_NOT_FOUND. This ties the first instance + // up so that no other instances can be used. This would have been + // cleaner if the Win32 API matched CreateFile with ConnectNamedPipe + // instead of CreateNamedPipe. (Apparently created named pipes are + // considered to be in listening state regardless of whether any + // active calls to ConnectNamedPipe are outstanding.) h2, err := createFile(path, 0, 0, nil, syscall.OPEN_EXISTING, cSECURITY_SQOS_PRESENT|cSECURITY_ANONYMOUS, 0) if err != nil { syscall.Close(h) return nil, err } + // Close the client handle. The server side of the instance will + // still be busy, leading to ERROR_PIPE_BUSY instead of + // ERROR_NOT_FOUND, as long as we don't close the server handle, + // or disconnect the client with DisconnectNamedPipe. syscall.Close(h2) l := &win32PipeListener{ firstHandle: h, diff --git a/vendor/vendor.json b/vendor/vendor.json index 4d1aa5e8..b3700ddb 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -15,10 +15,10 @@ "revisionTime": "2017-09-29T23:40:23Z" }, { - "checksumSHA1": "oEpUpU8ASfBWIRGesd9fBG1ar40=", + "checksumSHA1": "PbR6ZKoLeSZl8aXxDQqXih0wSgE=", "path": "github.com/Microsoft/go-winio", - "revision": "7da180ee92d8bd8bb8c37fc560e673e6557c392f", - "revisionTime": "2018-01-16T22:35:03Z" + "revision": "97e4973ce50b2ff5f09635a57e2b88a037aae829", + "revisionTime": "2018-08-23T22:24:21Z" }, { "checksumSHA1": "Aqy8/FoAIidY/DeQ5oTYSZ4YFVc=",