From d6708505b951bcb9c6b27c9f696a3a9684e39b96 Mon Sep 17 00:00:00 2001 From: aneesh-n <99904+aneesh-n@users.noreply.github.com> Date: Fri, 17 May 2024 14:15:03 -0600 Subject: [PATCH 1/7] Add helpers for Windows Extended Attributes --- internal/fs/ea_windows.go | 284 +++++++++++++++++++++++++++++++++ internal/fs/ea_windows_test.go | 259 ++++++++++++++++++++++++++++++ 2 files changed, 543 insertions(+) create mode 100644 internal/fs/ea_windows.go create mode 100644 internal/fs/ea_windows_test.go diff --git a/internal/fs/ea_windows.go b/internal/fs/ea_windows.go new file mode 100644 index 000000000..e4b23d35a --- /dev/null +++ b/internal/fs/ea_windows.go @@ -0,0 +1,284 @@ +//go:build windows +// +build windows + +package fs + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +// The code below was adapted from https://github.com/microsoft/go-winio under MIT license. + +// The MIT License (MIT) + +// Copyright (c) 2015 Microsoft + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +// The code below was copied over from https://github.com/microsoft/go-winio/blob/main/ea.go under MIT license. + +type fileFullEaInformation struct { + NextEntryOffset uint32 + Flags uint8 + NameLength uint8 + ValueLength uint16 +} + +var ( + fileFullEaInformationSize = binary.Size(&fileFullEaInformation{}) + + errInvalidEaBuffer = errors.New("invalid extended attribute buffer") + errEaNameTooLarge = errors.New("extended attribute name too large") + errEaValueTooLarge = errors.New("extended attribute value too large") +) + +// ExtendedAttribute represents a single Windows EA. +type ExtendedAttribute struct { + Name string + Value []byte + Flags uint8 +} + +func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) { + var info fileFullEaInformation + err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info) + if err != nil { + err = errInvalidEaBuffer + return ea, nb, err + } + + nameOffset := fileFullEaInformationSize + nameLen := int(info.NameLength) + valueOffset := nameOffset + int(info.NameLength) + 1 + valueLen := int(info.ValueLength) + nextOffset := int(info.NextEntryOffset) + if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) { + err = errInvalidEaBuffer + return ea, nb, err + } + + ea.Name = string(b[nameOffset : nameOffset+nameLen]) + ea.Value = b[valueOffset : valueOffset+valueLen] + ea.Flags = info.Flags + if info.NextEntryOffset != 0 { + nb = b[info.NextEntryOffset:] + } + return ea, nb, err +} + +// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION +// buffer retrieved from BackupRead, ZwQueryEaFile, etc. +func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) { + for len(b) != 0 { + ea, nb, err := parseEa(b) + if err != nil { + return nil, err + } + + eas = append(eas, ea) + b = nb + } + return eas, err +} + +func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error { + if int(uint8(len(ea.Name))) != len(ea.Name) { + return errEaNameTooLarge + } + if int(uint16(len(ea.Value))) != len(ea.Value) { + return errEaValueTooLarge + } + entrySize := uint32(fileFullEaInformationSize + len(ea.Name) + 1 + len(ea.Value)) + withPadding := (entrySize + 3) &^ 3 + nextOffset := uint32(0) + if !last { + nextOffset = withPadding + } + info := fileFullEaInformation{ + NextEntryOffset: nextOffset, + Flags: ea.Flags, + NameLength: uint8(len(ea.Name)), + ValueLength: uint16(len(ea.Value)), + } + + err := binary.Write(buf, binary.LittleEndian, &info) + if err != nil { + return err + } + + _, err = buf.Write([]byte(ea.Name)) + if err != nil { + return err + } + + err = buf.WriteByte(0) + if err != nil { + return err + } + + _, err = buf.Write(ea.Value) + if err != nil { + return err + } + + _, err = buf.Write([]byte{0, 0, 0}[0 : withPadding-entrySize]) + if err != nil { + return err + } + + return nil +} + +// EncodeExtendedAttributes encodes a list of EAs into a FILE_FULL_EA_INFORMATION +// buffer for use with BackupWrite, ZwSetEaFile, etc. +func EncodeExtendedAttributes(eas []ExtendedAttribute) ([]byte, error) { + var buf bytes.Buffer + for i := range eas { + last := false + if i == len(eas)-1 { + last = true + } + + err := writeEa(&buf, &eas[i], last) + if err != nil { + return nil, err + } + } + return buf.Bytes(), nil +} + +// The code below was copied over from https://github.com/microsoft/go-winio/blob/main/pipe.go under MIT license. + +type ntStatus int32 + +func (status ntStatus) Err() error { + if status >= 0 { + return nil + } + return rtlNtStatusToDosError(status) +} + +// The code below was copied over from https://github.com/microsoft/go-winio/blob/main/zsyscall_windows.go under MIT license. + +// ioStatusBlock represents the IO_STATUS_BLOCK struct defined here: +// https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ns-wdm-_io_status_block +type ioStatusBlock struct { + Status, Information uintptr +} + +var ( + modntdll = windows.NewLazySystemDLL("ntdll.dll") + procRtlNtStatusToDosErrorNoTeb = modntdll.NewProc("RtlNtStatusToDosErrorNoTeb") +) + +func rtlNtStatusToDosError(status ntStatus) (winerr error) { + r0, _, _ := syscall.SyscallN(procRtlNtStatusToDosErrorNoTeb.Addr(), uintptr(status)) + if r0 != 0 { + winerr = syscall.Errno(r0) + } + return +} + +// The code below was adapted from https://github.com/ambarve/go-winio/blob/a7564fd482feb903f9562a135f1317fd3b480739/ea.go +// under MIT license. + +var ( + procNtQueryEaFile = modntdll.NewProc("NtQueryEaFile") + procNtSetEaFile = modntdll.NewProc("NtSetEaFile") +) + +const ( + // noExtendedAttribsStatus is a constant value which indicates no extended attributes were found + noExtendedAttribsStatus = -1073741742 +) + +// GetFileEA retrieves the extended attributes for the file represented by `handle`. The +// `handle` must have been opened with file access flag FILE_READ_EA (0x8). +// The extended file attribute names in windows are case-insensitive and when fetching +// the attributes the names are generally returned in UPPER case. +func GetFileEA(handle windows.Handle) ([]ExtendedAttribute, error) { + // default buffer size to start with + bufLen := 1024 + buf := make([]byte, bufLen) + var iosb ioStatusBlock + // keep increasing the buffer size until it is large enough + for { + status := getFileEA(handle, &iosb, &buf[0], uint32(bufLen), false, 0, 0, nil, true) + + if status == noExtendedAttribsStatus { + //If status is -1073741742, no extended attributes were found + return nil, nil + } + err := status.Err() + if err != nil { + // convert ntstatus code to windows error + if err == windows.ERROR_INSUFFICIENT_BUFFER || err == windows.ERROR_MORE_DATA { + bufLen *= 2 + buf = make([]byte, bufLen) + continue + } + return nil, fmt.Errorf("get file EA failed with: %w", err) + } + break + } + return DecodeExtendedAttributes(buf) +} + +// SetFileEA sets the extended attributes for the file represented by `handle`. The +// handle must have been opened with the file access flag FILE_WRITE_EA(0x10). +func SetFileEA(handle windows.Handle, attrs []ExtendedAttribute) error { + encodedEA, err := EncodeExtendedAttributes(attrs) + if err != nil { + return fmt.Errorf("failed to encoded extended attributes: %w", err) + } + + var iosb ioStatusBlock + + return setFileEA(handle, &iosb, &encodedEA[0], uint32(len(encodedEA))).Err() +} + +// The code below was adapted from https://github.com/ambarve/go-winio/blob/a7564fd482feb903f9562a135f1317fd3b480739/zsyscall_windows.go +// under MIT license. + +func getFileEA(handle windows.Handle, iosb *ioStatusBlock, buf *uint8, bufLen uint32, returnSingleEntry bool, eaList uintptr, eaListLen uint32, eaIndex *uint32, restartScan bool) (status ntStatus) { + var _p0 uint32 + if returnSingleEntry { + _p0 = 1 + } + var _p1 uint32 + if restartScan { + _p1 = 1 + } + r0, _, _ := syscall.SyscallN(procNtQueryEaFile.Addr(), uintptr(handle), uintptr(unsafe.Pointer(iosb)), uintptr(unsafe.Pointer(buf)), uintptr(bufLen), uintptr(_p0), uintptr(eaList), uintptr(eaListLen), uintptr(unsafe.Pointer(eaIndex)), uintptr(_p1)) + status = ntStatus(r0) + return +} + +func setFileEA(handle windows.Handle, iosb *ioStatusBlock, buf *uint8, bufLen uint32) (status ntStatus) { + r0, _, _ := syscall.SyscallN(procNtSetEaFile.Addr(), uintptr(handle), uintptr(unsafe.Pointer(iosb)), uintptr(unsafe.Pointer(buf)), uintptr(bufLen)) + status = ntStatus(r0) + return +} diff --git a/internal/fs/ea_windows_test.go b/internal/fs/ea_windows_test.go new file mode 100644 index 000000000..f9a4d9adf --- /dev/null +++ b/internal/fs/ea_windows_test.go @@ -0,0 +1,259 @@ +//go:build windows +// +build windows + +package fs + +import ( + "crypto/rand" + "fmt" + "math/big" + "os" + "path/filepath" + "reflect" + "syscall" + "testing" + "unsafe" + + "golang.org/x/sys/windows" +) + +// The code below was adapted from github.com/Microsoft/go-winio under MIT license. + +// The MIT License (MIT) + +// Copyright (c) 2015 Microsoft + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +// The code below was adapted from https://github.com/ambarve/go-winio/blob/a7564fd482feb903f9562a135f1317fd3b480739/ea_test.go +// under MIT license. + +var ( + testEas = []ExtendedAttribute{ + {Name: "foo", Value: []byte("bar")}, + {Name: "fizz", Value: []byte("buzz")}, + } + + testEasEncoded = []byte{16, 0, 0, 0, 0, 3, 3, 0, 102, 111, 111, 0, 98, 97, 114, 0, 0, + 0, 0, 0, 0, 4, 4, 0, 102, 105, 122, 122, 0, 98, 117, 122, 122, 0, 0, 0} + testEasNotPadded = testEasEncoded[0 : len(testEasEncoded)-3] + testEasTruncated = testEasEncoded[0:20] +) + +func TestRoundTripEas(t *testing.T) { + b, err := EncodeExtendedAttributes(testEas) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(testEasEncoded, b) { + t.Fatalf("Encoded mismatch %v %v", testEasEncoded, b) + } + eas, err := DecodeExtendedAttributes(b) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(testEas, eas) { + t.Fatalf("mismatch %+v %+v", testEas, eas) + } +} + +func TestEasDontNeedPaddingAtEnd(t *testing.T) { + eas, err := DecodeExtendedAttributes(testEasNotPadded) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(testEas, eas) { + t.Fatalf("mismatch %+v %+v", testEas, eas) + } +} + +func TestTruncatedEasFailCorrectly(t *testing.T) { + _, err := DecodeExtendedAttributes(testEasTruncated) + if err == nil { + t.Fatal("expected error") + } +} + +func TestNilEasEncodeAndDecodeAsNil(t *testing.T) { + b, err := EncodeExtendedAttributes(nil) + if err != nil { + t.Fatal(err) + } + if len(b) != 0 { + t.Fatal("expected empty") + } + eas, err := DecodeExtendedAttributes(nil) + if err != nil { + t.Fatal(err) + } + if len(eas) != 0 { + t.Fatal("expected empty") + } +} + +// TestSetFileEa makes sure that the test buffer is actually parsable by NtSetEaFile. +func TestSetFileEa(t *testing.T) { + f, err := os.CreateTemp("", "testea") + if err != nil { + t.Fatal(err) + } + defer func() { + err := os.Remove(f.Name()) + if err != nil { + t.Logf("Error removing file %s: %v\n", f.Name(), err) + } + err = f.Close() + if err != nil { + t.Logf("Error closing file %s: %v\n", f.Name(), err) + } + }() + ntdll := syscall.MustLoadDLL("ntdll.dll") + ntSetEaFile := ntdll.MustFindProc("NtSetEaFile") + var iosb [2]uintptr + r, _, _ := ntSetEaFile.Call(f.Fd(), + uintptr(unsafe.Pointer(&iosb[0])), + uintptr(unsafe.Pointer(&testEasEncoded[0])), + uintptr(len(testEasEncoded))) + if r != 0 { + t.Fatalf("NtSetEaFile failed with %08x", r) + } +} + +func TestSetGetFileEA(t *testing.T) { + tempDir := t.TempDir() + testfilePath := filepath.Join(tempDir, "testfile.txt") + // create temp file + testfile, err := os.Create(testfilePath) + if err != nil { + t.Fatalf("failed to create temporary file: %s", err) + } + defer func() { + err := testfile.Close() + if err != nil { + t.Logf("Error closing file %s: %v\n", testfile.Name(), err) + } + }() + + nAttrs := 3 + testEAs := make([]ExtendedAttribute, 3) + // generate random extended attributes for test + for i := 0; i < nAttrs; i++ { + // EA name is automatically converted to upper case before storing, so + // when reading it back it returns the upper case name. To avoid test + // failures because of that keep the name upper cased. + testEAs[i].Name = fmt.Sprintf("TESTEA%d", i+1) + testEAs[i].Value = make([]byte, getRandomInt()) + _, err := rand.Read(testEAs[i].Value) + if err != nil { + t.Logf("Error reading rand for file %s: %v\n", testfilePath, err) + } + } + + utf16Path := windows.StringToUTF16Ptr(testfilePath) + fileAccessRightReadWriteEA := (0x8 | 0x10) + fileHandle, err := windows.CreateFile(utf16Path, uint32(fileAccessRightReadWriteEA), 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0) + if err != nil { + t.Fatalf("open file failed with: %s", err) + } + defer func() { + err := windows.Close(fileHandle) + if err != nil { + t.Logf("Error closing file handle %s: %v\n", testfilePath, err) + } + }() + + if err := SetFileEA(fileHandle, testEAs); err != nil { + t.Fatalf("set EA for file failed: %s", err) + } + + var readEAs []ExtendedAttribute + if readEAs, err = GetFileEA(fileHandle); err != nil { + t.Fatalf("get EA for file failed: %s", err) + } + + if !reflect.DeepEqual(readEAs, testEAs) { + t.Logf("expected: %+v, found: %+v\n", testEAs, readEAs) + t.Fatalf("EAs read from testfile don't match") + } +} + +func TestSetGetFolderEA(t *testing.T) { + tempDir := t.TempDir() + testfolderPath := filepath.Join(tempDir, "testfolder") + // create temp folder + err := os.Mkdir(testfolderPath, os.ModeDir) + if err != nil { + t.Fatalf("failed to create temporary file: %s", err) + } + + nAttrs := 3 + testEAs := make([]ExtendedAttribute, 3) + // generate random extended attributes for test + for i := 0; i < nAttrs; i++ { + // EA name is automatically converted to upper case before storing, so + // when reading it back it returns the upper case name. To avoid test + // failures because of that keep the name upper cased. + testEAs[i].Name = fmt.Sprintf("TESTEA%d", i+1) + testEAs[i].Value = make([]byte, getRandomInt()) + _, err := rand.Read(testEAs[i].Value) + if err != nil { + t.Logf("Error reading rand for file %s: %v\n", testfolderPath, err) + } + } + + utf16Path := windows.StringToUTF16Ptr(testfolderPath) + fileAccessRightReadWriteEA := (0x8 | 0x10) + fileHandle, err := windows.CreateFile(utf16Path, uint32(fileAccessRightReadWriteEA), 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_BACKUP_SEMANTICS, 0) + + if err != nil { + t.Fatalf("open folder failed with: %s", err) + } + defer func() { + err := windows.Close(fileHandle) + if err != nil { + t.Logf("Error closing file handle %s: %v\n", testfolderPath, err) + } + }() + + if err := SetFileEA(fileHandle, testEAs); err != nil { + t.Fatalf("set EA for folder failed: %s", err) + } + + var readEAs []ExtendedAttribute + if readEAs, err = GetFileEA(fileHandle); err != nil { + t.Fatalf("get EA for folder failed: %s", err) + } + + if !reflect.DeepEqual(readEAs, testEAs) { + t.Logf("expected: %+v, found: %+v\n", testEAs, readEAs) + t.Fatalf("EAs read from test folder don't match") + } +} + +func getRandomInt() int64 { + nBig, err := rand.Int(rand.Reader, big.NewInt(27)) + if err != nil { + panic(err) + } + n := nBig.Int64() + if n == 0 { + n = getRandomInt() + } + return n +} From 5cff6e084e28c5d8fcff65cf344977df36575db3 Mon Sep 17 00:00:00 2001 From: aneesh-n <99904+aneesh-n@users.noreply.github.com> Date: Fri, 17 May 2024 14:18:20 -0600 Subject: [PATCH 2/7] Add support for Windows EA in node Refactor Extended Attribute related functions in node files as windows apis get and set EA in bulk --- internal/restic/node.go | 38 --------- internal/restic/node_aix.go | 18 ++--- internal/restic/node_netbsd.go | 18 ++--- internal/restic/node_openbsd.go | 18 ++--- internal/restic/node_test.go | 15 +++- internal/restic/node_windows.go | 117 +++++++++++++++++++++++---- internal/restic/node_windows_test.go | 66 +++++++++++++++ internal/restic/node_xattr.go | 52 ++++++++++-- 8 files changed, 246 insertions(+), 96 deletions(-) diff --git a/internal/restic/node.go b/internal/restic/node.go index 807ee0c0f..5bdc5ba27 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -284,16 +284,6 @@ func (node Node) restoreMetadata(path string, warn func(msg string)) error { return firsterr } -func (node Node) restoreExtendedAttributes(path string) error { - for _, attr := range node.ExtendedAttributes { - err := Setxattr(path, attr.Name, attr.Value) - if err != nil { - return err - } - } - return nil -} - func (node Node) RestoreTimestamps(path string) error { var utimes = [...]syscall.Timespec{ syscall.NsecToTimespec(node.AccessTime.UnixNano()), @@ -726,34 +716,6 @@ func (node *Node) fillExtra(path string, fi os.FileInfo, ignoreXattrListError bo return err } -func (node *Node) fillExtendedAttributes(path string, ignoreListError bool) error { - xattrs, err := Listxattr(path) - debug.Log("fillExtendedAttributes(%v) %v %v", path, xattrs, err) - if err != nil { - if ignoreListError && IsListxattrPermissionError(err) { - return nil - } - return err - } - - node.ExtendedAttributes = make([]ExtendedAttribute, 0, len(xattrs)) - for _, attr := range xattrs { - attrVal, err := Getxattr(path, attr) - if err != nil { - fmt.Fprintf(os.Stderr, "can not obtain extended attribute %v for %v:\n", attr, path) - continue - } - attr := ExtendedAttribute{ - Name: attr, - Value: attrVal, - } - - node.ExtendedAttributes = append(node.ExtendedAttributes, attr) - } - - return nil -} - func mkfifo(path string, mode uint32) (err error) { return mknod(path, mode|syscall.S_IFIFO, 0) } diff --git a/internal/restic/node_aix.go b/internal/restic/node_aix.go index 8ee9022c9..32f63af15 100644 --- a/internal/restic/node_aix.go +++ b/internal/restic/node_aix.go @@ -23,25 +23,21 @@ func (s statT) atim() syscall.Timespec { return toTimespec(s.Atim) } func (s statT) mtim() syscall.Timespec { return toTimespec(s.Mtim) } func (s statT) ctim() syscall.Timespec { return toTimespec(s.Ctim) } -// Getxattr is a no-op on AIX. -func Getxattr(path, name string) ([]byte, error) { - return nil, nil +// restoreExtendedAttributes is a no-op on AIX. +func (node Node) restoreExtendedAttributes(_ string) error { + return nil } -// Listxattr is a no-op on AIX. -func Listxattr(path string) ([]string, error) { - return nil, nil +// fillExtendedAttributes is a no-op on AIX. +func (node *Node) fillExtendedAttributes(_ string, _ bool) error { + return nil } +// IsListxattrPermissionError is a no-op on AIX. func IsListxattrPermissionError(_ error) bool { return false } -// Setxattr is a no-op on AIX. -func Setxattr(path, name string, data []byte) error { - return nil -} - // restoreGenericAttributes is no-op on AIX. func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { return node.handleAllUnknownGenericAttributesFound(warn) diff --git a/internal/restic/node_netbsd.go b/internal/restic/node_netbsd.go index cf1fa36bd..0fe46a3f2 100644 --- a/internal/restic/node_netbsd.go +++ b/internal/restic/node_netbsd.go @@ -13,25 +13,21 @@ func (s statT) atim() syscall.Timespec { return s.Atimespec } func (s statT) mtim() syscall.Timespec { return s.Mtimespec } func (s statT) ctim() syscall.Timespec { return s.Ctimespec } -// Getxattr is a no-op on netbsd. -func Getxattr(path, name string) ([]byte, error) { - return nil, nil +// restoreExtendedAttributes is a no-op on netbsd. +func (node Node) restoreExtendedAttributes(_ string) error { + return nil } -// Listxattr is a no-op on netbsd. -func Listxattr(path string) ([]string, error) { - return nil, nil +// fillExtendedAttributes is a no-op on netbsd. +func (node *Node) fillExtendedAttributes(_ string, _ bool) error { + return nil } +// IsListxattrPermissionError is a no-op on netbsd. func IsListxattrPermissionError(_ error) bool { return false } -// Setxattr is a no-op on netbsd. -func Setxattr(path, name string, data []byte) error { - return nil -} - // restoreGenericAttributes is no-op on netbsd. func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { return node.handleAllUnknownGenericAttributesFound(warn) diff --git a/internal/restic/node_openbsd.go b/internal/restic/node_openbsd.go index 4f1c0dacb..71841f59f 100644 --- a/internal/restic/node_openbsd.go +++ b/internal/restic/node_openbsd.go @@ -13,25 +13,21 @@ func (s statT) atim() syscall.Timespec { return s.Atim } func (s statT) mtim() syscall.Timespec { return s.Mtim } func (s statT) ctim() syscall.Timespec { return s.Ctim } -// Getxattr is a no-op on openbsd. -func Getxattr(path, name string) ([]byte, error) { - return nil, nil +// restoreExtendedAttributes is a no-op on openbsd. +func (node Node) restoreExtendedAttributes(_ string) error { + return nil } -// Listxattr is a no-op on openbsd. -func Listxattr(path string) ([]string, error) { - return nil, nil +// fillExtendedAttributes is a no-op on openbsd. +func (node *Node) fillExtendedAttributes(_ string, _ bool) error { + return nil } +// IsListxattrPermissionError is a no-op on openbsd. func IsListxattrPermissionError(_ error) bool { return false } -// Setxattr is a no-op on openbsd. -func Setxattr(path, name string, data []byte) error { - return nil -} - // restoreGenericAttributes is no-op on openbsd. func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { return node.handleAllUnknownGenericAttributesFound(warn) diff --git a/internal/restic/node_test.go b/internal/restic/node_test.go index ea271faab..99ea48bbb 100644 --- a/internal/restic/node_test.go +++ b/internal/restic/node_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "reflect" "runtime" + "strings" "testing" "time" @@ -205,8 +206,18 @@ func TestNodeRestoreAt(t *testing.T) { var nodePath string if test.ExtendedAttributes != nil { if runtime.GOOS == "windows" { - // restic does not support xattrs on windows - return + // In windows extended attributes are case insensitive and windows returns + // the extended attributes in UPPER case. + // Update the tests to use UPPER case xattr names for windows. + extAttrArr := test.ExtendedAttributes + // Iterate through the array using pointers + for i := 0; i < len(extAttrArr); i++ { + // Get the pointer to the current element + namePtr := &extAttrArr[i].Name + + // Modify the value through the pointer + *namePtr = strings.ToUpper(*namePtr) + } } // tempdir might be backed by a filesystem that does not support diff --git a/internal/restic/node_windows.go b/internal/restic/node_windows.go index 0c6d3775e..881c394be 100644 --- a/internal/restic/node_windows.go +++ b/internal/restic/node_windows.go @@ -70,26 +70,109 @@ func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespe return syscall.SetFileTime(h, nil, &a, &w) } -// Getxattr retrieves extended attribute data associated with path. -func Getxattr(path, name string) ([]byte, error) { - return nil, nil -} - -// Listxattr retrieves a list of names of extended attributes associated with the -// given path in the file system. -func Listxattr(path string) ([]string, error) { - return nil, nil -} - -func IsListxattrPermissionError(_ error) bool { - return false -} - -// Setxattr associates name and data together as an attribute of path. -func Setxattr(path, name string, data []byte) error { +// restore extended attributes for windows +func (node Node) restoreExtendedAttributes(path string) (err error) { + eas := []fs.ExtendedAttribute{} + for _, attr := range node.ExtendedAttributes { + extr := new(fs.ExtendedAttribute) + extr.Name = attr.Name + extr.Value = attr.Value + eas = append(eas, *extr) + } + if len(eas) > 0 { + if errExt := restoreExtendedAttributes(node.Type, path, eas); errExt != nil { + return errExt + } + } return nil } +// fill extended attributes in the node. This also includes the Generic attributes for windows. +func (node *Node) fillExtendedAttributes(path string, _ bool) (err error) { + var fileHandle windows.Handle + + //Get file handle for file or dir + if node.Type == "file" { + if strings.HasSuffix(filepath.Clean(path), `\`) { + return nil + } + utf16Path := windows.StringToUTF16Ptr(path) + fileAccessRightReadWriteEA := (0x8 | 0x10) + fileHandle, err = windows.CreateFile(utf16Path, uint32(fileAccessRightReadWriteEA), 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0) + } else if node.Type == "dir" { + utf16Path := windows.StringToUTF16Ptr(path) + fileAccessRightReadWriteEA := (0x8 | 0x10) + fileHandle, err = windows.CreateFile(utf16Path, uint32(fileAccessRightReadWriteEA), 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_BACKUP_SEMANTICS, 0) + } else { + return nil + } + if err != nil { + err = errors.Errorf("open file failed for path: %s, with: %v", path, err) + return err + } + defer func() { + err := windows.CloseHandle(fileHandle) + if err != nil { + debug.Log("Error closing file handle for %s: %v\n", path, err) + } + }() + + //Get the windows Extended Attributes using the file handle + extAtts, err := fs.GetFileEA(fileHandle) + debug.Log("fillExtendedAttributes(%v) %v", path, extAtts) + if err != nil { + debug.Log("open file failed for path: %s : %v", path, err) + return err + } else if len(extAtts) == 0 { + return nil + } + + //Fill the ExtendedAttributes in the node using the name/value pairs in the windows EA + for _, attr := range extAtts { + if err != nil { + err = errors.Errorf("can not obtain extended attribute for path %v, attr: %v, err: %v\n,", path, attr, err) + continue + } + extendedAttr := ExtendedAttribute{ + Name: attr.Name, + Value: attr.Value, + } + + node.ExtendedAttributes = append(node.ExtendedAttributes, extendedAttr) + } + return nil +} + +// restoreExtendedAttributes handles restore of the Windows Extended Attributes to the specified path. +// The Windows API requires setting of all the Extended Attributes in one call. +func restoreExtendedAttributes(nodeType, path string, eas []fs.ExtendedAttribute) (err error) { + var fileHandle windows.Handle + switch nodeType { + case "file": + utf16Path := windows.StringToUTF16Ptr(path) + fileAccessRightReadWriteEA := (0x8 | 0x10) + fileHandle, err = windows.CreateFile(utf16Path, uint32(fileAccessRightReadWriteEA), 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0) + case "dir": + utf16Path := windows.StringToUTF16Ptr(path) + fileAccessRightReadWriteEA := (0x8 | 0x10) + fileHandle, err = windows.CreateFile(utf16Path, uint32(fileAccessRightReadWriteEA), 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_BACKUP_SEMANTICS, 0) + default: + return nil + } + defer func() { + err := windows.CloseHandle(fileHandle) + if err != nil { + debug.Log("Error closing file handle for %s: %v\n", path, err) + } + }() + if err != nil { + err = errors.Errorf("open file failed for path %v, with: %v:\n", path, err) + } else if err = fs.SetFileEA(fileHandle, eas); err != nil { + err = errors.Errorf("set EA failed for path %v, with: %v:\n", path, err) + } + return err +} + type statT syscall.Win32FileAttributeData func toStatT(i interface{}) (*statT, bool) { diff --git a/internal/restic/node_windows_test.go b/internal/restic/node_windows_test.go index 57fc51e07..f89e2aeee 100644 --- a/internal/restic/node_windows_test.go +++ b/internal/restic/node_windows_test.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "syscall" "testing" @@ -265,3 +266,68 @@ func TestNewGenericAttributeType(t *testing.T) { test.Assert(t, len(ua) == 0, "Unkown attributes: %s found for path: %s", ua, testPath) } } + +func TestRestoreExtendedAttributes(t *testing.T) { + t.Parallel() + tempDir := t.TempDir() + expectedNodes := []Node{ + { + Name: "testfile", + Type: "file", + Mode: 0644, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + ExtendedAttributes: []ExtendedAttribute{ + {"user.foo", []byte("bar")}, + }, + }, + { + Name: "testdirectory", + Type: "dir", + Mode: 0755, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + ExtendedAttributes: []ExtendedAttribute{ + {"user.foo", []byte("bar")}, + }, + }, + } + for _, testNode := range expectedNodes { + testPath, node := restoreAndGetNode(t, tempDir, testNode, false) + + var handle windows.Handle + var err error + utf16Path := windows.StringToUTF16Ptr(testPath) + if node.Type == "file" { + handle, err = windows.CreateFile(utf16Path, windows.FILE_READ_EA, 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0) + } else if node.Type == "dir" { + handle, err = windows.CreateFile(utf16Path, windows.FILE_READ_EA, 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_BACKUP_SEMANTICS, 0) + } + test.OK(t, errors.Wrapf(err, "Error opening file/directory for: %s", testPath)) + defer func() { + err := windows.Close(handle) + test.OK(t, errors.Wrapf(err, "Error closing file for: %s", testPath)) + }() + + if len(node.ExtendedAttributes) > 0 { + extAttr, err := fs.GetFileEA(handle) + test.OK(t, errors.Wrapf(err, "Error getting extended attributes for: %s", testPath)) + test.Equals(t, len(node.ExtendedAttributes), len(extAttr)) + + for _, expectedExtAttr := range node.ExtendedAttributes { + var foundExtAttr *fs.ExtendedAttribute + for _, ea := range extAttr { + if strings.EqualFold(ea.Name, expectedExtAttr.Name) { + foundExtAttr = &ea + break + + } + } + test.Assert(t, foundExtAttr != nil, "Expected extended attribute not found") + test.Equals(t, expectedExtAttr.Value, foundExtAttr.Value) + } + } + } +} diff --git a/internal/restic/node_xattr.go b/internal/restic/node_xattr.go index 8b080e74f..a55fcb2db 100644 --- a/internal/restic/node_xattr.go +++ b/internal/restic/node_xattr.go @@ -4,23 +4,25 @@ package restic import ( + "fmt" "os" "syscall" + "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/pkg/xattr" ) -// Getxattr retrieves extended attribute data associated with path. -func Getxattr(path, name string) ([]byte, error) { +// getxattr retrieves extended attribute data associated with path. +func getxattr(path, name string) ([]byte, error) { b, err := xattr.LGet(path, name) return b, handleXattrErr(err) } -// Listxattr retrieves a list of names of extended attributes associated with the +// listxattr retrieves a list of names of extended attributes associated with the // given path in the file system. -func Listxattr(path string) ([]string, error) { +func listxattr(path string) ([]string, error) { l, err := xattr.LList(path) return l, handleXattrErr(err) } @@ -33,8 +35,8 @@ func IsListxattrPermissionError(err error) bool { return false } -// Setxattr associates name and data together as an attribute of path. -func Setxattr(path, name string, data []byte) error { +// setxattr associates name and data together as an attribute of path. +func setxattr(path, name string, data []byte) error { return handleXattrErr(xattr.LSet(path, name, data)) } @@ -66,3 +68,41 @@ func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) erro func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { return true, nil } + +func (node Node) restoreExtendedAttributes(path string) error { + for _, attr := range node.ExtendedAttributes { + err := setxattr(path, attr.Name, attr.Value) + if err != nil { + return err + } + } + return nil +} + +func (node *Node) fillExtendedAttributes(path string, ignoreListError bool) error { + xattrs, err := listxattr(path) + debug.Log("fillExtendedAttributes(%v) %v %v", path, xattrs, err) + if err != nil { + if ignoreListError && IsListxattrPermissionError(err) { + return nil + } + return err + } + + node.ExtendedAttributes = make([]ExtendedAttribute, 0, len(xattrs)) + for _, attr := range xattrs { + attrVal, err := getxattr(path, attr) + if err != nil { + fmt.Fprintf(os.Stderr, "can not obtain extended attribute %v for %v:\n", attr, path) + continue + } + attr := ExtendedAttribute{ + Name: attr, + Value: attrVal, + } + + node.ExtendedAttributes = append(node.ExtendedAttributes, attr) + } + + return nil +} From 9d1b23964d7f718af19ad906514adf8cf5812965 Mon Sep 17 00:00:00 2001 From: aneesh-n <99904+aneesh-n@users.noreply.github.com> Date: Fri, 17 May 2024 14:19:05 -0600 Subject: [PATCH 3/7] Update comment for licensing --- internal/fs/sd_windows.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/fs/sd_windows.go b/internal/fs/sd_windows.go index cc44433c3..5d98b4ef4 100644 --- a/internal/fs/sd_windows.go +++ b/internal/fs/sd_windows.go @@ -346,7 +346,7 @@ func getPrivilegeName(luid uint64) string { return string(utf16.Decode(displayNameBuffer[:displayBufSize])) } -// The functions below are copied over from https://github.com/microsoft/go-winio/blob/main/zsyscall_windows.go +// The functions below are copied over from https://github.com/microsoft/go-winio/blob/main/zsyscall_windows.go under MIT license. // This windows api always returns an error even in case of success, warnings (partial success) and error cases. // @@ -424,7 +424,7 @@ func _lookupPrivilegeValue(systemName *uint16, name *uint16, luid *uint64) (err return } -// The code below was copied from https://github.com/microsoft/go-winio/blob/main/tools/mkwinsyscall/mkwinsyscall.go +// The code below was copied from https://github.com/microsoft/go-winio/blob/main/tools/mkwinsyscall/mkwinsyscall.go under MIT license. // errnoErr returns common boxed Errno values, to prevent // allocations at runtime. From 43bc304e423341c0255b2e6172ef494657a7b15f Mon Sep 17 00:00:00 2001 From: aneesh-n <99904+aneesh-n@users.noreply.github.com> Date: Fri, 17 May 2024 14:54:25 -0600 Subject: [PATCH 4/7] Add unreleased changelog --- changelog/unreleased/pull-4807 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/pull-4807 diff --git a/changelog/unreleased/pull-4807 b/changelog/unreleased/pull-4807 new file mode 100644 index 000000000..12f8391e3 --- /dev/null +++ b/changelog/unreleased/pull-4807 @@ -0,0 +1,5 @@ +Enhancement: Back up and restore Extended Attributes on Windows NTFS + +Restic now backs up and restores Extended Attributes on Windows NTFS when backing up files and folders. + +https://github.com/restic/restic/pull/4807 \ No newline at end of file From 7a48c9ebd7191124ce13b4c73804840f0e2c15c1 Mon Sep 17 00:00:00 2001 From: aneesh-n <99904+aneesh-n@users.noreply.github.com> Date: Wed, 5 Jun 2024 16:06:57 -0600 Subject: [PATCH 5/7] Fix review comments --- internal/fs/ea_windows.go | 7 +- internal/fs/ea_windows_test.go | 157 +++++++++++++-------------- internal/restic/node_test.go | 6 +- internal/restic/node_windows.go | 83 ++++++-------- internal/restic/node_windows_test.go | 24 ++-- 5 files changed, 123 insertions(+), 154 deletions(-) diff --git a/internal/fs/ea_windows.go b/internal/fs/ea_windows.go index e4b23d35a..08466c33f 100644 --- a/internal/fs/ea_windows.go +++ b/internal/fs/ea_windows.go @@ -211,8 +211,9 @@ var ( ) const ( - // noExtendedAttribsStatus is a constant value which indicates no extended attributes were found - noExtendedAttribsStatus = -1073741742 + // STATUS_NO_EAS_ON_FILE is a constant value which indicates EAs were requested for the file but it has no EAs. + // Windows NTSTATUS value: STATUS_NO_EAS_ON_FILE=0xC0000052 + STATUS_NO_EAS_ON_FILE = -1073741742 ) // GetFileEA retrieves the extended attributes for the file represented by `handle`. The @@ -228,7 +229,7 @@ func GetFileEA(handle windows.Handle) ([]ExtendedAttribute, error) { for { status := getFileEA(handle, &iosb, &buf[0], uint32(bufLen), false, 0, 0, nil, true) - if status == noExtendedAttribsStatus { + if status == STATUS_NO_EAS_ON_FILE { //If status is -1073741742, no extended attributes were found return nil, nil } diff --git a/internal/fs/ea_windows_test.go b/internal/fs/ea_windows_test.go index f9a4d9adf..14b7a2452 100644 --- a/internal/fs/ea_windows_test.go +++ b/internal/fs/ea_windows_test.go @@ -135,114 +135,103 @@ func TestSetFileEa(t *testing.T) { } } +// The code below was refactored from github.com/Microsoft/go-winio/blob/a7564fd482feb903f9562a135f1317fd3b480739/ea_test.go +// under MIT license. func TestSetGetFileEA(t *testing.T) { + testfilePath := setupTestFile(t) + defer cleanupTestFile(t, testfilePath) + + testEAs := generateTestEAs(t, 3, testfilePath) + fileHandle := openFile(t, testfilePath, windows.FILE_ATTRIBUTE_NORMAL) + defer closeFileHandle(t, fileHandle, testfilePath) + + testSetGetEA(t, fileHandle, testEAs, testfilePath) +} + +// The code is new code and reuses code refactored from github.com/Microsoft/go-winio/blob/a7564fd482feb903f9562a135f1317fd3b480739/ea_test.go +// under MIT license. +func TestSetGetFolderEA(t *testing.T) { + testfolderPath := setupTestFolder(t) + defer cleanupTestFolder(t, testfolderPath) + + testEAs := generateTestEAs(t, 3, testfolderPath) + fileHandle := openFile(t, testfolderPath, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_BACKUP_SEMANTICS) + defer closeFileHandle(t, fileHandle, testfolderPath) + + testSetGetEA(t, fileHandle, testEAs, testfolderPath) +} + +func setupTestFile(t *testing.T) string { tempDir := t.TempDir() testfilePath := filepath.Join(tempDir, "testfile.txt") - // create temp file - testfile, err := os.Create(testfilePath) - if err != nil { + if _, err := os.Create(testfilePath); err != nil { t.Fatalf("failed to create temporary file: %s", err) } - defer func() { - err := testfile.Close() - if err != nil { - t.Logf("Error closing file %s: %v\n", testfile.Name(), err) - } - }() + return testfilePath +} - nAttrs := 3 - testEAs := make([]ExtendedAttribute, 3) - // generate random extended attributes for test +func setupTestFolder(t *testing.T) string { + tempDir := t.TempDir() + testfolderPath := filepath.Join(tempDir, "testfolder") + if err := os.Mkdir(testfolderPath, os.ModeDir); err != nil { + t.Fatalf("failed to create temporary folder: %s", err) + } + return testfolderPath +} + +func generateTestEAs(t *testing.T, nAttrs int, path string) []ExtendedAttribute { + testEAs := make([]ExtendedAttribute, nAttrs) for i := 0; i < nAttrs; i++ { - // EA name is automatically converted to upper case before storing, so - // when reading it back it returns the upper case name. To avoid test - // failures because of that keep the name upper cased. testEAs[i].Name = fmt.Sprintf("TESTEA%d", i+1) testEAs[i].Value = make([]byte, getRandomInt()) - _, err := rand.Read(testEAs[i].Value) - if err != nil { - t.Logf("Error reading rand for file %s: %v\n", testfilePath, err) + if _, err := rand.Read(testEAs[i].Value); err != nil { + t.Logf("Error reading rand for path %s: %v\n", path, err) } } + return testEAs +} - utf16Path := windows.StringToUTF16Ptr(testfilePath) - fileAccessRightReadWriteEA := (0x8 | 0x10) - fileHandle, err := windows.CreateFile(utf16Path, uint32(fileAccessRightReadWriteEA), 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0) +func openFile(t *testing.T, path string, attributes uint32) windows.Handle { + utf16Path := windows.StringToUTF16Ptr(path) + fileAccessRightReadWriteEA := uint32(0x8 | 0x10) + fileHandle, err := windows.CreateFile(utf16Path, fileAccessRightReadWriteEA, 0, nil, windows.OPEN_EXISTING, attributes, 0) if err != nil { t.Fatalf("open file failed with: %s", err) } - defer func() { - err := windows.Close(fileHandle) - if err != nil { - t.Logf("Error closing file handle %s: %v\n", testfilePath, err) - } - }() + return fileHandle +} - if err := SetFileEA(fileHandle, testEAs); err != nil { - t.Fatalf("set EA for file failed: %s", err) - } - - var readEAs []ExtendedAttribute - if readEAs, err = GetFileEA(fileHandle); err != nil { - t.Fatalf("get EA for file failed: %s", err) - } - - if !reflect.DeepEqual(readEAs, testEAs) { - t.Logf("expected: %+v, found: %+v\n", testEAs, readEAs) - t.Fatalf("EAs read from testfile don't match") +func closeFileHandle(t *testing.T, handle windows.Handle, path string) { + if err := windows.Close(handle); err != nil { + t.Logf("Error closing file handle %s: %v\n", path, err) } } -func TestSetGetFolderEA(t *testing.T) { - tempDir := t.TempDir() - testfolderPath := filepath.Join(tempDir, "testfolder") - // create temp folder - err := os.Mkdir(testfolderPath, os.ModeDir) +func testSetGetEA(t *testing.T, handle windows.Handle, testEAs []ExtendedAttribute, path string) { + if err := SetFileEA(handle, testEAs); err != nil { + t.Fatalf("set EA for path %s failed: %s", path, err) + } + + readEAs, err := GetFileEA(handle) if err != nil { - t.Fatalf("failed to create temporary file: %s", err) - } - - nAttrs := 3 - testEAs := make([]ExtendedAttribute, 3) - // generate random extended attributes for test - for i := 0; i < nAttrs; i++ { - // EA name is automatically converted to upper case before storing, so - // when reading it back it returns the upper case name. To avoid test - // failures because of that keep the name upper cased. - testEAs[i].Name = fmt.Sprintf("TESTEA%d", i+1) - testEAs[i].Value = make([]byte, getRandomInt()) - _, err := rand.Read(testEAs[i].Value) - if err != nil { - t.Logf("Error reading rand for file %s: %v\n", testfolderPath, err) - } - } - - utf16Path := windows.StringToUTF16Ptr(testfolderPath) - fileAccessRightReadWriteEA := (0x8 | 0x10) - fileHandle, err := windows.CreateFile(utf16Path, uint32(fileAccessRightReadWriteEA), 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_BACKUP_SEMANTICS, 0) - - if err != nil { - t.Fatalf("open folder failed with: %s", err) - } - defer func() { - err := windows.Close(fileHandle) - if err != nil { - t.Logf("Error closing file handle %s: %v\n", testfolderPath, err) - } - }() - - if err := SetFileEA(fileHandle, testEAs); err != nil { - t.Fatalf("set EA for folder failed: %s", err) - } - - var readEAs []ExtendedAttribute - if readEAs, err = GetFileEA(fileHandle); err != nil { - t.Fatalf("get EA for folder failed: %s", err) + t.Fatalf("get EA for path %s failed: %s", path, err) } if !reflect.DeepEqual(readEAs, testEAs) { t.Logf("expected: %+v, found: %+v\n", testEAs, readEAs) - t.Fatalf("EAs read from test folder don't match") + t.Fatalf("EAs read from path %s don't match", path) + } +} + +func cleanupTestFile(t *testing.T, path string) { + if err := os.Remove(path); err != nil { + t.Logf("Error removing file %s: %v\n", path, err) + } +} + +func cleanupTestFolder(t *testing.T, path string) { + if err := os.Remove(path); err != nil { + t.Logf("Error removing folder %s: %v\n", path, err) } } diff --git a/internal/restic/node_test.go b/internal/restic/node_test.go index 99ea48bbb..6e0f31e21 100644 --- a/internal/restic/node_test.go +++ b/internal/restic/node_test.go @@ -212,11 +212,7 @@ func TestNodeRestoreAt(t *testing.T) { extAttrArr := test.ExtendedAttributes // Iterate through the array using pointers for i := 0; i < len(extAttrArr); i++ { - // Get the pointer to the current element - namePtr := &extAttrArr[i].Name - - // Modify the value through the pointer - *namePtr = strings.ToUpper(*namePtr) + extAttrArr[i].Name = strings.ToUpper(extAttrArr[i].Name) } } diff --git a/internal/restic/node_windows.go b/internal/restic/node_windows.go index 881c394be..3e1dcae18 100644 --- a/internal/restic/node_windows.go +++ b/internal/restic/node_windows.go @@ -35,12 +35,12 @@ var ( ) // mknod is not supported on Windows. -func mknod(_ string, mode uint32, dev uint64) (err error) { +func mknod(_ string, _ uint32, _ uint64) (err error) { return errors.New("device nodes cannot be created on windows") } // Windows doesn't need lchown -func lchown(_ string, uid int, gid int) (err error) { +func lchown(_ string, _ int, _ int) (err error) { return nil } @@ -72,14 +72,12 @@ func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespe // restore extended attributes for windows func (node Node) restoreExtendedAttributes(path string) (err error) { - eas := []fs.ExtendedAttribute{} - for _, attr := range node.ExtendedAttributes { - extr := new(fs.ExtendedAttribute) - extr.Name = attr.Name - extr.Value = attr.Value - eas = append(eas, *extr) - } - if len(eas) > 0 { + count := len(node.ExtendedAttributes) + if count > 0 { + eas := make([]fs.ExtendedAttribute, count) + for i, attr := range node.ExtendedAttributes { + eas[i] = fs.ExtendedAttribute{Name: attr.Name, Value: attr.Value} + } if errExt := restoreExtendedAttributes(node.Type, path, eas); errExt != nil { return errExt } @@ -90,25 +88,9 @@ func (node Node) restoreExtendedAttributes(path string) (err error) { // fill extended attributes in the node. This also includes the Generic attributes for windows. func (node *Node) fillExtendedAttributes(path string, _ bool) (err error) { var fileHandle windows.Handle - - //Get file handle for file or dir - if node.Type == "file" { - if strings.HasSuffix(filepath.Clean(path), `\`) { - return nil - } - utf16Path := windows.StringToUTF16Ptr(path) - fileAccessRightReadWriteEA := (0x8 | 0x10) - fileHandle, err = windows.CreateFile(utf16Path, uint32(fileAccessRightReadWriteEA), 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0) - } else if node.Type == "dir" { - utf16Path := windows.StringToUTF16Ptr(path) - fileAccessRightReadWriteEA := (0x8 | 0x10) - fileHandle, err = windows.CreateFile(utf16Path, uint32(fileAccessRightReadWriteEA), 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_BACKUP_SEMANTICS, 0) - } else { - return nil - } + fileHandle, err = getFileHandleForEA(node.Type, path) if err != nil { - err = errors.Errorf("open file failed for path: %s, with: %v", path, err) - return err + return errors.Errorf("get EA failed while opening file handle for path %v, with: %v", path, err) } defer func() { err := windows.CloseHandle(fileHandle) @@ -116,23 +98,19 @@ func (node *Node) fillExtendedAttributes(path string, _ bool) (err error) { debug.Log("Error closing file handle for %s: %v\n", path, err) } }() - //Get the windows Extended Attributes using the file handle - extAtts, err := fs.GetFileEA(fileHandle) + var extAtts []fs.ExtendedAttribute + extAtts, err = fs.GetFileEA(fileHandle) debug.Log("fillExtendedAttributes(%v) %v", path, extAtts) if err != nil { - debug.Log("open file failed for path: %s : %v", path, err) - return err - } else if len(extAtts) == 0 { + return errors.Errorf("get EA failed for path %v, with: %v", path, err) + } + if len(extAtts) == 0 { return nil } //Fill the ExtendedAttributes in the node using the name/value pairs in the windows EA for _, attr := range extAtts { - if err != nil { - err = errors.Errorf("can not obtain extended attribute for path %v, attr: %v, err: %v\n,", path, attr, err) - continue - } extendedAttr := ExtendedAttribute{ Name: attr.Name, Value: attr.Value, @@ -143,21 +121,30 @@ func (node *Node) fillExtendedAttributes(path string, _ bool) (err error) { return nil } -// restoreExtendedAttributes handles restore of the Windows Extended Attributes to the specified path. -// The Windows API requires setting of all the Extended Attributes in one call. -func restoreExtendedAttributes(nodeType, path string, eas []fs.ExtendedAttribute) (err error) { - var fileHandle windows.Handle +// Get file handle for file or dir for setting/getting EAs +func getFileHandleForEA(nodeType, path string) (handle windows.Handle, err error) { switch nodeType { case "file": utf16Path := windows.StringToUTF16Ptr(path) fileAccessRightReadWriteEA := (0x8 | 0x10) - fileHandle, err = windows.CreateFile(utf16Path, uint32(fileAccessRightReadWriteEA), 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0) + handle, err = windows.CreateFile(utf16Path, uint32(fileAccessRightReadWriteEA), 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0) case "dir": utf16Path := windows.StringToUTF16Ptr(path) fileAccessRightReadWriteEA := (0x8 | 0x10) - fileHandle, err = windows.CreateFile(utf16Path, uint32(fileAccessRightReadWriteEA), 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_BACKUP_SEMANTICS, 0) + handle, err = windows.CreateFile(utf16Path, uint32(fileAccessRightReadWriteEA), 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_BACKUP_SEMANTICS, 0) default: - return nil + return 0, nil + } + return handle, err +} + +// restoreExtendedAttributes handles restore of the Windows Extended Attributes to the specified path. +// The Windows API requires setting of all the Extended Attributes in one call. +func restoreExtendedAttributes(nodeType, path string, eas []fs.ExtendedAttribute) (err error) { + var fileHandle windows.Handle + fileHandle, err = getFileHandleForEA(nodeType, path) + if err != nil { + return errors.Errorf("set EA failed while opening file handle for path %v, with: %v", path, err) } defer func() { err := windows.CloseHandle(fileHandle) @@ -165,12 +152,10 @@ func restoreExtendedAttributes(nodeType, path string, eas []fs.ExtendedAttribute debug.Log("Error closing file handle for %s: %v\n", path, err) } }() - if err != nil { - err = errors.Errorf("open file failed for path %v, with: %v:\n", path, err) - } else if err = fs.SetFileEA(fileHandle, eas); err != nil { - err = errors.Errorf("set EA failed for path %v, with: %v:\n", path, err) + if err = fs.SetFileEA(fileHandle, eas); err != nil { + return errors.Errorf("set EA failed for path %v, with: %v", path, err) } - return err + return nil } type statT syscall.Win32FileAttributeData diff --git a/internal/restic/node_windows_test.go b/internal/restic/node_windows_test.go index f89e2aeee..29a42e9e2 100644 --- a/internal/restic/node_windows_test.go +++ b/internal/restic/node_windows_test.go @@ -311,23 +311,21 @@ func TestRestoreExtendedAttributes(t *testing.T) { test.OK(t, errors.Wrapf(err, "Error closing file for: %s", testPath)) }() - if len(node.ExtendedAttributes) > 0 { - extAttr, err := fs.GetFileEA(handle) - test.OK(t, errors.Wrapf(err, "Error getting extended attributes for: %s", testPath)) - test.Equals(t, len(node.ExtendedAttributes), len(extAttr)) + extAttr, err := fs.GetFileEA(handle) + test.OK(t, errors.Wrapf(err, "Error getting extended attributes for: %s", testPath)) + test.Equals(t, len(node.ExtendedAttributes), len(extAttr)) - for _, expectedExtAttr := range node.ExtendedAttributes { - var foundExtAttr *fs.ExtendedAttribute - for _, ea := range extAttr { - if strings.EqualFold(ea.Name, expectedExtAttr.Name) { - foundExtAttr = &ea - break + for _, expectedExtAttr := range node.ExtendedAttributes { + var foundExtAttr *fs.ExtendedAttribute + for _, ea := range extAttr { + if strings.EqualFold(ea.Name, expectedExtAttr.Name) { + foundExtAttr = &ea + break - } } - test.Assert(t, foundExtAttr != nil, "Expected extended attribute not found") - test.Equals(t, expectedExtAttr.Value, foundExtAttr.Value) } + test.Assert(t, foundExtAttr != nil, "Expected extended attribute not found") + test.Equals(t, expectedExtAttr.Value, foundExtAttr.Value) } } } From 2101dfe448b739fc284c29718447036edf991179 Mon Sep 17 00:00:00 2001 From: aneesh-n <99904+aneesh-n@users.noreply.github.com> Date: Wed, 5 Jun 2024 22:40:21 -0600 Subject: [PATCH 6/7] Add missing return to fix failing test --- internal/restic/node_windows.go | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/internal/restic/node_windows.go b/internal/restic/node_windows.go index 3e1dcae18..9117c6a10 100644 --- a/internal/restic/node_windows.go +++ b/internal/restic/node_windows.go @@ -88,16 +88,13 @@ func (node Node) restoreExtendedAttributes(path string) (err error) { // fill extended attributes in the node. This also includes the Generic attributes for windows. func (node *Node) fillExtendedAttributes(path string, _ bool) (err error) { var fileHandle windows.Handle - fileHandle, err = getFileHandleForEA(node.Type, path) + if fileHandle, err = getFileHandleForEA(node.Type, path); fileHandle == 0 { + return nil + } if err != nil { return errors.Errorf("get EA failed while opening file handle for path %v, with: %v", path, err) } - defer func() { - err := windows.CloseHandle(fileHandle) - if err != nil { - debug.Log("Error closing file handle for %s: %v\n", path, err) - } - }() + defer closeFileHandle(fileHandle, path) // Replaced inline defer with named function call //Get the windows Extended Attributes using the file handle var extAtts []fs.ExtendedAttribute extAtts, err = fs.GetFileEA(fileHandle) @@ -138,20 +135,26 @@ func getFileHandleForEA(nodeType, path string) (handle windows.Handle, err error return handle, err } +// closeFileHandle safely closes a file handle and logs any errors. +func closeFileHandle(fileHandle windows.Handle, path string) { + err := windows.CloseHandle(fileHandle) + if err != nil { + debug.Log("Error closing file handle for %s: %v\n", path, err) + } +} + // restoreExtendedAttributes handles restore of the Windows Extended Attributes to the specified path. // The Windows API requires setting of all the Extended Attributes in one call. func restoreExtendedAttributes(nodeType, path string, eas []fs.ExtendedAttribute) (err error) { var fileHandle windows.Handle - fileHandle, err = getFileHandleForEA(nodeType, path) + if fileHandle, err = getFileHandleForEA(nodeType, path); fileHandle == 0 { + return nil + } if err != nil { return errors.Errorf("set EA failed while opening file handle for path %v, with: %v", path, err) } - defer func() { - err := windows.CloseHandle(fileHandle) - if err != nil { - debug.Log("Error closing file handle for %s: %v\n", path, err) - } - }() + defer closeFileHandle(fileHandle, path) // Replaced inline defer with named function call + if err = fs.SetFileEA(fileHandle, eas); err != nil { return errors.Errorf("set EA failed for path %v, with: %v", path, err) } From 4f053da06a8c69767e3783e1affe2849836e4777 Mon Sep 17 00:00:00 2001 From: aneesh-n <99904+aneesh-n@users.noreply.github.com> Date: Wed, 5 Jun 2024 23:10:29 -0600 Subject: [PATCH 7/7] Fix test case by correcting cleanup order --- internal/fs/ea_windows_test.go | 87 +++++++++++++++++----------------- 1 file changed, 43 insertions(+), 44 deletions(-) diff --git a/internal/fs/ea_windows_test.go b/internal/fs/ea_windows_test.go index 14b7a2452..b249f43c4 100644 --- a/internal/fs/ea_windows_test.go +++ b/internal/fs/ea_windows_test.go @@ -138,36 +138,34 @@ func TestSetFileEa(t *testing.T) { // The code below was refactored from github.com/Microsoft/go-winio/blob/a7564fd482feb903f9562a135f1317fd3b480739/ea_test.go // under MIT license. func TestSetGetFileEA(t *testing.T) { - testfilePath := setupTestFile(t) - defer cleanupTestFile(t, testfilePath) + testFilePath, testFile := setupTestFile(t) + testEAs := generateTestEAs(t, 3, testFilePath) + fileHandle := openFile(t, testFilePath, windows.FILE_ATTRIBUTE_NORMAL) + defer closeFileHandle(t, testFilePath, testFile, fileHandle) - testEAs := generateTestEAs(t, 3, testfilePath) - fileHandle := openFile(t, testfilePath, windows.FILE_ATTRIBUTE_NORMAL) - defer closeFileHandle(t, fileHandle, testfilePath) - - testSetGetEA(t, fileHandle, testEAs, testfilePath) + testSetGetEA(t, testFilePath, fileHandle, testEAs) } // The code is new code and reuses code refactored from github.com/Microsoft/go-winio/blob/a7564fd482feb903f9562a135f1317fd3b480739/ea_test.go // under MIT license. func TestSetGetFolderEA(t *testing.T) { - testfolderPath := setupTestFolder(t) - defer cleanupTestFolder(t, testfolderPath) + testFolderPath := setupTestFolder(t) - testEAs := generateTestEAs(t, 3, testfolderPath) - fileHandle := openFile(t, testfolderPath, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_BACKUP_SEMANTICS) - defer closeFileHandle(t, fileHandle, testfolderPath) + testEAs := generateTestEAs(t, 3, testFolderPath) + fileHandle := openFile(t, testFolderPath, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_BACKUP_SEMANTICS) + defer closeFileHandle(t, testFolderPath, nil, fileHandle) - testSetGetEA(t, fileHandle, testEAs, testfolderPath) + testSetGetEA(t, testFolderPath, fileHandle, testEAs) } -func setupTestFile(t *testing.T) string { +func setupTestFile(t *testing.T) (testFilePath string, testFile *os.File) { tempDir := t.TempDir() - testfilePath := filepath.Join(tempDir, "testfile.txt") - if _, err := os.Create(testfilePath); err != nil { + testFilePath = filepath.Join(tempDir, "testfile.txt") + var err error + if testFile, err = os.Create(testFilePath); err != nil { t.Fatalf("failed to create temporary file: %s", err) } - return testfilePath + return testFilePath, testFile } func setupTestFolder(t *testing.T) string { @@ -191,6 +189,18 @@ func generateTestEAs(t *testing.T, nAttrs int, path string) []ExtendedAttribute return testEAs } +func getRandomInt() int64 { + nBig, err := rand.Int(rand.Reader, big.NewInt(27)) + if err != nil { + panic(err) + } + n := nBig.Int64() + if n == 0 { + n = getRandomInt() + } + return n +} + func openFile(t *testing.T, path string, attributes uint32) windows.Handle { utf16Path := windows.StringToUTF16Ptr(path) fileAccessRightReadWriteEA := uint32(0x8 | 0x10) @@ -201,13 +211,26 @@ func openFile(t *testing.T, path string, attributes uint32) windows.Handle { return fileHandle } -func closeFileHandle(t *testing.T, handle windows.Handle, path string) { +func closeFileHandle(t *testing.T, testfilePath string, testFile *os.File, handle windows.Handle) { + if testFile != nil { + err := testFile.Close() + if err != nil { + t.Logf("Error closing file %s: %v\n", testFile.Name(), err) + } + } if err := windows.Close(handle); err != nil { - t.Logf("Error closing file handle %s: %v\n", path, err) + t.Logf("Error closing file handle %s: %v\n", testfilePath, err) + } + cleanupTestFile(t, testfilePath) +} + +func cleanupTestFile(t *testing.T, path string) { + if err := os.Remove(path); err != nil { + t.Logf("Error removing file/folder %s: %v\n", path, err) } } -func testSetGetEA(t *testing.T, handle windows.Handle, testEAs []ExtendedAttribute, path string) { +func testSetGetEA(t *testing.T, path string, handle windows.Handle, testEAs []ExtendedAttribute) { if err := SetFileEA(handle, testEAs); err != nil { t.Fatalf("set EA for path %s failed: %s", path, err) } @@ -222,27 +245,3 @@ func testSetGetEA(t *testing.T, handle windows.Handle, testEAs []ExtendedAttribu t.Fatalf("EAs read from path %s don't match", path) } } - -func cleanupTestFile(t *testing.T, path string) { - if err := os.Remove(path); err != nil { - t.Logf("Error removing file %s: %v\n", path, err) - } -} - -func cleanupTestFolder(t *testing.T, path string) { - if err := os.Remove(path); err != nil { - t.Logf("Error removing folder %s: %v\n", path, err) - } -} - -func getRandomInt() int64 { - nBig, err := rand.Int(rand.Reader, big.NewInt(27)) - if err != nil { - panic(err) - } - n := nBig.Int64() - if n == 0 { - n = getRandomInt() - } - return n -}