From b1f60357fad075e78edcb90201195c2d0b602d23 Mon Sep 17 00:00:00 2001 From: Florian Maury Date: Mon, 10 Jan 2022 09:07:36 +0100 Subject: [PATCH] initial --- README.md | 2 +- decrypt/decrypt.go | 100 ++++++++++++++++++++++++++++++++++++ decrypt/decrypt_test.go | 45 ++++++++++++++++ encrypt/encrypt.go | 102 ++++++++++++++++++++++++++++++++++++ encrypt/encrypt_test.go | 44 ++++++++++++++++ go.mod | 9 ++++ go.sum | 15 ++++++ utils/utils.go | 111 ++++++++++++++++++++++++++++++++++++++++ 8 files changed, 427 insertions(+), 1 deletion(-) create mode 100644 decrypt/decrypt.go create mode 100644 decrypt/decrypt_test.go create mode 100644 encrypt/encrypt.go create mode 100644 encrypt/encrypt_test.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 utils/utils.go diff --git a/README.md b/README.md index 53e1832..79591eb 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ -# large-file-decrypt +# Large File Decrypt Example code for hopefully secure large file decryption diff --git a/decrypt/decrypt.go b/decrypt/decrypt.go new file mode 100644 index 0000000..fc739cc --- /dev/null +++ b/decrypt/decrypt.go @@ -0,0 +1,100 @@ +package decrypt + +import ( + "errors" + "fmt" + "path/filepath" + "syscall" + + "github.com/X-Cli/large-file-decrypt/utils" + "github.com/hashicorp/go-multierror" + "golang.org/x/crypto/nacl/box" + "golang.org/x/sys/unix" +) + +func DecryptFile(inputPath, outputPath string, pubKey, privKey *[32]byte) error { + return decryptFile(inputPath, outputPath, pubKey, privKey).ErrorOrNil() +} + +func decryptFile(inputPath, outputPath string, pubKey, privKey *[32]byte) (errStack *multierror.Error) { + inputTempDir := filepath.Dir(inputPath) + privateEncryptedFile, errs := utils.CreatePrivateCopyOf(inputPath, inputTempDir) + if errs.ErrorOrNil() != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to create the private copy of the encrypted file: %w", errs)) + return + } + privateEncryptedFileClosed := false + defer func() { + if !privateEncryptedFileClosed { + if err := privateEncryptedFile.Close(); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to close private copy of encrypted file: %w", err)) + } + } + }() + + encryptedData, err := utils.GetDataFromFile(privateEncryptedFile) + if err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to get encrypted data: %w", err)) + return + } + privateEncryptedDataFreed := false + defer func() { + if !privateEncryptedDataFreed { + if err := syscall.Munmap(encryptedData); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to munmap encrypted data: %w", err)) + } + } + }() + + decryptedSize := len(encryptedData) - box.AnonymousOverhead + outputTempDir := filepath.Dir(outputPath) + privateDecryptedFile, err := utils.CreatePrivateFile(outputTempDir, int64(decryptedSize)) + if err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to create private decrypted file: %w", err)) + return + } + defer func() { + if err := privateDecryptedFile.Close(); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to close private decrypted file: %w", err)) + } + }() + + decryptedData, err := utils.GetDataFromFile(privateDecryptedFile) + if err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to mmap decrypted file: %w", err)) + return + } + defer func() { + if err := syscall.Munmap(decryptedData); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to munmap decrypted data: %w", err)) + } + }() + + if _, ok := box.OpenAnonymous(decryptedData[:0], encryptedData, pubKey, privKey); !ok { + errStack = multierror.Append(errStack, errors.New("failed to decrypt file")) + return + } + + if err := unix.Msync(decryptedData, unix.MS_SYNC); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to flush changes to disk: %w", err)) + return + } + + // Since in the worst case, we can only copy the private decrypted file to the public file, we release the private encrypted file now, so that the algorithm only uses a maximum of 3 times the size of the encrypted file + privateEncryptedDataFreed = true + if err := syscall.Munmap(encryptedData); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to munmap encrypted data: %w", err)) + return + } + privateEncryptedFileClosed = true + if err := privateEncryptedFile.Close(); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to close private copy of encrypted file: %w", err)) + return + } + + if err := utils.PublishFile(privateDecryptedFile, outputPath).ErrorOrNil(); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to publish decrypted file: %w", err)) + return + } + return +} diff --git a/decrypt/decrypt_test.go b/decrypt/decrypt_test.go new file mode 100644 index 0000000..fb6b581 --- /dev/null +++ b/decrypt/decrypt_test.go @@ -0,0 +1,45 @@ +package decrypt + +import ( + "bytes" + "crypto/rand" + "io/ioutil" + "path" + "testing" + + "golang.org/x/crypto/nacl/box" +) + +func TestDecrypt1(t *testing.T) { + tmpdir := t.TempDir() + + pubKey, privKey, err := box.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("failed to generate keys: %v", err) + } + + expectedContent := []byte("Geronimo!") + encryptedContent, err := box.SealAnonymous(nil, expectedContent, pubKey, rand.Reader) + if err != nil { + t.Fatalf("failed to encrypt message: %v", err) + } + + encryptedFilePath := path.Join(tmpdir, "encryptedFile") + if err := ioutil.WriteFile(encryptedFilePath, encryptedContent, 0o600); err != nil { + t.Fatalf("failed to initialize encrypted file: %v", err) + } + + decryptedFilePath := path.Join(tmpdir, "decryptedFile") + if err := DecryptFile(encryptedFilePath, decryptedFilePath, pubKey, privKey); err != nil { + t.Fatalf("failed to decrypt file: %v", err) + } + + decryptedContent, err := ioutil.ReadFile(decryptedFilePath) + if err != nil { + t.Fatalf("failed to read decrypted file: %v", err) + } + + if !bytes.Equal(expectedContent, decryptedContent) { + t.Fatalf("mismatched content: expected %v, found %v", expectedContent, decryptedContent) + } +} diff --git a/encrypt/encrypt.go b/encrypt/encrypt.go new file mode 100644 index 0000000..149acbd --- /dev/null +++ b/encrypt/encrypt.go @@ -0,0 +1,102 @@ +package encrypt + +import ( + "crypto/rand" + "fmt" + "io" + "path/filepath" + "syscall" + + "github.com/X-Cli/large-file-decrypt/utils" + "github.com/hashicorp/go-multierror" + "golang.org/x/crypto/nacl/box" + "golang.org/x/sys/unix" +) + +func EncryptFile(inputPath, outputPath string, pubKey *[32]byte) error { + return encryptFile(inputPath, outputPath, pubKey, rand.Reader).ErrorOrNil() +} + +func encryptFile(inputPath, outputPath string, pubKey *[32]byte, cryptoReader io.Reader) (errStack *multierror.Error) { + inputTempDir := filepath.Dir(inputPath) + + privateClearFile, errs := utils.CreatePrivateCopyOf(inputPath, inputTempDir) + if errs.ErrorOrNil() != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to acquire a private copy of %q: %w", inputPath, errs)) + return + } + privateClearFileClosed := false + defer func() { + if !privateClearFileClosed { + if err := privateClearFile.Close(); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to close private copy of %q: %w", inputPath, err)) + } + } + }() + + clearData, err := utils.GetDataFromFile(privateClearFile) + if err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to get data from private copy: %w", err)) + return + } + clearDataFreed := false + defer func() { + if !clearDataFreed { + if err := syscall.Munmap(clearData); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to munmap clear data: %w", err)) + } + } + }() + + outputTempDir := filepath.Dir(outputPath) + encryptedSize := len(clearData) + box.AnonymousOverhead + privateEncryptedFile, err := utils.CreatePrivateFile(outputTempDir, int64(encryptedSize)) + if err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to create private encrypted file: %w", err)) + return + } + defer func() { + if err := privateEncryptedFile.Close(); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to close private encrypted file: %w", err)) + } + }() + + encryptedData, err := utils.GetDataFromFile(privateEncryptedFile) + if err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to get data from encrypted file: %w", err)) + return + } + defer func() { + if err := syscall.Munmap(encryptedData); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to release encrypted data: %w", err)) + } + }() + + if _, err := box.SealAnonymous(encryptedData[:0], clearData, pubKey, cryptoReader); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to seal data: %w", err)) + return + } + if err := unix.Msync(encryptedData, unix.MS_SYNC); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to flush encrypted data on disk: %w", err)) + return + } + + // Releasing private clear text resources since they are no longer needed and they may occupy resources if file clone was not possible + clearDataFreed = true + if err := syscall.Munmap(clearData); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to get data from private copy: %w", err)) + return + } + privateClearFileClosed = true + if err := privateClearFile.Close(); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to close private copy of %q: %w", inputPath, err)) + return + } + + if err := utils.PublishFile(privateEncryptedFile, outputPath); err.ErrorOrNil() != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to publish encrypted file: %w", err)) + return + } + return + +} diff --git a/encrypt/encrypt_test.go b/encrypt/encrypt_test.go new file mode 100644 index 0000000..aea3758 --- /dev/null +++ b/encrypt/encrypt_test.go @@ -0,0 +1,44 @@ +package encrypt + +import ( + "bytes" + "io/ioutil" + "path" + "testing" + + "golang.org/x/crypto/nacl/box" +) + +func TestEncrypt(t *testing.T) { + tmpdir := t.TempDir() + + var constantRandom [512]byte + randReader := bytes.NewBuffer(constantRandom[:]) + + clearFile := path.Join(tmpdir, "clear") + encryptedFile := path.Join(tmpdir, "encrypted") + + clearText := []byte("Geronimo!") + if err := ioutil.WriteFile(clearFile, clearText, 0o600); err != nil { + t.Fatalf("failed to write cleartext file: %v", err) + } + + pubKey, _, err := box.GenerateKey(randReader) + if err != nil { + t.Fatalf("failed to generate keys: %v", err) + } + + if err := encryptFile(clearFile, encryptedFile, pubKey, randReader); err != nil { + t.Fatalf("failed to encrypt file: %v", err) + } + + cipherText, err := ioutil.ReadFile(encryptedFile) + if err != nil { + t.Fatalf("failed to read encrypted file: %v", err) + } + + expectedCiphertext := []byte{47, 229, 125, 163, 71, 205, 98, 67, 21, 40, 218, 172, 95, 187, 41, 7, 48, 255, 246, 132, 175, 196, 207, 194, 237, 144, 153, 95, 88, 203, 59, 116, 83, 218, 223, 164, 117, 110, 80, 197, 138, 128, 169, 16, 149, 74, 20, 245, 10, 169, 53, 61, 119, 218, 240, 50, 92} + if !bytes.Equal(expectedCiphertext, cipherText) { + t.Fatalf("unexpected ciphertext: expected %v; found %v", expectedCiphertext, cipherText) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..885ca31 --- /dev/null +++ b/go.mod @@ -0,0 +1,9 @@ +module github.com/X-Cli/large-file-decrypt + +go 1.15 + +require ( + github.com/hashicorp/go-multierror v1.1.1 + golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 + golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..834c46c --- /dev/null +++ b/go.sum @@ -0,0 +1,15 @@ +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 h1:0es+/5331RGQPcXlMfP+WrnIIS6dNnNRe0WB02W0F4M= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/utils/utils.go b/utils/utils.go new file mode 100644 index 0000000..05bf6c9 --- /dev/null +++ b/utils/utils.go @@ -0,0 +1,111 @@ +package utils + +import ( + "fmt" + "io" + "os" + "syscall" + + "github.com/hashicorp/go-multierror" + "golang.org/x/sys/unix" +) + +func CreatePrivateCopyOf(inputPath, tempDir string) (privateFile *os.File, errStack *multierror.Error) { + inputFile, err := os.Open(inputPath) + if err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to open file %q: %w", inputPath, err)) + return + } + defer func() { + if err := inputFile.Close(); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to close file %q: %w", inputPath, err)) + } + }() + + privateFileFd, err := unix.Open(tempDir, unix.O_RDWR|unix.O_TMPFILE, 0o600) + if err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to open private file: %w", err)) + return + } + + privateFile = os.NewFile(uintptr(privateFileFd), "") + defer func() { + if errStack.ErrorOrNil() != nil { + if err := privateFile.Close(); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to close private copy: %w", err)) + } + } + }() + + if err := unix.IoctlFileClone(int(privateFile.Fd()), int(inputFile.Fd())); err != nil { + if err != syscall.EOPNOTSUPP && err != syscall.EINVAL { + errStack = multierror.Append(errStack, fmt.Errorf("failed to clone file: %w", err)) + return + } + if _, err := io.Copy(privateFile, inputFile); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to copy file: %w", err)) + return + } + } + return +} + +func GetDataFromFile(privateEncryptedFile *os.File) (data []byte, err error) { + fs, err := privateEncryptedFile.Stat() + if err != nil { + return nil, fmt.Errorf("failed to stat(2) private copy of encrypted file: %w", err) + } + data, err = syscall.Mmap(int(privateEncryptedFile.Fd()), 0, int(fs.Size()), syscall.PROT_WRITE|syscall.PROT_READ, syscall.MAP_SHARED) + if err != nil { + return nil, fmt.Errorf("failed to mmap(2) private copy of encrypted file: %w", err) + } + return data, nil +} + +func CreatePrivateFile(tempDir string, size int64) (*os.File, error) { + privateFileFd, err := unix.Open(tempDir, unix.O_RDWR|unix.O_TMPFILE, 0o600) + if err != nil { + return nil, fmt.Errorf("failed to create private file: %w", err) + } + privateFile := os.NewFile(uintptr(privateFileFd), "") + + if err := syscall.Fallocate(int(privateFile.Fd()), 0, 0, size); err != nil { + var errStack error = err + if err := privateFile.Close(); err != nil { + errStack = multierror.Append(errStack, err) + } + return nil, errStack + } + return privateFile, nil +} + +func PublishFile(privateDecryptedFile *os.File, outputPath string) (errStack *multierror.Error) { + if err := unix.Linkat(int(privateDecryptedFile.Fd()), "", 0, outputPath, unix.AT_EMPTY_PATH); err == nil { + return + } else if err != syscall.ENOENT { + // ENOENT is returned if CAP_DAC_READ_SEARCH is not effective + errStack = multierror.Append(errStack, fmt.Errorf("failed to call linkat: %w", err)) + return + } + outputFile, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE, 0o600) + if err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to create output file: %w", err)) + return + } + defer func() { + if err := outputFile.Close(); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to close file %q: %w", outputPath, err)) + } + }() + if err := unix.IoctlFileClone(int(outputFile.Fd()), int(privateDecryptedFile.Fd())); err == nil { + return + } else if err != syscall.EOPNOTSUPP && err != syscall.EINVAL { + errStack = multierror.Append(errStack, fmt.Errorf("failed to clone file %w", err)) + return + } + if _, err := io.Copy(outputFile, privateDecryptedFile); err != nil { + errStack = multierror.Append(errStack, fmt.Errorf("failed to copy into public decrypted file: %w", err)) + return + } + return +}