initial

README.md

@@ -1,2 +1,2 @@
-# large-file-decrypt
+# Large File Decrypt
 Example code for hopefully secure large file decryption

decrypt/decrypt.go

@@ -0,0 +1,100 @@
+package decrypt
+
+import (
+	"errors"
+	"fmt"
+	"path/filepath"
+	"syscall"
+
+	"github.com/X-Cli/large-file-decrypt/utils"
+	"github.com/hashicorp/go-multierror"
+	"golang.org/x/crypto/nacl/box"
+	"golang.org/x/sys/unix"
+)
+
+func DecryptFile(inputPath, outputPath string, pubKey, privKey *[32]byte) error {
+	return decryptFile(inputPath, outputPath, pubKey, privKey).ErrorOrNil()
+}
+
+func decryptFile(inputPath, outputPath string, pubKey, privKey *[32]byte) (errStack *multierror.Error) {
+	inputTempDir := filepath.Dir(inputPath)
+	privateEncryptedFile, errs := utils.CreatePrivateCopyOf(inputPath, inputTempDir)
+	if errs.ErrorOrNil() != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to create the private copy of the encrypted file: %w", errs))
+		return
+	}
+	privateEncryptedFileClosed := false
+	defer func() {
+		if !privateEncryptedFileClosed {
+			if err := privateEncryptedFile.Close(); err != nil {
+				errStack = multierror.Append(errStack, fmt.Errorf("failed to close private copy of encrypted file: %w", err))
+			}
+		}
+	}()
+
+	encryptedData, err := utils.GetDataFromFile(privateEncryptedFile)
+	if err != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to get encrypted data: %w", err))
+		return
+	}
+	privateEncryptedDataFreed := false
+	defer func() {
+		if !privateEncryptedDataFreed {
+			if err := syscall.Munmap(encryptedData); err != nil {
+				errStack = multierror.Append(errStack, fmt.Errorf("failed to munmap encrypted data: %w", err))
+			}
+		}
+	}()
+
+	decryptedSize := len(encryptedData) - box.AnonymousOverhead
+	outputTempDir := filepath.Dir(outputPath)
+	privateDecryptedFile, err := utils.CreatePrivateFile(outputTempDir, int64(decryptedSize))
+	if err != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to create private decrypted file: %w", err))
+		return
+	}
+	defer func() {
+		if err := privateDecryptedFile.Close(); err != nil {
+			errStack = multierror.Append(errStack, fmt.Errorf("failed to close private decrypted file: %w", err))
+		}
+	}()
+
+	decryptedData, err := utils.GetDataFromFile(privateDecryptedFile)
+	if err != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to mmap decrypted file: %w", err))
+		return
+	}
+	defer func() {
+		if err := syscall.Munmap(decryptedData); err != nil {
+			errStack = multierror.Append(errStack, fmt.Errorf("failed to munmap decrypted data: %w", err))
+		}
+	}()
+
+	if _, ok := box.OpenAnonymous(decryptedData[:0], encryptedData, pubKey, privKey); !ok {
+		errStack = multierror.Append(errStack, errors.New("failed to decrypt file"))
+		return
+	}
+
+	if err := unix.Msync(decryptedData, unix.MS_SYNC); err != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to flush changes to disk: %w", err))
+		return
+	}
+
+	// Since in the worst case, we can only copy the private decrypted file to the public file, we release the private encrypted file now, so that the algorithm only uses a maximum of 3 times the size of the encrypted file
+	privateEncryptedDataFreed = true
+	if err := syscall.Munmap(encryptedData); err != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to munmap encrypted data: %w", err))
+		return
+	}
+	privateEncryptedFileClosed = true
+	if err := privateEncryptedFile.Close(); err != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to close private copy of encrypted file: %w", err))
+		return
+	}
+
+	if err := utils.PublishFile(privateDecryptedFile, outputPath).ErrorOrNil(); err != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to publish decrypted file: %w", err))
+		return
+	}
+	return
+}

decrypt/decrypt_test.go

@@ -0,0 +1,45 @@
+package decrypt
+
+import (
+	"bytes"
+	"crypto/rand"
+	"io/ioutil"
+	"path"
+	"testing"
+
+	"golang.org/x/crypto/nacl/box"
+)
+
+func TestDecrypt1(t *testing.T) {
+	tmpdir := t.TempDir()
+
+	pubKey, privKey, err := box.GenerateKey(rand.Reader)
+	if err != nil {
+		t.Fatalf("failed to generate keys: %v", err)
+	}
+
+	expectedContent := []byte("Geronimo!")
+	encryptedContent, err := box.SealAnonymous(nil, expectedContent, pubKey, rand.Reader)
+	if err != nil {
+		t.Fatalf("failed to encrypt message: %v", err)
+	}
+
+	encryptedFilePath := path.Join(tmpdir, "encryptedFile")
+	if err := ioutil.WriteFile(encryptedFilePath, encryptedContent, 0o600); err != nil {
+		t.Fatalf("failed to initialize encrypted file: %v", err)
+	}
+
+	decryptedFilePath := path.Join(tmpdir, "decryptedFile")
+	if err := DecryptFile(encryptedFilePath, decryptedFilePath, pubKey, privKey); err != nil {
+		t.Fatalf("failed to decrypt file: %v", err)
+	}
+
+	decryptedContent, err := ioutil.ReadFile(decryptedFilePath)
+	if err != nil {
+		t.Fatalf("failed to read decrypted file: %v", err)
+	}
+
+	if !bytes.Equal(expectedContent, decryptedContent) {
+		t.Fatalf("mismatched content: expected %v, found %v", expectedContent, decryptedContent)
+	}
+}

encrypt/encrypt.go

@@ -0,0 +1,102 @@
+package encrypt
+
+import (
+	"crypto/rand"
+	"fmt"
+	"io"
+	"path/filepath"
+	"syscall"
+
+	"github.com/X-Cli/large-file-decrypt/utils"
+	"github.com/hashicorp/go-multierror"
+	"golang.org/x/crypto/nacl/box"
+	"golang.org/x/sys/unix"
+)
+
+func EncryptFile(inputPath, outputPath string, pubKey *[32]byte) error {
+	return encryptFile(inputPath, outputPath, pubKey, rand.Reader).ErrorOrNil()
+}
+
+func encryptFile(inputPath, outputPath string, pubKey *[32]byte, cryptoReader io.Reader) (errStack *multierror.Error) {
+	inputTempDir := filepath.Dir(inputPath)
+
+	privateClearFile, errs := utils.CreatePrivateCopyOf(inputPath, inputTempDir)
+	if errs.ErrorOrNil() != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to acquire a private copy of %q: %w", inputPath, errs))
+		return
+	}
+	privateClearFileClosed := false
+	defer func() {
+		if !privateClearFileClosed {
+			if err := privateClearFile.Close(); err != nil {
+				errStack = multierror.Append(errStack, fmt.Errorf("failed to close private copy of %q: %w", inputPath, err))
+			}
+		}
+	}()
+
+	clearData, err := utils.GetDataFromFile(privateClearFile)
+	if err != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to get data from private copy: %w", err))
+		return
+	}
+	clearDataFreed := false
+	defer func() {
+		if !clearDataFreed {
+			if err := syscall.Munmap(clearData); err != nil {
+				errStack = multierror.Append(errStack, fmt.Errorf("failed to munmap clear data: %w", err))
+			}
+		}
+	}()
+
+	outputTempDir := filepath.Dir(outputPath)
+	encryptedSize := len(clearData) + box.AnonymousOverhead
+	privateEncryptedFile, err := utils.CreatePrivateFile(outputTempDir, int64(encryptedSize))
+	if err != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to create private encrypted file: %w", err))
+		return
+	}
+	defer func() {
+		if err := privateEncryptedFile.Close(); err != nil {
+			errStack = multierror.Append(errStack, fmt.Errorf("failed to close private encrypted file: %w", err))
+		}
+	}()
+
+	encryptedData, err := utils.GetDataFromFile(privateEncryptedFile)
+	if err != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to get data from encrypted file: %w", err))
+		return
+	}
+	defer func() {
+		if err := syscall.Munmap(encryptedData); err != nil {
+			errStack = multierror.Append(errStack, fmt.Errorf("failed to release encrypted data: %w", err))
+		}
+	}()
+
+	if _, err := box.SealAnonymous(encryptedData[:0], clearData, pubKey, cryptoReader); err != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to seal data: %w", err))
+		return
+	}
+	if err := unix.Msync(encryptedData, unix.MS_SYNC); err != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to flush encrypted data on disk: %w", err))
+		return
+	}
+
+	// Releasing private clear text resources since they are no longer needed and they may occupy resources if file clone was not possible
+	clearDataFreed = true
+	if err := syscall.Munmap(clearData); err != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to get data from private copy: %w", err))
+		return
+	}
+	privateClearFileClosed = true
+	if err := privateClearFile.Close(); err != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to close private copy of %q: %w", inputPath, err))
+		return
+	}
+
+	if err := utils.PublishFile(privateEncryptedFile, outputPath); err.ErrorOrNil() != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to publish encrypted file: %w", err))
+		return
+	}
+	return
+
+}

encrypt/encrypt_test.go

@@ -0,0 +1,44 @@
+package encrypt
+
+import (
+	"bytes"
+	"io/ioutil"
+	"path"
+	"testing"
+
+	"golang.org/x/crypto/nacl/box"
+)
+
+func TestEncrypt(t *testing.T) {
+	tmpdir := t.TempDir()
+
+	var constantRandom [512]byte
+	randReader := bytes.NewBuffer(constantRandom[:])
+
+	clearFile := path.Join(tmpdir, "clear")
+	encryptedFile := path.Join(tmpdir, "encrypted")
+
+	clearText := []byte("Geronimo!")
+	if err := ioutil.WriteFile(clearFile, clearText, 0o600); err != nil {
+		t.Fatalf("failed to write cleartext file: %v", err)
+	}
+
+	pubKey, _, err := box.GenerateKey(randReader)
+	if err != nil {
+		t.Fatalf("failed to generate keys: %v", err)
+	}
+
+	if err := encryptFile(clearFile, encryptedFile, pubKey, randReader); err != nil {
+		t.Fatalf("failed to encrypt file: %v", err)
+	}
+
+	cipherText, err := ioutil.ReadFile(encryptedFile)
+	if err != nil {
+		t.Fatalf("failed to read encrypted file: %v", err)
+	}
+
+	expectedCiphertext := []byte{47, 229, 125, 163, 71, 205, 98, 67, 21, 40, 218, 172, 95, 187, 41, 7, 48, 255, 246, 132, 175, 196, 207, 194, 237, 144, 153, 95, 88, 203, 59, 116, 83, 218, 223, 164, 117, 110, 80, 197, 138, 128, 169, 16, 149, 74, 20, 245, 10, 169, 53, 61, 119, 218, 240, 50, 92}
+	if !bytes.Equal(expectedCiphertext, cipherText) {
+		t.Fatalf("unexpected ciphertext: expected %v; found %v", expectedCiphertext, cipherText)
+	}
+}

go.mod

@@ -0,0 +1,9 @@
+module github.com/X-Cli/large-file-decrypt
+
+go 1.15
+
+require (
+	github.com/hashicorp/go-multierror v1.1.1
+	golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3
+	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
+)

go.sum

@@ -0,0 +1,15 @@
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 h1:0es+/5331RGQPcXlMfP+WrnIIS6dNnNRe0WB02W0F4M=
+golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

utils/utils.go

@@ -0,0 +1,111 @@
+package utils
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"syscall"
+
+	"github.com/hashicorp/go-multierror"
+	"golang.org/x/sys/unix"
+)
+
+func CreatePrivateCopyOf(inputPath, tempDir string) (privateFile *os.File, errStack *multierror.Error) {
+	inputFile, err := os.Open(inputPath)
+	if err != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to open file %q: %w", inputPath, err))
+		return
+	}
+	defer func() {
+		if err := inputFile.Close(); err != nil {
+			errStack = multierror.Append(errStack, fmt.Errorf("failed to close file %q: %w", inputPath, err))
+		}
+	}()
+
+	privateFileFd, err := unix.Open(tempDir, unix.O_RDWR|unix.O_TMPFILE, 0o600)
+	if err != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to open private file: %w", err))
+		return
+	}
+
+	privateFile = os.NewFile(uintptr(privateFileFd), "")
+	defer func() {
+		if errStack.ErrorOrNil() != nil {
+			if err := privateFile.Close(); err != nil {
+				errStack = multierror.Append(errStack, fmt.Errorf("failed to close private copy: %w", err))
+			}
+		}
+	}()
+
+	if err := unix.IoctlFileClone(int(privateFile.Fd()), int(inputFile.Fd())); err != nil {
+		if err != syscall.EOPNOTSUPP && err != syscall.EINVAL {
+			errStack = multierror.Append(errStack, fmt.Errorf("failed to clone file: %w", err))
+			return
+		}
+		if _, err := io.Copy(privateFile, inputFile); err != nil {
+			errStack = multierror.Append(errStack, fmt.Errorf("failed to copy file: %w", err))
+			return
+		}
+	}
+	return
+}
+
+func GetDataFromFile(privateEncryptedFile *os.File) (data []byte, err error) {
+	fs, err := privateEncryptedFile.Stat()
+	if err != nil {
+		return nil, fmt.Errorf("failed to stat(2) private copy of encrypted file: %w", err)
+	}
+	data, err = syscall.Mmap(int(privateEncryptedFile.Fd()), 0, int(fs.Size()), syscall.PROT_WRITE|syscall.PROT_READ, syscall.MAP_SHARED)
+	if err != nil {
+		return nil, fmt.Errorf("failed to mmap(2) private copy of encrypted file: %w", err)
+	}
+	return data, nil
+}
+
+func CreatePrivateFile(tempDir string, size int64) (*os.File, error) {
+	privateFileFd, err := unix.Open(tempDir, unix.O_RDWR|unix.O_TMPFILE, 0o600)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create private file: %w", err)
+	}
+	privateFile := os.NewFile(uintptr(privateFileFd), "")
+
+	if err := syscall.Fallocate(int(privateFile.Fd()), 0, 0, size); err != nil {
+		var errStack error = err
+		if err := privateFile.Close(); err != nil {
+			errStack = multierror.Append(errStack, err)
+		}
+		return nil, errStack
+	}
+	return privateFile, nil
+}
+
+func PublishFile(privateDecryptedFile *os.File, outputPath string) (errStack *multierror.Error) {
+	if err := unix.Linkat(int(privateDecryptedFile.Fd()), "", 0, outputPath, unix.AT_EMPTY_PATH); err == nil {
+		return
+	} else if err != syscall.ENOENT {
+		// ENOENT is returned if CAP_DAC_READ_SEARCH is not effective
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to call linkat: %w", err))
+		return
+	}
+	outputFile, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE, 0o600)
+	if err != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to create output file: %w", err))
+		return
+	}
+	defer func() {
+		if err := outputFile.Close(); err != nil {
+			errStack = multierror.Append(errStack, fmt.Errorf("failed to close file %q: %w", outputPath, err))
+		}
+	}()
+	if err := unix.IoctlFileClone(int(outputFile.Fd()), int(privateDecryptedFile.Fd())); err == nil {
+		return
+	} else if err != syscall.EOPNOTSUPP && err != syscall.EINVAL {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to clone file %w", err))
+		return
+	}
+	if _, err := io.Copy(outputFile, privateDecryptedFile); err != nil {
+		errStack = multierror.Append(errStack, fmt.Errorf("failed to copy into public decrypted file: %w", err))
+		return
+	}
+	return
+}