standalone client-side remote state encryption

internal/states/remote/state.go

@@ -3,10 +3,12 @@ package remote
 import (
 	"bytes"
 	"fmt"
+	"log"
 	"sync"
 
 	uuid "github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/terraform/internal/states"
+	"github.com/hashicorp/terraform/internal/states/statecrypto"
 	"github.com/hashicorp/terraform/internal/states/statefile"
 	"github.com/hashicorp/terraform/internal/states/statemgr"
 )
@@ -121,7 +123,13 @@ func (s *State) refreshState() error {
 		return nil
 	}
 
-	stateFile, err := statefile.Read(bytes.NewReader(payload.Data))
+	decrypted, err := statecrypto.StateCryptoWrapper().Decrypt(payload.Data)
+	if err != nil {
+		log.Printf("[ERROR] remote state decryption failed: %s", err.Error())
+		return err
+	}
+
+	stateFile, err := statefile.Read(bytes.NewReader(decrypted))
 	if err != nil {
 		return err
 	}
@@ -178,7 +186,13 @@ func (s *State) PersistState() error {
 		return err
 	}
 
-	err = s.Client.Put(buf.Bytes())
+	maybeEncrypted, err := statecrypto.StateCryptoWrapper().Encrypt(buf.Bytes())
+	if err != nil {
+		log.Printf("[ERROR] remote state encryption failed: %s", err.Error())
+		return err
+	}
+
+	err = s.Client.Put(maybeEncrypted)
 	if err != nil {
 		return err
 	}

internal/states/statecrypto/cryptoconfig/cryptoconfig.go

@@ -0,0 +1,76 @@
+package cryptoconfig
+
+import (
+	"bytes"
+	"encoding/json"
+	"log"
+	"os"
+)
+
+const ClientSide_Aes256cfb_Sha256 = "client-side/AES256-CFB/SHA256"
+
+// StateCryptoConfig holds the configuration for transparent client-side remote state encryption
+type StateCryptoConfig struct {
+	// Implementation selects the implementation to use
+	//
+	// supported values are
+	//   "client-side/AES256-CFB/SHA256"
+	//   "" (means not encrypted, the default)
+	//
+	// supplying an unsupported value raises an error
+	Implementation string `json:"implementation"`
+
+	// Parameters contains implementation-specific parameters, such as the key
+	Parameters map[string]string `json:"parameters"`
+}
+
+// ConfigEnvName configures the name of the environment variable used to configure encryption and decryption
+//
+// Set this environment variable to a json representation of StateCryptoConfig, or leave it unset/blank
+// to disable encryption.
+var ConfigEnvName = "TF_REMOTE_STATE_ENCRYPTION"
+
+// FallbackConfigEnvName configures the name of the environment variable used to configure fallback decryption
+//
+// Set this environment variable to a json representation of StateCryptoConfig, or leave it unset/blank
+// in order to not supply a fallback.
+//
+// Note that decryption will always try the configuration specified in TF_REMOTE_STATE_ENCRYPTION first.
+// Only if decryption fails with that, it will try this configuration.
+//
+// Why is this useful?
+// - key rotation (put the old key here until all state has been migrated)
+// - decryption (leave TF_REMOTE_STATE_ENCRYPTION blank/unset, but set this variable, and your state will be decrypted on next write)
+var FallbackConfigEnvName = "TF_REMOTE_STATE_DECRYPTION_FALLBACK"
+
+func Configuration() StateCryptoConfig {
+	return configFromEnv(ConfigEnvName)
+}
+
+func FallbackConfiguration() StateCryptoConfig {
+	return configFromEnv(FallbackConfigEnvName)
+}
+
+var logFatalf = log.Fatalf
+
+func configFromEnv(envName string) StateCryptoConfig {
+	config, err := Parse(os.Getenv(envName))
+	if err != nil {
+		logFatalf("[ERROR] failed to parse remote state encryption configuration from environment variable %s: %s", envName, err.Error())
+	}
+	return config
+}
+
+func Parse(jsonConfig string) (StateCryptoConfig, error) {
+	if jsonConfig == "" {
+		return StateCryptoConfig{}, nil
+	}
+
+	config := StateCryptoConfig{}
+
+	dec := json.NewDecoder(bytes.NewReader([]byte(jsonConfig)))
+	dec.DisallowUnknownFields()
+	err := dec.Decode(&config)
+
+	return config, err
+}

internal/states/statecrypto/cryptoconfig/cryptoconfig_test.go

@@ -0,0 +1,43 @@
+package cryptoconfig
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"strings"
+	"testing"
+)
+
+func resetLogFatalf() {
+	logFatalf = log.Fatalf
+}
+
+func TestBlankConfigurationProducesNoErrors(t *testing.T) {
+	logFatalf = func(format string, v ...interface{}) {
+		t.Errorf("received unexpected error: "+format, v...)
+	}
+	defer resetLogFatalf()
+
+	_ = Configuration()
+	_ = FallbackConfiguration()
+}
+
+func TestUnexpectedJsonInConfigurationProducesError(t *testing.T) {
+	lastError := ""
+	logFatalf = func(format string, v ...interface{}) {
+		lastError = fmt.Sprintf(format, v...)
+	}
+	defer resetLogFatalf()
+
+	envName := "TEST_CRYPTOCONFIG_TestInvalidJsonInConfigurationProducesError"
+	configInvalid := `{"implementation":"something", "unexpectedField":"another thing"}`
+	_ = os.Setenv(envName, configInvalid)
+	defer os.Unsetenv(envName)
+
+	_ = configFromEnv(envName)
+
+	expected := "[ERROR] failed to parse remote state encryption configuration from environment variable TEST_CRYPTOCONFIG_TestInvalidJsonInConfigurationProducesError: "
+	if !strings.HasPrefix(lastError, expected) {
+		t.Error("did not receive expected error")
+	}
+}

internal/states/statecrypto/fallbackretry.go

@@ -0,0 +1,68 @@
+package statecrypto
+
+import (
+	"github.com/hashicorp/terraform/internal/states/statecrypto/implementations/passthrough"
+	"log"
+)
+
+// FallbackRetryStateWrapper is a StateCryptoProvider that contains two other StateCryptoProviders,
+// the first choice, and an optional fallback.
+//
+// encryption always uses the first choice.
+//
+// decryption first tries the first choice, if an error occurs and a fallback has been provided, the fallback
+// is also tried, and a message is logged about this fact.
+//
+// exception: if the first choice is PassthroughStateWrapper and a fallback is configured,
+// ONLY the fallback is tried for decryption. This is because PassthroughStateWrapper would have no way to determine
+// if it got an unencrypted state or an encrypted state (any json could be valid state).
+//
+// Example use case: key rotation - first choice is encryption with the new key, fallback knows how to decrypt with the old key.
+type FallbackRetryStateWrapper struct {
+	firstChoice StateCryptoProvider
+	fallback    StateCryptoProvider
+}
+
+func (f *FallbackRetryStateWrapper) Encrypt(data []byte) ([]byte, error) {
+	return f.firstChoice.Encrypt(data)
+}
+
+func (f *FallbackRetryStateWrapper) Decrypt(data []byte) ([]byte, error) {
+	_, firstChoiceIsPassthrough := f.firstChoice.(*passthrough.PassthroughStateWrapper)
+	if firstChoiceIsPassthrough && f.fallback != nil {
+		// try only the fallback, so encrypted state can be successfully decrypted using it
+		// (note that all StateCryptoProviders are required to be able to pass through unencrypted state during decryption)
+		candidate, err := f.fallback.Decrypt(data)
+		if err != nil {
+			log.Printf("[ERROR] failed to decrypt state with fallback configuration and main configuration is passthrough, bailing out")
+			return []byte{}, err
+		}
+		log.Printf("[TRACE] successfully decrypted state using fallback configuration, input %d bytes, output %d bytes", len(data), len(candidate))
+		return candidate, nil
+	} else {
+		candidate, err := f.firstChoice.Decrypt(data)
+		if err != nil {
+			if f.fallback != nil {
+				log.Printf("[INFO] failed to decrypt state with main encryption configuration, trying fallback configuration")
+				candidate2, err := f.fallback.Decrypt(data)
+				if err != nil {
+					log.Printf("[ERROR] failed to decrypt state with fallback configuration as well, bailing out")
+					return []byte{}, err
+				}
+				log.Printf("[TRACE] successfully decrypted state using fallback configuration, input %d bytes, output %d bytes", len(data), len(candidate2))
+				return candidate2, nil
+			}
+			log.Print("[TRACE] failed to decrypt state with first choice configuration and no fallback available")
+			return []byte{}, err
+		}
+		log.Printf("[TRACE] successfully decrypted state using first choice configuration, input %d bytes, output %d bytes", len(data), len(candidate))
+		return candidate, nil
+	}
+}
+
+func fallbackRetryInstance(firstChoice StateCryptoProvider, fallback StateCryptoProvider) StateCryptoProvider {
+	return &FallbackRetryStateWrapper{
+		firstChoice: firstChoice,
+		fallback:    fallback,
+	}
+}

internal/states/statecrypto/implementations/aes256state/aes256state.go

@@ -0,0 +1,161 @@
+package aes256state
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"github.com/hashicorp/terraform/internal/states/statecrypto/cryptoconfig"
+	"io"
+	"log"
+	"regexp"
+)
+
+type AES256StateWrapper struct {
+	key []byte
+}
+
+func parseKey(hexKey string) ([]byte, error) {
+	validator := regexp.MustCompile("^[0-9a-f]{64}$")
+	if !validator.MatchString(hexKey) {
+		return []byte{}, fmt.Errorf("key was not a hex string representing 32 bytes, must match [0-9a-f]{64}")
+	}
+
+	key, _ := hex.DecodeString(hexKey)
+
+	return key, nil
+}
+
+func (a *AES256StateWrapper) parseKeyFromConfiguration(config cryptoconfig.StateCryptoConfig) error {
+	hexkey, ok := config.Parameters["key"]
+	if !ok {
+		return fmt.Errorf("configuration for AES256 needs the parameter 'key' set to a 32 byte lower case hexadecimal value")
+	}
+
+	key, err := parseKey(hexkey)
+	if err != nil {
+		return err
+	}
+
+	a.key = []byte(key)
+	return nil
+}
+
+// determine if data (which is a []byte containing a json structure) is encrypted, that is, of the following form:
+//     {"crypted":"<hex containing iv and payload>"}
+func (a *AES256StateWrapper) isEncrypted(data []byte) bool {
+	validator := regexp.MustCompile(`^{"crypted":".*$`)
+	return validator.Match(data)
+}
+
+func (a *AES256StateWrapper) isSyntacticallyValidEncrypted(data []byte) bool {
+	validator := regexp.MustCompile(`^{"crypted":"[0-9a-f]+"}$`)
+	return validator.Match(data)
+}
+
+func (a *AES256StateWrapper) decodeFromEncryptedJsonWithChecks(jsonCryptedData []byte) ([]byte, error) {
+	if !a.isSyntacticallyValidEncrypted(jsonCryptedData) {
+		return []byte{}, fmt.Errorf("ciphertext contains invalid characters, possibly cut off or garbled")
+	}
+
+	// extract the hex part only, cutting off {"crypted":" (12 characters) and "} (2 characters)
+	src := jsonCryptedData[12 : len(jsonCryptedData)-2]
+
+	ciphertext := make([]byte, hex.DecodedLen(len(src)))
+	n, err := hex.Decode(ciphertext, src)
+	if err != nil {
+		return []byte{}, err
+	}
+	if n != hex.DecodedLen(len(src)) {
+		return []byte{}, fmt.Errorf("did not fully decode, only read %d characters before encountering an error", n)
+	}
+	return ciphertext, nil
+}
+
+func (a *AES256StateWrapper) encodeToEncryptedJson(ciphertext []byte) []byte {
+	prefix := []byte(`{"crypted":"`)
+	postfix := []byte(`"}`)
+	encryptedHex := make([]byte, hex.EncodedLen(len(ciphertext)))
+	_ = hex.Encode(encryptedHex, ciphertext)
+
+	return append(append(prefix, encryptedHex...), postfix...)
+}
+
+func (a *AES256StateWrapper) attemptDecryption(jsonCryptedData []byte, key []byte) ([]byte, error) {
+	ciphertext, err := a.decodeFromEncryptedJsonWithChecks(jsonCryptedData)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	if len(ciphertext) < aes.BlockSize {
+		return []byte{}, fmt.Errorf("ciphertext too short, did not contain initial vector")
+	}
+	iv := ciphertext[:aes.BlockSize]
+	payloadWithHash := ciphertext[aes.BlockSize:]
+
+	stream := cipher.NewCFBDecrypter(block, iv)
+
+	// XORKeyStream can work in-place if the two arguments are the same.
+	stream.XORKeyStream(payloadWithHash, payloadWithHash)
+
+	plaintextPayload := payloadWithHash[:len(payloadWithHash)-sha256.Size]
+	hashRead := payloadWithHash[len(payloadWithHash)-sha256.Size:]
+
+	hashComputed := sha256.Sum256(plaintextPayload)
+	for i, v := range hashComputed {
+		if v != hashRead[i] {
+			return []byte{}, fmt.Errorf("hash of decrypted payload did not match at position %d", i)
+		}
+	}
+
+	// payloadWithHash is now decrypted
+	return plaintextPayload, nil
+}
+
+// Encrypt data (which is a []byte containing a json structure) into a json structure
+//      {"crypted":"<hex-encoded random iv + hex-encoded CFB encrypted data including hash>"}
+// fail if encryption is not possible to prevent writing unencrypted state
+func (a *AES256StateWrapper) Encrypt(plaintextPayload []byte) ([]byte, error) {
+	block, err := aes.NewCipher(a.key)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	ciphertext := make([]byte, aes.BlockSize+len(plaintextPayload)+sha256.Size)
+	iv := ciphertext[:aes.BlockSize]
+	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
+		return []byte{}, err
+	}
+
+	// add hash over plaintext to end of plaintext (allows integrity check when decrypting)
+	hashArray := sha256.Sum256(plaintextPayload)
+	plaintextWithHash := append(plaintextPayload, hashArray[0:sha256.Size]...)
+
+	stream := cipher.NewCFBEncrypter(block, iv)
+	stream.XORKeyStream(ciphertext[aes.BlockSize:], plaintextWithHash)
+
+	return a.encodeToEncryptedJson(ciphertext), nil
+}
+
+// Decrypt the hex-encoded contents of data, which is expected to be of the form
+//         {"crypted":"<hex containing iv and payload>"}
+// supports reading unencrypted state as well but logs a warning
+func (a *AES256StateWrapper) Decrypt(data []byte) ([]byte, error) {
+	if a.isEncrypted(data) {
+		candidate, err := a.attemptDecryption(data, a.key)
+		if err != nil {
+			return []byte{}, err
+		}
+		return candidate, nil
+	} else {
+		log.Printf("[WARN] found unencrypted state, transparently reading it anyway")
+		return data, nil
+	}
+}