diff --git a/internal/states/remote/state.go b/internal/states/remote/state.go index ca939a96a312..147aafdc0528 100644 --- a/internal/states/remote/state.go +++ b/internal/states/remote/state.go @@ -3,10 +3,12 @@ package remote import ( "bytes" "fmt" + "log" "sync" uuid "github.com/hashicorp/go-uuid" "github.com/hashicorp/terraform/internal/states" + "github.com/hashicorp/terraform/internal/states/statecrypto" "github.com/hashicorp/terraform/internal/states/statefile" "github.com/hashicorp/terraform/internal/states/statemgr" ) @@ -121,7 +123,13 @@ func (s *State) refreshState() error { return nil } - stateFile, err := statefile.Read(bytes.NewReader(payload.Data)) + decrypted, err := statecrypto.StateCryptoWrapper().Decrypt(payload.Data) + if err != nil { + log.Printf("[ERROR] remote state decryption failed: %s", err.Error()) + return err + } + + stateFile, err := statefile.Read(bytes.NewReader(decrypted)) if err != nil { return err } @@ -178,7 +186,13 @@ func (s *State) PersistState() error { return err } - err = s.Client.Put(buf.Bytes()) + maybeEncrypted, err := statecrypto.StateCryptoWrapper().Encrypt(buf.Bytes()) + if err != nil { + log.Printf("[ERROR] remote state encryption failed: %s", err.Error()) + return err + } + + err = s.Client.Put(maybeEncrypted) if err != nil { return err } diff --git a/internal/states/statecrypto/cryptoconfig/cryptoconfig.go b/internal/states/statecrypto/cryptoconfig/cryptoconfig.go new file mode 100644 index 000000000000..2e3be75aa8dd --- /dev/null +++ b/internal/states/statecrypto/cryptoconfig/cryptoconfig.go @@ -0,0 +1,76 @@ +package cryptoconfig + +import ( + "bytes" + "encoding/json" + "log" + "os" +) + +const ClientSide_Aes256cfb_Sha256 = "client-side/AES256-CFB/SHA256" + +// StateCryptoConfig holds the configuration for transparent client-side remote state encryption +type StateCryptoConfig struct { + // Implementation selects the implementation to use + // + // supported values are + // "client-side/AES256-CFB/SHA256" + // "" (means not encrypted, the default) + // + // supplying an unsupported value raises an error + Implementation string `json:"implementation"` + + // Parameters contains implementation-specific parameters, such as the key + Parameters map[string]string `json:"parameters"` +} + +// ConfigEnvName configures the name of the environment variable used to configure encryption and decryption +// +// Set this environment variable to a json representation of StateCryptoConfig, or leave it unset/blank +// to disable encryption. +var ConfigEnvName = "TF_REMOTE_STATE_ENCRYPTION" + +// FallbackConfigEnvName configures the name of the environment variable used to configure fallback decryption +// +// Set this environment variable to a json representation of StateCryptoConfig, or leave it unset/blank +// in order to not supply a fallback. +// +// Note that decryption will always try the configuration specified in TF_REMOTE_STATE_ENCRYPTION first. +// Only if decryption fails with that, it will try this configuration. +// +// Why is this useful? +// - key rotation (put the old key here until all state has been migrated) +// - decryption (leave TF_REMOTE_STATE_ENCRYPTION blank/unset, but set this variable, and your state will be decrypted on next write) +var FallbackConfigEnvName = "TF_REMOTE_STATE_DECRYPTION_FALLBACK" + +func Configuration() StateCryptoConfig { + return configFromEnv(ConfigEnvName) +} + +func FallbackConfiguration() StateCryptoConfig { + return configFromEnv(FallbackConfigEnvName) +} + +var logFatalf = log.Fatalf + +func configFromEnv(envName string) StateCryptoConfig { + config, err := Parse(os.Getenv(envName)) + if err != nil { + logFatalf("[ERROR] failed to parse remote state encryption configuration from environment variable %s: %s", envName, err.Error()) + } + return config +} + +func Parse(jsonConfig string) (StateCryptoConfig, error) { + if jsonConfig == "" { + return StateCryptoConfig{}, nil + } + + config := StateCryptoConfig{} + + dec := json.NewDecoder(bytes.NewReader([]byte(jsonConfig))) + dec.DisallowUnknownFields() + err := dec.Decode(&config) + + return config, err +} diff --git a/internal/states/statecrypto/cryptoconfig/cryptoconfig_test.go b/internal/states/statecrypto/cryptoconfig/cryptoconfig_test.go new file mode 100644 index 000000000000..b455e61ec4f8 --- /dev/null +++ b/internal/states/statecrypto/cryptoconfig/cryptoconfig_test.go @@ -0,0 +1,43 @@ +package cryptoconfig + +import ( + "fmt" + "log" + "os" + "strings" + "testing" +) + +func resetLogFatalf() { + logFatalf = log.Fatalf +} + +func TestBlankConfigurationProducesNoErrors(t *testing.T) { + logFatalf = func(format string, v ...interface{}) { + t.Errorf("received unexpected error: "+format, v...) + } + defer resetLogFatalf() + + _ = Configuration() + _ = FallbackConfiguration() +} + +func TestUnexpectedJsonInConfigurationProducesError(t *testing.T) { + lastError := "" + logFatalf = func(format string, v ...interface{}) { + lastError = fmt.Sprintf(format, v...) + } + defer resetLogFatalf() + + envName := "TEST_CRYPTOCONFIG_TestInvalidJsonInConfigurationProducesError" + configInvalid := `{"implementation":"something", "unexpectedField":"another thing"}` + _ = os.Setenv(envName, configInvalid) + defer os.Unsetenv(envName) + + _ = configFromEnv(envName) + + expected := "[ERROR] failed to parse remote state encryption configuration from environment variable TEST_CRYPTOCONFIG_TestInvalidJsonInConfigurationProducesError: " + if !strings.HasPrefix(lastError, expected) { + t.Error("did not receive expected error") + } +} diff --git a/internal/states/statecrypto/fallbackretry.go b/internal/states/statecrypto/fallbackretry.go new file mode 100644 index 000000000000..f3738b0f8c6f --- /dev/null +++ b/internal/states/statecrypto/fallbackretry.go @@ -0,0 +1,68 @@ +package statecrypto + +import ( + "github.com/hashicorp/terraform/internal/states/statecrypto/implementations/passthrough" + "log" +) + +// FallbackRetryStateWrapper is a StateCryptoProvider that contains two other StateCryptoProviders, +// the first choice, and an optional fallback. +// +// encryption always uses the first choice. +// +// decryption first tries the first choice, if an error occurs and a fallback has been provided, the fallback +// is also tried, and a message is logged about this fact. +// +// exception: if the first choice is PassthroughStateWrapper and a fallback is configured, +// ONLY the fallback is tried for decryption. This is because PassthroughStateWrapper would have no way to determine +// if it got an unencrypted state or an encrypted state (any json could be valid state). +// +// Example use case: key rotation - first choice is encryption with the new key, fallback knows how to decrypt with the old key. +type FallbackRetryStateWrapper struct { + firstChoice StateCryptoProvider + fallback StateCryptoProvider +} + +func (f *FallbackRetryStateWrapper) Encrypt(data []byte) ([]byte, error) { + return f.firstChoice.Encrypt(data) +} + +func (f *FallbackRetryStateWrapper) Decrypt(data []byte) ([]byte, error) { + _, firstChoiceIsPassthrough := f.firstChoice.(*passthrough.PassthroughStateWrapper) + if firstChoiceIsPassthrough && f.fallback != nil { + // try only the fallback, so encrypted state can be successfully decrypted using it + // (note that all StateCryptoProviders are required to be able to pass through unencrypted state during decryption) + candidate, err := f.fallback.Decrypt(data) + if err != nil { + log.Printf("[ERROR] failed to decrypt state with fallback configuration and main configuration is passthrough, bailing out") + return []byte{}, err + } + log.Printf("[TRACE] successfully decrypted state using fallback configuration, input %d bytes, output %d bytes", len(data), len(candidate)) + return candidate, nil + } else { + candidate, err := f.firstChoice.Decrypt(data) + if err != nil { + if f.fallback != nil { + log.Printf("[INFO] failed to decrypt state with main encryption configuration, trying fallback configuration") + candidate2, err := f.fallback.Decrypt(data) + if err != nil { + log.Printf("[ERROR] failed to decrypt state with fallback configuration as well, bailing out") + return []byte{}, err + } + log.Printf("[TRACE] successfully decrypted state using fallback configuration, input %d bytes, output %d bytes", len(data), len(candidate2)) + return candidate2, nil + } + log.Print("[TRACE] failed to decrypt state with first choice configuration and no fallback available") + return []byte{}, err + } + log.Printf("[TRACE] successfully decrypted state using first choice configuration, input %d bytes, output %d bytes", len(data), len(candidate)) + return candidate, nil + } +} + +func fallbackRetryInstance(firstChoice StateCryptoProvider, fallback StateCryptoProvider) StateCryptoProvider { + return &FallbackRetryStateWrapper{ + firstChoice: firstChoice, + fallback: fallback, + } +} diff --git a/internal/states/statecrypto/implementations/aes256state/aes256state.go b/internal/states/statecrypto/implementations/aes256state/aes256state.go new file mode 100644 index 000000000000..84d05e35ed27 --- /dev/null +++ b/internal/states/statecrypto/implementations/aes256state/aes256state.go @@ -0,0 +1,161 @@ +package aes256state + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "github.com/hashicorp/terraform/internal/states/statecrypto/cryptoconfig" + "io" + "log" + "regexp" +) + +type AES256StateWrapper struct { + key []byte +} + +func parseKey(hexKey string) ([]byte, error) { + validator := regexp.MustCompile("^[0-9a-f]{64}$") + if !validator.MatchString(hexKey) { + return []byte{}, fmt.Errorf("key was not a hex string representing 32 bytes, must match [0-9a-f]{64}") + } + + key, _ := hex.DecodeString(hexKey) + + return key, nil +} + +func (a *AES256StateWrapper) parseKeyFromConfiguration(config cryptoconfig.StateCryptoConfig) error { + hexkey, ok := config.Parameters["key"] + if !ok { + return fmt.Errorf("configuration for AES256 needs the parameter 'key' set to a 32 byte lower case hexadecimal value") + } + + key, err := parseKey(hexkey) + if err != nil { + return err + } + + a.key = []byte(key) + return nil +} + +// determine if data (which is a []byte containing a json structure) is encrypted, that is, of the following form: +// {"crypted":""} +func (a *AES256StateWrapper) isEncrypted(data []byte) bool { + validator := regexp.MustCompile(`^{"crypted":".*$`) + return validator.Match(data) +} + +func (a *AES256StateWrapper) isSyntacticallyValidEncrypted(data []byte) bool { + validator := regexp.MustCompile(`^{"crypted":"[0-9a-f]+"}$`) + return validator.Match(data) +} + +func (a *AES256StateWrapper) decodeFromEncryptedJsonWithChecks(jsonCryptedData []byte) ([]byte, error) { + if !a.isSyntacticallyValidEncrypted(jsonCryptedData) { + return []byte{}, fmt.Errorf("ciphertext contains invalid characters, possibly cut off or garbled") + } + + // extract the hex part only, cutting off {"crypted":" (12 characters) and "} (2 characters) + src := jsonCryptedData[12 : len(jsonCryptedData)-2] + + ciphertext := make([]byte, hex.DecodedLen(len(src))) + n, err := hex.Decode(ciphertext, src) + if err != nil { + return []byte{}, err + } + if n != hex.DecodedLen(len(src)) { + return []byte{}, fmt.Errorf("did not fully decode, only read %d characters before encountering an error", n) + } + return ciphertext, nil +} + +func (a *AES256StateWrapper) encodeToEncryptedJson(ciphertext []byte) []byte { + prefix := []byte(`{"crypted":"`) + postfix := []byte(`"}`) + encryptedHex := make([]byte, hex.EncodedLen(len(ciphertext))) + _ = hex.Encode(encryptedHex, ciphertext) + + return append(append(prefix, encryptedHex...), postfix...) +} + +func (a *AES256StateWrapper) attemptDecryption(jsonCryptedData []byte, key []byte) ([]byte, error) { + ciphertext, err := a.decodeFromEncryptedJsonWithChecks(jsonCryptedData) + if err != nil { + return []byte{}, err + } + + block, err := aes.NewCipher(key) + if err != nil { + return []byte{}, err + } + + if len(ciphertext) < aes.BlockSize { + return []byte{}, fmt.Errorf("ciphertext too short, did not contain initial vector") + } + iv := ciphertext[:aes.BlockSize] + payloadWithHash := ciphertext[aes.BlockSize:] + + stream := cipher.NewCFBDecrypter(block, iv) + + // XORKeyStream can work in-place if the two arguments are the same. + stream.XORKeyStream(payloadWithHash, payloadWithHash) + + plaintextPayload := payloadWithHash[:len(payloadWithHash)-sha256.Size] + hashRead := payloadWithHash[len(payloadWithHash)-sha256.Size:] + + hashComputed := sha256.Sum256(plaintextPayload) + for i, v := range hashComputed { + if v != hashRead[i] { + return []byte{}, fmt.Errorf("hash of decrypted payload did not match at position %d", i) + } + } + + // payloadWithHash is now decrypted + return plaintextPayload, nil +} + +// Encrypt data (which is a []byte containing a json structure) into a json structure +// {"crypted":""} +// fail if encryption is not possible to prevent writing unencrypted state +func (a *AES256StateWrapper) Encrypt(plaintextPayload []byte) ([]byte, error) { + block, err := aes.NewCipher(a.key) + if err != nil { + return []byte{}, err + } + + ciphertext := make([]byte, aes.BlockSize+len(plaintextPayload)+sha256.Size) + iv := ciphertext[:aes.BlockSize] + if _, err := io.ReadFull(rand.Reader, iv); err != nil { + return []byte{}, err + } + + // add hash over plaintext to end of plaintext (allows integrity check when decrypting) + hashArray := sha256.Sum256(plaintextPayload) + plaintextWithHash := append(plaintextPayload, hashArray[0:sha256.Size]...) + + stream := cipher.NewCFBEncrypter(block, iv) + stream.XORKeyStream(ciphertext[aes.BlockSize:], plaintextWithHash) + + return a.encodeToEncryptedJson(ciphertext), nil +} + +// Decrypt the hex-encoded contents of data, which is expected to be of the form +// {"crypted":""} +// supports reading unencrypted state as well but logs a warning +func (a *AES256StateWrapper) Decrypt(data []byte) ([]byte, error) { + if a.isEncrypted(data) { + candidate, err := a.attemptDecryption(data, a.key) + if err != nil { + return []byte{}, err + } + return candidate, nil + } else { + log.Printf("[WARN] found unencrypted state, transparently reading it anyway") + return data, nil + } +} diff --git a/internal/states/statecrypto/implementations/aes256state/aes256state_test.go b/internal/states/statecrypto/implementations/aes256state/aes256state_test.go new file mode 100644 index 000000000000..ac01a20831d9 --- /dev/null +++ b/internal/states/statecrypto/implementations/aes256state/aes256state_test.go @@ -0,0 +1,257 @@ +package aes256state + +import ( + "encoding/hex" + "fmt" + "github.com/hashicorp/terraform/internal/states/statecrypto/cryptoconfig" + "testing" +) + +const validKey1 = "a0a1a2a3a4a5a6a7a8a9b0b1b2b3b4b5b6b7b8b9c0c1c2c3c4c5c6c7c8c9d0d1" + +const tooShortKey = "a0a1a2a3a4a5a6a7a8a9b0b1b2b3b4b5b6b7b8b9c0c1c2c3c4c5c6c7c8c9" +const tooLongKey = "a0a1a2a3a4a5a6a7a8a9b0b1b2b3b4b5b6b7b8b9c0c1c2c3c4c5c6c7c8c9d0d1d2d3d4d5" +const invalidChars = "somethingsomethinga9b0b1b2b3b4b5b6b7b8b9c0c1c2c3c4c5c6c7c8c9d0d1" + +const validPlaintext = `{"animals":[{"species":"cheetah","genus":"acinonyx"}]}` +const validEncryptedKey1 = `{"crypted":"e93e3e7ad3434055251f695865a13c11744b97e54cb7dee8f8fb40d1fb096b728f2a00606e7109f0720aacb15008b410cf2f92dd7989c2ff10b9712b6ef7d69ecdad1dccd2f1bddd127f0f0d87c79c3c062e03c2297614e2effa2fb1f4072d86df0dda4fc061"}` +const invalidEncryptedHash = `{"crypted":"a6625332f6e3061e1202cea86d2ddf7cf6d5f296a9856fe989cd20b18c8522f670d368f523481876bb2b98eea1e8cf845b4e003de11153bc47b884ce907b1e6a075f515ddd2aa4fbdbc7bbab1b411e153d164f84990e9c6fa82d7cacde7401546b47b2f30000"}` +const invalidEncryptedCutoff = `{"crypted":"447c2fc8982ed203681298be9f1b03ed30dbfe794a68e4ad873fb68c34f10394ffddd9c76b2d3fdb006d75068453854af63766fc059a569d243eb7d8c92ec3a00535ccaab769bdafb534d5471ed01ca36f640d1f` +const invalidEncryptedChars = `{"crypted":"447c2fc8982ed203681298be9f1b03ed30dbfe794a68e4ad873fb68c34 SOMETHING WEIRD d3fdb006d75068453854af63766fc059a569d243eb7d8c92ec3a00535ccaab769bdafb534d5471ed01ca36f640d1f720c9a2bf0aa4e0a40496dacee92325a9f86"}` +const invalidEncryptedTooShort = `{"crypted":"a6625332"}` +const invalidEncryptedOddNumberCharacters = `{"crypted":"e93e3e7ad3434055251f695865a13c11744b97e54cb7dee8f8fb40d1fb096b728f2a00606e7109f0720aacb15008b410cf2f92dd7989c2ff10b9712b6ef7d69ecdad1dccd2f1bddd127f0f0d87c79c3c062e03c2297614e2effa2fb1f4072d86df0dda4fc06"}` + +type parseKeyTestCase struct { + description string + configuration cryptoconfig.StateCryptoConfig + expectedError string + expectedKey []byte +} + +func compareSlices(got []byte, expected []byte) bool { + eEmpty := len(expected) == 0 + gEmpty := len(got) == 0 + if eEmpty != gEmpty { + return false + } + if eEmpty { + return true + } + if len(expected) != len(got) { + return false + } + for i, v := range expected { + if v != got[i] { + return false + } + } + return true +} + +func compareErrors(got error, expected string) string { + if got != nil { + if got.Error() != expected { + return fmt.Sprintf("unexpected error '%s'; want '%s'", got.Error(), expected) + } + } else { + if expected != "" { + return fmt.Sprintf("did not get expected error '%s'", expected) + } + } + return "" +} + +func conf(key string) cryptoconfig.StateCryptoConfig { + return cryptoconfig.StateCryptoConfig{ + Implementation: "client-side/AES256-CFB/SHA256", + Parameters: map[string]string{ + "key": key, + }, + } +} + +func TestParseKeysFromConfiguration(t *testing.T) { + k1, _ := hex.DecodeString(validKey1) + + testCases := []parseKeyTestCase{ + // happy cases + { + description: "work on encrypted state files, no previous key", + configuration: conf(validKey1), + expectedKey: k1, + }, + { + description: "work on encrypted state files, empty previous key", + configuration: conf(validKey1), + expectedKey: k1, + }, + + // error cases + { + description: "key missing", + configuration: conf(""), + expectedError: "key was not a hex string representing 32 bytes, must match [0-9a-f]{64}", + }, + { + description: "too short key", + configuration: conf(tooShortKey), + expectedError: "key was not a hex string representing 32 bytes, must match [0-9a-f]{64}", + }, + { + description: "too long key", + configuration: conf(tooLongKey), + expectedError: "key was not a hex string representing 32 bytes, must match [0-9a-f]{64}", + }, + { + description: "invalid chars in main key", + configuration: conf(invalidChars), + expectedError: "key was not a hex string representing 32 bytes, must match [0-9a-f]{64}", + }, + { + description: "parse error", + configuration: conf(`"`), + expectedError: "key was not a hex string representing 32 bytes, must match [0-9a-f]{64}", + }, + } + + for _, tc := range testCases { + cut := &AES256StateWrapper{} + err := cut.parseKeyFromConfiguration(tc.configuration) + if comp := compareErrors(err, tc.expectedError); comp != "" { + t.Error(comp) + } + if !compareSlices(cut.key, tc.expectedKey) { + t.Errorf("unexpected key %#v; want %#v", cut.key, tc.expectedKey) + } + } +} + +type roundtripTestCase struct { + description string + configuration cryptoconfig.StateCryptoConfig + input string + injectOutput string + expectedNewError string + expectedEncError string + expectedDecError string +} + +func TestEncryptDecrypt(t *testing.T) { + testCases := []roundtripTestCase{ + // happy path cases + { + description: "standard work on encrypted data", + configuration: conf(validKey1), + input: validPlaintext, + }, + { + description: "standard work on unencrypted data", + configuration: conf(validKey1), + input: validPlaintext, + injectOutput: validPlaintext, + }, + + // error cases + { + description: "invalid hash received on decrypt", + configuration: conf(validKey1), + input: validPlaintext, + injectOutput: invalidEncryptedHash, + expectedDecError: "hash of decrypted payload did not match at position 30", + }, + { + description: "decrypt received incomplete crypted json", + configuration: conf(validKey1), + input: validPlaintext, + injectOutput: invalidEncryptedCutoff, + expectedDecError: "ciphertext contains invalid characters, possibly cut off or garbled", + }, + { + description: "decrypt received invalid crypted json", + configuration: conf(validKey1), + input: validPlaintext, + injectOutput: invalidEncryptedChars, + expectedDecError: "ciphertext contains invalid characters, possibly cut off or garbled", + }, + { + description: "decrypt received crypted json too short even for iv", + configuration: conf(validKey1), + input: validPlaintext, + injectOutput: invalidEncryptedTooShort, + expectedDecError: "ciphertext too short, did not contain initial vector", + }, + } + for _, tc := range testCases { + cut, err := New(tc.configuration) + if comp := compareErrors(err, tc.expectedNewError); comp != "" { + t.Error(comp) + } + if err == nil { + if cut == nil { + t.Error("got unexpected nil implementation") + } else { + encOutput, err := cut.Encrypt([]byte(tc.input)) + if comp := compareErrors(err, tc.expectedEncError); comp != "" { + t.Error(comp) + } else { + // log.Printf("crypted json is %s", string(encOutput)) + + if tc.injectOutput != "" { + encOutput = []byte(tc.injectOutput) + } + + decOutput, err := cut.Decrypt(encOutput) + if comp := compareErrors(err, tc.expectedDecError); comp != "" { + t.Error(comp) + } else { + if err == nil && !compareSlices(decOutput, []byte(tc.input)) { + t.Errorf("round trip error, got %#v; want %#v", decOutput, []byte(tc.input)) + } + } + } + } + } + } +} + +func TestEncryptDoesNotUseSameIV(t *testing.T) { + cut, _ := New(conf(validKey1)) + encOutput1, _ := cut.Encrypt([]byte(validPlaintext)) + if len(encOutput1) != len([]byte(validEncryptedKey1)) { + t.Error("encryption output 1 did not have the expected length") + } + encOutput2, _ := cut.Encrypt([]byte(validPlaintext)) + if len(encOutput2) != len([]byte(validEncryptedKey1)) { + t.Error("encryption output 2 did not have the expected length") + } + if compareSlices(encOutput1, []byte(validEncryptedKey1)) { + t.Error("random iv created same vector as in recorded run! SECURITY PROBLEM!") + } + if compareSlices(encOutput1, encOutput2) { + t.Error("random iv created same vector as in previous call! SECURITY PROBLEM!") + } +} + +func TestEncrypt_FailingCipherCreation(t *testing.T) { + cut := &AES256StateWrapper{key: []byte{127, 42}} + _, err := cut.Encrypt([]byte(validPlaintext)) + if comp := compareErrors(err, "crypto/aes: invalid key size 2"); comp != "" { + t.Error(comp) + } +} + +func TestAttemptDecryption_FailingCipherCreation(t *testing.T) { + cut := &AES256StateWrapper{key: []byte{127, 42}} + _, err := cut.attemptDecryption([]byte(validEncryptedKey1), cut.key) + if comp := compareErrors(err, "crypto/aes: invalid key size 2"); comp != "" { + t.Error(comp) + } +} + +func TestAttemptDecryption_InvalidHexadecimal(t *testing.T) { + cut := &AES256StateWrapper{} + _, err := cut.attemptDecryption([]byte(invalidEncryptedOddNumberCharacters), cut.key) + if comp := compareErrors(err, "encoding/hex: odd length hex string"); comp != "" { + t.Error(comp) + } +} diff --git a/internal/states/statecrypto/implementations/aes256state/instance.go b/internal/states/statecrypto/implementations/aes256state/instance.go new file mode 100644 index 000000000000..fbba00742a73 --- /dev/null +++ b/internal/states/statecrypto/implementations/aes256state/instance.go @@ -0,0 +1,10 @@ +package aes256state + +import "github.com/hashicorp/terraform/internal/states/statecrypto/cryptoconfig" + +// New creates a new client-side/AES256-CFB/SHA256 state encryption wrapper. +func New(configuration cryptoconfig.StateCryptoConfig) (*AES256StateWrapper, error) { + instance := &AES256StateWrapper{} + err := instance.parseKeyFromConfiguration(configuration) + return instance, err +} diff --git a/internal/states/statecrypto/implementations/passthrough/instance.go b/internal/states/statecrypto/implementations/passthrough/instance.go new file mode 100644 index 000000000000..ab260b8223c0 --- /dev/null +++ b/internal/states/statecrypto/implementations/passthrough/instance.go @@ -0,0 +1,7 @@ +package passthrough + +import "github.com/hashicorp/terraform/internal/states/statecrypto/cryptoconfig" + +func New(_ cryptoconfig.StateCryptoConfig) (*PassthroughStateWrapper, error) { + return &PassthroughStateWrapper{}, nil +} diff --git a/internal/states/statecrypto/implementations/passthrough/passthrough.go b/internal/states/statecrypto/implementations/passthrough/passthrough.go new file mode 100644 index 000000000000..df5c8b33d449 --- /dev/null +++ b/internal/states/statecrypto/implementations/passthrough/passthrough.go @@ -0,0 +1,12 @@ +package passthrough + +type PassthroughStateWrapper struct { +} + +func (p *PassthroughStateWrapper) Encrypt(data []byte) ([]byte, error) { + return data, nil +} + +func (p *PassthroughStateWrapper) Decrypt(data []byte) ([]byte, error) { + return data, nil +} diff --git a/internal/states/statecrypto/implementations/passthrough/passthrough_test.go b/internal/states/statecrypto/implementations/passthrough/passthrough_test.go new file mode 100644 index 000000000000..91f595b25a5d --- /dev/null +++ b/internal/states/statecrypto/implementations/passthrough/passthrough_test.go @@ -0,0 +1,51 @@ +package passthrough + +import ( + "github.com/hashicorp/terraform/internal/states/statecrypto/cryptoconfig" + "testing" +) + +func TestPassthroughWorks(t *testing.T) { + cut, err := New(cryptoconfig.StateCryptoConfig{}) + if err != nil { + t.Fatalf("got unexpected error during passthrough instantiate: %s", err.Error()) + } + + data := []byte(`{"some":"json","document":[{"with":"an"},{"array","inside"}]}`) + + dataPassthroughEncrypted, err := cut.Encrypt(data) + if err != nil { + t.Errorf("got unexpected error during passthrough encryption: %s", err.Error()) + } + if !compareSlices(dataPassthroughEncrypted, data) { + t.Error("passthrough encryption changed the data") + } + + dataPassthroughDecrypted, err := cut.Decrypt(data) + if err != nil { + t.Errorf("got unexpected error during passthrough decryption: %s", err.Error()) + } + if !compareSlices(dataPassthroughDecrypted, data) { + t.Error("passthrough decryption changed the data") + } +} + +func compareSlices(got []byte, expected []byte) bool { + eEmpty := len(expected) == 0 + gEmpty := len(got) == 0 + if eEmpty != gEmpty { + return false + } + if eEmpty { + return true + } + if len(expected) != len(got) { + return false + } + for i, v := range expected { + if v != got[i] { + return false + } + } + return true +} diff --git a/internal/states/statecrypto/interface.go b/internal/states/statecrypto/interface.go new file mode 100644 index 000000000000..618d3290a875 --- /dev/null +++ b/internal/states/statecrypto/interface.go @@ -0,0 +1,28 @@ +package statecrypto + +// StateCryptoProvider is the interface that must be implemented for a transparent client side +// remote state encryption wrapper. It is used to encrypt/decrypt the state payload before writing +// to or after reading from the remote state backend. +// +// Note that the encrypted payload must still be valid json, because some remote state backends +// expect valid json. +// +// Also note that all implementations must gracefully handle unencrypted state being passed into Decrypt(), +// because this will inevitably happen when first encrypting previously unencrypted state. +// You should log a warning, though. +type StateCryptoProvider interface { + // Decrypt the state if encrypted, otherwise pass through unmodified. + // + // encryptedPayload is a json document passed in as a []byte. + // + // if you do not return an error, you must ensure you return a json document as a []byte. + Decrypt(encryptedPayload []byte) ([]byte, error) + + // Encrypt the plaintext state. + // + // plaintextPayload is a json document passed in as a []byte. + // + // if you do not return an error, you must ensure you return a json document as + // a []byte, because some remote state storage backends rely on this. + Encrypt(plaintextPayload []byte) ([]byte, error) +} diff --git a/internal/states/statecrypto/statecrypto.go b/internal/states/statecrypto/statecrypto.go new file mode 100644 index 000000000000..17c2f61c2fc4 --- /dev/null +++ b/internal/states/statecrypto/statecrypto.go @@ -0,0 +1,48 @@ +package statecrypto + +import ( + "github.com/hashicorp/terraform/internal/states/statecrypto/cryptoconfig" + "github.com/hashicorp/terraform/internal/states/statecrypto/implementations/aes256state" + "github.com/hashicorp/terraform/internal/states/statecrypto/implementations/passthrough" + "log" +) + +func StateCryptoWrapper() StateCryptoProvider { + return fallbackRetryInstance(firstChoice(), fallback()) +} + +func firstChoice() StateCryptoProvider { + return instanceFromConfig(cryptoconfig.Configuration(), true) +} + +func fallback() StateCryptoProvider { + return instanceFromConfig(cryptoconfig.FallbackConfiguration(), false) +} + +var logFatalf = log.Fatalf + +func instanceFromConfig(config cryptoconfig.StateCryptoConfig, allowPassthrough bool) StateCryptoProvider { + var implementation StateCryptoProvider + var err error = nil + + switch config.Implementation { + case cryptoconfig.ClientSide_Aes256cfb_Sha256: + implementation, err = aes256state.New(config) + // add additional implementations here + case "": + if allowPassthrough { + implementation, err = passthrough.New(config) + } else { + // valid case for fallback - means no fallback available + return nil + } + default: + logFatalf("[ERROR] failed to configure remote state encryption: unsupported implementation '%s'", config.Implementation) + } + + if err != nil { + logFatalf("[ERROR] failed to configure remote state encryption: %s", err.Error()) + } + + return implementation +} diff --git a/internal/states/statecrypto/statecrypto_test.go b/internal/states/statecrypto/statecrypto_test.go new file mode 100644 index 000000000000..071148b66d5d --- /dev/null +++ b/internal/states/statecrypto/statecrypto_test.go @@ -0,0 +1,248 @@ +package statecrypto + +import ( + "fmt" + "github.com/hashicorp/terraform/internal/states/statecrypto/cryptoconfig" + "github.com/hashicorp/terraform/internal/states/statecrypto/implementations/passthrough" + "log" + "testing" +) + +func resetLogFatalf() { + logFatalf = log.Fatalf +} + +// instance creation + +func TestCreation(t *testing.T) { + cut := StateCryptoWrapper() + if cut == nil { + t.Fatal("instance creation failed") + } + cutTypecast, ok := cut.(*FallbackRetryStateWrapper) + if !ok { + t.Fatal("did not create a FallbackRetryStateWrapper") + } + if cutTypecast.fallback != nil { + t.Fatal("default configuration unexpectedly has a decryption fallback") + } + if cutTypecast.firstChoice == nil { + t.Fatal("default configuration unexpectedly has a nil first choice") + } + _, ok = cutTypecast.firstChoice.(*passthrough.PassthroughStateWrapper) + if !ok { + t.Fatal("default configuration unexpectedly created something other than passthrough as first choice") + } +} + +func creationErrorCase(t *testing.T, jsonConfig string, expectedError string) { + var lastError string + logFatalf = func(format string, v ...interface{}) { + lastError = fmt.Sprintf(format, v...) + } + defer resetLogFatalf() + + lastError = "" + mainConfig, err := cryptoconfig.Parse(jsonConfig) + if err != nil { + t.Fatal("error parsing configuration") + } + + _ = instanceFromConfig(mainConfig, true) + if lastError != expectedError { + t.Errorf("got wrong error during instance creation '%s', expected '%s'", lastError, expectedError) + } +} + +const invalidConfigUnknownImpl = `{"implementation":"something-unknown","parameters":{"key":"a0a1a2a3a4a5a6a7a8a9b0b1b2b3b4b5b6b7b8b9c0c1c2c3c4c5c6c7c8c9d0d1"}}` +const invalidConfigNoKey = `{"implementation":"client-side/AES256-CFB/SHA256","parameters":{}}` + +func TestCreation_invalidConfigUnknownImpl(t *testing.T) { + creationErrorCase(t, invalidConfigUnknownImpl, "[ERROR] failed to configure remote state encryption: unsupported implementation 'something-unknown'") +} + +func TestCreation_invalidConfigNoKey(t *testing.T) { + creationErrorCase(t, invalidConfigNoKey, "[ERROR] failed to configure remote state encryption: configuration for AES256 needs the parameter 'key' set to a 32 byte lower case hexadecimal value") +} + +// business scenarios + +const validConfigWithKey1 = `{"implementation":"client-side/AES256-CFB/SHA256","parameters":{"key":"a0a1a2a3a4a5a6a7a8a9b0b1b2b3b4b5b6b7b8b9c0c1c2c3c4c5c6c7c8c9d0d1"}}` +const validConfigWithKey2 = `{"implementation":"client-side/AES256-CFB/SHA256","parameters":{"key":"89346775897897a35892735ffd34723489734ee238748293741abcdef0123456"}}` +const validConfigWithKey3 = `{"implementation":"client-side/AES256-CFB/SHA256","parameters":{"key":"33336775897897a35892735ffd34723489734ee238748293741abcdef0123456"}}` + +const validPlaintext = `{"animals":[{"species":"cheetah","genus":"acinonyx"}]}` +const validEncryptedKey1 = `{"crypted":"e93e3e7ad3434055251f695865a13c11744b97e54cb7dee8f8fb40d1fb096b728f2a00606e7109f0720aacb15008b410cf2f92dd7989c2ff10b9712b6ef7d69ecdad1dccd2f1bddd127f0f0d87c79c3c062e03c2297614e2effa2fb1f4072d86df0dda4fc061"}` + +func compareSlices(got []byte, expected []byte) bool { + eEmpty := len(expected) == 0 + gEmpty := len(got) == 0 + if eEmpty != gEmpty { + return false + } + if eEmpty { + return true + } + if len(expected) != len(got) { + return false + } + for i, v := range expected { + if v != got[i] { + return false + } + } + return true +} + +func compareErrors(got error, expected string) string { + if got != nil { + if got.Error() != expected { + return fmt.Sprintf("unexpected error '%s'; want '%s'", got.Error(), expected) + } + } else { + if expected != "" { + return fmt.Sprintf("did not get expected error '%s'", expected) + } + } + return "" +} + +type roundtripTestCase struct { + description string + mainConfiguration string + fallbackConfiguration string + input string + injectOutput string + expectedEncError string + expectedDecError string +} + +func TestEncryptDecrypt(t *testing.T) { + // each test case first encrypts, then decrypts again + testCases := []roundtripTestCase{ + // happy path cases + { + description: "unencrypted operation - no encryption configuration present, no fallback", + input: validPlaintext, + }, + { + description: "normal operation on encrypted data - main configuration for aes256, no fallback", + mainConfiguration: validConfigWithKey1, + input: validPlaintext, + }, + { + description: "initial encryption - main configuration for aes256, no fallback - prints a warning but must work anyway", + mainConfiguration: validConfigWithKey1, + input: validPlaintext, + injectOutput: validPlaintext, + }, + { + description: "decryption - no main configuration, fallback aes256", + fallbackConfiguration: validConfigWithKey1, + input: validPlaintext, // exact value irrelevant for this test case + injectOutput: validEncryptedKey1, + }, + { + description: "unencrypted operation with fallback still present (decryption edge case) - no encryption configuration present, fallback aes256 - prints a warning but must still work anyway", + input: validPlaintext, + fallbackConfiguration: validConfigWithKey1, + }, + { + description: "key rotation - main configuration for aes256 key 2, fallback aes256 key 1, read state with key 1 encryption - prints a warning but must work anyway", + mainConfiguration: validConfigWithKey2, + fallbackConfiguration: validConfigWithKey1, + input: validPlaintext, // exact value irrelevant for this test case + injectOutput: validEncryptedKey1, + }, + { + description: "key rotation - main configuration for aes256 key 2, fallback aes256 key 1, read state with key 2 encryption", + mainConfiguration: validConfigWithKey2, + fallbackConfiguration: validConfigWithKey1, + input: validPlaintext, + }, + { + description: "initial encryption happens during key rotation (key rotation edge case) - main configuration for aes256 key 1, fallback for aes256 key 2 - prints a warning but must still work anyway", + mainConfiguration: validConfigWithKey1, + fallbackConfiguration: validConfigWithKey2, + input: validPlaintext, // exact value irrelevant for this test case + injectOutput: validPlaintext, + }, + + // error cases + { + description: "decryption fails due to wrong key - main configuration for aes256 key 3 - but state was encrypted with key 1", + mainConfiguration: validConfigWithKey3, + input: validPlaintext, // exact value irrelevant for this test case + injectOutput: validEncryptedKey1, + expectedDecError: "hash of decrypted payload did not match at position 0", + }, + { + description: "decryption fails due to wrong fallback key during decrypt lifecycle - no main configuration, fallback configuration for aes256 key 3 - but state was encrypted with key 1 - must fail and not use passthrough", + fallbackConfiguration: validConfigWithKey3, + input: validPlaintext, // exact value irrelevant for this test case + injectOutput: validEncryptedKey1, + expectedDecError: "hash of decrypted payload did not match at position 0", + }, + { + description: "decryption fails due to two wrong keys - main configuration for aes256 key 3, fallback for aes256 key 2 - but state was encrypted with key 1", + mainConfiguration: validConfigWithKey3, + fallbackConfiguration: validConfigWithKey2, + input: validPlaintext, // exact value irrelevant for this test case + injectOutput: validEncryptedKey1, + expectedDecError: "hash of decrypted payload did not match at position 0", + }, + } + + var lastError string + logFatalf = func(format string, v ...interface{}) { + lastError = fmt.Sprintf(format, v...) + } + defer resetLogFatalf() + + for _, tc := range testCases { + log.Printf("test case: %s", tc.description) + + lastError = "" + mainConfig, err := cryptoconfig.Parse(tc.mainConfiguration) + if err != nil { + t.Fatal("error parsing main configuration") + } + fallbackConfig, err := cryptoconfig.Parse(tc.fallbackConfiguration) + if err != nil { + t.Fatal("error parsing fallback configuration") + } + cut := fallbackRetryInstance( + instanceFromConfig(mainConfig, true), + instanceFromConfig(fallbackConfig, false), + ) + if lastError != "" { + t.Error("skipping test case, got error during instance creation: " + lastError) + } else { + if cut == nil { + t.Error("got unexpected nil implementation") + } else { + roundtripTestcase(t, cut, tc) + } + } + } +} + +func roundtripTestcase(t *testing.T, cut StateCryptoProvider, tc roundtripTestCase) { + encOutput, err := cut.Encrypt([]byte(tc.input)) + if comp := compareErrors(err, tc.expectedEncError); comp != "" { + t.Error(comp) + } else { + if tc.injectOutput != "" { + encOutput = []byte(tc.injectOutput) + } + + decOutput, err := cut.Decrypt(encOutput) + if comp := compareErrors(err, tc.expectedDecError); comp != "" { + t.Error(comp) + } else { + if err == nil && !compareSlices(decOutput, []byte(tc.input)) { + t.Errorf("round trip error, got %#v; want %#v", decOutput, []byte(tc.input)) + } + } + } +} diff --git a/website/docs/cli/config/environment-variables.mdx b/website/docs/cli/config/environment-variables.mdx index f03b4f633562..ed13c13d8dfc 100644 --- a/website/docs/cli/config/environment-variables.mdx +++ b/website/docs/cli/config/environment-variables.mdx @@ -161,6 +161,40 @@ export TF_IGNORE=trace For more details on `.terraformignore`, please see [Excluding Files from Upload with .terraformignore](/language/settings/backends/remote#excluding-files-from-upload-with-terraformignore). +## TF_REMOTE_STATE_ENCRYPTION + +**This is an experimental feature!** + +Set `TF_REMOTE_STATE_ENCRYPTION` to a valid json document with two fields + + * `implementation`: select a _state crypto provider_ by name + * `parameters`: configure the _state crypto provider_ + +to enable client-side remote state encryption. + +```shell +export TF_REMOTE_STATE_ENCRYPTION='{"implementation":"client-side/AES256-CFB/SHA256","parameters":{"key":"a0a1a2a3a4a5a6a7a8a9b0b1b2b3b4b5b6b7b8b9c0c1c2c3c4c5c6c7c8c9d0d1"}}' +``` + +Not setting this environment variable (or leaving it empty) disables this feature, that is Terraform sends +unencrypted state to the remote state backend as usual. + +For more details please see [Client-Side Remote State Encryption](/docs/language/state/encryption.html). + +## TF_REMOTE_STATE_DECRYPTION_FALLBACK + +**This is an experimental feature!** + +Set `TF_REMOTE_STATE_DECRYPTION_FALLBACK` to a fallback configuration for client-side remote state decryption. + +When decrypting remote state, Terraform will always try the configuration in `TF_REMOTE_STATE_ENCRYPTION` first, +then try this one, if provided. + +This is useful for key rotation, permanent decryption, and switching between state crypto providers. Just set this +variable to the old configuration, then cause a change in state. + +For more details please see [Client-Side Remote State Encryption](/docs/language/state/encryption.html). + ## Terraform Cloud CLI Integration The CLI integration with Terraform Cloud lets you use Terraform Cloud and Terraform Enterprise on the command line. The integration requires including a `cloud` block in your Terraform configuration. You can define its arguments directly in your configuration file or supply them through environment variables, which can be useful for non-interactive workflows like Continuous Integration (CI). diff --git a/website/docs/language/state/encryption.html.md b/website/docs/language/state/encryption.html.md new file mode 100644 index 000000000000..ea2d3e16b766 --- /dev/null +++ b/website/docs/language/state/encryption.html.md @@ -0,0 +1,137 @@ +--- +layout: "language" +page_title: "State: Client-Side Remote State Encryption" +sidebar_current: "docs-state-encryption" +description: |- + Client-Side Remote State Encryption. +--- + +# Client-Side Remote State Encryption + +State regularly contains mission-critical sensitive information such as credentials. + +Several remote state backends already support server-side encryption, but + +- many do not +- some implementations effectively turn encryption into nothing but a proxy for key access control +- you need to trust the remote backend operator +- you need to trust your communication channel to the remote backend + +This **experimental feature** lets you encrypt the complete state on the client before transferring it. + +Depending on your choice of state crypto provider, no third party ever sees the key. + +## Limitations + +Client-side state encryption will not work with [enhanced backends](/docs/language/settings/backends/index.html), as +those need access to the information in the state to function correctly. + +**Client-side remote state encryption is an experimental feature!** Implementation and configuration details are subject to change, +and not all remote state backends have been fully tested! + +## State Crypto Providers + +State crypto providers are a transparent encryption layer around the communication with remote state backends. + +**This is an experimental feature!** + +You can configure state crypto providers by using two environment variables: + +- `TF_REMOTE_STATE_ENCRYPTION`: configuration used for encryption and decryption +- `TF_REMOTE_STATE_DECRYPTION_FALLBACK`: fallback configuration for decryption, tried if decryption fails with the first choice + +You set each of these to a json document with two fields + + * `implementation`: select a _state crypto provider_ by name + * `parameters`: configure the _state crypto provider_ + +to enable client-side remote state encryption. To disable, either do not set the variable at all or set it to a blank value. + +Right now, there is only one value for `implementation`: `client-side/AES256-CFB/SHA256`. + +In the future, more state encryption providers may be added, such as: + +- asymmetric encryption with RSA public key cryptography +- key retrieval from [Vault](https://www.vaultproject.io/) +- ... + +### client-side/AES256-CFB/SHA256 + +This state crypto provider offers pure client-side symmetric encryption. + +The key is not transferred to any third party. Note that this places the burden of key management on you +and you alone. + +Encryption is performed with AES256-CFB, using a fresh random initialization vector every time. Payload integrity +is verified using a SHA256 hash over the plaintext, which is encrypted with the plaintext. + +_Implementation Name:_ `client-side/AES256-CFB/SHA256` + +_Parameters:_ + +- `key`: the 32 byte AES256 key represented in hexadecimal, must be exactly 64 characters, `0-9a-f` only. + +Example: + +```shell +export TF_REMOTE_STATE_ENCRYPTION='{"implementation":"client-side/AES256-CFB/SHA256","parameters":{"key":"a0a1a2a3a4a5a6a7a8a9b0b1b2b3b4b5b6b7b8b9c0c1c2c3c4c5c6c7c8c9d0d1"}}' +``` + +## State Encryption Lifecycle + +When transparently encrypting the state, one must consider these lifecycle events: + +- initial encryption +- the way out (decrypting state that is currently encrypted) +- key rotation +- switching state encryption providers (e.g. there is a security issue with one of them, or a wish to migrate) + +### Preparations + +All remote state encryption lifecycle events will require you to cause a change in state to trigger (re-)encryption +or decryption during `terraform apply`. + +The easiest way to achieve this is by putting a null resource in your state that you do not use anywhere else. +Then you can change its value to an arbitrary new value without needing to make an actual change to your infrastructure. + +### Initial Encryption + +Let us assume you currently have unencrypted remote state and wish to encrypt it going forward. + +The same approach also works when remote state is initially created. + +With the ability to force a change in state in place (see Preparations above) do this: + +1. Set `TF_REMOTE_STATE_ENCRYPTION` to the desired configuration +2. Leave `TF_REMOTE_STATE_DECRYPTION_FALLBACK` unset +3. Cause a change in state and run `terraform apply`. This will encrypt your state. + +From now on, you will need to run terraform with `TF_REMOTE_STATE_ENCRYPTION` set to the configuration you just used, +or it will not be able to read your state any more. + +### Permanent Decryption + +Now let us assume that you wish to move from encrypted remote state to the default unencrypted state. Do this: + +1. Leave `TF_REMOTE_STATE_ENCRYPTION` unset +2. Set `TF_REMOTE_STATE_DECRYPTION_FALLBACK` to the configuration previously used to encrypt your state +3. Cause a change in state and run `terraform apply`. This will decrypt your state. + +Once all your state has been decrypted, you should unset both environment variables, and as long as you +do not set them again, terraform will operate on unencrypted remote state. + +### Key Rotation + +Now let us assume that you wish to move from one encryption key to another. Do this: + +1. Set `TF_REMOTE_STATE_ENCRYPTION` to the configuration with the new key +2. Set `TF_REMOTE_STATE_DECRYPTION_FALLBACK` to the previous configuration with the old key +3. Cause a change in state and run `terraform apply`. This will re-encrypt your state. + +Once all your state has been migrated, you can then drop the fallback configuration and run +terraform with `TF_REMOTE_STATE_ENCRYPTION` set to the new configuration going forward. + +### Switching State Crypto Providers + +Just use the same approach as for key rotation. There is no requirement to use the same state crypto +provider for encryption and fallback.