diff --git a/README.rst b/README.rst index 6fdc2497f..586e256bc 100644 --- a/README.rst +++ b/README.rst @@ -1207,15 +1207,28 @@ This command requires a ``.sops.yaml`` configuration file. Below is an example: vault_kv_version: 2 # default path_regex: vault/* omit_extensions: true + - aws_secrets_manager_secret_name: "my-secret" + aws_region: "us-west-2" + path_regex: aws-secrets/* + - aws_parameter_store_path: "/sops/" + aws_region: "us-west-2" + path_regex: aws-params/* The above configuration will place all files under ``s3/*`` into the S3 bucket ``sops-secrets``, -all files under ``gcs/*`` into the GCS bucket ``sops-secrets``, and the contents of all files under -``vault/*`` into Vault's KV store under the path ``secrets/sops/``. For the files that will be -published to S3 and GCS, it will decrypt them and re-encrypt them using the -``F69E4901EDBAD2D1753F8C67A64535C4163FB307`` pgp key. +all files under ``gcs/*`` into the GCS bucket ``sops-secrets``, the contents of all files under +``vault/*`` into Vault's KV store under the path ``secrets/sops/``, files under ``aws-secrets/*`` +into AWS Secrets Manager as JSON secrets, and files under ``aws-params/*`` into AWS Parameter Store +as SecureString parameters. For the files that will be published to S3 and GCS, it will decrypt them +and re-encrypt them using the ``F69E4901EDBAD2D1753F8C67A64535C4163FB307`` pgp key. Files published to Vault +will be decrypted and stored as JSON data encrypted by Vault. Files published to AWS Secrets Manager and AWS Parameter Store +will be decrypted and stored as JSON data encrypted by AWS KMS. You would deploy a file to S3 with a command like: ``sops publish s3/app.yaml`` +Similarly, you can publish to AWS Secrets Manager: ``sops publish aws-secrets/database-config.yaml`` + +Or to AWS Parameter Store: ``sops publish aws-params/app-config.yaml`` + To publish all files in selected directory recursively, you need to specify ``--recursive`` flag. If you don't want file extension to appear in destination secret path, use ``--omit-extensions`` @@ -1267,6 +1280,77 @@ Below is an example of publishing to Vault (using token auth with a local dev in example_number 42 example_string bar +Publishing to AWS Secrets Manager +********************************** + +AWS Secrets Manager is a service that helps you protect secrets needed to access your applications, +services, and IT resources. SOPS can publish decrypted data directly to AWS Secrets Manager as JSON secrets. + +There are a few settings for AWS Secrets Manager that you can place in your destination rules: + +* ``aws_secrets_manager_secret_name`` - The name of the secret in AWS Secrets Manager. If not specified, the filename will be used as the secret name. +* ``aws_region`` - The AWS region where the secret should be stored. This is required. + +SOPS uses the AWS SDK for Go v2, which automatically uses your configured AWS credentials from the AWS CLI, +environment variables, or IAM roles. + +If the destination secret already exists in AWS Secrets Manager and contains the same data as the source +file, it will be skipped to avoid creating unnecessary versions. + +Note: Recreation rules (re-encryption with different keys) are not supported for AWS Secrets Manager. +The data is decrypted from the source file and stored as plaintext JSON in the secret. + +Below is an example of publishing to AWS Secrets Manager: + +.. code:: sh + + $ export AWS_REGION=us-west-2 + $ sops decrypt aws-secrets/database-config.yaml + database: + host: db.example.com + port: 5432 + username: myuser + password: mypassword + $ sops publish aws-secrets/database-config.yaml + uploading /home/user/sops_directory/aws-secrets/database-config.yaml to AWS Secrets Manager secret database-config in us-west-2 ? (y/n): y + Successfully created secret database-config + +Publishing to AWS Parameter Store +********************************** + +AWS Systems Manager Parameter Store provides secure, hierarchical storage for configuration data +and secrets management. SOPS can publish decrypted data directly to Parameter Store as JSON parameters encrypted by AWS KMS. + +There are a few settings for AWS Parameter Store that you can place in your destination rules: + +* ``aws_parameter_store_path`` - The parameter path in AWS Parameter Store. If it ends with ``/``, the filename will be appended. If not specified, the filename will be used as the parameter name with a leading ``/``. +* ``aws_region`` - The AWS region where the parameter should be stored. This is required. + +All parameters are stored as ``SecureString`` type for security, since SOPS files may contain sensitive data. + +SOPS uses the AWS SDK for Go v2, which automatically uses your configured AWS credentials from the AWS CLI, +environment variables, or IAM roles. + +If the destination parameter already exists in AWS Parameter Store and contains the same data as the source +file, it will be skipped to avoid creating unnecessary versions. + +Note: Recreation rules (re-encryption with different keys) are not supported for AWS Parameter Store. +The data is decrypted from the source file and stored as JSON in the SecureString parameter, encrypted by AWS KMS. + +Below is an example of publishing to AWS Parameter Store: + +.. code:: sh + + $ export AWS_REGION=us-west-2 + $ sops decrypt aws-params/app-config.yaml + app: + debug: false + database_url: postgres://user:pass@localhost/db + api_key: secret-api-key + $ sops publish aws-params/app-config.yaml + uploading /home/user/sops_directory/aws-params/app-config.yaml to AWS Parameter Store parameter /app-config in us-west-2 ? (y/n): y + Successfully created parameter /app-config + Important information on types ------------------------------ diff --git a/cmd/sops/subcommand/publish/publish.go b/cmd/sops/subcommand/publish/publish.go index aed1118de..fba541003 100644 --- a/cmd/sops/subcommand/publish/publish.go +++ b/cmd/sops/subcommand/publish/publish.go @@ -137,7 +137,7 @@ func Run(opts Opts) error { return fmt.Errorf("could not read file: %s", err) } } - case *publish.VaultDestination: + case *publish.VaultDestination, *publish.AWSSecretsManagerDestination, *publish.AWSParameterStoreDestination: _, err = common.DecryptTree(common.DecryptTreeOpts{ Cipher: opts.Cipher, IgnoreMac: false, @@ -177,7 +177,7 @@ func Run(opts Opts) error { switch dest := conf.Destination.(type) { case *publish.S3Destination, *publish.GCSDestination: err = dest.Upload(fileContents, destinationPath) - case *publish.VaultDestination: + case *publish.VaultDestination, *publish.AWSSecretsManagerDestination, *publish.AWSParameterStoreDestination: err = dest.UploadUnencrypted(data, destinationPath) } diff --git a/config/config.go b/config/config.go index 6a67e0619..63fc12247 100644 --- a/config/config.go +++ b/config/config.go @@ -156,17 +156,22 @@ type azureKVKey struct { } type destinationRule struct { - PathRegex string `yaml:"path_regex"` - S3Bucket string `yaml:"s3_bucket"` - S3Prefix string `yaml:"s3_prefix"` - GCSBucket string `yaml:"gcs_bucket"` - GCSPrefix string `yaml:"gcs_prefix"` - VaultPath string `yaml:"vault_path"` - VaultAddress string `yaml:"vault_address"` - VaultKVMountName string `yaml:"vault_kv_mount_name"` - VaultKVVersion int `yaml:"vault_kv_version"` - RecreationRule creationRule `yaml:"recreation_rule,omitempty"` - OmitExtensions bool `yaml:"omit_extensions"` + PathRegex string `yaml:"path_regex"` + S3Bucket string `yaml:"s3_bucket"` + S3Prefix string `yaml:"s3_prefix"` + GCSBucket string `yaml:"gcs_bucket"` + GCSPrefix string `yaml:"gcs_prefix"` + VaultPath string `yaml:"vault_path"` + VaultAddress string `yaml:"vault_address"` + VaultKVMountName string `yaml:"vault_kv_mount_name"` + VaultKVVersion int `yaml:"vault_kv_version"` + RecreationRule creationRule `yaml:"recreation_rule,omitempty"` + OmitExtensions bool `yaml:"omit_extensions"` + AWSSecretsManagerRegion string `yaml:"aws_secrets_manager_region"` + AWSSecretsManagerSecretName string `yaml:"aws_secrets_manager_secret_name"` + AWSParameterStoreRegion string `yaml:"aws_parameter_store_region"` + AWSParameterStorePath string `yaml:"aws_parameter_store_path"` + AWSParameterStoreType string `yaml:"aws_parameter_store_type"` } type creationRule struct { @@ -523,6 +528,13 @@ func parseDestinationRuleForFile(conf *configFile, filePath string, kmsEncryptio destinationCount++ } + if dRule.AWSSecretsManagerRegion != "" { + destinationCount++ + } + if dRule.AWSParameterStoreRegion != "" { + destinationCount++ + } + if destinationCount > 1 { return nil, fmt.Errorf("error loading config: more than one destinations were found in a single destination rule, you can only use one per rule") } @@ -535,6 +547,12 @@ func parseDestinationRuleForFile(conf *configFile, filePath string, kmsEncryptio if dRule.VaultPath != "" { dest = publish.NewVaultDestination(dRule.VaultAddress, dRule.VaultPath, dRule.VaultKVMountName, dRule.VaultKVVersion) } + if dRule.AWSSecretsManagerRegion != "" { + dest = publish.NewAWSSecretsManagerDestination(dRule.AWSSecretsManagerRegion, dRule.AWSSecretsManagerSecretName) + } + if dRule.AWSParameterStoreRegion != "" { + dest = publish.NewAWSParameterStoreDestination(dRule.AWSParameterStoreRegion, dRule.AWSParameterStorePath) + } config, err := configFromRule(rule, kmsEncryptionContext) if err != nil { diff --git a/config/config_aws_test.go b/config/config_aws_test.go new file mode 100644 index 000000000..3b52ee9ac --- /dev/null +++ b/config/config_aws_test.go @@ -0,0 +1,123 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +var sampleConfigWithAWSSecretsManagerDestinationRules = []byte(` +creation_rules: + - path_regex: foobar* + kms: "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012" +destination_rules: + - aws_secrets_manager_region: "us-east-1" + aws_secrets_manager_secret_name: "myapp/database" + path_regex: "^secrets/.*" + - aws_secrets_manager_region: "us-west-2" + path_regex: "^west-secrets/.*" +`) + +var sampleConfigWithAWSParameterStoreDestinationRules = []byte(` +creation_rules: + - path_regex: foobar* + kms: "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012" +destination_rules: + - aws_parameter_store_region: "us-east-1" + aws_parameter_store_path: "/myapp/config" + aws_parameter_store_type: "SecureString" + path_regex: "^parameters/.*" + - aws_parameter_store_region: "us-west-2" + aws_parameter_store_path: "/myapp/west/" + aws_parameter_store_type: "String" + path_regex: "^west-parameters/.*" +`) + +var sampleConfigWithMixedAWSDestinationRules = []byte(` +creation_rules: + - path_regex: foobar* + kms: "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012" +destination_rules: + - aws_secrets_manager_region: "us-east-1" + aws_secrets_manager_secret_name: "myapp/database" + path_regex: "^secrets/.*" + - aws_parameter_store_region: "us-east-1" + aws_parameter_store_path: "/myapp/config" + path_regex: "^parameters/.*" + - s3_bucket: "mybucket" + path_regex: "^s3/.*" +`) + +func TestLoadConfigFileWithAWSSecretsManagerDestinationRules(t *testing.T) { + conf, err := parseDestinationRuleForFile(parseConfigFile(sampleConfigWithAWSSecretsManagerDestinationRules, t), "secrets/database.yaml", nil) + assert.Nil(t, err) + assert.NotNil(t, conf.Destination) + path := conf.Destination.Path("database.yaml") + assert.Contains(t, path, "arn:aws:secretsmanager:us-east-1:*:secret:myapp/database") + + // Test with region but no specific secret name - this should match the second rule + conf, err = parseDestinationRuleForFile(parseConfigFile(sampleConfigWithAWSSecretsManagerDestinationRules, t), "west-secrets/api.yaml", nil) + assert.Nil(t, err) + assert.NotNil(t, conf.Destination) + path = conf.Destination.Path("api.yaml") + assert.Contains(t, path, "arn:aws:secretsmanager:us-west-2:*:secret:api.yaml") +} + +func TestLoadConfigFileWithAWSParameterStoreDestinationRules(t *testing.T) { + conf, err := parseDestinationRuleForFile(parseConfigFile(sampleConfigWithAWSParameterStoreDestinationRules, t), "parameters/app.yaml", nil) + assert.Nil(t, err) + assert.NotNil(t, conf.Destination) + assert.Equal(t, "/myapp/config", conf.Destination.Path("app.yaml")) + + // Test with path ending with slash + conf, err = parseDestinationRuleForFile(parseConfigFile(sampleConfigWithAWSParameterStoreDestinationRules, t), "west-parameters/config.yaml", nil) + assert.Nil(t, err) + assert.NotNil(t, conf.Destination) + assert.Equal(t, "/myapp/west/config.yaml", conf.Destination.Path("config.yaml")) +} + +func TestLoadConfigFileWithMixedAWSDestinationRules(t *testing.T) { + // Test AWS Secrets Manager + conf, err := parseDestinationRuleForFile(parseConfigFile(sampleConfigWithMixedAWSDestinationRules, t), "secrets/database.yaml", nil) + assert.Nil(t, err) + assert.NotNil(t, conf.Destination) + assert.Contains(t, conf.Destination.Path("database.yaml"), "arn:aws:secretsmanager:us-east-1:*:secret:myapp/database") + + // Test AWS Parameter Store + conf, err = parseDestinationRuleForFile(parseConfigFile(sampleConfigWithMixedAWSDestinationRules, t), "parameters/config.yaml", nil) + assert.Nil(t, err) + assert.NotNil(t, conf.Destination) + assert.Equal(t, "/myapp/config", conf.Destination.Path("config.yaml")) + + // Test S3 + conf, err = parseDestinationRuleForFile(parseConfigFile(sampleConfigWithMixedAWSDestinationRules, t), "s3/backup.yaml", nil) + assert.Nil(t, err) + assert.NotNil(t, conf.Destination) + assert.Contains(t, conf.Destination.Path("backup.yaml"), "s3://mybucket/backup.yaml") +} + +func TestValidateMultipleDestinationsInRule(t *testing.T) { + invalidConfig := []byte(` +destination_rules: + - aws_secrets_manager_region: "us-east-1" + aws_parameter_store_region: "us-east-1" + path_regex: "^invalid/.*" +`) + + _, err := parseDestinationRuleForFile(parseConfigFile(invalidConfig, t), "invalid/test.yaml", nil) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "more than one destinations were found") +} + +func TestValidateConflictingAWSDestinations(t *testing.T) { + invalidConfig := []byte(` +destination_rules: + - aws_secrets_manager_region: "us-east-1" + s3_bucket: "mybucket" + path_regex: "^invalid/.*" +`) + + _, err := parseDestinationRuleForFile(parseConfigFile(invalidConfig, t), "invalid/test.yaml", nil) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "more than one destinations were found") +} diff --git a/config/config_test.go b/config/config_test.go index 753f870b1..b11787316 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -879,3 +879,114 @@ destination_rules: assert.NotNil(t, conf.Destination) assert.Contains(t, conf.Destination.Path("secrets.yaml"), "https://vault.example.com/v1/secret/data/secret/sops/secrets.yaml") } + +func TestDestinationValidationAWSSecretsManagerConflicts(t *testing.T) { + testCases := []struct { + name string + config []byte + }{ + { + name: "AWS Secrets Manager + GCS conflict", + config: []byte(` +destination_rules: + - aws_secrets_manager_region: "us-east-1" + gcs_bucket: "my-gcs-bucket" + path_regex: "^test/.*" +`), + }, + { + name: "AWS Secrets Manager + Vault conflict", + config: []byte(` +destination_rules: + - aws_secrets_manager_region: "us-east-1" + vault_path: "secret/sops" + vault_address: "https://vault.example.com" + path_regex: "^test/.*" +`), + }, + { + name: "AWS Secrets Manager + AWS Parameter Store conflict", + config: []byte(` +destination_rules: + - aws_secrets_manager_region: "us-east-1" + aws_parameter_store_region: "us-east-1" + path_regex: "^test/.*" +`), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := parseDestinationRuleForFile(parseConfigFile(tc.config, t), "test/secrets.yaml", nil) + assert.NotNil(t, err, "Expected error for %s", tc.name) + if err != nil { + assert.Contains(t, err.Error(), "more than one destinations were found") + } + }) + } +} + +func TestDestinationValidationAWSParameterStoreConflicts(t *testing.T) { + testCases := []struct { + name string + config []byte + }{ + { + name: "AWS Parameter Store + S3 conflict", + config: []byte(` +destination_rules: + - aws_parameter_store_region: "us-east-1" + s3_bucket: "my-s3-bucket" + path_regex: "^test/.*" +`), + }, + { + name: "AWS Parameter Store + GCS conflict", + config: []byte(` +destination_rules: + - aws_parameter_store_region: "us-east-1" + gcs_bucket: "my-gcs-bucket" + path_regex: "^test/.*" +`), + }, + { + name: "AWS Parameter Store + Vault conflict", + config: []byte(` +destination_rules: + - aws_parameter_store_region: "us-east-1" + vault_path: "secret/sops" + vault_address: "https://vault.example.com" + path_regex: "^test/.*" +`), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := parseDestinationRuleForFile(parseConfigFile(tc.config, t), "test/secrets.yaml", nil) + assert.NotNil(t, err, "Expected error for %s", tc.name) + if err != nil { + assert.Contains(t, err.Error(), "more than one destinations were found") + } + }) + } +} + +func TestDestinationValidationAllFiveDestinationsConflict(t *testing.T) { + invalidConfig := []byte(` +destination_rules: + - aws_secrets_manager_region: "us-east-1" + aws_parameter_store_region: "us-east-1" + s3_bucket: "my-s3-bucket" + gcs_bucket: "my-gcs-bucket" + vault_path: "secret/sops" + vault_address: "https://vault.example.com" + path_regex: "^test/.*" +`) + + _, err := parseDestinationRuleForFile(parseConfigFile(invalidConfig, t), "test/secrets.yaml", nil) + assert.NotNil(t, err, "Expected error when all five destinations are specified") + if err != nil { + assert.Contains(t, err.Error(), "more than one destinations were found") + } +} diff --git a/go.mod b/go.mod index aa15fcaaa..a0aba6190 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,10 @@ require ( github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.15 github.com/aws/aws-sdk-go-v2/service/kms v1.46.2 github.com/aws/aws-sdk-go-v2/service/s3 v1.88.7 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.39.2 + github.com/aws/aws-sdk-go-v2/service/ssm v1.64.2 github.com/aws/aws-sdk-go-v2/service/sts v1.38.9 + github.com/aws/smithy-go v1.23.1 github.com/blang/semver v3.5.1+incompatible github.com/fatih/color v1.18.0 github.com/getsops/gopgagent v0.0.0-20241224165529-7044f28e491e @@ -78,7 +81,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.11 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.29.8 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3 // indirect - github.com/aws/smithy-go v1.23.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/circl v1.6.1 // indirect diff --git a/go.sum b/go.sum index afee6f0b5..b6211e724 100644 --- a/go.sum +++ b/go.sum @@ -95,6 +95,10 @@ github.com/aws/aws-sdk-go-v2/service/kms v1.46.2 h1:hz2rJseQXnVQtVbByFpeSCNJBBU7 github.com/aws/aws-sdk-go-v2/service/kms v1.46.2/go.mod h1:E4ink1KCQgqIe2pHFD9E+b5CNXovm50rQbWFuh0cM+I= github.com/aws/aws-sdk-go-v2/service/s3 v1.88.7 h1:Wer3W0GuaedWT7dv/PiWNZGSQFSTcBY2rZpbiUp5xcA= github.com/aws/aws-sdk-go-v2/service/s3 v1.88.7/go.mod h1:UHKgcRSx8PVtvsc1Poxb/Co3PD3wL7P+f49P0+cWtuY= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.39.2 h1:QMayWWWmfWyQwP4nZf3qdIVS39Pm65Yi5waYj1euCzo= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.39.2/go.mod h1:4eAXC8WdO1rRt01ZKKq57z8oTzzLkkIo5IReQ+b8hEU= +github.com/aws/aws-sdk-go-v2/service/ssm v1.64.2 h1:6P4W42RUTZixRG6TgfRB8KlsqNzHtvBhs6sTbkVPZvk= +github.com/aws/aws-sdk-go-v2/service/ssm v1.64.2/go.mod h1:wtxdacy3oO5sHO03uOtk8HMGfgo1gBHKwuJdYM220i0= github.com/aws/aws-sdk-go-v2/service/sso v1.29.8 h1:M5nimZmugcZUO9wG7iVtROxPhiqyZX6ejS1lxlDPbTU= github.com/aws/aws-sdk-go-v2/service/sso v1.29.8/go.mod h1:mbef/pgKhtKRwrigPPs7SSSKZgytzP8PQ6P6JAAdqyM= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3 h1:S5GuJZpYxE0lKeMHKn+BRTz6PTFpgThyJ+5mYfux7BM= diff --git a/publish/aws_integration_test.go b/publish/aws_integration_test.go new file mode 100644 index 000000000..8b88ba24b --- /dev/null +++ b/publish/aws_integration_test.go @@ -0,0 +1,225 @@ +//go:build integration + +package publish + +import ( + "context" + "encoding/json" + "os" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/aws-sdk-go-v2/service/ssm" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Integration tests for AWS Secrets Manager and Parameter Store publishing. +// These tests require real AWS credentials and resources. +// +// To run these tests: +// 1. Set up AWS credentials (AWS_PROFILE, AWS_ACCESS_KEY_ID, etc.) +// 2. Set required environment variables: +// - SOPS_TEST_AWS_REGION (default: us-east-1) +// - SOPS_TEST_AWS_SECRET_NAME (secret for testing) +// - SOPS_TEST_AWS_PARAMETER_NAME (parameter for testing) +// 3. Run with: go test -tags=integration ./publish -run TestAWS -v +// +// Prerequisites: +// - AWS credentials with Secrets Manager and Parameter Store permissions +// - Test secret and parameter resources should already exist or be creatable + +var ( + testAWSRegion = getEnvOrDefault("SOPS_TEST_AWS_REGION", "us-east-1") + testSecretName = os.Getenv("SOPS_TEST_AWS_SECRET_NAME") // e.g., "sops-test-secret" + testParameterName = os.Getenv("SOPS_TEST_AWS_PARAMETER_NAME") // e.g., "/sops-test/parameter" +) + +func getEnvOrDefault(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue +} + +func TestAWSSecretsManagerDestination_PlainText_Integration(t *testing.T) { + if testSecretName == "" { + t.Skip("Skipping integration test: SOPS_TEST_AWS_SECRET_NAME not set") + } + + ctx := context.Background() + dest := NewAWSSecretsManagerDestination(testAWSRegion, testSecretName) + + // Test data with complex nested structure + // Note: This format stores as Plain Text JSON and does NOT enable key/value editor in AWS console + // For key/value format, see TestAWSSecretsManagerDestination_KeyValue_Integration + testData := map[string]interface{}{ + "database": map[string]interface{}{ + "host": "localhost", + "port": float64(5432), + "username": "testuser", + "password": "supersecret", + }, + "api_keys": map[string]interface{}{ + "stripe": "sk_test_123456", + "github": "ghp_987654321", + }, + } + + // Upload test data + err := dest.UploadUnencrypted(testData, "test-secret") + require.NoError(t, err, "Failed to upload secret to Secrets Manager") + + // Verify the secret was stored correctly + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(testAWSRegion)) + require.NoError(t, err, "Failed to load AWS config") + + client := secretsmanager.NewFromConfig(cfg) + result, err := client.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(testSecretName), + }) + require.NoError(t, err, "Failed to retrieve secret from Secrets Manager") + + // Parse and verify the stored data + var storedData map[string]interface{} + err = json.Unmarshal([]byte(*result.SecretString), &storedData) + require.NoError(t, err, "Failed to parse stored secret JSON") + + assert.Equal(t, testData, storedData, "Stored data doesn't match original") + + // Test no-op behavior (upload same data again) + err = dest.UploadUnencrypted(testData, "test-secret") + assert.NoError(t, err, "No-op upload should succeed") +} + +func TestAWSSecretsManagerDestination_KeyValue_Integration(t *testing.T) { + if testSecretName == "" { + t.Skip("Skipping integration test: SOPS_TEST_AWS_SECRET_NAME not set") + } + + ctx := context.Background() + // Use a different secret name for key/value test to avoid conflicts + keyValueSecretName := testSecretName + "-keyvalue" + dest := NewAWSSecretsManagerDestination(testAWSRegion, keyValueSecretName) + + // Test data with simple key/value pairs (no nested objects) + // This format enables the key/value editor in AWS Secrets Manager console + testData := map[string]interface{}{ + "database_host": "db.example.com", + "database_port": "5432", + "database_username": "app_user", + "database_password": "super_secret_password", + "api_key_stripe": "sk_live_abcdef123456", + "api_key_github": "ghp_xyz789012345", + "debug_mode": "false", + "log_level": "info", + "max_connections": "100", + "timeout_seconds": "30", + } + + // Upload test data + err := dest.UploadUnencrypted(testData, "test-keyvalue-secret") + require.NoError(t, err, "Failed to upload key/value secret to Secrets Manager") + + // Verify the secret was stored correctly + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(testAWSRegion)) + require.NoError(t, err, "Failed to load AWS config") + + client := secretsmanager.NewFromConfig(cfg) + result, err := client.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(keyValueSecretName), + }) + require.NoError(t, err, "Failed to retrieve key/value secret from Secrets Manager") + + // Parse and verify the stored data + var storedData map[string]interface{} + err = json.Unmarshal([]byte(*result.SecretString), &storedData) + require.NoError(t, err, "Failed to parse stored key/value secret JSON") + + assert.Equal(t, testData, storedData, "Stored key/value data doesn't match original") + + // Verify that all values are stored as strings (important for key/value format) + for key, value := range storedData { + assert.IsType(t, "", value, "Value for key %s should be a string for key/value format", key) + } + + // Test no-op behavior (upload same data again) + err = dest.UploadUnencrypted(testData, "test-keyvalue-secret") + assert.NoError(t, err, "No-op upload should succeed for key/value format") +} + +func TestAWSParameterStoreDestination_Integration(t *testing.T) { + if testParameterName == "" { + t.Skip("Skipping integration test: SOPS_TEST_AWS_PARAMETER_NAME not set") + } + + ctx := context.Background() + dest := NewAWSParameterStoreDestination(testAWSRegion, testParameterName) + + // Test data + testData := map[string]interface{}{ + "app_config": map[string]interface{}{ + "debug": false, + "log_level": "info", + "max_workers": float64(10), + "features": map[string]interface{}{ + "new_ui": true, + "beta_feature": false, + }, + }, + } + + // Upload test data (this is the method used by the publish command) + err := dest.UploadUnencrypted(testData, "test-config") + require.NoError(t, err, "Failed to upload parameter to Parameter Store") + + // Verify the parameter was stored correctly + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(testAWSRegion)) + require.NoError(t, err, "Failed to load AWS config") + + client := ssm.NewFromConfig(cfg) + result, err := client.GetParameter(ctx, &ssm.GetParameterInput{ + Name: aws.String(testParameterName), + WithDecryption: aws.Bool(true), + }) + require.NoError(t, err, "Failed to retrieve parameter from Parameter Store") + + // Parse and verify the stored data + var storedData map[string]interface{} + err = json.Unmarshal([]byte(*result.Parameter.Value), &storedData) + require.NoError(t, err, "Failed to parse stored parameter JSON") + + assert.Equal(t, testData, storedData, "Stored data doesn't match original") + + // Verify parameter type is always SecureString + assert.Equal(t, "SecureString", string(result.Parameter.Type), "Parameter type should always be SecureString") + + // Test no-op behavior (upload same data again) + err = dest.UploadUnencrypted(testData, "test-config") + assert.NoError(t, err, "No-op upload should succeed") +} + +func TestAWSParameterStoreDestination_EncryptedFile_Integration(t *testing.T) { + if testParameterName == "" { + t.Skip("Skipping integration test: SOPS_TEST_AWS_PARAMETER_NAME not set") + } + + dest := NewAWSParameterStoreDestination(testAWSRegion, testParameterName+"-file") + + encryptedContent := []byte(`# SOPS encrypted file +database: + host: ENC[AES256_GCM,data:xyz123,type:str] + password: ENC[AES256_GCM,data:abc456,type:str] +sops: + kms: + - arn: arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012 + version: 3.8.1`) + + // Upload should return NotImplementedError + err := dest.Upload(encryptedContent, "encrypted-test") + require.NotNil(t, err, "Upload should return an error") + assert.IsType(t, &NotImplementedError{}, err, "Should return NotImplementedError") + assert.Contains(t, err.Error(), "AWS Parameter Store does not support uploading encrypted sops files directly") +} diff --git a/publish/aws_parameter_store.go b/publish/aws_parameter_store.go new file mode 100644 index 000000000..648264e20 --- /dev/null +++ b/publish/aws_parameter_store.go @@ -0,0 +1,130 @@ +package publish + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ssm" + "github.com/aws/aws-sdk-go-v2/service/ssm/types" + "github.com/aws/smithy-go" + "github.com/getsops/sops/v3/logging" + "github.com/sirupsen/logrus" +) + +var parameterLog *logrus.Logger + +func init() { + parameterLog = logging.NewLogger("PUBLISH") +} + +// AWSParameterStoreDestination is the AWS Parameter Store implementation of the Destination interface. +type AWSParameterStoreDestination struct { + region string + parameterPath string +} + +// NewAWSParameterStoreDestination creates a new AWS Parameter Store destination. +func NewAWSParameterStoreDestination(region, parameterPath string) *AWSParameterStoreDestination { + // Ensure parameter path starts with / + if parameterPath != "" && !strings.HasPrefix(parameterPath, "/") { + parameterPath = "/" + parameterPath + } + + return &AWSParameterStoreDestination{region, parameterPath} +} + +// Path returns the AWS Parameter Store path for the given fileName. +func (awspsd *AWSParameterStoreDestination) Path(fileName string) string { + if awspsd.parameterPath != "" { + // If path ends with /, append filename; otherwise use path as-is + if strings.HasSuffix(awspsd.parameterPath, "/") { + return awspsd.parameterPath + fileName + } + return awspsd.parameterPath + } + // Default: use filename as parameter name + if !strings.HasPrefix(fileName, "/") { + return "/" + fileName + } + return fileName +} + +// Upload returns NotImplementedError as AWS Parameter Store does not support uploading encrypted files directly. +func (awspsd *AWSParameterStoreDestination) Upload(fileContents []byte, fileName string) error { + return &NotImplementedError{"AWS Parameter Store does not support uploading encrypted sops files directly. Use UploadUnencrypted instead."} +} + +// UploadUnencrypted uploads unencrypted data to AWS Parameter Store as JSON. +func (awspsd *AWSParameterStoreDestination) UploadUnencrypted(data map[string]interface{}, fileName string) error { + ctx := context.TODO() + + // Load AWS config + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(awspsd.region)) + if err != nil { + return fmt.Errorf("unable to load AWS SDK config: %w", err) + } + + client := ssm.NewFromConfig(cfg) + parameterName := awspsd.Path(fileName) + + // Convert data to JSON string for storage + jsonData, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("failed to marshal data to JSON: %w", err) + } + parameterValue := string(jsonData) + + // Check if parameter already exists and compare content + getParamOutput, err := client.GetParameter(ctx, &ssm.GetParameterInput{ + Name: aws.String(parameterName), + WithDecryption: aws.Bool(true), // Decrypt for comparison if it's a SecureString + }) + + parameterExists := true + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "ParameterNotFound" { + parameterExists = false + parameterLog.Infof("Parameter %s does not exist, will create new parameter", parameterName) + } else { + parameterLog.Warnf("Cannot check if destination parameter already exists in %s. New version will be created even if the data has not been changed.", parameterName) + } + } + + // If parameter exists, check if content is identical + if parameterExists && getParamOutput.Parameter.Value != nil { + if *getParamOutput.Parameter.Value == parameterValue { + parameterLog.Infof("Parameter %s is already up-to-date.", parameterName) + return nil + } + } + + // Always use SecureString for security - SOPS files may contain secrets + paramType := types.ParameterTypeSecureString + + // Put parameter (creates or updates) + _, err = client.PutParameter(ctx, &ssm.PutParameterInput{ + Name: aws.String(parameterName), + Value: aws.String(parameterValue), + Type: paramType, + Overwrite: aws.Bool(true), + Description: aws.String("Parameter created/updated by SOPS publish command"), + }) + + if err != nil { + return fmt.Errorf("failed to put parameter %s: %w", parameterName, err) + } + + if parameterExists { + parameterLog.Infof("Successfully updated parameter %s", parameterName) + } else { + parameterLog.Infof("Successfully created parameter %s", parameterName) + } + + return nil +} diff --git a/publish/aws_parameter_store_test.go b/publish/aws_parameter_store_test.go new file mode 100644 index 000000000..9591dc364 --- /dev/null +++ b/publish/aws_parameter_store_test.go @@ -0,0 +1,49 @@ +package publish + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewAWSParameterStoreDestination(t *testing.T) { + dest := NewAWSParameterStoreDestination("us-east-1", "/myapp/config") + assert.NotNil(t, dest) + assert.Equal(t, "us-east-1", dest.region) + assert.Equal(t, "/myapp/config", dest.parameterPath) + + // Test path normalization (should add leading slash) + dest = NewAWSParameterStoreDestination("us-east-1", "myapp/config") + assert.Equal(t, "/myapp/config", dest.parameterPath) +} + +func TestAWSParameterStoreDestination_Path(t *testing.T) { + // Test with specific parameter path (no trailing slash) + dest := NewAWSParameterStoreDestination("us-east-1", "/myapp/database") + path := dest.Path("config.yaml") + assert.Equal(t, "/myapp/database", path) + + // Test with parameter path ending with slash + dest = NewAWSParameterStoreDestination("us-east-1", "/myapp/configs/") + path = dest.Path("api.yaml") + assert.Equal(t, "/myapp/configs/api.yaml", path) + + // Test with empty parameter path (uses filename) + dest = NewAWSParameterStoreDestination("us-east-1", "") + path = dest.Path("standalone.yaml") + assert.Equal(t, "/standalone.yaml", path) + + // Test with filename that already has leading slash + dest = NewAWSParameterStoreDestination("us-east-1", "") + path = dest.Path("/already-prefixed.yaml") + assert.Equal(t, "/already-prefixed.yaml", path) +} + +func TestAWSParameterStoreDestination_Upload(t *testing.T) { + dest := NewAWSParameterStoreDestination("us-east-1", "/test-parameter") + err := dest.Upload([]byte("test content"), "test.yaml") + + assert.NotNil(t, err) + assert.IsType(t, &NotImplementedError{}, err) + assert.Contains(t, err.Error(), "AWS Parameter Store does not support uploading encrypted sops files directly") +} diff --git a/publish/aws_secrets_manager.go b/publish/aws_secrets_manager.go new file mode 100644 index 000000000..b53f1ba50 --- /dev/null +++ b/publish/aws_secrets_manager.go @@ -0,0 +1,141 @@ +package publish + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/smithy-go" + "github.com/getsops/sops/v3/logging" + "github.com/sirupsen/logrus" +) + +var awsSecretsLog *logrus.Logger + +func init() { + awsSecretsLog = logging.NewLogger("PUBLISH") +} + +// AWSSecretsManagerDestination is the AWS Secrets Manager implementation of the Destination interface +type AWSSecretsManagerDestination struct { + region string + secretName string +} + +// NewAWSSecretsManagerDestination is the constructor for an AWS Secrets Manager Destination +func NewAWSSecretsManagerDestination(region, secretName string) *AWSSecretsManagerDestination { + return &AWSSecretsManagerDestination{region, secretName} +} + +// Path returns the AWS Secrets Manager path/ARN of a secret +func (awssmsd *AWSSecretsManagerDestination) Path(fileName string) string { + if awssmsd.secretName != "" { + return fmt.Sprintf("arn:aws:secretsmanager:%s:*:secret:%s", awssmsd.region, awssmsd.secretName) + } + return fmt.Sprintf("arn:aws:secretsmanager:%s:*:secret:%s", awssmsd.region, fileName) +} + +// Returns NotImplementedError +func (awssmsd *AWSSecretsManagerDestination) Upload(fileContents []byte, fileName string) error { + return &NotImplementedError{"AWS Secrets Manager does not support uploading encrypted sops files directly. Use UploadUnencrypted instead."} +} + +// UploadUnencrypted uploads unencrypted data to AWS Secrets Manager as JSON +func (awssmsd *AWSSecretsManagerDestination) UploadUnencrypted(data map[string]interface{}, fileName string) error { + ctx := context.TODO() + + // Load AWS config + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(awssmsd.region)) + if err != nil { + return fmt.Errorf("unable to load AWS SDK config: %w", err) + } + + client := secretsmanager.NewFromConfig(cfg) + + // Determine secret name - use configured name or derive from filename + secretName := awssmsd.secretName + if secretName == "" { + secretName = fileName + } + + // Convert data to JSON string for storage + jsonData, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("failed to marshal data to JSON: %w", err) + } + secretString := string(jsonData) + + // Check if secret metadata exists first + _, err = client.DescribeSecret(ctx, &secretsmanager.DescribeSecretInput{ + SecretId: aws.String(secretName), + }) + + secretExists := true + hasValue := false + var getSecretOutput *secretsmanager.GetSecretValueOutput + + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "ResourceNotFoundException" { + secretExists = false + awsSecretsLog.Infof("Secret %s does not exist, will create new secret", secretName) + } else { + awsSecretsLog.Warnf("Cannot check if destination secret already exists in %s. New version will be created even if the data has not been changed.", secretName) + } + } else { + // Secret exists, now check if it has a value + getSecretOutput, err = client.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(secretName), + }) + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "ResourceNotFoundException" { + hasValue = false + awsSecretsLog.Infof("Secret %s exists but has no value, will add initial value", secretName) + } else { + awsSecretsLog.Warnf("Cannot retrieve current value of secret %s: %v", secretName, err) + hasValue = false + } + } else { + hasValue = true + } + } + + // If secret exists and has value, check if content is identical + if secretExists && hasValue && getSecretOutput.SecretString != nil { + if *getSecretOutput.SecretString == secretString { + awsSecretsLog.Infof("Secret %s is already up-to-date.", secretName) + return nil + } + } + + // Create or update secret + if secretExists { + // Update existing secret + _, err = client.PutSecretValue(ctx, &secretsmanager.PutSecretValueInput{ + SecretId: aws.String(secretName), + SecretString: aws.String(secretString), + }) + if err != nil { + return fmt.Errorf("failed to update secret %s: %w", secretName, err) + } + awsSecretsLog.Infof("Successfully updated secret %s", secretName) + } else { + // Create new secret + _, err = client.CreateSecret(ctx, &secretsmanager.CreateSecretInput{ + Name: aws.String(secretName), + SecretString: aws.String(secretString), + Description: aws.String("Secret created by SOPS publish command"), + }) + if err != nil { + return fmt.Errorf("failed to create secret %s: %w", secretName, err) + } + awsSecretsLog.Infof("Successfully created secret %s", secretName) + } + + return nil +} diff --git a/publish/aws_secrets_manager_test.go b/publish/aws_secrets_manager_test.go new file mode 100644 index 000000000..1f0576460 --- /dev/null +++ b/publish/aws_secrets_manager_test.go @@ -0,0 +1,38 @@ +package publish + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewAWSSecretsManagerDestination(t *testing.T) { + dest := NewAWSSecretsManagerDestination("us-east-1", "myapp/database") + assert.NotNil(t, dest) + assert.Equal(t, "us-east-1", dest.region) + assert.Equal(t, "myapp/database", dest.secretName) +} + +func TestAWSSecretsManagerDestination_Path(t *testing.T) { + // Test with specified secret name + dest := NewAWSSecretsManagerDestination("us-east-1", "myapp/database") + path := dest.Path("config.yaml") + expected := "arn:aws:secretsmanager:us-east-1:*:secret:myapp/database" + assert.Equal(t, expected, path) + + // Test without specified secret name (uses filename) + dest = NewAWSSecretsManagerDestination("us-west-2", "") + path = dest.Path("api-keys.yaml") + expected = "arn:aws:secretsmanager:us-west-2:*:secret:api-keys.yaml" + assert.Equal(t, expected, path) +} + +func TestAWSSecretsManagerDestination_Upload(t *testing.T) { + dest := NewAWSSecretsManagerDestination("us-east-1", "test-secret") + err := dest.Upload([]byte("test content"), "test.yaml") + + // Should return NotImplementedError + assert.NotNil(t, err) + assert.IsType(t, &NotImplementedError{}, err) + assert.Contains(t, err.Error(), "AWS Secrets Manager does not support uploading encrypted sops files directly") +}