Add sync support for gsm (tellerops#105)

Add sync support for gsm

Author: guylev008

README.md

@@ -691,18 +691,22 @@ You should populate `GOOGLE_APPLICATION_CREDENTIALS=account.json` in your enviro
 
 ### Features
 
-* Sync - `no`
+* Sync - `yes`
 * Mapping - `yes`
-* Modes - `read`, [write: accepting PR](https://github.com/spectralops/teller)
+* Modes - `read+write+delete`
 * Key format 
   * `env` - path based, needs to include a version
+  * `env_sync` - your project's path (gets the secrets latest version), when using --sync a new secret version will be created
   * `decrypt` - available in this provider, will use KMS automatically
 
 
 ### Example Config
 
 ```yaml
 google_secretmanager:
+  env_sync:
+    # secrets version is not relevant here since we are getting the latest version
+    path: projects/44882
   env:
     MG_KEY:
       # need to supply the relevant version (versions/1)

go.mod

@@ -39,6 +39,7 @@ require (
 	go.etcd.io/etcd/pkg/v3 v3.5.0-alpha.0
 	golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
+	google.golang.org/api v0.40.0
 	google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
@@ -151,7 +152,6 @@ require (
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
-	google.golang.org/api v0.40.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/grpc v1.35.0 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect

pkg/providers/google_secretmanager.go

@@ -3,17 +3,24 @@ package providers
 import (
 	"context"
 	"fmt"
+	"regexp"
+	"sort"
+	"strings"
 
 	secretmanager "cloud.google.com/go/secretmanager/apiv1"
 	secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1"
 
 	"github.com/googleapis/gax-go/v2"
 	"github.com/spectralops/teller/pkg/core"
 	"github.com/spectralops/teller/pkg/logging"
+	"google.golang.org/api/iterator"
 )
 
 type GoogleSMClient interface {
 	AccessSecretVersion(ctx context.Context, req *secretmanagerpb.AccessSecretVersionRequest, opts ...gax.CallOption) (*secretmanagerpb.AccessSecretVersionResponse, error)
+	DestroySecretVersion(ctx context.Context, req *secretmanagerpb.DestroySecretVersionRequest, opts ...gax.CallOption) (*secretmanagerpb.SecretVersion, error)
+	ListSecrets(ctx context.Context, in *secretmanagerpb.ListSecretsRequest, opts ...gax.CallOption) *secretmanager.SecretIterator
+	AddSecretVersion(ctx context.Context, req *secretmanagerpb.AddSecretVersionRequest, opts ...gax.CallOption) (*secretmanagerpb.SecretVersion, error)
 }
 type GoogleSecretManager struct {
 	client GoogleSMClient
@@ -33,18 +40,46 @@ func (a *GoogleSecretManager) Name() string {
 }
 
 func (a *GoogleSecretManager) Put(p core.KeyPath, val string) error {
-	return fmt.Errorf("provider %q does not implement write yet", a.Name())
+	reg := regexp.MustCompile(`(?i)/versions/\d+$`)
+	res := reg.ReplaceAllString(p.Path, "")
+	return a.addSecret(res, val)
 }
 func (a *GoogleSecretManager) PutMapping(p core.KeyPath, m map[string]string) error {
-	return fmt.Errorf("provider %q does not implement write yet", a.Name())
+	for k, v := range m {
+		path := fmt.Sprintf("%v/secrets/%v", p.Path, k)
+		err := a.addSecret(path, v)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
 func (a *GoogleSecretManager) GetMapping(kp core.KeyPath) ([]core.EnvEntry, error) {
-	return nil, fmt.Errorf("does not support full env sync (path: %s)", kp.Path)
+	secrets, err := a.getSecrets(kp.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	entries := []core.EnvEntry{}
+
+	for _, val := range secrets {
+		path := fmt.Sprintf("%s/%s", val, "versions/latest")
+		secretVal, err := a.getSecret(path)
+		if err != nil {
+			return nil, err
+		}
+		key := strings.TrimPrefix(val, kp.Path)
+		entries = append(entries, kp.FoundWithKey(key, secretVal))
+	}
+	sort.Sort(core.EntriesByKey(entries))
+
+	return entries, nil
 }
 
 func (a *GoogleSecretManager) Get(p core.KeyPath) (*core.EnvEntry, error) {
-	secret, err := a.getSecret(p)
+	secret, err := a.getSecret(p.Path)
 	if err != nil {
 		return nil, err
 	}
@@ -54,16 +89,16 @@ func (a *GoogleSecretManager) Get(p core.KeyPath) (*core.EnvEntry, error) {
 }
 
 func (a *GoogleSecretManager) Delete(kp core.KeyPath) error {
-	return fmt.Errorf("%s does not implement delete yet", a.Name())
+	return a.deleteSecret(kp.Path)
 }
 
 func (a *GoogleSecretManager) DeleteMapping(kp core.KeyPath) error {
 	return fmt.Errorf("%s does not implement delete yet", a.Name())
 }
 
-func (a *GoogleSecretManager) getSecret(kp core.KeyPath) (string, error) {
+func (a *GoogleSecretManager) getSecret(path string) (string, error) {
 	r := secretmanagerpb.AccessSecretVersionRequest{
-		Name: kp.Path,
+		Name: path,
 	}
 	a.logger.WithField("path", r.Name).Debug("get secret")
 
@@ -73,3 +108,44 @@ func (a *GoogleSecretManager) getSecret(kp core.KeyPath) (string, error) {
 	}
 	return string(secret.Payload.Data), nil
 }
+
+func (a *GoogleSecretManager) deleteSecret(path string) error {
+	req := &secretmanagerpb.DestroySecretVersionRequest{
+		Name: path,
+	}
+	_, err := a.client.DestroySecretVersion(context.TODO(), req)
+	return err
+}
+
+func (a *GoogleSecretManager) addSecret(path, val string) error {
+	req := &secretmanagerpb.AddSecretVersionRequest{
+		Parent: path,
+		Payload: &secretmanagerpb.SecretPayload{
+			Data: []byte(val),
+		},
+	}
+
+	_, err := a.client.AddSecretVersion(context.TODO(), req)
+	return err
+}
+
+func (a *GoogleSecretManager) getSecrets(path string) ([]string, error) {
+	req := &secretmanagerpb.ListSecretsRequest{
+		Parent: path,
+	}
+	entries := []string{}
+
+	it := a.client.ListSecrets(context.TODO(), req)
+	for {
+		resp, err := it.Next()
+		if err == iterator.Done {
+			break
+		}
+
+		if err != nil {
+			return nil, err
+		}
+		entries = append(entries, resp.Name)
+	}
+	return entries, nil
+}

pkg/providers/google_secretmanager_test.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"testing"
 
+	secretmanager "cloud.google.com/go/secretmanager/apiv1"
 	"github.com/alecthomas/assert"
 	"github.com/golang/mock/gomock"
 	secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1"
@@ -24,10 +25,34 @@ func TestGoogleSM(t *testing.T) {
 	out := &secretmanagerpb.AccessSecretVersionResponse{
 		Payload: sec,
 	}
+	outDelete := &secretmanagerpb.SecretVersion{
+		Name: string(sec.Data),
+	}
+	outAdd := &secretmanagerpb.SecretVersion{
+		Name: string(sec.Data),
+	}
+	outList := &secretmanager.SecretIterator{
+		Response: string(sec.Data),
+	}
 	in := secretmanagerpb.AccessSecretVersionRequest{
 		Name: path,
 	}
+	inDelete := secretmanagerpb.DestroySecretVersionRequest{
+		Name: path,
+	}
+	inList := secretmanagerpb.ListSecretsRequest{
+		Parent: path,
+	}
+	inAdd := secretmanagerpb.AddSecretVersionRequest{
+		Parent: path,
+		Payload: &secretmanagerpb.SecretPayload{
+			Data: []byte("some value"),
+		},
+	}
 	client.EXPECT().AccessSecretVersion(gomock.Any(), gomock.Eq(&in)).Return(out, nil).AnyTimes()
+	client.EXPECT().DestroySecretVersion(gomock.Any(), gomock.Eq(&inDelete)).Return(outDelete, nil).AnyTimes()
+	client.EXPECT().ListSecrets(gomock.Any(), gomock.Eq(&inList)).Return(outList).AnyTimes()
+	client.EXPECT().AddSecretVersion(gomock.Any(), gomock.Eq(&inAdd)).Return(outAdd, nil).AnyTimes()
 	s := GoogleSecretManager{
 		client: client,
 		logger: GetTestLogger(),