Add copy sync, mirror-drift use cases

Author: jondot

.gitignore

@@ -6,3 +6,4 @@ node_modules/
 todo.txt
 .teller.writecase.yml
 coverage.out
+fixtures/sync/target.env

README.md

@@ -208,7 +208,43 @@ If you omit `--in` Teller will take `stdin`, and if you omit `--out` Teller will
 
 You can detect _secret drift_ by comparing values from different providers against each other. It might be that you want to pin a set of keys in different providers to always be the same value; when they aren't -- that means you have a drift.
 
-For this, you first need to label values as `source` and couple with the appropriate sink as `sink` (use same label for both to couple them). Then, source keys will be compared against other keys in your configuration:
+In most cases, keys in providers would be similar which we call _mirrored_ providers. Example:
+
+```
+Provider1:
+  MG_PASS=foo***
+
+Provider2:
+  MG_PASS=foo***   # Both keys are called MG_PASS
+```
+
+To detected mirror drifts, we use `teller mirror-drift`.
+
+```bash
+$ teller mirror-drift --from global-dotenv --to my-dotenv
+
+Drifts detected: 2
+
+changed [] global-dotenv FOO_BAR "n***** != my-dotenv FOO_BAR ne*****
+missing [] global-dotenv FB 3***** ??
+```
+
+As always, the specific provider definitions are in your `teller.yml` file.
+## :beetle: Detect secrets and value drift (non-mirrored providers)
+
+Some times you want to check drift between two providers, and two unrelated keys. For example:
+
+```
+Provider1:
+  MG_PASS=foo***
+
+Provider2:
+  MAILGUN_PASS=foo***
+```
+
+This poses a challenge. We need some way to "wire" the keys `MG_PASS` and `MAILGUN_PASS` and declare a relationship of source (`MG_PASS`) and destination, or sink (`MAILGUN_PASS`).
+
+For this, you can label mappings as `source` and couple with the appropriate sink as `sink` (use same label value for both to wire them together). Then, source values will be compared against sink values in your configuration:
 
 ```yaml
 providers:
@@ -259,9 +295,42 @@ Will get you, assuming `FOO_BAR=Spock`:
 Hello, Spock!
 ```
 
-## :bike: Multi-write, rotation & sync
+## :arrows_counterclockwise: Copy/sync data between providers
+
+In cases where you want to sync between providers, you can do that with `teller copy`.
+
+
+**Specific mapping key sync**
+
+```bash
+$ teller copy --from dotenv1 --to dotenv2,heroku1
+```
+
+This will:
+
+1. Grab all mapped values from source (`dotenv1`)
+2. For each target provider, find the matching mapped key, and copy the value from source into it
+
+**Full copy sync**
+
+```bash
+$ teller copy --sync --from dotenv1 --to dotenv2,heroku1
+```
+This will:
+
+1. Grab all mapped values from source (`dotenv1`)
+2. For each target provider, perform a full copy of values from source into the mapped `env_sync` key
+
+
+Notes:
+
+* The mapping per provider is as configured in your `teller.yaml` file, in the `env_sync` or `env` properties.
+* This sync will try to copy _all_ values from the source.
+
+
+## :bike: Write and multi-write to providers
 
-Teller providers supporting _write_ use cases which allow writing values _into_ the providers: **putting new values, synchronizing values, synchronizing providers, and fixing drift**.
+Teller providers supporting _write_ use cases which allow writing values _into_ providers.
 
 Remember, for this feature it still revolves around definitions in your `teller.yml` file:
 

fixtures/mirror-drift/source.env

@@ -0,0 +1,3 @@
+ONE=1
+TWO=2
+THREE=3

fixtures/mirror-drift/target.env

@@ -0,0 +1,2 @@
+ONE=5
+TWO=2

fixtures/mirror-drift/teller.yml

@@ -0,0 +1,11 @@
+project: mirror-drift
+
+providers:
+  source:
+    kind: dotenv
+    env_sync:
+      path: ../fixtures/mirror-drift/source.env
+  target:
+    kind: dotenv
+    env_sync:
+      path: ../fixtures/mirror-drift/target.env

fixtures/sync/source.env

@@ -0,0 +1,3 @@
+ONE=1
+TWO=2
+THREE=3

fixtures/sync/target2.env

@@ -0,0 +1,4 @@
+FOO="2"
+ONE="1"
+THREE="3"
+TWO="2"

fixtures/sync/teller.yml

@@ -0,0 +1,15 @@
+project: sync
+
+providers:
+  source:
+    kind: dotenv
+    env_sync:
+      path: ../fixtures/sync/source.env
+  target:
+    kind: dotenv
+    env_sync:
+      path: ../fixtures/sync/target.env
+  target2:
+    kind: dotenv
+    env_sync:
+      path: ../fixtures/sync/target2.env

main.go

@@ -57,6 +57,17 @@ var CLI struct {
 		Sync      bool              `optional name:"sync" help:"Sync all given k/vs to the env_sync key"`
 		Path      string            `optional name:"path" help:"Take literal path and not from config"`
 	} `cmd help:"Put a new value"`
+
+	Copy struct {
+		From string   `name:"from" help:"A provider name to sync from"`
+		To   []string `name:"to" help:"A list of provider names to copy values from the source provider to"`
+		Sync bool     `optional name:"sync" help:"Sync all given k/vs to the env_sync key"`
+	} `cmd help:"Sync data from a source provider directly to multiple target providers"`
+
+	MirrorDrift struct {
+		Source string `name:"source" help:"A source to check drift against"`
+		Target string `name:"target" help:"A target to check against source"`
+	} `cmd help:"Check same-key (mirror) value drift between source and target"`
 }
 
 var (
@@ -116,6 +127,24 @@ func main() {
 			os.Exit(1)
 		}
 		os.Exit(0)
+	case "copy":
+		err := teller.Sync(CLI.Copy.From, CLI.Copy.To, CLI.Copy.Sync)
+		if err != nil {
+			fmt.Printf("Error: %v\n", err)
+			os.Exit(1)
+		}
+		os.Exit(0)
+	case "mirror-drift":
+		drifts, err := teller.MirrorDrift(CLI.MirrorDrift.Source, CLI.MirrorDrift.Target)
+		if err != nil {
+			fmt.Printf("Error: %v\n", err)
+			os.Exit(1)
+		}
+		if len(drifts) > 0 {
+			teller.Porcelain.PrintDrift(drifts)
+			os.Exit(1)
+		}
+		os.Exit(0)
 	}
 
 	// collecting

pkg/teller.go

@@ -319,56 +319,67 @@ func updateParams(ent *core.EnvEntry, from *core.KeyPath, pname string) {
 	}
 }
 
-func (tl *Teller) CollectFromProviderMap(ps *ProvidersMap) ([]core.EnvEntry, error) {
+func (tl *Teller) CollectFromProvider(pname string) ([]core.EnvEntry, error) {
 	entries := []core.EnvEntry{}
-	for pname, conf := range *ps {
-		p, err := tl.Providers.GetProvider(pname)
-		if err != nil {
-			// ok, maybe same provider, with 'kind'?
-			p, err = tl.Providers.GetProvider(conf.Kind)
-		}
+	conf := tl.Config.Providers[pname]
+	p, err := tl.Providers.GetProvider(pname)
+	if err != nil {
+		// ok, maybe same provider, with 'kind'?
+		p, err = tl.Providers.GetProvider(conf.Kind)
+	}
 
-		// still no provider? bail.
+	// still no provider? bail.
+	if err != nil {
+		return nil, err
+	}
+
+	if conf.EnvMapping != nil {
+		es, err := p.GetMapping(tl.Populate.KeyPath(*conf.EnvMapping))
 		if err != nil {
 			return nil, err
 		}
 
-		if conf.EnvMapping != nil {
-			es, err := p.GetMapping(tl.Populate.KeyPath(*conf.EnvMapping))
-			if err != nil {
-				return nil, err
-			}
-
-			//nolint
-			for k, v := range es {
-				// optionally remap environment variables synced from the provider
-				if val, ok := conf.EnvMapping.Remap[v.Key]; ok {
-					es[k].Key = val
-				}
-				updateParams(&es[k], conf.EnvMapping, pname)
+		//nolint
+		for k, v := range es {
+			// optionally remap environment variables synced from the provider
+			if val, ok := conf.EnvMapping.Remap[v.Key]; ok {
+				es[k].Key = val
 			}
-
-			entries = append(entries, es...)
+			updateParams(&es[k], conf.EnvMapping, pname)
 		}
 
-		if conf.Env != nil {
-			//nolint
-			for k, v := range *conf.Env {
-				ent, err := p.Get(tl.Populate.KeyPath(v.WithEnv(k)))
-				if err != nil {
-					if v.Optional {
-						continue
-					} else {
-						return nil, err
-					}
+		entries = append(entries, es...)
+	}
+
+	if conf.Env != nil {
+		//nolint
+		for k, v := range *conf.Env {
+			ent, err := p.Get(tl.Populate.KeyPath(v.WithEnv(k)))
+			if err != nil {
+				if v.Optional {
+					continue
 				} else {
-					//nolint
-					updateParams(ent, &v, pname)
-					entries = append(entries, *ent)
+					return nil, err
 				}
+			} else {
+				//nolint
+				updateParams(ent, &v, pname)
+				entries = append(entries, *ent)
 			}
 		}
 	}
+	return entries, nil
+}
+
+func (tl *Teller) CollectFromProviderMap(ps *ProvidersMap) ([]core.EnvEntry, error) {
+	entries := []core.EnvEntry{}
+	for pname := range *ps {
+		pents, err := tl.CollectFromProvider(pname)
+		if err != nil {
+			return nil, err
+		}
+		entries = append(entries, pents...)
+	}
 
 	sort.Sort(core.EntriesByKey(entries))
 	return entries, nil
@@ -499,3 +510,48 @@ func (tl *Teller) Put(kvmap map[string]string, providerNames []string, sync bool
 
 	return nil
 }
+
+func (tl *Teller) Sync(from string, to []string, sync bool) error {
+	entries, err := tl.CollectFromProvider(from)
+	if err != nil {
+		return err
+	}
+	kvmap := map[string]string{}
+	for i := range entries {
+		ent := entries[i]
+		kvmap[ent.Key] = ent.Value
+	}
+
+	err = tl.Put(kvmap, to, sync, "")
+	return err
+}
+
+func (tl *Teller) MirrorDrift(source, target string) ([]core.DriftedEntry, error) {
+	drifts := []core.DriftedEntry{}
+	sourceEntries, err := tl.CollectFromProvider(source)
+	if err != nil {
+		return nil, err
+	}
+
+	targetEntries, err := tl.CollectFromProvider(target)
+	if err != nil {
+		return nil, err
+	}
+
+	for i := range sourceEntries {
+		sent := sourceEntries[i]
+		tent := funk.Find(targetEntries, func(ent core.EnvEntry) bool {
+			return sent.Key == ent.Key
+		})
+		if tent == nil {
+			drifts = append(drifts, core.DriftedEntry{Diff: "missing", Source: sent})
+			continue
+		}
+		tentry := tent.(core.EnvEntry)
+		if sent.Value != tentry.Value {
+			drifts = append(drifts, core.DriftedEntry{Diff: "changed", Source: sent, Target: tentry})
+		}
+	}
+
+	return drifts, nil
+}