Merge pull request #623 from nginx-proxy/filter-containers

feat: filter containers seen by docker-gen

Author: buchdag

README.md

@@ -87,67 +87,75 @@ Usage: docker-gen [options] template [dest]
 Generate files from docker container meta-data
 
 Options:
-  -config value
-      config files with template directives. Config files will be merged if this option is specified multiple times. (default [])
+  -config path
+      config files with template directives.
+      Config files will be merged if this option is specified multiple times. (default [])
+  -container-filter key=value
+      container filter for inclusion by docker-gen.
+      You can pass this option multiple times to combine filters with AND.
+      https://docs.docker.com/engine/reference/commandline/ps/#filter
   -endpoint string
       docker api endpoint (tcp|unix://..). Default unix:///var/run/docker.sock
-  -event-filter value
+  -event-filter key=value
       additional filter for event watched by docker-gen (e.g -event-filter event=connect -event-filter event=disconnect).
       You can pass this option multiple times to combine filters.
       By default docker-gen listen for container events start, stop, die and health_status.
       https://docs.docker.com/engine/reference/commandline/events/#filtering-events
+  -include-stopped
+      include stopped containers.
+      Bypassed when providing a container status filter (-container-filter status=foo).
   -interval int
       notify command interval (secs)
   -keep-blank-lines
       keep blank lines in the output file
   -notify restart xyz
       run command after template is regenerated (e.g restart xyz)
-  -notify-output
-      log the output(stdout/stderr) of notify command
-  -notify-sighup container-ID
-      send HUP signal to container.
-      Equivalent to 'docker kill -s HUP container-ID', or `-notify-container container-ID -notify-signal 1`.
-      You can pass this option multiple times to send HUP to multiple containers.
   -notify-container container-ID
       send -notify-signal signal (defaults to 1 / HUP) to container.
       You can pass this option multiple times to notify multiple containers.
   -notify-filter key=value
       container filter for notification (e.g -notify-filter name=foo).
       You can pass this option multiple times to combine filters with AND.
       https://docs.docker.com/engine/reference/commandline/ps/#filter
+  -notify-output
+      log the output(stdout/stderr) of notify command
+  -notify-sighup container-ID
+      send HUP signal to container.
+      Equivalent to 'docker kill -s HUP container-ID', or `-notify-container container-ID -notify-signal 1`.
+      You can pass this option multiple times to send HUP to multiple containers.
   -notify-signal signal
       signal to send to the -notify-container and -notify-filter. -1 to call docker restart. Defaults to 1 aka. HUP.
       All available signals available on the dockerclient
       https://github.com/fsouza/go-dockerclient/blob/main/signal.go
   -only-exposed
-      only include containers with exposed ports
+      only include containers with exposed ports.
+      Bypassed when using the exposed filter with (-container-filter exposed=foo).
   -only-published
-      only include containers with published ports (implies -only-exposed)
-  -include-stopped
-      include stopped containers
+      only include containers with published ports (implies -only-exposed).
+      Bypassed when providing a container published filter (-container-filter published=foo).
   -tlscacert string
-      path to TLS CA certificate file (default "~/.docker/machine/machines/default/ca.pem")
+      path to TLS CA certificate file (default "~/.docker/ca.pem")
   -tlscert string
-      path to TLS client certificate file (default "~/.docker/machine/machines/default/cert.pem")
+      path to TLS client certificate file (default "~/.docker/cert.pem")
   -tlskey string
-      path to TLS client key file (default "~/.docker/machine/machines/default/key.pem")
+      path to TLS client key file (default "~/.docker/key.pem")
   -tlsverify
-      verify docker daemon's TLS certicate (default true)
+      verify docker daemon's TLS certicate
   -version
       show version
+  -wait string
+      minimum and maximum durations to wait (e.g. "500ms:2s") before triggering generate
   -watch
       watch for container changes
-  -wait
-      minimum (and/or maximum) duration to wait after each container change before triggering
 
 Arguments:
   template - path to a template to generate
-  dest - path to write the template. If not specfied, STDOUT is used
+  dest - path to write the template to. If not specfied, STDOUT is used
 
 Environment Variables:
   DOCKER_HOST - default value for -endpoint
-  DOCKER_CERT_PATH - directory path containing key.pem, cert.pm and ca.pem
-  DOCKER_TLS_VERIFY - enable client TLS verification]
+  DOCKER_CERT_PATH - directory path containing key.pem, cert.pem and ca.pem
+  DOCKER_TLS_VERIFY - enable client TLS verification
 ```
 
 If no `<dest>` file is specified, the output is sent to stdout. Mainly useful for debugging.

cmd/docker-gen/main.go

@@ -34,6 +34,7 @@ var (
 	onlyExposed           bool
 	onlyPublished         bool
 	includeStopped        bool
+	containerFilter       mapstringslice = make(mapstringslice)
 	configFiles           stringslice
 	configs               config.ConfigFile
 	eventFilter           mapstringslice = mapstringslice{"event": {"start", "stop", "die", "health_status"}}
@@ -78,7 +79,7 @@ Options:`)
 	println(`
 Arguments:
   template - path to a template to generate
-  dest - path to a write the template.  If not specfied, STDOUT is used`)
+  dest - path to write the template to. If not specfied, STDOUT is used`)
 
 	println(`
 Environment Variables:
@@ -98,21 +99,35 @@ func loadConfig(file string) error {
 }
 
 func initFlags() {
-
 	certPath := filepath.Join(os.Getenv("DOCKER_CERT_PATH"))
 	if certPath == "" {
 		certPath = filepath.Join(os.Getenv("HOME"), ".docker")
 	}
+
 	flag.BoolVar(&version, "version", false, "show version")
+
+	// General configuration options
 	flag.BoolVar(&watch, "watch", false, "watch for container changes")
 	flag.StringVar(&wait, "wait", "", "minimum and maximum durations to wait (e.g. \"500ms:2s\") before triggering generate")
-	flag.BoolVar(&onlyExposed, "only-exposed", false, "only include containers with exposed ports")
+	flag.Var(&configFiles, "config", "config files with template directives. Config files will be merged if this option is specified multiple times.")
+	flag.BoolVar(&keepBlankLines, "keep-blank-lines", false, "keep blank lines in the output file")
 
+	// Containers filtering options
+	flag.BoolVar(&onlyExposed, "only-exposed", false,
+		"only include containers with exposed ports. Bypassed when providing a container exposed filter (-container-filter exposed=foo).")
 	flag.BoolVar(&onlyPublished, "only-published", false,
-		"only include containers with published ports (implies -only-exposed)")
-	flag.BoolVar(&includeStopped, "include-stopped", false, "include stopped containers")
-	flag.BoolVar(&notifyOutput, "notify-output", false, "log the output(stdout/stderr) of notify command")
+		"only include containers with published ports (implies -only-exposed). Bypassed when providing a container published filter (-container-filter published=foo).")
+	flag.BoolVar(&includeStopped, "include-stopped", false,
+		"include stopped containers. Bypassed when providing a container status filter (-container-filter status=foo).")
+	flag.Var(&containerFilter, "container-filter",
+		"container filter for inclusion by docker-gen. You can pass this option multiple times to combine filters with AND. https://docs.docker.com/engine/reference/commandline/ps/#filter")
+
+	// Command notification options
 	flag.StringVar(&notifyCmd, "notify", "", "run command after template is regenerated (e.g `restart xyz`)")
+	flag.BoolVar(&notifyOutput, "notify-output", false, "log the output(stdout/stderr) of notify command")
+	flag.IntVar(&interval, "interval", 0, "notify command interval (secs)")
+
+	// Containers notification options
 	flag.Var(&sighupContainerID, "notify-sighup",
 		"send HUP signal to container. Equivalent to docker kill -s HUP `container-ID`. You can pass this option multiple times to send HUP to multiple containers.")
 	flag.Var(&notifyContainerID, "notify-container",
@@ -121,9 +136,8 @@ func initFlags() {
 		"container filter for notification (e.g -notify-filter name=foo). You can pass this option multiple times to combine filters with AND. https://docs.docker.com/engine/reference/commandline/ps/#filter")
 	flag.IntVar(&notifyContainerSignal, "notify-signal", int(docker.SIGHUP),
 		"signal to send to the notify-container and notify-filter. Defaults to SIGHUP")
-	flag.Var(&configFiles, "config", "config files with template directives. Config files will be merged if this option is specified multiple times.")
-	flag.IntVar(&interval, "interval", 0, "notify command interval (secs)")
-	flag.BoolVar(&keepBlankLines, "keep-blank-lines", false, "keep blank lines in the output file")
+
+	// Docker API endpoint configuration options
 	flag.StringVar(&endpoint, "endpoint", "", "docker api endpoint (tcp|unix://..). Default unix:///var/run/docker.sock")
 	flag.StringVar(&tlsCert, "tlscert", filepath.Join(certPath, "cert.pem"), "path to TLS client certificate file")
 	flag.StringVar(&tlsKey, "tlskey", filepath.Join(certPath, "key.pem"), "path to TLS client key file")
@@ -177,9 +191,7 @@ func main() {
 			NotifyCmd:        notifyCmd,
 			NotifyOutput:     notifyOutput,
 			NotifyContainers: make(map[string]int),
-			OnlyExposed:      onlyExposed,
-			OnlyPublished:    onlyPublished,
-			IncludeStopped:   includeStopped,
+			ContainerFilter:  containerFilter,
 			Interval:         interval,
 			KeepBlankLines:   keepBlankLines,
 		}
@@ -193,25 +205,37 @@ func main() {
 			cfg.NotifyContainersFilter = notifyContainerFilter
 			cfg.NotifyContainersSignal = notifyContainerSignal
 		}
+		if len(cfg.ContainerFilter["status"]) == 0 {
+			if includeStopped {
+				cfg.ContainerFilter["status"] = []string{
+					"created",
+					"restarting",
+					"running",
+					"removing",
+					"paused",
+					"exited",
+					"dead",
+				}
+			} else {
+				cfg.ContainerFilter["status"] = []string{"running"}
+			}
+		}
+		if onlyPublished && len(cfg.ContainerFilter["publish"]) == 0 {
+			cfg.ContainerFilter["publish"] = []string{"1-65535"}
+		} else if onlyExposed && len(cfg.ContainerFilter["expose"]) == 0 {
+			cfg.ContainerFilter["expose"] = []string{"1-65535"}
+		}
 		configs = config.ConfigFile{
 			Config: []config.Config{cfg},
 		}
 	}
 
-	all := false
-	for _, config := range configs.Config {
-		if config.IncludeStopped {
-			all = true
-		}
-	}
-
 	generator, err := generator.NewGenerator(generator.GeneratorConfig{
 		Endpoint:    endpoint,
 		TLSKey:      tlsKey,
 		TLSCert:     tlsCert,
 		TLSCACert:   tlsCaCert,
 		TLSVerify:   tlsVerify,
-		All:         all,
 		EventFilter: eventFilter,
 		ConfigFile:  configs,
 	})

internal/config/config.go

@@ -16,9 +16,7 @@ type Config struct {
 	NotifyContainers       map[string]int
 	NotifyContainersFilter map[string][]string
 	NotifyContainersSignal int
-	OnlyExposed            bool
-	OnlyPublished          bool
-	IncludeStopped         bool
+	ContainerFilter        map[string][]string
 	Interval               int
 	KeepBlankLines         bool
 }

internal/context/context.go

@@ -105,16 +105,6 @@ func (r *RuntimeContainer) Equals(o RuntimeContainer) bool {
 	return r.ID == o.ID && r.Image == o.Image
 }
 
-func (r *RuntimeContainer) PublishedAddresses() []Address {
-	mapped := []Address{}
-	for _, address := range r.Addresses {
-		if address.HostPort != "" {
-			mapped = append(mapped, address)
-		}
-	}
-	return mapped
-}
-
 type DockerImage struct {
 	Registry   string
 	Repository string

internal/context/context_test.go

@@ -132,37 +132,6 @@ func TestGetCurrentContainerEmpty(t *testing.T) {
 	assert.Equal(t, "", GetCurrentContainerID())
 }
 
-func TestPublishedAddresses(t *testing.T) {
-	container := &RuntimeContainer{
-		Addresses: []Address{
-			{
-				IP:       "172.19.0.1",
-				HostPort: "80",
-			},
-			{
-				IP: "172.19.0.2",
-			},
-			{
-				IP:       "172.19.0.3",
-				HostPort: "8080",
-			},
-		},
-	}
-
-	expected := []Address{
-		{
-			IP:       "172.19.0.1",
-			HostPort: "80",
-		},
-		{
-			IP:       "172.19.0.3",
-			HostPort: "8080",
-		},
-	}
-
-	assert.ElementsMatch(t, expected, container.PublishedAddresses())
-}
-
 func TestRuntimeContainerEquals(t *testing.T) {
 	rc1 := &RuntimeContainer{
 		ID: "baz",

internal/generator/generator.go

@@ -39,7 +39,6 @@ type GeneratorConfig struct {
 	TLSKey    string
 	TLSCACert string
 	TLSVerify bool
-	All       bool
 
 	EventFilter map[string][]string
 
@@ -72,7 +71,6 @@ func NewGenerator(gc GeneratorConfig) (*generator, error) {
 		TLSCert:     gc.TLSCert,
 		TLSCaCert:   gc.TLSCACert,
 		TLSKey:      gc.TLSKey,
-		All:         gc.All,
 		EventFilter: gc.EventFilter,
 		Configs:     gc.ConfigFile,
 		retry:       true,
@@ -124,12 +122,13 @@ func (g *generator) generateFromSignals() {
 }
 
 func (g *generator) generateFromContainers() {
-	containers, err := g.getContainers()
-	if err != nil {
-		log.Printf("Error listing containers: %s\n", err)
-		return
-	}
 	for _, config := range g.Configs.Config {
+		containers, err := g.getContainers(config)
+		if err != nil {
+			log.Printf("Error listing containers: %s\n", err)
+			return
+		}
+
 		changed := template.GenerateFile(config, containers)
 		if !changed {
 			log.Printf("Contents of %s did not change. Skipping notification '%s'", config.Dest, config.NotifyCmd)
@@ -159,7 +158,7 @@ func (g *generator) generateAtInterval() {
 			for {
 				select {
 				case <-ticker.C:
-					containers, err := g.getContainers()
+					containers, err := g.getContainers(cfg)
 					if err != nil {
 						log.Printf("Error listing containers: %s\n", err)
 						continue
@@ -205,7 +204,7 @@ func (g *generator) generateFromEvents() {
 			defer g.wg.Done()
 			debouncedChan := newDebounceChannel(watcher, cfg.Wait)
 			for range debouncedChan {
-				containers, err := g.getContainers()
+				containers, err := g.getContainers(cfg)
 				if err != nil {
 					log.Printf("Error listing containers: %s\n", err)
 					continue
@@ -389,7 +388,7 @@ func (g *generator) sendSignalToFilteredContainers(config config.Config) {
 	}
 }
 
-func (g *generator) getContainers() ([]*context.RuntimeContainer, error) {
+func (g *generator) getContainers(config config.Config) ([]*context.RuntimeContainer, error) {
 	apiInfo, err := g.Client.Info()
 	if err != nil {
 		log.Printf("Error retrieving docker server info: %s\n", err)
@@ -398,8 +397,9 @@ func (g *generator) getContainers() ([]*context.RuntimeContainer, error) {
 	}
 
 	apiContainers, err := g.Client.ListContainers(docker.ListContainersOptions{
-		All:  g.All,
-		Size: false,
+		All:     true,
+		Size:    false,
+		Filters: config.ContainerFilter,
 	})
 	if err != nil {
 		return nil, err

internal/generator/generator_test.go

@@ -199,14 +199,14 @@ func TestGenerateFromEvents(t *testing.T) {
 	//       init   150ms  200ms  250ms  300ms  350ms  400ms  450ms  500ms  550ms  600ms  650ms  700ms
 	//       ├──────╫──────┼──────┼──────╫──────┼──────┼──────╫──────┼──────┼──────┼──────┼──────┤
 	// File0 ├─ 1   ║                    ║                    ║
-	// File1 ├─ 1   ╟─ 2                 ╟─ 3                 ╟─ 5
-	// File2 ├─ 1   ╟───── max (250ms) ──║───────────> 4      ╟─────── min (200ms) ─────> 6
-	// File3 └─ 1   ╟──────────────────> ╟──────────────────> ╟─────────── min (250ms) ────────> 7
+	// File1 ├─ 2   ╟─ 5                 ╟─ 6                 ╟─ 8
+	// File2 ├─ 3   ╟───── max (250ms) ──║───────────> 7      ╟─────── min (200ms) ─────> 9
+	// File3 └─ 4   ╟──────────────────> ╟──────────────────> ╟─────────── min (250ms) ────────> 10
 	//          ┌───╨───┐            ┌───╨──┐             ┌───╨───┐
 	//          │ start │            │ stop │             │ start │
 	//          └───────┘            └──────┘             └───────┘
 
-	expectedCounters := []int{1, 5, 6, 7}
+	expectedCounters := []int{1, 8, 9, 10}
 
 	for i, counter := range expectedCounters {
 		value, _ = os.ReadFile(destFiles[i].Name())