Fix health_monitor NATS Authorization Violation on fresh deployments

On a fresh bosh create-env, pre-start writes a fooBar token placeholder to
auth.json. The old startup code only sent a SIGHUP to reload this placeholder,
switching NATS to token-based auth and causing every health_monitor connection
attempt to fail with "Authorization Violation" until the first periodic sync ran.

Add UsersSync.Bootstrap() which reads the director-subject and hm-subject files
written by pre-start and immediately writes a proper user-based NATS config,
then sends SIGHUP — all without querying the director. Runner.Run() now calls
bootstrapNATSConfig() at startup instead of a bare ReloadNATSServerConfig,
ensuring health_monitor can authenticate against NATS before the director
finishes initializing. Bootstrap failures are non-fatal and logged; the periodic
sync loop continues so the full user list is populated once the director is up.

Author: colins

src/bosh-nats-sync/pkg/runner/runner.go

@@ -46,13 +46,11 @@ func (r *Runner) Run() error {
 		cmdRunner = userssync.DefaultCommandRunner
 	}
 
-	if err := userssync.ReloadNATSServerConfig(
-		r.config.NATS.NATSServerExecutable,
-		r.config.NATS.NATSServerPIDFile,
-		cmdRunner,
-	); err != nil {
-		return fmt.Errorf("failed to reload NATS server config on startup: %w", err)
-	}
+	// Bootstrap: write the initial NATS config from on-disk subject files
+	// immediately, before the director is queried.  This replaces the
+	// placeholder token written by pre-start so that health_monitor and the
+	// director can authenticate against NATS during director startup.
+	r.bootstrapNATSConfig(cmdRunner)
 
 	interval := time.Duration(r.config.Intervals.PollUserSync) * time.Second
 	if interval <= 0 {
@@ -80,6 +78,20 @@ func (r *Runner) Stop() {
 	<-r.stopped
 }
 
+func (r *Runner) bootstrapNATSConfig(cmdRunner userssync.CommandRunner) {
+	sync := &userssync.UsersSync{
+		NATSConfigFilePath:   r.config.NATS.ConfigFilePath,
+		BoshConfig:           r.config.Director,
+		NATSServerExecutable: r.config.NATS.NATSServerExecutable,
+		NATSServerPIDFile:    r.config.NATS.NATSServerPIDFile,
+		Logger:               r.logger,
+		CommandRunner:        cmdRunner,
+	}
+	if err := sync.Bootstrap(); err != nil {
+		r.logger.Error("Bootstrap failed, health_monitor may not connect to NATS until next sync", "error", err)
+	}
+}
+
 func (r *Runner) syncNATSUsers(cmdRunner userssync.CommandRunner) {
 	sync := &userssync.UsersSync{
 		NATSConfigFilePath:   r.config.NATS.ConfigFilePath,

src/bosh-nats-sync/pkg/runner/runner_test.go

@@ -138,28 +138,91 @@ var _ = Describe("Runner", func() {
 		})
 	})
 
-	Describe("exception handling", func() {
-		Context("when startup reload fails", func() {
-			It("returns an error and exits Run() without starting the sync loop", func() {
-				startupFailRunner := func(executable string, args ...string) ([]byte, error) {
-					return nil, fmt.Errorf("cannot execute: reload failed on startup")
+	Describe("bootstrap on startup", func() {
+		It("writes the initial NATS config from subject files before the first sync tick", func() {
+			// Override natsConfigFile with the fooBar placeholder that pre-start creates.
+			os.WriteFile(natsConfigFile.Name(), []byte(`{"authorization":{"token":"f0oBar"}}`), 0644)
+
+			r := runner.NewWithCommandRunner(cfg, logger, cmdRunner)
+
+			go r.Run()
+			// Sleep well under PollUserSync (1s) — bootstrap must fire synchronously.
+			time.Sleep(200 * time.Millisecond)
+			r.Stop()
+
+			data, err := os.ReadFile(natsConfigFile.Name())
+			Expect(err).NotTo(HaveOccurred())
+			var result map[string]interface{}
+			Expect(json.Unmarshal(data, &result)).To(Succeed())
+
+			auth := result["authorization"].(map[string]interface{})
+			users := auth["users"].([]interface{})
+			Expect(len(users)).To(BeNumerically(">=", 2), "expected director and HM users in bootstrap config")
+
+			subjects := make([]string, 0, len(users))
+			for _, u := range users {
+				subjects = append(subjects, u.(map[string]interface{})["user"].(string))
+			}
+			Expect(subjects).To(ContainElement(ContainSubstring("director.bosh-internal")))
+			Expect(subjects).To(ContainElement(ContainSubstring("hm.bosh-internal")))
+
+			// cmdRunner must have been called at least once for the bootstrap SIGHUP.
+			Expect(len(commandRunnerCalls)).To(BeNumerically(">=", 1))
+		})
+
+		It("does not contact the director during bootstrap", func() {
+			var directorCalled bool
+			isolatedMux := http.NewServeMux()
+			isolatedMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+				directorCalled = true
+				w.WriteHeader(http.StatusOK)
+			})
+			isolatedServer := httptest.NewServer(isolatedMux)
+			defer isolatedServer.Close()
+
+			isolatedCfg := *cfg
+			isolatedCfg.Director.URL = isolatedServer.URL
+
+			r := runner.NewWithCommandRunner(&isolatedCfg, logger, cmdRunner)
+			go r.Run()
+			time.Sleep(200 * time.Millisecond)
+			r.Stop()
+
+			Expect(directorCalled).To(BeFalse(), "bootstrap must not query the director")
+		})
+
+		Context("when bootstrap fails (e.g. NATS SIGHUP error)", func() {
+			It("logs the error but continues running the sync loop", func() {
+				var callCount int32
+				nonFatalRunner := func(executable string, args ...string) ([]byte, error) {
+					n := atomic.AddInt32(&callCount, 1)
+					if n == 1 {
+						// bootstrap reload fails
+						return nil, fmt.Errorf("cannot execute: bootstrap reload failed")
+					}
+					return []byte("ok"), nil
 				}
 
-				r := runner.NewWithCommandRunner(cfg, logger, startupFailRunner)
+				r := runner.NewWithCommandRunner(cfg, logger, nonFatalRunner)
 
 				done := make(chan struct{})
-				var runErr error
 				go func() {
-					runErr = r.Run()
+					r.Run()
 					close(done)
 				}()
 
+				// Run() must NOT exit immediately after a bootstrap failure.
+				Consistently(done, 500*time.Millisecond).ShouldNot(BeClosed())
+				r.Stop()
 				Eventually(done, 2*time.Second).Should(BeClosed())
-				Expect(runErr).To(HaveOccurred())
-				Expect(runErr.Error()).To(ContainSubstring("reload failed on startup"))
+
+				Expect(logBuf.String()).To(ContainSubstring("Bootstrap failed"))
+				Expect(logBuf.String()).To(ContainSubstring("bootstrap reload failed"))
 			})
 		})
+	})
 
+	Describe("exception handling", func() {
 		Context("when an error occurs during periodic sync", func() {
 			It("stops the runner and logs the error", func() {
 				var syncCount int32
@@ -194,10 +257,7 @@ var _ = Describe("Runner", func() {
 
 				var reloadCount int32
 				failCmdRunner := func(executable string, args ...string) ([]byte, error) {
-					n := atomic.AddInt32(&reloadCount, 1)
-					if n == 1 {
-						return []byte("ok"), nil // startup reload succeeds
-					}
+					atomic.AddInt32(&reloadCount, 1)
 					return nil, fmt.Errorf("cannot execute: reload failed")
 				}
 

src/bosh-nats-sync/pkg/userssync/users_sync.go

@@ -104,6 +104,31 @@ func ReloadNATSServerConfig(executable, pidFile string, runner CommandRunner) er
 	return err
 }
 
+// Bootstrap writes the initial NATS authorization config with only the
+// director and health-monitor subjects read from their subject files on disk.
+// It is called once at startup before the periodic sync loop so that
+// health_monitor and the director can authenticate against NATS immediately,
+// without waiting for bosh_nats_sync to successfully query the director API.
+func (u *UsersSync) Bootstrap() error {
+	directorSubject := readSubjectFile(u.BoshConfig.DirectorSubjectFile)
+	hmSubject := readSubjectFile(u.BoshConfig.HMSubjectFile)
+
+	if directorSubject == nil && hmSubject == nil {
+		u.Logger.Info("Bootstrap: no subject files found, skipping initial NATS config write")
+		return nil
+	}
+
+	u.Logger.Info("Bootstrap: writing initial NATS config with director/HM subjects")
+	if err := u.writeNATSConfigFile(nil, directorSubject, hmSubject); err != nil {
+		return fmt.Errorf("bootstrap: failed to write NATS config: %w", err)
+	}
+	if err := ReloadNATSServerConfig(u.NATSServerExecutable, u.NATSServerPIDFile, u.getCommandRunner()); err != nil {
+		return fmt.Errorf("bootstrap: failed to reload NATS server config: %w", err)
+	}
+	u.Logger.Info("Bootstrap: NATS config written and server reloaded")
+	return nil
+}
+
 func (u *UsersSync) withDirectorConnection(fn func() error) error {
 	timeout := u.BoshConfig.ConnectionWaitTimeout
 	if timeout <= 0 {

src/bosh-nats-sync/pkg/userssync/users_sync_test.go

@@ -775,6 +775,113 @@ var _ = Describe("UsersSync", func() {
 			Expect(err).To(HaveOccurred())
 		})
 	})
+
+	// Mirrors the BOSH startup race: pre-start writes a fooBar token placeholder
+	// to auth.json and the hm-subject / director-subject files, then bosh-nats-sync
+	// starts. Bootstrap() must overwrite the placeholder with real user credentials
+	// immediately, without querying the director, so that health_monitor can
+	// authenticate against NATS before the director finishes starting up.
+	Describe("Bootstrap", func() {
+		var (
+			bootstrapSync     *userssync.UsersSync
+			bootstrapCmdCalls []string
+			bootstrapCmdErr   error
+		)
+
+		BeforeEach(func() {
+			bootstrapCmdCalls = nil
+			bootstrapCmdErr = nil
+
+			// Simulate the fooBar token placeholder written by pre-start.
+			os.WriteFile(natsConfigFilePath, []byte(`{"authorization":{"token":"f0oBar"}}`), 0644)
+
+			// boshConfig is only populated in inner BeforeEach blocks elsewhere,
+			// so we build it explicitly here with the subject files from the
+			// outer BeforeEach.
+			bootstrapSync = &userssync.UsersSync{
+				NATSConfigFilePath: natsConfigFilePath,
+				BoshConfig: config.DirectorConfig{
+					URL:                 "http://127.0.0.1:1", // unreachable; Bootstrap must not contact it
+					DirectorSubjectFile: directorSubjectFile,
+					HMSubjectFile:       hmSubjectFile,
+				},
+				NATSServerExecutable: natsExecutable,
+				NATSServerPIDFile:    natsServerPIDFile,
+				Logger:               logger,
+				CommandRunner: func(executable string, args ...string) ([]byte, error) {
+					bootstrapCmdCalls = append(bootstrapCmdCalls, fmt.Sprintf("%s %s", executable, strings.Join(args, " ")))
+					return []byte("ok"), bootstrapCmdErr
+				},
+			}
+		})
+
+		It("writes the director and HM users to the NATS config without querying the director", func() {
+			err := bootstrapSync.Bootstrap()
+			Expect(err).NotTo(HaveOccurred())
+
+			data, readErr := os.ReadFile(natsConfigFilePath)
+			Expect(readErr).NotTo(HaveOccurred())
+
+			var cfg natsauthconfig.AuthorizationConfig
+			Expect(json.Unmarshal(data, &cfg)).To(Succeed())
+
+			subjects := make([]string, 0, len(cfg.Authorization.Users))
+			for _, u := range cfg.Authorization.Users {
+				subjects = append(subjects, u.User)
+			}
+			Expect(subjects).To(ContainElement(ContainSubstring("director.bosh-internal")))
+			Expect(subjects).To(ContainElement(ContainSubstring("hm.bosh-internal")))
+		})
+
+		It("overwrites the fooBar token placeholder left by pre-start", func() {
+			Expect(bootstrapSync.Bootstrap()).To(Succeed())
+
+			data, _ := os.ReadFile(natsConfigFilePath)
+			Expect(string(data)).NotTo(ContainSubstring("f0oBar"))
+			Expect(string(data)).To(ContainSubstring("users"))
+		})
+
+		It("sends a SIGHUP to reload the NATS server after writing the config", func() {
+			Expect(bootstrapSync.Bootstrap()).To(Succeed())
+
+			Expect(bootstrapCmdCalls).To(HaveLen(1))
+			Expect(bootstrapCmdCalls[0]).To(ContainSubstring("--signal"))
+			Expect(bootstrapCmdCalls[0]).To(ContainSubstring("reload="))
+		})
+
+		It("skips the write when neither subject file exists", func() {
+			bootstrapSync.BoshConfig.DirectorSubjectFile = "/nonexistent"
+			bootstrapSync.BoshConfig.HMSubjectFile = "/nonexistent"
+
+			err := bootstrapSync.Bootstrap()
+			Expect(err).NotTo(HaveOccurred())
+
+			// Config must remain unchanged (no SIGHUP either).
+			data, _ := os.ReadFile(natsConfigFilePath)
+			Expect(string(data)).To(ContainSubstring("f0oBar"))
+			Expect(bootstrapCmdCalls).To(BeEmpty())
+		})
+
+		It("returns an error when the NATS reload fails", func() {
+			bootstrapCmdErr = fmt.Errorf("reload failed")
+
+			err := bootstrapSync.Bootstrap()
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("reload failed"))
+		})
+
+		It("includes only the HM user when the director subject file is missing", func() {
+			bootstrapSync.BoshConfig.DirectorSubjectFile = "/nonexistent"
+
+			Expect(bootstrapSync.Bootstrap()).To(Succeed())
+
+			data, _ := os.ReadFile(natsConfigFilePath)
+			var cfg natsauthconfig.AuthorizationConfig
+			Expect(json.Unmarshal(data, &cfg)).To(Succeed())
+			Expect(cfg.Authorization.Users).To(HaveLen(1))
+			Expect(cfg.Authorization.Users[0].User).To(ContainSubstring("hm.bosh-internal"))
+		})
+	})
 })
 
 // Mirrors Ruby spec: spec/nats_sync/users_sync_spec.rb