feature: Add Job API + CLI support for setting the working directory from polyglot hooks

clicommand/commands.go

@@ -149,4 +149,12 @@ var BuildkiteAgentCommands = []cli.Command{
 			ToolSignCommand,
 		},
 	},
+	{
+		Name:     "workdir",
+		Category: categoryJobCommands,
+		Usage:    "Interact with the working directory of the currently running job",
+		Subcommands: []cli.Command{
+			WorkdirSetCommand,
+		},
+	},
 }

clicommand/workdir_set.go

@@ -0,0 +1,126 @@
+package clicommand
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/buildkite/agent/v3/jobapi"
+	"github.com/urfave/cli"
+)
+
+const workdirSetHelpDescription = `Usage:
+
+    buildkite-agent workdir set <path>
+
+Description:
+
+Sets the working directory for subsequent phases of the current job. The change
+persists across later hooks and the command phase.
+
+This is intended for binary and polyglot (non-shell) hooks, which run in a child
+process whose own directory changes are otherwise lost. Wrapped POSIX shell
+hooks can change the working directory simply by ′cd′-ing.
+
+Relative paths are resolved against the current working directory of this command
+(i.e. the hook's actual working directory). The path must exist and be a
+directory.
+
+Examples:
+
+Setting the working directory to a subdirectory of the checkout:
+
+    $ buildkite-agent workdir set ./subdir
+
+Setting the working directory to an absolute path:
+
+    $ buildkite-agent workdir set /tmp/build-scratch`
+
+type WorkdirSetConfig struct {
+	GlobalConfig
+
+	OutputFormat string `cli:"output-format"`
+}
+
+var WorkdirSetCommand = cli.Command{
+	Name:        "set",
+	Usage:       "Sets the working directory for subsequent phases of the job",
+	Description: workdirSetHelpDescription,
+	Flags: append(globalFlags(),
+		cli.StringFlag{
+			Name:   "output-format",
+			Usage:  "Output format: quiet (no output), plain, or json",
+			EnvVar: "BUILDKITE_AGENT_WORKDIR_SET_OUTPUT_FORMAT",
+			Value:  "plain",
+		},
+	),
+	Action: workdirSetAction,
+}
+
+func workdirSetAction(c *cli.Context) error {
+	ctx := context.Background()
+	ctx, cfg, _, _, done := setupLoggerAndConfig[WorkdirSetConfig](ctx, c)
+	defer done()
+
+	args := c.Args()
+	if len(args) != 1 {
+		return fmt.Errorf("expected exactly one argument (the working directory), got %d", len(args))
+	}
+
+	abs, err := resolveWorkdir(args[0])
+	if err != nil {
+		return err
+	}
+
+	client, err := jobapi.NewDefaultClient(ctx)
+	if err != nil {
+		return fmt.Errorf(envClientErrMessage, err)
+	}
+
+	workdir, err := client.SetWorkdir(ctx, abs)
+	if err != nil {
+		return fmt.Errorf("setting the job working directory: %w", err)
+	}
+
+	switch cfg.OutputFormat {
+	case "quiet":
+		return nil
+
+	case "plain":
+		_, _ = fmt.Fprintln(c.App.Writer, workdir)
+
+	case "json":
+		enc := json.NewEncoder(c.App.Writer)
+		if err := enc.Encode(jobapi.WorkdirSetResponse{Workdir: workdir}); err != nil {
+			return fmt.Errorf("error marshalling JSON: %w", err)
+		}
+
+	default:
+		return fmt.Errorf("invalid output format %q", cfg.OutputFormat)
+	}
+
+	return nil
+}
+
+// resolveWorkdir resolves path to an absolute path against the current working
+// directory (the hook's actual working directory) and validates that it exists
+// and is a directory. The Job API endpoint only accepts absolute paths and does
+// not stat the filesystem, so this resolution and validation happens client-side.
+func resolveWorkdir(path string) (string, error) {
+	abs, err := filepath.Abs(path)
+	if err != nil {
+		return "", fmt.Errorf("resolving absolute path of %q: %w", path, err)
+	}
+
+	info, err := os.Stat(abs)
+	if err != nil {
+		return "", fmt.Errorf("checking working directory %q: %w", abs, err)
+	}
+	if !info.IsDir() {
+		return "", fmt.Errorf("working directory %q is not a directory", abs)
+	}
+
+	return abs, nil
+}

clicommand/workdir_set_test.go

@@ -0,0 +1,75 @@
+package clicommand
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// TestResolveWorkdir is deliberately not parallel: the "relative path" subtest
+// changes the process working directory, which would corrupt other tests if run
+// concurrently. Non-parallel tests run during testing's sequential phase, when
+// no parallel test is active.
+func TestResolveWorkdir(t *testing.T) {
+	dir := t.TempDir()
+
+	t.Run("absolute existing directory", func(t *testing.T) {
+		got, err := resolveWorkdir(dir)
+		if err != nil {
+			t.Fatalf("resolveWorkdir(%q) error = %v, want nil", dir, err)
+		}
+		if got != dir {
+			t.Fatalf("resolveWorkdir(%q) = %q, want %q", dir, got, dir)
+		}
+	})
+
+	t.Run("relative path is resolved against cwd", func(t *testing.T) {
+		// Not parallel: changes the process working directory.
+		sub := "workdir-sub"
+		if err := os.Mkdir(filepath.Join(dir, sub), 0o700); err != nil {
+			t.Fatalf("os.Mkdir error = %v", err)
+		}
+
+		prev, err := os.Getwd()
+		if err != nil {
+			t.Fatalf("os.Getwd() error = %v", err)
+		}
+		t.Cleanup(func() { _ = os.Chdir(prev) })
+		if err := os.Chdir(dir); err != nil {
+			t.Fatalf("os.Chdir(%q) error = %v", dir, err)
+		}
+
+		// Compute want from the actual cwd, since os.Getwd (used by filepath.Abs)
+		// may resolve symlinks (e.g. /tmp -> /private/tmp on macOS).
+		cwd, err := os.Getwd()
+		if err != nil {
+			t.Fatalf("os.Getwd() error = %v", err)
+		}
+
+		got, err := resolveWorkdir(sub)
+		if err != nil {
+			t.Fatalf("resolveWorkdir(%q) error = %v, want nil", sub, err)
+		}
+		want := filepath.Join(cwd, sub)
+		if got != want {
+			t.Fatalf("resolveWorkdir(%q) = %q, want %q", sub, got, want)
+		}
+	})
+
+	t.Run("nonexistent path errors", func(t *testing.T) {
+		missing := filepath.Join(dir, "does-not-exist")
+		if _, err := resolveWorkdir(missing); err == nil {
+			t.Fatalf("resolveWorkdir(%q) error = nil, want non-nil", missing)
+		}
+	})
+
+	t.Run("file (not a directory) errors", func(t *testing.T) {
+		file := filepath.Join(dir, "a-file")
+		if err := os.WriteFile(file, []byte("hi"), 0o600); err != nil {
+			t.Fatalf("os.WriteFile error = %v", err)
+		}
+		if _, err := resolveWorkdir(file); err == nil {
+			t.Fatalf("resolveWorkdir(%q) error = nil, want non-nil", file)
+		}
+	})
+}

internal/job/api.go

@@ -34,6 +34,7 @@ We'll continue to run your job, but you won't be able to use the Job API`)
 	if err != nil {
 		return cleanup, fmt.Errorf("creating job API server: %w", err)
 	}
+	e.jobAPI = srv
 
 	e.shell.Env.Set("BUILDKITE_AGENT_JOB_API_SOCKET", socketPath)
 	e.shell.Env.Set("BUILDKITE_AGENT_JOB_API_TOKEN", token)

internal/job/executor.go

@@ -35,6 +35,7 @@ import (
 	"github.com/buildkite/agent/v3/internal/shell"
 	"github.com/buildkite/agent/v3/internal/shellscript"
 	"github.com/buildkite/agent/v3/internal/tempfile"
+	"github.com/buildkite/agent/v3/jobapi"
 	"github.com/buildkite/agent/v3/logger"
 	"github.com/buildkite/agent/v3/tracetools"
 	"github.com/buildkite/go-pipeline"
@@ -74,6 +75,11 @@ type Executor struct {
 	// redactors for the job logs. The will be populated with values both from environment variable and through the Job API.
 	// In order for the latter to happen, a reference is passed into the the Job API server as well
 	redactors *replacer.Mux
+
+	// jobAPI is the Job API server for this job. It's retained so the executor
+	// can consume out-of-band requests made by hooks, such as a working
+	// directory set by an unwrapped hook via the /workdir endpoint.
+	jobAPI *jobapi.Server
 }
 
 // New returns a new executor instance
@@ -465,7 +471,19 @@ func (e *Executor) runUnwrappedHook(ctx context.Context, _ string, hookCfg HookC
 	// Passing an empty env changes through because in polyglot hook we can't detect
 	// env change.
 	// But we call this method anyway because a hook might use buildkite-agent env set to update environment.
-	e.applyEnvironmentChanges(hook.EnvChanges{})
+	//
+	// Unwrapped hooks can also request a working directory change via
+	// `buildkite-agent workdir set`, which the Job API records as a pending
+	// workdir. Consume it here and apply it via applyEnvironmentChanges (which
+	// calls shell.Chdir). Once applied it persists in shell.wd, so subsequent
+	// hooks and the command phase run in the new directory.
+	changes := hook.EnvChanges{}
+	if e.jobAPI != nil {
+		if wd, ok := e.jobAPI.TakePendingWorkdir(); ok {
+			changes = hook.EnvChangesForWorkdir(wd)
+		}
+	}
+	e.applyEnvironmentChanges(changes)
 	return nil
 }
 

internal/job/hook/wrapper.go

@@ -87,6 +87,13 @@ type EnvChanges struct {
 	afterWd string
 }
 
+// EnvChangesForWorkdir returns EnvChanges that only request a working directory
+// change (no env var diff). It's used to apply a working directory set by an
+// unwrapped hook via the Job API /workdir endpoint.
+func EnvChangesForWorkdir(wd string) EnvChanges {
+	return EnvChanges{afterWd: wd}
+}
+
 func (changes *EnvChanges) GetAfterWd() (string, error) {
 	if changes.afterWd == "" {
 		return "", fmt.Errorf("%q was not present in the hook after environment", hookWorkingDirEnv)

internal/job/integration/hooks_integration_test.go

@@ -191,6 +191,59 @@ func TestDirectoryPassesBetweenHooksIgnoredUnderExit(t *testing.T) {
 	tester.RunAndCheck(t, "MY_CUSTOM_ENV=1")
 }
 
+func TestBinaryHookCanSetWorkdir(t *testing.T) {
+	t.Parallel()
+
+	if runtime.GOOS == "windows" {
+		t.Skip("Binary hooks set the workdir via the Job API, which behaves the same on all platforms, but this test relies on POSIX-y path handling")
+	}
+
+	ctx := mainCtx
+
+	tester, err := NewExecutorTester(ctx)
+	if err != nil {
+		t.Fatalf("NewExecutorTester() error = %v", err)
+	}
+	defer tester.Close()
+
+	// Build a binary pre-command hook that calls the Job API to set the working
+	// directory for subsequent phases. The source is in
+	// ./test-binary-hook-workdir/main.go.
+	t.Logf("Building test-binary-hook-workdir")
+	hookPath := filepath.Join(tester.HooksDir, "pre-command")
+	output, err := exec.Command("go", "build", "-o", hookPath, "./test-binary-hook-workdir").CombinedOutput()
+	if err != nil {
+		t.Fatalf("Failed to build test-binary-hook-workdir: %v, output: %s", err, string(output))
+	}
+
+	// The command hook should run in the directory the binary hook requested.
+	tester.ExpectGlobalHook("command").Once().AndExitWith(0).AndCallFunc(func(c *bintest.Call) {
+		if want := c.GetEnv("EXPECTED_WORKDIR"); want != c.Dir {
+			_, _ = fmt.Fprintf(c.Stderr, "Expected command hook dir to be %q, got %q\n", want, c.Dir)
+			c.Exit(1)
+		} else {
+			c.Exit(0)
+		}
+	})
+
+	// A subsequent hook should also run in the requested directory, proving the
+	// change persists across hooks.
+	tester.ExpectGlobalHook("post-command").Once().AndExitWith(0).AndCallFunc(func(c *bintest.Call) {
+		if want := c.GetEnv("EXPECTED_WORKDIR"); want != c.Dir {
+			_, _ = fmt.Fprintf(c.Stderr, "Expected post-command hook dir to be %q, got %q\n", want, c.Dir)
+			c.Exit(1)
+		} else {
+			c.Exit(0)
+		}
+	})
+
+	tester.RunAndCheck(t)
+
+	if !strings.Contains(tester.Output, "hi there from the workdir-setting binary hook 📂") {
+		t.Fatalf("tester.Output %q does not contain expected output: %q", tester.Output, "hi there from the workdir-setting binary hook 📂")
+	}
+}
+
 func TestCheckingOutFiresCorrectHooks(t *testing.T) {
 	t.Parallel()
 

internal/job/integration/test-binary-hook-workdir/main.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/buildkite/agent/v3/jobapi"
+)
+
+// This file gets built and used as part of the hooks integration test to ensure
+// that an unwrapped (binary) hook can change the working directory of subsequent
+// phases via the Job API /workdir endpoint.
+//
+// It creates a subdirectory of its own working directory (the checkout dir),
+// resolves it to an absolute, symlink-free path, then asks the executor to use
+// it as the working directory for subsequent hooks and the command. It also
+// records the expected directory in EXPECTED_WORKDIR so the test's command and
+// post-command hooks can assert they run there.
+func main() {
+	ctx := context.TODO()
+
+	c, err := jobapi.NewDefaultClient(ctx)
+	if err != nil {
+		log.Fatalf("error: %v", fmt.Errorf("creating job api client: %w", err))
+	}
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		log.Fatalf("error: %v", fmt.Errorf("getting working directory: %w", err))
+	}
+
+	target := filepath.Join(cwd, "binary-hook-subdir")
+	if err := os.MkdirAll(target, 0o777); err != nil {
+		log.Fatalf("error: %v", fmt.Errorf("creating target directory: %w", err))
+	}
+
+	// Resolve symlinks so the path matches what the command hook observes as its
+	// working directory (on macOS /tmp and /var are symlinks).
+	resolved, err := filepath.EvalSymlinks(target)
+	if err != nil {
+		log.Fatalf("error: %v", fmt.Errorf("resolving target directory: %w", err))
+	}
+
+	if _, err := c.SetWorkdir(ctx, resolved); err != nil {
+		log.Fatalf("error: %v", fmt.Errorf("setting workdir: %w", err))
+	}
+
+	if _, err := c.EnvUpdate(ctx, &jobapi.EnvUpdateRequest{
+		Env: map[string]string{"EXPECTED_WORKDIR": resolved},
+	}); err != nil {
+		log.Fatalf("error: %v", fmt.Errorf("updating env: %w", err))
+	}
+
+	fmt.Println("hi there from the workdir-setting binary hook 📂")
+}