diff --git a/config/module.go b/config/module.go index 604e9ea0bd4..7040e8a77ae 100644 --- a/config/module.go +++ b/config/module.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "encoding/json" + stderrors "errors" "fmt" "os" "os/exec" @@ -267,7 +268,7 @@ func (m *Module) FirstRun( // Load the module's meta.json. If it doesn't exist DEBUG log and exit quietly. // For all other errors WARN log and exit. - meta, err := m.getJSONManifest(unpackedModDir) + meta, moduleWorkingDirectory, err := m.getJSONManifest(unpackedModDir, env) var pathErr *os.PathError switch { case errors.As(err, &pathErr): @@ -282,7 +283,7 @@ func (m *Module) FirstRun( logger.Debug("no first run script specified, skipping first run") return nil } - relFirstRunPath, err := utils.SafeJoinDir(unpackedModDir, meta.FirstRun) + relFirstRunPath, err := utils.SafeJoinDir(moduleWorkingDirectory, meta.FirstRun) if err != nil { logger.Errorw("failed to build path to first run script, skipping first run", "error", err) return nil @@ -377,42 +378,116 @@ func (m *Module) FirstRun( return nil } -// getJSONManifest returns a loaded meta.json from one of two sources (in order of precedence): -// 1. if there is a meta.json in the exe dir, use that, except in local non-tarball case. -// 2. if this is a local tarball and there's a meta.json next to the tarball, use that. +// getJSONManifest returns a loaded meta.json from one of three sources (in order of precedence): +// 1. if this is a registry module and there is a meta.json in its top level directory, use that. +// 2. if there is a meta.json in the exe dir, use that, except in local non-tarball case. +// 3. if this is a local tarball, use the meta.json in unpackedModDir. // Note: the working directory must be the unpacked tarball directory or local exec directory. -func (m Module) getJSONManifest(unpackedModDir string) (*JSONManifest, error) { - // note: we don't look at internal meta.json in local non-tarball case because user has explicitly requested a binary. - localNonTarball := m.Type == ModuleTypeLocal && !m.NeedsSyntheticPackage() - if !localNonTarball { - // this is case 1, meta.json in exe folder. - metaPath, err := utils.SafeJoinDir(unpackedModDir, "meta.json") - if err != nil { - return nil, err +func (m Module) getJSONManifest(unpackedModDir string, env map[string]string) (*JSONManifest, string, error) { + // note: all registry modules qualify for cases 1 & 2; local tarballs for cases 2 & 3; and local non-tarballs for none. We don't look at + // internal meta.json in local non-tarball case because user has explicitly requested a binary. + + // note: each case is exited iff no errors occur but the meta.json file is not found + + var ok bool + var moduleWorkingDirectory string + var registryErr error + + online := m.Type == ModuleTypeRegistry + + // case 1: registry + if online { + moduleWorkingDirectory, ok = env["VIAM_MODULE_ROOT"] + if ok { + var meta *JSONManifest + meta, registryErr = findMetaJSONFile(moduleWorkingDirectory) + if registryErr != nil { + // return from getJSONManifest() if the error returned does NOT indicate that the file wasn't found + if !os.IsNotExist(registryErr) { + return nil, "", errors.Wrap(registryErr, "registry module") + } + } + + if meta != nil { + return meta, moduleWorkingDirectory, nil + } } - _, err = os.Stat(metaPath) - if err == nil { - // this is case 1, meta.json in exe dir - meta, err := parseJSONFile[JSONManifest](metaPath) - if err != nil { - return nil, err + } + + var registryTarballErr error + + localNonTarball := m.Type == ModuleTypeLocal && !m.NeedsSyntheticPackage() + + // case 2: registry OR tarball + if !localNonTarball && unpackedModDir != moduleWorkingDirectory { + var meta *JSONManifest + meta, registryTarballErr = findMetaJSONFile(unpackedModDir) + if registryTarballErr != nil { + if !os.IsNotExist(registryTarballErr) { + if online { + return nil, "", errors.Wrap(registryTarballErr, "registry module") + } + + return nil, "", errors.Wrap(registryTarballErr, "local tarball") } - return meta, nil + } + + if meta != nil { + return meta, unpackedModDir, nil } } + + var exeDir string + var localTarballErr error + + // TODO(RSDK-7848): remove this case once java sdk supports internal meta.json. + // case 3: local AND tarball if m.NeedsSyntheticPackage() { - // this is case 2, side-by-side - // TODO(RSDK-7848): remove this case once java sdk supports internal meta.json. - metaPath, err := utils.SafeJoinDir(filepath.Dir(m.ExePath), "meta.json") - if err != nil { - return nil, err + exeDir = filepath.Dir(m.ExePath) + + var meta *JSONManifest + meta, localTarballErr = findMetaJSONFile(exeDir) + if localTarballErr != nil { + if !os.IsNotExist(localTarballErr) { + return nil, "", errors.Wrap(localTarballErr, "local tarball") + } } - meta, err := parseJSONFile[JSONManifest](metaPath) - if err != nil { - // note: this error deprecates the side-by-side case because the side-by-side case is deprecated. - return nil, errors.Wrapf(err, "couldn't find meta.json inside tarball %s (or next to it)", m.ExePath) + + if meta != nil { + return meta, exeDir, nil + } + } + + if online { + if !ok { + return nil, "", errors.Wrap(registryTarballErr, "registry module: failed to find meta.json. VIAM_MODULE_ROOT not set") } - return meta, err + + return nil, "", errors.Wrap(stderrors.Join(registryErr, registryTarballErr), "registry module: failed to find meta.json") + } + + if !localNonTarball { + return nil, "", errors.Wrap(stderrors.Join(registryTarballErr, localTarballErr), "local tarball: failed to find meta.json") + } + + return nil, "", errors.New("local non-tarball: did not search for meta.json") +} + +func findMetaJSONFile(dir string) (*JSONManifest, error) { + metaPath, err := utils.SafeJoinDir(dir, "meta.json") + if err != nil { + return nil, err + } + + _, err = os.Stat(metaPath) + if err != nil { + return nil, err } - return nil, errors.New("failed to find meta.json") + + meta, err := parseJSONFile[JSONManifest](metaPath) + if err != nil { + return nil, err + } + + return meta, nil } diff --git a/config/module_test.go b/config/module_test.go index efe192edf70..8329dad5c23 100644 --- a/config/module_test.go +++ b/config/module_test.go @@ -1,12 +1,17 @@ package config import ( + "context" "encoding/json" "os" "path/filepath" "testing" + "github.com/pkg/errors" + "go.uber.org/zap/zaptest/observer" "go.viam.com/test" + + "go.viam.com/rdk/logging" ) // testChdir is a helper that cleans up an os.Chdir. @@ -102,6 +107,296 @@ func TestSyntheticModule(t *testing.T) { }) } +func TestRegistryModuleFirstRun(t *testing.T) { + ctx := context.Background() + localPackagesDir := "" + dataDir := "" + + t.Run("MetaFileNotFound", func(t *testing.T) { + module, _, env, logger, observedLogs := testSetUpRegistryModule(t) + + err := module.FirstRun(ctx, localPackagesDir, dataDir, env, logger) + test.That(t, err, test.ShouldBeNil) + test.That(t, observedLogs.FilterMessage("meta.json not found, skipping first run").Len(), test.ShouldEqual, 1) + }) + + t.Run("MetaFileInvalid", func(t *testing.T) { + module, metaJSONFilepath, env, logger, observedLogs := testSetUpRegistryModule(t) + + metaJSONFile, err := os.Create(metaJSONFilepath) + test.That(t, err, test.ShouldBeNil) + defer metaJSONFile.Close() + + err = module.FirstRun(ctx, localPackagesDir, dataDir, env, logger) + test.That(t, err, test.ShouldBeNil) + test.That(t, observedLogs.FilterMessage("failed to parse meta.json, skipping first run").Len(), test.ShouldEqual, 1) + }) + + t.Run("NoFirstRunScript", func(t *testing.T) { + module, metaJSONFilepath, env, logger, observedLogs := testSetUpRegistryModule(t) + + testWriteJSON(t, metaJSONFilepath, JSONManifest{}) + + err := module.FirstRun(ctx, localPackagesDir, dataDir, env, logger) + test.That(t, err, test.ShouldBeNil) + test.That(t, observedLogs.FilterMessage("no first run script specified, skipping first run").Len(), test.ShouldEqual, 1) + }) + + t.Run("InvalidFirstRunPath", func(t *testing.T) { + module, metaJSONFilepath, env, logger, observedLogs := testSetUpRegistryModule(t) + + testWriteJSON(t, metaJSONFilepath, JSONManifest{FirstRun: "../firstrun.sh"}) + + err := module.FirstRun(ctx, localPackagesDir, dataDir, env, logger) + test.That(t, err, test.ShouldBeNil) + test.That(t, observedLogs.FilterMessage("failed to build path to first run script, skipping first run").Len(), test.ShouldEqual, 1) + }) + + // the executable is one level deep, and the meta.json file is in the same directory + t.Run("NoFirstRunScriptOneLevelExe", func(t *testing.T) { + module := Module{Type: ModuleTypeRegistry} + + tmp := t.TempDir() + exeDir := filepath.Join(tmp, "executable-directory") + exePath := filepath.Join(exeDir, "whatever.sh") + module.ExePath = exePath + exeMetaJSONFilepath := filepath.Join(exeDir, "meta.json") + + env := map[string]string{"VIAM_MODULE_ROOT": tmp} + + logger, observedLogs := logging.NewObservedTestLogger(t) + + err := os.Mkdir(exeDir, 0o700) + test.That(t, err, test.ShouldBeNil) + + testWriteJSON(t, exeMetaJSONFilepath, JSONManifest{}) + + err = module.FirstRun(ctx, localPackagesDir, dataDir, env, logger) + test.That(t, err, test.ShouldBeNil) + test.That(t, observedLogs.FilterMessage("no first run script specified, skipping first run").Len(), test.ShouldEqual, 1) + }) + + // the executable is one level deep, and the meta.json file is in the top level directory + t.Run("NoFirstRunScriptOneLevelTopLevel", func(t *testing.T) { + module := Module{Type: ModuleTypeRegistry} + + tmp := t.TempDir() + topLevelDir := tmp + exeDir := filepath.Join(tmp, "executable-directory") + exePath := filepath.Join(exeDir, "whatever.sh") + module.ExePath = exePath + exeMetaJSONFilepath := filepath.Join(topLevelDir, "meta.json") + + env := map[string]string{"VIAM_MODULE_ROOT": tmp} + + logger, observedLogs := logging.NewObservedTestLogger(t) + + err := os.Mkdir(exeDir, 0o700) + test.That(t, err, test.ShouldBeNil) + + testWriteJSON(t, exeMetaJSONFilepath, JSONManifest{}) + + err = module.FirstRun(ctx, localPackagesDir, dataDir, env, logger) + test.That(t, err, test.ShouldBeNil) + test.That(t, observedLogs.FilterMessage("no first run script specified, skipping first run").Len(), test.ShouldEqual, 1) + }) +} + +func TestGetJSONManifest(t *testing.T) { + validJSONManifest := JSONManifest{Entrypoint: "entry"} + + t.Run("RegistryModule", func(t *testing.T) { + tmp := t.TempDir() + + topLevelDir := tmp + topLevelMetaJSONFilepath := filepath.Join(topLevelDir, "meta.json") + unpackedModDir := filepath.Join(tmp, "unpacked-mod-dir") + unpackedModMetaJSONFilepath := filepath.Join(unpackedModDir, "meta.json") + env := make(map[string]string, 1) + modRegistry := Module{Type: ModuleTypeRegistry} + + err := os.Mkdir(unpackedModDir, 0o700) + test.That(t, err, test.ShouldBeNil) + + // meta.json not found; only unpacked module directory searched + meta, moduleWorkingDirectory, err := modRegistry.getJSONManifest(unpackedModDir, env) + test.That(t, meta, test.ShouldBeNil) + test.That(t, moduleWorkingDirectory, test.ShouldBeEmpty) + test.That(t, err, test.ShouldNotBeNil) + test.That(t, err.Error(), test.ShouldContainSubstring, "registry module") + test.That(t, errors.Is(err, os.ErrNotExist), test.ShouldBeTrue) + test.That(t, err.Error(), test.ShouldContainSubstring, unpackedModMetaJSONFilepath) + test.That(t, err.Error(), test.ShouldNotContainSubstring, topLevelMetaJSONFilepath) + + // meta.json not found; top level module directory and unpacked module directories searched + + // setting the "VIAM_MODULE_ROOT" environment variable allows getJSONManifest() to search in a registry module's top level directory + // for the meta.json file. The variable is accessed through the 'env' function parameter + env["VIAM_MODULE_ROOT"] = topLevelDir + + meta, moduleWorkingDirectory, err = modRegistry.getJSONManifest(unpackedModDir, env) + test.That(t, meta, test.ShouldBeNil) + test.That(t, moduleWorkingDirectory, test.ShouldBeEmpty) + test.That(t, err, test.ShouldNotBeNil) + test.That(t, err.Error(), test.ShouldContainSubstring, "registry module") + test.That(t, errors.Is(err, os.ErrNotExist), test.ShouldBeTrue) + test.That(t, err.Error(), test.ShouldContainSubstring, unpackedModMetaJSONFilepath) + test.That(t, err.Error(), test.ShouldContainSubstring, topLevelMetaJSONFilepath) + + // meta.json found in unpacked modular directory; parsing fails + unpackedModMetaJSONFile, err := os.Create(unpackedModMetaJSONFilepath) + test.That(t, err, test.ShouldBeNil) + defer unpackedModMetaJSONFile.Close() + + meta, moduleWorkingDirectory, err = modRegistry.getJSONManifest(unpackedModDir, env) + test.That(t, meta, test.ShouldBeNil) + test.That(t, moduleWorkingDirectory, test.ShouldBeEmpty) + test.That(t, err, test.ShouldNotBeNil) + test.That(t, err.Error(), test.ShouldContainSubstring, "registry module") + test.That(t, errors.Is(err, os.ErrNotExist), test.ShouldBeFalse) + + // meta.json found in unpacked modular directory; parsing succeeds + testWriteJSON(t, unpackedModMetaJSONFilepath, validJSONManifest) + + meta, moduleWorkingDirectory, err = modRegistry.getJSONManifest(unpackedModDir, env) + test.That(t, *meta, test.ShouldResemble, validJSONManifest) + test.That(t, moduleWorkingDirectory, test.ShouldEqual, unpackedModDir) + test.That(t, err, test.ShouldBeNil) + + // meta.json found in top level modular directory; parsing fails + topLevelMetaJSONFile, err := os.Create(topLevelMetaJSONFilepath) + test.That(t, err, test.ShouldBeNil) + defer topLevelMetaJSONFile.Close() + + meta, moduleWorkingDirectory, err = modRegistry.getJSONManifest(unpackedModDir, env) + test.That(t, meta, test.ShouldBeNil) + test.That(t, moduleWorkingDirectory, test.ShouldBeEmpty) + test.That(t, err, test.ShouldNotBeNil) + test.That(t, err.Error(), test.ShouldContainSubstring, "registry module") + test.That(t, errors.Is(err, os.ErrNotExist), test.ShouldBeFalse) + + // meta.json found in top level modular directory; parsing succeeds + testWriteJSON(t, topLevelMetaJSONFilepath, validJSONManifest) + + meta, moduleWorkingDirectory, err = modRegistry.getJSONManifest(unpackedModDir, env) + test.That(t, *meta, test.ShouldResemble, validJSONManifest) + test.That(t, moduleWorkingDirectory, test.ShouldEqual, topLevelDir) + test.That(t, err, test.ShouldBeNil) + }) + + t.Run("LocalTarball", func(t *testing.T) { + tmp := t.TempDir() + + exePath := filepath.Join(tmp, "module.tgz") + exeDir := filepath.Dir(exePath) + exeMetaJSONFilepath := filepath.Join(exeDir, "meta.json") + unpackedModDir := filepath.Join(tmp, "unpacked-mod-dir") + unpackedModMetaJSONFilepath := filepath.Join(unpackedModDir, "meta.json") + env := map[string]string{} + modLocalTar := Module{Type: ModuleTypeLocal, ExePath: exePath} + + err := os.Mkdir(unpackedModDir, 0o700) + test.That(t, err, test.ShouldBeNil) + + // meta.json not found; unpacked module and executable directories searched + meta, moduleWorkingDirectory, err := modLocalTar.getJSONManifest(unpackedModDir, env) + test.That(t, meta, test.ShouldBeNil) + test.That(t, moduleWorkingDirectory, test.ShouldBeEmpty) + test.That(t, err, test.ShouldNotBeNil) + test.That(t, err.Error(), test.ShouldContainSubstring, "local tarball") + test.That(t, errors.Is(err, os.ErrNotExist), test.ShouldBeTrue) + test.That(t, err.Error(), test.ShouldContainSubstring, unpackedModDir) + test.That(t, err.Error(), test.ShouldContainSubstring, exeDir) + + // meta.json found in executable directory; parsing fails + exeMetaJSONFile, err := os.Create(exeMetaJSONFilepath) + test.That(t, err, test.ShouldBeNil) + defer exeMetaJSONFile.Close() + + meta, moduleWorkingDirectory, err = modLocalTar.getJSONManifest(unpackedModDir, env) + test.That(t, meta, test.ShouldBeNil) + test.That(t, moduleWorkingDirectory, test.ShouldBeEmpty) + test.That(t, err, test.ShouldNotBeNil) + test.That(t, err.Error(), test.ShouldContainSubstring, "local tarball") + test.That(t, errors.Is(err, os.ErrNotExist), test.ShouldBeFalse) + + // meta.json found in executable directory; parsing succeeds + testWriteJSON(t, exeMetaJSONFilepath, validJSONManifest) + + meta, moduleWorkingDirectory, err = modLocalTar.getJSONManifest(unpackedModDir, env) + test.That(t, *meta, test.ShouldResemble, validJSONManifest) + test.That(t, moduleWorkingDirectory, test.ShouldEqual, exeDir) + test.That(t, err, test.ShouldBeNil) + + // meta.json found in unpacked modular directory; parsing fails + unpackedModMetaJSONFile, err := os.Create(unpackedModMetaJSONFilepath) + test.That(t, err, test.ShouldBeNil) + defer unpackedModMetaJSONFile.Close() + + meta, moduleWorkingDirectory, err = modLocalTar.getJSONManifest(unpackedModDir, env) + test.That(t, meta, test.ShouldBeNil) + test.That(t, moduleWorkingDirectory, test.ShouldBeEmpty) + test.That(t, err, test.ShouldNotBeNil) + test.That(t, err.Error(), test.ShouldContainSubstring, "local tarball") + test.That(t, errors.Is(err, os.ErrNotExist), test.ShouldBeFalse) + + // meta.json found in unpacked module directory; parsing succeeds + testWriteJSON(t, unpackedModMetaJSONFilepath, validJSONManifest) + + meta, moduleWorkingDirectory, err = modLocalTar.getJSONManifest(unpackedModDir, env) + test.That(t, *meta, test.ShouldResemble, validJSONManifest) + test.That(t, moduleWorkingDirectory, test.ShouldEqual, unpackedModDir) + test.That(t, err, test.ShouldBeNil) + }) + + t.Run("LocalNontarball", func(t *testing.T) { + tmp := t.TempDir() + + unpackedModDir := filepath.Join(tmp, "unpacked-mod-dir") + env := map[string]string{} + modLocalNontar := Module{Type: ModuleTypeLocal} + + err := os.Mkdir(unpackedModDir, 0o700) + test.That(t, err, test.ShouldBeNil) + + meta, moduleWorkingDirectory, err := modLocalNontar.getJSONManifest(unpackedModDir, env) + test.That(t, meta, test.ShouldBeNil) + test.That(t, moduleWorkingDirectory, test.ShouldBeEmpty) + test.That(t, err, test.ShouldNotBeNil) + test.That(t, err.Error(), test.ShouldContainSubstring, "local non-tarball") + test.That(t, errors.Is(err, os.ErrNotExist), test.ShouldBeFalse) + }) +} + +func TestFindMetaJSONFile(t *testing.T) { + tmp := t.TempDir() + metaJSONFilePath := filepath.Join(tmp, "meta.json") + + t.Run("MissingMetaFile", func(t *testing.T) { + meta, err := findMetaJSONFile(tmp) + test.That(t, meta, test.ShouldBeNil) + test.That(t, os.IsNotExist(err), test.ShouldBeTrue) + }) + + file, err := os.Create(metaJSONFilePath) + test.That(t, err, test.ShouldBeNil) + defer file.Close() + t.Run("InvalidMetaFile", func(t *testing.T) { + meta, err := findMetaJSONFile(tmp) + test.That(t, meta, test.ShouldBeNil) + test.That(t, err, test.ShouldNotBeNil) + test.That(t, err, test.ShouldNotEqual, os.IsNotExist) + }) + + validMeta := JSONManifest{Entrypoint: "entry"} + testWriteJSON(t, metaJSONFilePath, &validMeta) + t.Run("ValidMetaFileFound", func(t *testing.T) { + meta, err := findMetaJSONFile(tmp) + test.That(t, *meta, test.ShouldResemble, validMeta) + test.That(t, err, test.ShouldBeNil) + }) +} + // testWriteJSON is a t.Helper that serializes `value` to `path` as json. func testWriteJSON(t *testing.T, path string, value any) { t.Helper() @@ -112,3 +407,23 @@ func testWriteJSON(t *testing.T, path string, value any) { err = encoder.Encode(value) test.That(t, err, test.ShouldBeNil) } + +// testSetUpRegistryModule is a t.Helper that creates a registry module with a meta.json file and an executable file in its top level +// directory. It also returns a logger and its observed logs for testing. +func testSetUpRegistryModule(t *testing.T) (module Module, metaJSONFilepath string, env map[string]string, logger logging.Logger, + observedLogs *observer.ObservedLogs, +) { + t.Helper() + module = Module{Type: ModuleTypeRegistry} + tmp := t.TempDir() + exePath := filepath.Join(tmp, "whatever.sh") + module.ExePath = exePath + metaJSONFilepath = filepath.Join(tmp, "meta.json") + + // getJSONManifest() uses the "VIAM_MODUE_ROOT" environment variable to find the top level directory of a registry module + env = make(map[string]string, 1) + env["VIAM_MODULE_ROOT"] = tmp + + logger, observedLogs = logging.NewObservedTestLogger(t) + return +}