Track tool definitions by defining artifact (Velocidex#2439)

This allows the admin to upgrade tool definitions by resetting to the
definition in another artifact.

.github/workflows/go.yml

@@ -16,42 +16,26 @@ jobs:
     steps:
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
-    - uses: actions/setup-go@v2
+    - uses: actions/setup-go@v3
       with:
         go-version: '^1.19'
 
     - run: go version
 
-#    - uses: actions/cache@v2
-#      with:
-#        # In order:
-#        # * Module download cache
-#        # * Build cache (Linux)
-#        # * Build cache (Mac)
-#        # * Build cache (Windows)
-#        path: |
-#          ~/go/pkg/mod
-#          ~/.cache/go-build
-#          ~/Library/Caches/go-build
-#          %LocalAppData%\go-build
-#        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-#        restore-keys: |
-#          ${{ runner.os }}-go-
-
     - name: Get dependencies
       run: |
         go get -v -t -d ./...
         sudo apt-get install mingw-w64-x86-64-dev gcc-mingw-w64-x86-64 gcc-mingw-w64
 
     - name: Use Node.js v16
-      uses: actions/setup-node@v1
+      uses: actions/setup-node@v3
       with:
         node-version: 16
 
     - name: Cache node-modules
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       env:
         cache-name: cache-node-modules
       with:
@@ -82,7 +66,7 @@ jobs:
         go run make.go -v DarwinBase
 
     - name: StoreBinaries
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v3
       with:
         name: Binaries
         path: output

.github/workflows/musl.yaml

@@ -12,9 +12,9 @@ jobs:
     steps:
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
-    - uses: actions/setup-go@v2
+    - uses: actions/setup-go@v3
       with:
         go-version: '^1.19'
 
@@ -35,7 +35,7 @@ jobs:
         cd ..
 
     - name: Use Node.js v16
-      uses: actions/setup-node@v1
+      uses: actions/setup-node@v3
       with:
         node-version: 16
 
@@ -53,7 +53,7 @@ jobs:
         go run make.go -v LinuxMusl
 
     - name: StoreBinaries
-      uses: actions/upload-artifact@v1
+      uses: actions/upload-artifact@v3
       with:
         name: Binaries
         path: output

.github/workflows/windows.yml

@@ -6,29 +6,13 @@ jobs:
     runs-on: windows-2019
     steps:
     - name: Set up Go 1.19
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v3
       with:
         go-version: 1.19
       id: go
 
-#    - uses: actions/cache@v2
-#      with:
-        # In order:
-        # * Module download cache
-        # * Build cache (Linux)
-        # * Build cache (Mac)
-        # * Build cache (Windows)
-#        path: |
-#          ~/go/pkg/mod
-#          ~/.cache/go-build
-#          ~/Library/Caches/go-build
-#          %LocalAppData%\go-build
-#        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-#        restore-keys: |
-#          ${{ runner.os }}-go-
-
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     - name: Configure test environment
       shell: cmd
@@ -123,7 +107,7 @@ jobs:
         mkdir -p artifact_output/server/
         cp artifacts/testdata/server/testcases/*.out* artifact_output/server/
 
-    - uses: actions/upload-artifact@master
+    - uses: actions/upload-artifact@v3
       if: always()
       with:
         name: artifact

accessors/vql_arg_parser.go

@@ -70,33 +70,6 @@ func parseOSPath(ctx context.Context,
 		// Initializer can be a list of components. In this case we
 		// take the base pathspec (which is accessor determined) and
 		// add the components to it.
-	case []types.Any:
-		components := make([]string, 0, len(t))
-		for _, i := range t {
-			i_str, ok := i.(string)
-			if ok {
-				components = append(components, i_str)
-			}
-		}
-
-		// Build a pathspec from the accessor and the components.
-		base, err := accessor.ParsePath("")
-		if err != nil {
-			return nil, err
-		}
-
-		base.Components = append(base.Components, components...)
-		return base, nil
-
-	case []string:
-		// Build a pathspec from the accessor and the components.
-		base, err := accessor.ParsePath("")
-		if err != nil {
-			return nil, err
-		}
-
-		base.Components = append(base.Components, t...)
-		return base, nil
 
 	case string:
 		return accessor.ParsePath(t)
@@ -105,6 +78,28 @@ func parseOSPath(ctx context.Context,
 		return accessor.ParsePath(string(t))
 
 	default:
+		result, _ := accessor.ParsePath("")
+
+		// Is it an array? Generic code to handle arrays - just append
+		// each element together to form a single path. This allows
+		// joining components directly:
+		//     ["bin", "ls"] or ["/usr/bin", "ls"]
+		a_value := reflect.Indirect(reflect.ValueOf(value))
+		if a_value.Type().Kind() == reflect.Slice {
+			for idx := 0; idx < a_value.Len(); idx++ {
+				item, err := parseOSPath(ctx, scope, args,
+					a_value.Index(int(idx)).Interface())
+				if err != nil {
+					continue
+				}
+				item_os_path, ok := item.(*OSPath)
+				if ok {
+					result = result.Append(item_os_path.Components...)
+				}
+			}
+			return result, nil
+		}
+
 		// This is a fatal error on the client.
 		return nil, fmt.Errorf("Expecting a path arg type, not %T", t)
 	}

api/api.go

@@ -290,7 +290,7 @@ func (self *ApiServer) NotifyClients(
 
 	if in.ClientId != "" {
 		self.server_obj.Info("sending notification to %s", in.ClientId)
-		err = notifier.NotifyListener(org_config_obj, in.ClientId,
+		err = notifier.NotifyListener(ctx, org_config_obj, in.ClientId,
 			"API.NotifyClients")
 	} else {
 		return nil, status.Error(codes.InvalidArgument,
@@ -694,7 +694,7 @@ func (self *ApiServer) GetArtifacts(
 		}
 
 		for _, name := range in.Names {
-			artifact, pres := repository.Get(org_config_obj, name)
+			artifact, pres := repository.Get(ctx, org_config_obj, name)
 			if pres {
 				result.Items = append(result.Items, artifact)
 			}
@@ -734,7 +734,7 @@ func (self *ApiServer) GetArtifactFile(
 			"User is not allowed to view custom artifacts.")
 	}
 
-	artifact, err := getArtifactFile(org_config_obj, in.Name)
+	artifact, err := getArtifactFile(ctx, org_config_obj, in.Name)
 	if err != nil {
 		return nil, Status(self.verbose, err)
 	}
@@ -787,7 +787,7 @@ func (self *ApiServer) SetArtifactFile(
 			"User is not allowed to modify artifacts (%v).", permissions))
 	}
 
-	definition, err := setArtifactFile(org_config_obj, principal, in, "")
+	definition, err := setArtifactFile(ctx, org_config_obj, principal, in, "")
 	if err != nil {
 		message := &api_proto.APIResponse{
 			Error:        true,

api/artifacts.go

@@ -67,7 +67,7 @@ sources:
 )
 
 func getArtifactFile(
-	config_obj *config_proto.Config,
+	ctx context.Context, config_obj *config_proto.Config,
 	name string) (string, error) {
 
 	manager, err := services.GetRepositoryManager(config_obj)
@@ -80,7 +80,7 @@ func getArtifactFile(
 		return "", err
 	}
 
-	artifact, pres := repository.Get(config_obj, name)
+	artifact, pres := repository.Get(ctx, config_obj, name)
 	if !pres {
 		return default_artifact, nil
 	}
@@ -107,9 +107,9 @@ func ensureArtifactPrefix(definition, prefix string) string {
 		})
 }
 
-func setArtifactFile(config_obj *config_proto.Config, principal string,
-	in *api_proto.SetArtifactRequest,
-	required_prefix string) (
+func setArtifactFile(
+	ctx context.Context, config_obj *config_proto.Config, principal string,
+	in *api_proto.SetArtifactRequest, required_prefix string) (
 	*artifacts_proto.Artifact, error) {
 
 	manager, err := services.GetRepositoryManager(config_obj)
@@ -134,11 +134,11 @@ func setArtifactFile(config_obj *config_proto.Config, principal string,
 					required_prefix + "'")
 		}
 
-		return artifact_definition, manager.DeleteArtifactFile(config_obj,
+		return artifact_definition, manager.DeleteArtifactFile(ctx, config_obj,
 			principal, artifact_definition.Name)
 
 	case api_proto.SetArtifactRequest_SET:
-		return manager.SetArtifactFile(
+		return manager.SetArtifactFile(ctx,
 			config_obj, principal, in.Artifact, required_prefix)
 	}
 
@@ -171,7 +171,7 @@ func getReportArtifacts(
 		return nil, Status(config_obj.Verbose, err)
 	}
 	for _, name := range names {
-		artifact, pres := repository.Get(config_obj, name)
+		artifact, pres := repository.Get(ctx, config_obj, name)
 		if pres {
 			for _, report := range artifact.Reports {
 				if report.Type == report_type {
@@ -258,7 +258,7 @@ func searchArtifact(
 			continue
 		}
 
-		artifact, pres := repository.Get(config_obj, name)
+		artifact, pres := repository.Get(ctx, config_obj, name)
 		if pres {
 			// Skip non matching types
 			if artifact_type != "" &&
@@ -352,7 +352,7 @@ func (self *ApiServer) LoadArtifactPack(
 				Artifact: artifact_definition,
 			}
 
-			definition, err := setArtifactFile(
+			definition, err := setArtifactFile(ctx,
 				org_config_obj, principal, request, prefix)
 			if err == nil {
 				logging.LogAudit(org_config_obj, principal, "LoadArtifactPack",

api/download.go

@@ -272,7 +272,7 @@ func getRows(
 
 	// We want an event table.
 	if request.Type == "CLIENT_EVENT" || request.Type == "SERVER_EVENT" {
-		path_manager, err := artifacts.NewArtifactPathManager(
+		path_manager, err := artifacts.NewArtifactPathManager(ctx,
 			config_obj, request.ClientId, request.FlowId,
 			request.Artifact)
 		if err != nil {
@@ -290,7 +290,7 @@ func getRows(
 		return rs_reader.Rows(ctx), rs_reader.Close, log_path, err
 
 	} else {
-		log_path, err := tables.GetPathSpec(config_obj, request)
+		log_path, err := tables.GetPathSpec(ctx, config_obj, request)
 		if err != nil {
 			return nil, nil, nil, err
 		}
@@ -314,9 +314,10 @@ func getTransformer(
 			client_id := utils.GetString(row, "ClientId")
 			flow_id := utils.GetString(row, "FlowId")
 
-			flow, err := flows.LoadCollectionContext(config_obj, client_id, flow_id)
+			flow, err := flows.LoadCollectionContext(
+				ctx, config_obj, client_id, flow_id)
 			if err != nil {
-				flow = flows.NewCollectionContext(config_obj)
+				flow = flows.NewCollectionContext(ctx, config_obj)
 			}
 
 			return ordereddict.NewDict().

api/events.go

@@ -69,7 +69,7 @@ func (self *ApiServer) PushEvents(
 	// only broadcast the events for local listeners. Minions
 	// write the events themselves, so we just need to broadcast
 	// for any server event artifacts that occur.
-	journal.Broadcast(org_config_obj,
+	journal.Broadcast(ctx, org_config_obj,
 		rows, in.Artifact, in.ClientId, in.FlowId)
 	return &emptypb.Empty{}, err
 }
@@ -127,7 +127,7 @@ func (self *ApiServer) WriteEvent(
 		return nil, Status(self.verbose, err)
 	}
 
-	err = journal.PushRowsToArtifact(org_config_obj,
+	err = journal.PushRowsToArtifact(ctx, org_config_obj,
 		rows, in.Query.Name, user_name, "")
 	return &emptypb.Empty{}, err
 }

api/reflect.go

@@ -123,7 +123,7 @@ func (self *ApiServer) GetKeywordCompletions(
 	}
 
 	for _, name := range names {
-		artifact, pres := repository.Get(org_config_obj, name)
+		artifact, pres := repository.Get(ctx, org_config_obj, name)
 		if !pres {
 			continue
 		}

api/reports.go

@@ -61,9 +61,10 @@ func getReport(ctx context.Context,
 	var template_data string
 
 	if in.Type == "" {
-		definition, pres := repository.Get(config_obj, "Custom."+in.Artifact)
+		definition, pres := repository.Get(
+			ctx, config_obj, "Custom."+in.Artifact)
 		if !pres {
-			definition, pres = repository.Get(config_obj, in.Artifact)
+			definition, pres = repository.Get(ctx, config_obj, in.Artifact)
 		}
 		if pres {
 			for _, report := range definition.Reports {
@@ -105,7 +106,7 @@ func getReport(ctx context.Context,
 			template_engine, in.ClientId, in.StartTime, in.EndTime)
 
 	case "ARTIFACT_DESCRIPTION":
-		template_data, err = reporting.GenerateArtifactDescriptionReport(
+		template_data, err = reporting.GenerateArtifactDescriptionReport(ctx,
 			template_engine, config_obj)
 	}