From b908093df0c70bf819d6fbfe89e05a07088ee609 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Sun, 12 Feb 2023 15:45:01 +1000 Subject: [PATCH] Track tool definitions by defining artifact (#2439) This allows the admin to upgrade tool definitions by resetting to the definition in another artifact. --- .github/workflows/go.yml | 26 +----- .github/workflows/musl.yaml | 8 +- .github/workflows/windows.yml | 22 +---- accessors/vql_arg_parser.go | 49 +++++----- api/api.go | 8 +- api/artifacts.go | 20 ++-- api/download.go | 9 +- api/events.go | 4 +- api/reflect.go | 2 +- api/reports.go | 7 +- api/tables/table.go | 18 ++-- api/tools.go | 4 +- api/upload.go | 3 +- .../definitions/Generic/Collectors/File.yaml | 8 ++ artifacts/proto/artifact.pb.go | 83 +++++++++++------ artifacts/proto/artifact.proto | 7 ++ .../server/testcases/binary_blobs.out.yaml | 8 +- .../testdata/server/testcases/tools.out.yaml | 12 ++- artifacts/testdata/windows/autoexec.out.yaml | 8 +- bin/artifacts.go | 4 +- bin/tools.go | 5 +- file_store/directory/queue_test.go | 4 +- flows/artifacts.go | 32 ++++--- flows/artifacts_test.go | 33 ++++--- flows/client_flow_runner.go | 32 +++---- flows/housekeeping.go | 2 +- flows/logs.go | 5 +- flows/monitoring.go | 12 ++- .../src/components/artifacts/artifacts.jsx | 15 ++- .../src/components/artifacts/reporting.jsx | 5 +- .../src/components/tools/tool-viewer.jsx | 91 ++++++++++++++++++- .../components/widgets/preview_uploads.jsx | 2 +- paths/artifacts/logs.go | 4 +- paths/artifacts/paths.go | 11 ++- paths/artifacts/paths_test.go | 2 +- reporting/expand.go | 6 +- reporting/gui.go | 5 +- reporting/html.go | 4 +- reporting/report.go | 7 +- reporting/text_expander.go | 3 +- result_sets/simple/simple_test.go | 2 +- result_sets/timed/reader_test.go | 2 +- result_sets/timed/writer_test.go | 6 +- server/comms.go | 4 +- server/enroll.go | 2 +- server/server_test.go | 5 +- services/client_info/client_info.go | 2 +- services/client_info/tasks.go | 4 +- .../client_monitoring/client_monitoring.go | 2 +- .../client_monitoring_test.go | 11 ++- services/client_monitoring/events.go | 16 ++-- services/frontend/frontend.go | 4 +- services/hunt_dispatcher.go | 9 +- services/hunt_dispatcher/hunt_dispatcher.go | 17 ++-- .../hunt_dispatcher/hunt_dispatcher_test.go | 7 +- services/hunt_dispatcher/list.go | 12 +-- services/hunt_dispatcher/modify.go | 4 +- services/hunt_dispatcher/utils.go | 6 +- services/hunt_manager/hunt_manager.go | 14 +-- services/hunt_manager/hunt_manager_test.go | 28 +++--- services/interrogation/interrogation.go | 8 +- services/interrogation/interrogation_test.go | 10 +- services/inventory.go | 5 +- services/inventory/dummy.go | 10 +- .../TestGihubToolServedLocally.golden | 12 ++- .../inventory/fixtures/TestGihubTools.golden | 9 +- .../TestGihubToolsUninitialized.golden | 11 ++- services/inventory/inventory.go | 78 +++++++++++++++- services/inventory/inventory_test.go | 39 +++++--- services/journal.go | 9 +- services/journal/journal.go | 30 +++--- services/journal/journal_test.go | 5 +- services/journal/replication.go | 31 ++++--- services/journal/replication_test.go | 4 +- services/labels/labels.go | 8 +- services/launcher.go | 2 +- services/launcher/compiler.go | 36 ++++---- services/launcher/delete.go | 6 +- services/launcher/launcher.go | 9 +- services/launcher/launcher_test.go | 6 +- services/notebook/initial.go | 15 +-- services/notebook/notebook.go | 2 +- services/notifications.go | 6 +- services/notifications/notifications.go | 15 +-- services/repository.go | 14 +-- services/repository/manager.go | 14 +-- services/repository/manager_test.go | 29 +++--- services/repository/plugin.go | 5 +- services/repository/plugin_test.go | 2 +- services/repository/repository.go | 37 +++++--- services/repository/repository_test.go | 2 +- .../sanity/fixtures/TestUpgradeTools.golden | 1 + services/sanity/sanity.go | 7 +- services/sanity/sanity_test.go | 3 +- services/server_artifacts/api.go | 5 +- .../server_artifacts/collection_context.go | 12 +-- services/server_artifacts/server_artifacts.go | 13 +-- .../server_artifacts/server_artifacts_test.go | 3 +- services/server_artifacts/server_uploader.go | 2 +- .../server_monitoring/server_monitoring.go | 4 +- .../server_monitoring_test.go | 4 +- services/vfs_service/vfs_service.go | 2 +- services/vfs_service/vfs_service_test.go | 12 +-- vql/server/clients/delete.go | 4 +- vql/server/downloads/downloads.go | 2 +- vql/server/events.go | 2 +- vql/server/flows/create.go | 4 +- vql/server/flows/monitoring.go | 4 +- vql/server/flows/parallel_test.go | 4 +- vql/server/flows/results.go | 12 +-- vql/server/hunts/create.go | 6 +- vql/server/hunts/delete.go | 2 +- vql/server/hunts/hunts.go | 6 +- vql/server/inventory.go | 2 +- vql/server/monitoring/add_monitoring.go | 4 +- vql/server/repository.go | 14 +-- vql/tools/collector/collector.go | 15 +-- vql/tools/collector/collector_manager.go | 2 +- 118 files changed, 827 insertions(+), 540 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 08141a2b946..51714255d35 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -16,42 +16,26 @@ jobs: steps: - name: Check out code into the Go module directory - uses: actions/checkout@v2 + uses: actions/checkout@v3 - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: '^1.19' - run: go version -# - uses: actions/cache@v2 -# with: -# # In order: -# # * Module download cache -# # * Build cache (Linux) -# # * Build cache (Mac) -# # * Build cache (Windows) -# path: | -# ~/go/pkg/mod -# ~/.cache/go-build -# ~/Library/Caches/go-build -# %LocalAppData%\go-build -# key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} -# restore-keys: | -# ${{ runner.os }}-go- - - name: Get dependencies run: | go get -v -t -d ./... sudo apt-get install mingw-w64-x86-64-dev gcc-mingw-w64-x86-64 gcc-mingw-w64 - name: Use Node.js v16 - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: 16 - name: Cache node-modules - uses: actions/cache@v2 + uses: actions/cache@v3 env: cache-name: cache-node-modules with: @@ -82,7 +66,7 @@ jobs: go run make.go -v DarwinBase - name: StoreBinaries - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: Binaries path: output diff --git a/.github/workflows/musl.yaml b/.github/workflows/musl.yaml index 44fd6ff3f2d..03a22404999 100644 --- a/.github/workflows/musl.yaml +++ b/.github/workflows/musl.yaml @@ -12,9 +12,9 @@ jobs: steps: - name: Check out code into the Go module directory - uses: actions/checkout@v2 + uses: actions/checkout@v3 - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: go-version: '^1.19' @@ -35,7 +35,7 @@ jobs: cd .. - name: Use Node.js v16 - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: 16 @@ -53,7 +53,7 @@ jobs: go run make.go -v LinuxMusl - name: StoreBinaries - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v3 with: name: Binaries path: output diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index e0ff65acc15..302571975eb 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -6,29 +6,13 @@ jobs: runs-on: windows-2019 steps: - name: Set up Go 1.19 - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: 1.19 id: go -# - uses: actions/cache@v2 -# with: - # In order: - # * Module download cache - # * Build cache (Linux) - # * Build cache (Mac) - # * Build cache (Windows) -# path: | -# ~/go/pkg/mod -# ~/.cache/go-build -# ~/Library/Caches/go-build -# %LocalAppData%\go-build -# key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} -# restore-keys: | -# ${{ runner.os }}-go- - - name: Check out code into the Go module directory - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Configure test environment shell: cmd @@ -123,7 +107,7 @@ jobs: mkdir -p artifact_output/server/ cp artifacts/testdata/server/testcases/*.out* artifact_output/server/ - - uses: actions/upload-artifact@master + - uses: actions/upload-artifact@v3 if: always() with: name: artifact diff --git a/accessors/vql_arg_parser.go b/accessors/vql_arg_parser.go index 2c3fb9895fc..94faac00365 100644 --- a/accessors/vql_arg_parser.go +++ b/accessors/vql_arg_parser.go @@ -70,33 +70,6 @@ func parseOSPath(ctx context.Context, // Initializer can be a list of components. In this case we // take the base pathspec (which is accessor determined) and // add the components to it. - case []types.Any: - components := make([]string, 0, len(t)) - for _, i := range t { - i_str, ok := i.(string) - if ok { - components = append(components, i_str) - } - } - - // Build a pathspec from the accessor and the components. - base, err := accessor.ParsePath("") - if err != nil { - return nil, err - } - - base.Components = append(base.Components, components...) - return base, nil - - case []string: - // Build a pathspec from the accessor and the components. - base, err := accessor.ParsePath("") - if err != nil { - return nil, err - } - - base.Components = append(base.Components, t...) - return base, nil case string: return accessor.ParsePath(t) @@ -105,6 +78,28 @@ func parseOSPath(ctx context.Context, return accessor.ParsePath(string(t)) default: + result, _ := accessor.ParsePath("") + + // Is it an array? Generic code to handle arrays - just append + // each element together to form a single path. This allows + // joining components directly: + // ["bin", "ls"] or ["/usr/bin", "ls"] + a_value := reflect.Indirect(reflect.ValueOf(value)) + if a_value.Type().Kind() == reflect.Slice { + for idx := 0; idx < a_value.Len(); idx++ { + item, err := parseOSPath(ctx, scope, args, + a_value.Index(int(idx)).Interface()) + if err != nil { + continue + } + item_os_path, ok := item.(*OSPath) + if ok { + result = result.Append(item_os_path.Components...) + } + } + return result, nil + } + // This is a fatal error on the client. return nil, fmt.Errorf("Expecting a path arg type, not %T", t) } diff --git a/api/api.go b/api/api.go index b7821589181..14951e85534 100644 --- a/api/api.go +++ b/api/api.go @@ -290,7 +290,7 @@ func (self *ApiServer) NotifyClients( if in.ClientId != "" { self.server_obj.Info("sending notification to %s", in.ClientId) - err = notifier.NotifyListener(org_config_obj, in.ClientId, + err = notifier.NotifyListener(ctx, org_config_obj, in.ClientId, "API.NotifyClients") } else { return nil, status.Error(codes.InvalidArgument, @@ -694,7 +694,7 @@ func (self *ApiServer) GetArtifacts( } for _, name := range in.Names { - artifact, pres := repository.Get(org_config_obj, name) + artifact, pres := repository.Get(ctx, org_config_obj, name) if pres { result.Items = append(result.Items, artifact) } @@ -734,7 +734,7 @@ func (self *ApiServer) GetArtifactFile( "User is not allowed to view custom artifacts.") } - artifact, err := getArtifactFile(org_config_obj, in.Name) + artifact, err := getArtifactFile(ctx, org_config_obj, in.Name) if err != nil { return nil, Status(self.verbose, err) } @@ -787,7 +787,7 @@ func (self *ApiServer) SetArtifactFile( "User is not allowed to modify artifacts (%v).", permissions)) } - definition, err := setArtifactFile(org_config_obj, principal, in, "") + definition, err := setArtifactFile(ctx, org_config_obj, principal, in, "") if err != nil { message := &api_proto.APIResponse{ Error: true, diff --git a/api/artifacts.go b/api/artifacts.go index f11dfbe55f0..42f17fdd9ae 100644 --- a/api/artifacts.go +++ b/api/artifacts.go @@ -67,7 +67,7 @@ sources: ) func getArtifactFile( - config_obj *config_proto.Config, + ctx context.Context, config_obj *config_proto.Config, name string) (string, error) { manager, err := services.GetRepositoryManager(config_obj) @@ -80,7 +80,7 @@ func getArtifactFile( return "", err } - artifact, pres := repository.Get(config_obj, name) + artifact, pres := repository.Get(ctx, config_obj, name) if !pres { return default_artifact, nil } @@ -107,9 +107,9 @@ func ensureArtifactPrefix(definition, prefix string) string { }) } -func setArtifactFile(config_obj *config_proto.Config, principal string, - in *api_proto.SetArtifactRequest, - required_prefix string) ( +func setArtifactFile( + ctx context.Context, config_obj *config_proto.Config, principal string, + in *api_proto.SetArtifactRequest, required_prefix string) ( *artifacts_proto.Artifact, error) { manager, err := services.GetRepositoryManager(config_obj) @@ -134,11 +134,11 @@ func setArtifactFile(config_obj *config_proto.Config, principal string, required_prefix + "'") } - return artifact_definition, manager.DeleteArtifactFile(config_obj, + return artifact_definition, manager.DeleteArtifactFile(ctx, config_obj, principal, artifact_definition.Name) case api_proto.SetArtifactRequest_SET: - return manager.SetArtifactFile( + return manager.SetArtifactFile(ctx, config_obj, principal, in.Artifact, required_prefix) } @@ -171,7 +171,7 @@ func getReportArtifacts( return nil, Status(config_obj.Verbose, err) } for _, name := range names { - artifact, pres := repository.Get(config_obj, name) + artifact, pres := repository.Get(ctx, config_obj, name) if pres { for _, report := range artifact.Reports { if report.Type == report_type { @@ -258,7 +258,7 @@ func searchArtifact( continue } - artifact, pres := repository.Get(config_obj, name) + artifact, pres := repository.Get(ctx, config_obj, name) if pres { // Skip non matching types if artifact_type != "" && @@ -352,7 +352,7 @@ func (self *ApiServer) LoadArtifactPack( Artifact: artifact_definition, } - definition, err := setArtifactFile( + definition, err := setArtifactFile(ctx, org_config_obj, principal, request, prefix) if err == nil { logging.LogAudit(org_config_obj, principal, "LoadArtifactPack", diff --git a/api/download.go b/api/download.go index c9355a0f4fb..e0802ed3662 100644 --- a/api/download.go +++ b/api/download.go @@ -272,7 +272,7 @@ func getRows( // We want an event table. if request.Type == "CLIENT_EVENT" || request.Type == "SERVER_EVENT" { - path_manager, err := artifacts.NewArtifactPathManager( + path_manager, err := artifacts.NewArtifactPathManager(ctx, config_obj, request.ClientId, request.FlowId, request.Artifact) if err != nil { @@ -290,7 +290,7 @@ func getRows( return rs_reader.Rows(ctx), rs_reader.Close, log_path, err } else { - log_path, err := tables.GetPathSpec(config_obj, request) + log_path, err := tables.GetPathSpec(ctx, config_obj, request) if err != nil { return nil, nil, nil, err } @@ -314,9 +314,10 @@ func getTransformer( client_id := utils.GetString(row, "ClientId") flow_id := utils.GetString(row, "FlowId") - flow, err := flows.LoadCollectionContext(config_obj, client_id, flow_id) + flow, err := flows.LoadCollectionContext( + ctx, config_obj, client_id, flow_id) if err != nil { - flow = flows.NewCollectionContext(config_obj) + flow = flows.NewCollectionContext(ctx, config_obj) } return ordereddict.NewDict(). diff --git a/api/events.go b/api/events.go index d72b254241c..b28f164fd77 100644 --- a/api/events.go +++ b/api/events.go @@ -69,7 +69,7 @@ func (self *ApiServer) PushEvents( // only broadcast the events for local listeners. Minions // write the events themselves, so we just need to broadcast // for any server event artifacts that occur. - journal.Broadcast(org_config_obj, + journal.Broadcast(ctx, org_config_obj, rows, in.Artifact, in.ClientId, in.FlowId) return &emptypb.Empty{}, err } @@ -127,7 +127,7 @@ func (self *ApiServer) WriteEvent( return nil, Status(self.verbose, err) } - err = journal.PushRowsToArtifact(org_config_obj, + err = journal.PushRowsToArtifact(ctx, org_config_obj, rows, in.Query.Name, user_name, "") return &emptypb.Empty{}, err } diff --git a/api/reflect.go b/api/reflect.go index 1062260597a..2525616d9ed 100644 --- a/api/reflect.go +++ b/api/reflect.go @@ -123,7 +123,7 @@ func (self *ApiServer) GetKeywordCompletions( } for _, name := range names { - artifact, pres := repository.Get(org_config_obj, name) + artifact, pres := repository.Get(ctx, org_config_obj, name) if !pres { continue } diff --git a/api/reports.go b/api/reports.go index 591e6e0b471..36c7cb184aa 100644 --- a/api/reports.go +++ b/api/reports.go @@ -61,9 +61,10 @@ func getReport(ctx context.Context, var template_data string if in.Type == "" { - definition, pres := repository.Get(config_obj, "Custom."+in.Artifact) + definition, pres := repository.Get( + ctx, config_obj, "Custom."+in.Artifact) if !pres { - definition, pres = repository.Get(config_obj, in.Artifact) + definition, pres = repository.Get(ctx, config_obj, in.Artifact) } if pres { for _, report := range definition.Reports { @@ -105,7 +106,7 @@ func getReport(ctx context.Context, template_engine, in.ClientId, in.StartTime, in.EndTime) case "ARTIFACT_DESCRIPTION": - template_data, err = reporting.GenerateArtifactDescriptionReport( + template_data, err = reporting.GenerateArtifactDescriptionReport(ctx, template_engine, config_obj) } diff --git a/api/tables/table.go b/api/tables/table.go index d192f577672..1f073a43598 100644 --- a/api/tables/table.go +++ b/api/tables/table.go @@ -76,7 +76,7 @@ func GetTable( return nil, err } - artifact, pres := repository.Get(config_obj, in.Artifact) + artifact, pres := repository.Get(ctx, config_obj, in.Artifact) if pres { result.ColumnTypes = artifact.ColumnTypes } @@ -98,10 +98,10 @@ func getTable( } result := &api_proto.GetTableResponse{ - ColumnTypes: getColumnTypes(config_obj, in), + ColumnTypes: getColumnTypes(ctx, config_obj, in), } - path_spec, err := GetPathSpec(config_obj, in) + path_spec, err := GetPathSpec(ctx, config_obj, in) if err != nil { return result, err } @@ -188,7 +188,7 @@ func getTable( // The GUI is requesting table data. This function tries to figure out // the column types. func getColumnTypes( - config_obj *config_proto.Config, + ctx context.Context, config_obj *config_proto.Config, in *api_proto.GetTableRequest) []*artifacts_proto.ColumnType { // For artifacts column types are specified in the `column_types` @@ -204,7 +204,7 @@ func getColumnTypes( return nil } - artifact, pres := repository.Get(config_obj, in.Artifact) + artifact, pres := repository.Get(ctx, config_obj, in.Artifact) if pres { return artifact.ColumnTypes } @@ -234,11 +234,11 @@ func getColumnTypes( // switch to figure out where the result set we want to look at is // stored. func GetPathSpec( - config_obj *config_proto.Config, + ctx context.Context, config_obj *config_proto.Config, in *api_proto.GetTableRequest) (api.FSPathSpec, error) { if in.FlowId != "" && in.Artifact != "" { - path_manager, err := artifacts.NewArtifactPathManager( + path_manager, err := artifacts.NewArtifactPathManager(ctx, config_obj, in.ClientId, in.FlowId, in.Artifact) if err != nil { return nil, err @@ -289,7 +289,7 @@ func getEventTable( config_obj *config_proto.Config, in *api_proto.GetTableRequest) ( *api_proto.GetTableResponse, error) { - path_manager, err := artifacts.NewArtifactPathManager( + path_manager, err := artifacts.NewArtifactPathManager(ctx, config_obj, in.ClientId, in.FlowId, in.Artifact) if err != nil { return nil, err @@ -303,7 +303,7 @@ func getEventTableLogs( config_obj *config_proto.Config, in *api_proto.GetTableRequest) ( *api_proto.GetTableResponse, error) { - path_manager, err := artifacts.NewArtifactLogPathManager( + path_manager, err := artifacts.NewArtifactLogPathManager(ctx, config_obj, in.ClientId, "", in.Artifact) if err != nil { return nil, err diff --git a/api/tools.go b/api/tools.go index 7724bb22425..4bc598f9fb5 100644 --- a/api/tools.go +++ b/api/tools.go @@ -34,7 +34,7 @@ func (self *ApiServer) GetToolInfo(ctx context.Context, return inventory.GetToolInfo(ctx, org_config_obj, in.Name) } - return inventory.ProbeToolInfo(in.Name) + return inventory.ProbeToolInfo(ctx, org_config_obj, in.Name) } func (self *ApiServer) SetToolInfo(ctx context.Context, @@ -64,7 +64,7 @@ func (self *ApiServer) SetToolInfo(ctx context.Context, return nil, Status(self.verbose, err) } - err = inventory.AddTool(org_config_obj, in, + err = inventory.AddTool(ctx, org_config_obj, in, services.ToolOptions{ AdminOverride: true, }) diff --git a/api/upload.go b/api/upload.go index c7c1c63c203..999ba339525 100644 --- a/api/upload.go +++ b/api/upload.go @@ -125,7 +125,8 @@ func toolUploadHandler() http.Handler { return } - err = inventory.AddTool(org_config_obj, tool, + ctx := r.Context() + err = inventory.AddTool(ctx, org_config_obj, tool, services.ToolOptions{ AdminOverride: true, }) diff --git a/artifacts/definitions/Generic/Collectors/File.yaml b/artifacts/definitions/Generic/Collectors/File.yaml index 7dc0829698f..84c1b35a974 100755 --- a/artifacts/definitions/Generic/Collectors/File.yaml +++ b/artifacts/definitions/Generic/Collectors/File.yaml @@ -16,17 +16,25 @@ parameters: default: | Glob Users\*\NTUser.dat + - name: Root description: | On Windows, this is the device to apply all the glob on (e.g. `C:`). On *NIX, this should be a path to a subdirectory or /. default: "C:" + - name: Accessor default: auto description: | On Windows, this can be changed to `ntfs`. + - name: NTFS_CACHE_TIME + type: int + description: How often to flush the NTFS cache. (Default is never). + default: "1000000" + + sources: - name: All Matches Metadata query: | diff --git a/artifacts/proto/artifact.pb.go b/artifacts/proto/artifact.pb.go index dd1e7496b40..bd6339bcecb 100644 --- a/artifacts/proto/artifact.pb.go +++ b/artifacts/proto/artifact.pb.go @@ -796,6 +796,10 @@ type Tool struct { Hash string `protobuf:"bytes,6,opt,name=hash,proto3" json:"hash,omitempty"` // If set on a request we refresh the hash. Materialize bool `protobuf:"varint,11,opt,name=materialize,proto3" json:"materialize,omitempty"` + // The artifact this definition came from. + Artifact string `protobuf:"bytes,13,opt,name=artifact,proto3" json:"artifact,omitempty"` + // Additional versions of this tool. + Versions []*Tool `protobuf:"bytes,14,rep,name=versions,proto3" json:"versions,omitempty"` } func (x *Tool) Reset() { @@ -914,6 +918,20 @@ func (x *Tool) GetMaterialize() bool { return false } +func (x *Tool) GetArtifact() string { + if x != nil { + return x.Artifact + } + return "" +} + +func (x *Tool) GetVersions() []*Tool { + if x != nil { + return x.Versions + } + return nil +} + // Keep track of all the third party tools we know about. type ThirdParty struct { state protoimpl.MessageState @@ -1294,7 +1312,7 @@ var file_artifact_proto_rawDesc = []byte{ 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x25, 0x0a, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x41, 0x72, 0x74, 0x69, 0x66, - 0x61, 0x63, 0x74, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0x82, 0x03, 0x0a, 0x04, 0x54, + 0x61, 0x63, 0x74, 0x52, 0x05, 0x69, 0x74, 0x65, 0x6d, 0x73, 0x22, 0xc7, 0x03, 0x0a, 0x04, 0x54, 0x6f, 0x6f, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x12, 0x25, 0x0a, 0x0e, 0x67, 0x69, 0x74, @@ -1318,29 +1336,33 @@ var file_artifact_proto_rawDesc = []byte{ 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x20, 0x0a, 0x0b, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x18, 0x0b, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0b, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x22, - 0x4a, 0x0a, 0x0b, 0x74, 0x68, 0x69, 0x72, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x74, 0x79, 0x12, 0x21, - 0x0a, 0x05, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x54, 0x6f, 0x6f, 0x6c, 0x52, 0x05, 0x74, 0x6f, 0x6f, 0x6c, - 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0xcc, 0x01, 0x0a, 0x09, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x69, 0x6d, - 0x65, 0x6f, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, - 0x6f, 0x75, 0x74, 0x12, 0x24, 0x0a, 0x0e, 0x6f, 0x70, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x73, - 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x02, 0x52, 0x0c, 0x6f, 0x70, 0x73, - 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x70, 0x75, - 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x02, 0x52, 0x08, 0x63, 0x70, - 0x75, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x6f, 0x70, 0x73, 0x5f, 0x6c, - 0x69, 0x6d, 0x69, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x02, 0x52, 0x09, 0x69, 0x6f, 0x70, 0x73, - 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x6f, 0x77, - 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x6d, 0x61, 0x78, 0x52, 0x6f, 0x77, 0x73, - 0x12, 0x28, 0x0a, 0x10, 0x6d, 0x61, 0x78, 0x5f, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x62, - 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0e, 0x6d, 0x61, 0x78, 0x55, - 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x42, 0x79, 0x74, 0x65, 0x73, 0x42, 0x37, 0x5a, 0x35, 0x77, 0x77, - 0x77, 0x2e, 0x76, 0x65, 0x6c, 0x6f, 0x63, 0x69, 0x64, 0x65, 0x78, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2f, 0x76, 0x65, 0x6c, 0x6f, 0x63, 0x69, 0x72, 0x61, 0x70, - 0x74, 0x6f, 0x72, 0x2f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x73, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x28, 0x08, 0x52, 0x0b, 0x6d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x12, + 0x1a, 0x0a, 0x08, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x18, 0x0d, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x12, 0x27, 0x0a, 0x08, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0e, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x54, 0x6f, 0x6f, 0x6c, 0x52, 0x08, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x4a, 0x0a, 0x0b, 0x74, 0x68, 0x69, 0x72, 0x64, 0x5f, 0x70, 0x61, + 0x72, 0x74, 0x79, 0x12, 0x21, 0x0a, 0x05, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x54, 0x6f, 0x6f, 0x6c, 0x52, + 0x05, 0x74, 0x6f, 0x6f, 0x6c, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x22, 0xcc, 0x01, 0x0a, 0x09, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x18, + 0x0a, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x24, 0x0a, 0x0e, 0x6f, 0x70, 0x73, 0x5f, + 0x70, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x02, + 0x52, 0x0c, 0x6f, 0x70, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x12, 0x1b, + 0x0a, 0x09, 0x63, 0x70, 0x75, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x02, 0x52, 0x08, 0x63, 0x70, 0x75, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x69, + 0x6f, 0x70, 0x73, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x02, 0x52, + 0x09, 0x69, 0x6f, 0x70, 0x73, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x61, + 0x78, 0x5f, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x6d, 0x61, + 0x78, 0x52, 0x6f, 0x77, 0x73, 0x12, 0x28, 0x0a, 0x10, 0x6d, 0x61, 0x78, 0x5f, 0x75, 0x70, 0x6c, + 0x6f, 0x61, 0x64, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x0e, 0x6d, 0x61, 0x78, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x42, 0x79, 0x74, 0x65, 0x73, 0x42, + 0x37, 0x5a, 0x35, 0x77, 0x77, 0x77, 0x2e, 0x76, 0x65, 0x6c, 0x6f, 0x63, 0x69, 0x64, 0x65, 0x78, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2f, 0x76, 0x65, 0x6c, 0x6f, + 0x63, 0x69, 0x72, 0x61, 0x70, 0x74, 0x6f, 0x72, 0x2f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, + 0x74, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1380,12 +1402,13 @@ var file_artifact_proto_depIdxs = []int32{ 5, // 7: proto.Artifact.reports:type_name -> proto.Report 1, // 8: proto.Artifact.column_types:type_name -> proto.ColumnType 6, // 9: proto.ArtifactDescriptors.items:type_name -> proto.Artifact - 8, // 10: proto.third_party.tools:type_name -> proto.Tool - 11, // [11:11] is the sub-list for method output_type - 11, // [11:11] is the sub-list for method input_type - 11, // [11:11] is the sub-list for extension type_name - 11, // [11:11] is the sub-list for extension extendee - 0, // [0:11] is the sub-list for field type_name + 8, // 10: proto.Tool.versions:type_name -> proto.Tool + 8, // 11: proto.third_party.tools:type_name -> proto.Tool + 12, // [12:12] is the sub-list for method output_type + 12, // [12:12] is the sub-list for method input_type + 12, // [12:12] is the sub-list for extension type_name + 12, // [12:12] is the sub-list for extension extendee + 0, // [0:12] is the sub-list for field type_name } func init() { file_artifact_proto_init() } diff --git a/artifacts/proto/artifact.proto b/artifacts/proto/artifact.proto index 1ad9203e39e..9299a1c3a3a 100644 --- a/artifacts/proto/artifact.proto +++ b/artifacts/proto/artifact.proto @@ -271,6 +271,13 @@ message Tool { // If set on a request we refresh the hash. bool materialize = 11; + + // The artifact this definition came from. + string artifact = 13; + + // Additional versions of this tool. + repeated Tool versions = 14; + } // Keep track of all the third party tools we know about. diff --git a/artifacts/testdata/server/testcases/binary_blobs.out.yaml b/artifacts/testdata/server/testcases/binary_blobs.out.yaml index f2c5e9f6534..97332889117 100644 --- a/artifacts/testdata/server/testcases/binary_blobs.out.yaml +++ b/artifacts/testdata/server/testcases/binary_blobs.out.yaml @@ -16,7 +16,9 @@ SELECT * FROM switch( b={SELECT Complete FROM execve(argv=["rm", "-f", "/tmp/aut "filestore_path": "", "filename": "winpmem_v3.3.rc3.exe", "hash": "", - "materialize": false + "materialize": false, + "artifact": "", + "versions": [] } } ]SELECT * FROM inventory() WHERE name = "WinPmem"[ @@ -32,7 +34,9 @@ SELECT * FROM switch( b={SELECT Complete FROM execve(argv=["rm", "-f", "/tmp/aut "filestore_path": "351b4f6d59a4266cc7a2eab9cedf959eb6a4c924746044e6edeabdd1a477643e", "filename": "winpmem_v3.3.rc3.exe", "hash": "", - "materialize": false + "materialize": false, + "artifact": "", + "versions": [] } ]LET ToolInfo <= dict( Tool_WinPmem_URL="https://github.com/Velocidex/c-aff4/releases/download/v3.3.rc3/winpmem_v3.3.rc3.exe", Tool_WinPmem_FILENAME="winpmem_v3.3.rc3.exe", Tool_WinPmem_HASH="319f6c714d682505157cf72aa928c94ada3c839fb8eb0e503d8770624e897318")[]SELECT DownloadStatus, Hash FROM Artifact.Generic.Utils.FetchBinary( ToolName="WinPmem", SleepDuration=0, ToolInfo=ToolInfo)[ { diff --git a/artifacts/testdata/server/testcases/tools.out.yaml b/artifacts/testdata/server/testcases/tools.out.yaml index 33adec57686..065ec161db1 100644 --- a/artifacts/testdata/server/testcases/tools.out.yaml +++ b/artifacts/testdata/server/testcases/tools.out.yaml @@ -12,7 +12,9 @@ SELECT inventory_add(tool='Autorun_amd64', url='https://storage.googleapis.com/g "filestore_path": "", "filename": "autorunsc_x64.exe", "hash": "083d7eee4ed40a3e5a35675503b0b6be0cb627b4cb1009d185a558a805f64153", - "materialize": false + "materialize": false, + "artifact": "", + "versions": [] } } ]SELECT inventory_get(tool='Autorun_amd64') FROM scope()[ @@ -37,7 +39,9 @@ SELECT inventory_add(tool='Autorun_amd64', url='https://storage.googleapis.com/g "filestore_path": "", "filename": "autorunsc_x64.exe", "hash": "083d7eee4ed40a3e5a35675503b0b6be0cb627b4cb1009d185a558a805f64153", - "materialize": false + "materialize": false, + "artifact": "", + "versions": [] } } ]SELECT inventory_get(tool='Autorun_amd64') FROM scope()[ @@ -62,7 +66,9 @@ SELECT inventory_add(tool='Autorun_amd64', url='https://storage.googleapis.com/g "filestore_path": "1c21ee4d8609f81482dc0a78c641e4586488a9fd562ee28eec25e448a9d0b2e1", "filename": "yara_test.txt", "hash": "f03278c10a41adcc97f24a612a680e7aa43efb461b337fef3d2a3d47b51e77bb", - "materialize": false + "materialize": false, + "artifact": "", + "versions": [] } } ]SELECT inventory_get(tool="FooBar") FROM scope()[ diff --git a/artifacts/testdata/windows/autoexec.out.yaml b/artifacts/testdata/windows/autoexec.out.yaml index c767698847d..22f00a503e1 100644 --- a/artifacts/testdata/windows/autoexec.out.yaml +++ b/artifacts/testdata/windows/autoexec.out.yaml @@ -12,7 +12,9 @@ SELECT inventory_add(tool='Autorun_amd64', url='https://storage.googleapis.com/g "filestore_path": "", "filename": "autorunsc_x64.exe", "hash": "5f4cdd5cbc5aea49e007e35550eaac89a68efc409683730c722f6dd378ba81e9", - "materialize": false + "materialize": false, + "artifact": "", + "versions": [] }, "inventory_add(tool='Autorun_x86', url='https://storage.googleapis.com/go.velocidex.com/autorunsc.exe', hash='5f4cdd5cbc5aea49e007e35550eaac89a68efc409683730c722f6dd378ba81e9', filename='autorunsc_x86.exe')": { "name": "Autorun_x86", @@ -26,7 +28,9 @@ SELECT inventory_add(tool='Autorun_amd64', url='https://storage.googleapis.com/g "filestore_path": "", "filename": "autorunsc_x86.exe", "hash": "5f4cdd5cbc5aea49e007e35550eaac89a68efc409683730c722f6dd378ba81e9", - "materialize": false + "materialize": false, + "artifact": "", + "versions": [] } } ]SELECT Company FROM Artifact.Windows.Sysinternals.Autoruns( All=FALSE, `Boot execute`=TRUE, ToolInfo=inventory_get(tool='Autorun_amd64')) WHERE Company =~ 'Microsoft' LIMIT 1[ diff --git a/bin/artifacts.go b/bin/artifacts.go index 9d1bdfc3f38..4278c778271 100644 --- a/bin/artifacts.go +++ b/bin/artifacts.go @@ -302,7 +302,7 @@ func doArtifactShow() error { return err } - artifact, pres := repository.Get(config_obj, *artifact_command_show_name) + artifact, pres := repository.Get(ctx, config_obj, *artifact_command_show_name) if !pres { return fmt.Errorf("Artifact %s not found", *artifact_command_show_name) @@ -365,7 +365,7 @@ func doArtifactList() error { continue } - artifact, pres := repository.Get(config_obj, name) + artifact, pres := repository.Get(ctx, config_obj, name) if !pres { return fmt.Errorf("Artifact %s not found", name) } diff --git a/bin/tools.go b/bin/tools.go index dfc11ddf559..ca5984d2e86 100644 --- a/bin/tools.go +++ b/bin/tools.go @@ -83,7 +83,8 @@ func doThirdPartyShow() error { } fmt.Println(string(serialized)) } else { - tool, err := inventory_manager.ProbeToolInfo(*third_party_show_file) + tool, err := inventory_manager.ProbeToolInfo( + ctx, config_obj, *third_party_show_file) if err != nil { return fmt.Errorf("Tool not found: %w", err) } @@ -197,7 +198,7 @@ func doThirdPartyUpload() error { return err } - err = inventory_manager.AddTool( + err = inventory_manager.AddTool(ctx, config_obj, tool, services.ToolOptions{ AdminOverride: true, }) diff --git a/file_store/directory/queue_test.go b/file_store/directory/queue_test.go index cc9bf68e5be..48ebabe1f4c 100644 --- a/file_store/directory/queue_test.go +++ b/file_store/directory/queue_test.go @@ -94,7 +94,7 @@ func (self *TestSuite) TestQueueManager() { FileBufferLeaseSize: 1, }) - path_manager, err := artifacts.NewArtifactPathManager(self.ConfigObj, + path_manager, err := artifacts.NewArtifactPathManager(self.Ctx, self.ConfigObj, "C.123", "", "TestQueue") assert.NoError(self.T(), err) @@ -163,7 +163,7 @@ func (self *TestSuite) TestQueueManagerJsonl() { manager := directory.NewDirectoryQueueManager( self.ConfigObj, file_store).(*directory.DirectoryQueueManager) - path_manager, err := artifacts.NewArtifactPathManager(self.ConfigObj, + path_manager, err := artifacts.NewArtifactPathManager(self.Ctx, self.ConfigObj, "C.123", "", "TestQueue") assert.NoError(self.T(), err) diff --git a/flows/artifacts.go b/flows/artifacts.go index c370c11b2fd..4fce0c143d5 100644 --- a/flows/artifacts.go +++ b/flows/artifacts.go @@ -96,7 +96,8 @@ type CollectionContext struct { send_update bool } -func NewCollectionContext(config_obj *config_proto.Config) *CollectionContext { +func NewCollectionContext( + ctx context.Context, config_obj *config_proto.Config) *CollectionContext { self := &CollectionContext{ ArtifactCollectorContext: flows_proto.ArtifactCollectorContext{}, monitoring_batch: make(map[string]*jsonBatch), @@ -129,7 +130,7 @@ func NewCollectionContext(config_obj *config_proto.Config) *CollectionContext { journal, err := services.GetJournal(config_obj) if err == nil { - journal.PushRowsToArtifactAsync( + journal.PushRowsToArtifactAsync(ctx, config_obj, row, "System.Flow.Completion") } }) @@ -226,7 +227,7 @@ func closeContext( } if len(collection_context.Logs) > 0 { - err := flushContextLogs( + err := flushContextLogs(ctx, config_obj, collection_context, collection_context.completer) if err != nil { collection_context.State = flows_proto.ArtifactCollectorContext_ERROR @@ -246,7 +247,7 @@ func closeContext( } if len(collection_context.monitoring_batch) > 0 { - err = flushMonitoringLogs(config_obj, collection_context) + err = flushMonitoringLogs(ctx, config_obj, collection_context) if err != nil { collection_context.State = flows_proto.ArtifactCollectorContext_ERROR collection_context.Status = err.Error() @@ -309,11 +310,11 @@ func flushContextUploadedFiles( // Load the collector context from storage. func LoadCollectionContext( - config_obj *config_proto.Config, + ctx context.Context, config_obj *config_proto.Config, client_id, flow_id string) (*CollectionContext, error) { if flow_id == constants.MONITORING_WELL_KNOWN_FLOW { - result := NewCollectionContext(config_obj) + result := NewCollectionContext(ctx, config_obj) result.SessionId = flow_id result.ClientId = client_id @@ -321,7 +322,7 @@ func LoadCollectionContext( } flow_path_manager := paths.NewFlowPathManager(client_id, flow_id) - collection_context := NewCollectionContext(config_obj) + collection_context := NewCollectionContext(ctx, config_obj) db, err := datastore.GetDB(config_obj) if err != nil { return nil, err @@ -344,7 +345,7 @@ func LoadCollectionContext( // Process an incoming message from the client. func ArtifactCollectorProcessOneMessage( - config_obj *config_proto.Config, + ctx context.Context, config_obj *config_proto.Config, collection_context *CollectionContext, message *crypto_proto.VeloMessage) error { @@ -366,7 +367,7 @@ func ArtifactCollectorProcessOneMessage( // Handle the response depending on the RequestId switch message.RequestId { case constants.TransferWellKnownFlowId: - return appendUploadDataToFile( + return appendUploadDataToFile(ctx, config_obj, collection_context, message) case constants.ProcessVQLResponses: @@ -387,7 +388,7 @@ func ArtifactCollectorProcessOneMessage( rows_written := uint64(0) if response.Query.Name != "" { path_manager, err := artifact_paths.NewArtifactPathManager( - config_obj, + ctx, config_obj, collection_context.Request.ClientId, collection_context.SessionId, response.Query.Name) @@ -484,7 +485,7 @@ func CheckForStatus( } func appendUploadDataToFile( - config_obj *config_proto.Config, + ctx context.Context, config_obj *config_proto.Config, collection_context *CollectionContext, message *crypto_proto.VeloMessage) error { @@ -607,7 +608,7 @@ func appendUploadDataToFile( return err } - return journal.PushRowsToArtifact(config_obj, + return journal.PushRowsToArtifact(ctx, config_obj, []*ordereddict.Dict{row}, "System.Upload.Completion", message.Source, collection_context.SessionId, @@ -723,7 +724,7 @@ func (self *FlowRunner) ProcessSingleMessage( return invalidClientId } - collection_context, err = LoadCollectionContext( + collection_context, err = LoadCollectionContext(ctx, self.config_obj, job.Source, job.SessionId) if err != nil { // Ignore logs and status messages from the @@ -770,7 +771,8 @@ func (self *FlowRunner) ProcessSingleMessage( } if job.SessionId == constants.MONITORING_WELL_KNOWN_FLOW { - err := MonitoringProcessMessage(self.config_obj, collection_context, job) + err := MonitoringProcessMessage( + ctx, self.config_obj, collection_context, job) if err != nil { Log(self.config_obj, collection_context, fmt.Sprintf("MonitoringProcessMessage: %v", err)) @@ -778,7 +780,7 @@ func (self *FlowRunner) ProcessSingleMessage( return err } - err := ArtifactCollectorProcessOneMessage( + err := ArtifactCollectorProcessOneMessage(ctx, self.config_obj, collection_context, job) if err != nil { Log(self.config_obj, collection_context, diff --git a/flows/artifacts_test.go b/flows/artifacts_test.go index 44c90625839..1f530be08f4 100644 --- a/flows/artifacts_test.go +++ b/flows/artifacts_test.go @@ -204,7 +204,7 @@ func (self *TestSuite) TestRetransmission() { runner.Close(context.Background()) // Load the collection context and see what happened. - collection_context, err := LoadCollectionContext(self.ConfigObj, + collection_context, err := LoadCollectionContext(self.Ctx, self.ConfigObj, self.client_id, flow_id) assert.NoError(self.T(), err) @@ -216,8 +216,7 @@ func (self *TestSuite) TestRetransmission() { func (self *TestSuite) TestResourceLimits() { manager, err := services.GetRepositoryManager(self.ConfigObj) assert.NoError(self.T(), err) - repository, err := manager.GetGlobalRepository( - self.ConfigObj) + repository, err := manager.GetGlobalRepository(self.ConfigObj) assert.NoError(self.T(), err) request := &flows_proto.ArtifactCollectorArgs{ @@ -280,7 +279,7 @@ func (self *TestSuite) TestResourceLimits() { runner.Close(context.Background()) // Load the collection context and see what happened. - collection_context, err := LoadCollectionContext(self.ConfigObj, + collection_context, err := LoadCollectionContext(self.Ctx, self.ConfigObj, self.client_id, flow_id) assert.NoError(self.T(), err) @@ -296,7 +295,7 @@ func (self *TestSuite) TestResourceLimits() { runner.Close(context.Background()) // Load the collection context and see what happened. - collection_context, err = LoadCollectionContext(self.ConfigObj, + collection_context, err = LoadCollectionContext(self.Ctx, self.ConfigObj, self.client_id, flow_id) assert.NoError(self.T(), err) @@ -314,7 +313,7 @@ func (self *TestSuite) TestResourceLimits() { runner.Close(context.Background()) // Load the collection context and see what happened. - collection_context, err = LoadCollectionContext(self.ConfigObj, + collection_context, err = LoadCollectionContext(self.Ctx, self.ConfigObj, self.client_id, flow_id) assert.NoError(self.T(), err) @@ -342,7 +341,7 @@ func (self *TestSuite) TestResourceLimits() { // We still collect these rows but the flow is still in the // error state. We do this so we dont lose the last few // messages which are still in flight. - collection_context, err = LoadCollectionContext(self.ConfigObj, + collection_context, err = LoadCollectionContext(self.Ctx, self.ConfigObj, self.client_id, flow_id) assert.NoError(self.T(), err) @@ -375,7 +374,7 @@ func (self *TestSuite) TestClientUploaderStoreFile() { nilTime, nilTime, nilTime, nilTime, reader) // Get a new collection context. - collection_context := NewCollectionContext(self.ConfigObj) + collection_context := NewCollectionContext(self.Ctx, self.ConfigObj) collection_context.ArtifactCollectorContext = flows_proto.ArtifactCollectorContext{ SessionId: self.flow_id, ClientId: self.client_id, @@ -387,7 +386,7 @@ func (self *TestSuite) TestClientUploaderStoreFile() { for _, response := range resp.Drain.WaitForStatsMessage(self.T()) { response.Source = self.client_id - err := ArtifactCollectorProcessOneMessage( + err := ArtifactCollectorProcessOneMessage(self.Ctx, self.ConfigObj, collection_context, response) assert.NoError(self.T(), err) } @@ -437,7 +436,7 @@ func (self *TestSuite) TestClientUploaderStoreFile() { // Check the System.Upload.Completion event. artifact_path_manager, err := artifacts.NewArtifactPathManager( - self.ConfigObj, self.client_id, self.flow_id, + self.Ctx, self.ConfigObj, self.client_id, self.flow_id, "System.Upload.Completion") assert.NoError(self.T(), err) @@ -580,7 +579,7 @@ func (self *TestSuite) testCollectionCompletion( outstanding_requests int64, requests []*crypto_proto.VeloMessage) *flows_proto.ArtifactCollectorContext { // Get a new collection context. - collection_context := NewCollectionContext(self.ConfigObj) + collection_context := NewCollectionContext(self.Ctx, self.ConfigObj) collection_context.ArtifactCollectorContext = flows_proto.ArtifactCollectorContext{ SessionId: self.flow_id, ClientId: self.client_id, @@ -661,7 +660,7 @@ func (self *TestSuite) TestClientUploaderStoreSparseFile() { nilTime, nilTime, nilTime, nilTime, reader) // Get a new collection context. - collection_context := NewCollectionContext(self.ConfigObj) + collection_context := NewCollectionContext(self.Ctx, self.ConfigObj) collection_context.ArtifactCollectorContext = flows_proto.ArtifactCollectorContext{ SessionId: self.flow_id, ClientId: self.client_id, @@ -677,7 +676,7 @@ func (self *TestSuite) TestClientUploaderStoreSparseFile() { assert.Equal(self.T(), msg.FileBuffer.StoredSize, uint64(12)) } - ArtifactCollectorProcessOneMessage(self.ConfigObj, + ArtifactCollectorProcessOneMessage(self.Ctx, self.ConfigObj, collection_context, msg) } @@ -737,7 +736,7 @@ func (self *TestSuite) TestClientUploaderStoreSparseFile() { // Check the System.Upload.Completion event. artifact_path_manager, err := artifacts.NewArtifactPathManager( - self.ConfigObj, self.client_id, self.flow_id, + self.Ctx, self.ConfigObj, self.client_id, self.flow_id, "System.Upload.Completion") assert.NoError(self.T(), err) @@ -796,7 +795,7 @@ func (self *TestSuite) TestClientUploaderStoreSparseFileNTFS() { nilTime, nilTime, nilTime, nilTime, fd) // Get a new collection context. - collection_context := NewCollectionContext(self.ConfigObj) + collection_context := NewCollectionContext(self.Ctx, self.ConfigObj) collection_context.ArtifactCollectorContext = flows_proto.ArtifactCollectorContext{ SessionId: self.flow_id, ClientId: self.client_id, @@ -806,7 +805,7 @@ func (self *TestSuite) TestClientUploaderStoreSparseFileNTFS() { // Process it. for _, resp := range resp.Drain.WaitForStatsMessage(self.T()) { resp.Source = self.client_id - ArtifactCollectorProcessOneMessage(self.ConfigObj, + ArtifactCollectorProcessOneMessage(self.Ctx, self.ConfigObj, collection_context, resp) } @@ -866,7 +865,7 @@ func (self *TestSuite) TestClientUploaderStoreSparseFileNTFS() { // Check the System.Upload.Completion event. artifact_path_manager, err := artifacts.NewArtifactPathManager( - self.ConfigObj, self.client_id, self.flow_id, + self.Ctx, self.ConfigObj, self.client_id, self.flow_id, "System.Upload.Completion") assert.NoError(self.T(), err) diff --git a/flows/client_flow_runner.go b/flows/client_flow_runner.go index 6096414c8d1..08bccdf3ec8 100644 --- a/flows/client_flow_runner.go +++ b/flows/client_flow_runner.go @@ -67,7 +67,7 @@ func (self *ClientFlowRunner) ProcessMonitoringMessage( client_id := msg.Source if msg.VQLResponse != nil && msg.VQLResponse.Query != nil { - err := self.MonitoringVQLResponse(client_id, flow_id, msg.VQLResponse) + err := self.MonitoringVQLResponse(ctx, client_id, flow_id, msg.VQLResponse) if err != nil { return fmt.Errorf("MonitoringVQLResponse: %w", err) } @@ -75,7 +75,7 @@ func (self *ClientFlowRunner) ProcessMonitoringMessage( } if msg.LogMessage != nil { - err := self.MonitoringLogMessage(client_id, flow_id, msg.LogMessage) + err := self.MonitoringLogMessage(ctx, client_id, flow_id, msg.LogMessage) if err != nil { return fmt.Errorf("MonitoringLogMessage: %w", err) } @@ -83,7 +83,7 @@ func (self *ClientFlowRunner) ProcessMonitoringMessage( } if msg.FileBuffer != nil { - err := self.FileBuffer(client_id, flow_id, msg.FileBuffer) + err := self.FileBuffer(ctx, client_id, flow_id, msg.FileBuffer) if err != nil { return fmt.Errorf("FileBuffer: %w", err) } @@ -94,7 +94,7 @@ func (self *ClientFlowRunner) ProcessMonitoringMessage( } func (self *ClientFlowRunner) MonitoringLogMessage( - client_id, flow_id string, + ctx context.Context, client_id, flow_id string, response *crypto_proto.LogMessage) error { artifact_name := artifacts.DeobfuscateString( @@ -108,7 +108,7 @@ func (self *ClientFlowRunner) MonitoringLogMessage( return nil } - log_path_manager, err := artifact_paths.NewArtifactLogPathManager( + log_path_manager, err := artifact_paths.NewArtifactLogPathManager(ctx, self.config_obj, client_id, flow_id, artifact_name) if err != nil { return err @@ -133,7 +133,7 @@ func (self *ClientFlowRunner) MonitoringLogMessage( } func (self *ClientFlowRunner) MonitoringVQLResponse( - client_id, flow_id string, + ctx context.Context, client_id, flow_id string, response *actions_proto.VQLResponse) error { // Ignore empty responses @@ -158,7 +158,7 @@ func (self *ClientFlowRunner) MonitoringVQLResponse( return err } - return journal.PushJsonlToArtifact( + return journal.PushJsonlToArtifact(ctx, self.config_obj, []byte(response.JSONLResponse), int(response.TotalRows), query_name, client_id, flow_id) @@ -189,7 +189,7 @@ func (self *ClientFlowRunner) ProcessSingleMessage( } if msg.VQLResponse != nil { - err := self.VQLResponse(client_id, flow_id, msg.VQLResponse) + err := self.VQLResponse(ctx, client_id, flow_id, msg.VQLResponse) if err != nil { return fmt.Errorf("VQLResponse: %w", err) } @@ -197,7 +197,7 @@ func (self *ClientFlowRunner) ProcessSingleMessage( } if msg.FlowStats != nil { - err := self.FlowStats(client_id, flow_id, msg.FlowStats) + err := self.FlowStats(ctx, client_id, flow_id, msg.FlowStats) if err != nil { return fmt.Errorf("FlowStats: %w", err) } @@ -205,7 +205,7 @@ func (self *ClientFlowRunner) ProcessSingleMessage( } if msg.FileBuffer != nil { - err := self.FileBuffer(client_id, flow_id, msg.FileBuffer) + err := self.FileBuffer(ctx, client_id, flow_id, msg.FileBuffer) if err != nil { return fmt.Errorf("FileBuffer: %w", err) } @@ -216,7 +216,7 @@ func (self *ClientFlowRunner) ProcessSingleMessage( } func (self *ClientFlowRunner) FileBuffer( - client_id, flow_id string, + ctx context.Context, client_id, flow_id string, file_buffer *actions_proto.FileBuffer) error { if file_buffer == nil || file_buffer.Pathspec == nil { @@ -328,7 +328,7 @@ func (self *ClientFlowRunner) FileBuffer( return err } - return journal.PushRowsToArtifact(self.config_obj, + return journal.PushRowsToArtifact(ctx, self.config_obj, []*ordereddict.Dict{row}, "System.Upload.Completion", client_id, flow_id) @@ -341,7 +341,7 @@ func (self *ClientFlowRunner) Close(ctx context.Context) { } func (self *ClientFlowRunner) FlowStats( - client_id, flow_id string, + ctx context.Context, client_id, flow_id string, msg *crypto_proto.FlowStats) error { // Write a partial ArtifactCollectorContext protobuf containing @@ -394,7 +394,7 @@ func (self *ClientFlowRunner) FlowStats( journal, err := services.GetJournal(self.config_obj) if err == nil { - journal.PushRowsToArtifactAsync( + journal.PushRowsToArtifactAsync(ctx, self.config_obj, row, "System.Flow.Completion") } } @@ -403,7 +403,7 @@ func (self *ClientFlowRunner) FlowStats( } func (self *ClientFlowRunner) VQLResponse( - client_id, flow_id string, + ctx context.Context, client_id, flow_id string, response *actions_proto.VQLResponse) error { if response == nil || response.Query == nil || response.Query.Name == "" { @@ -420,7 +420,7 @@ func (self *ClientFlowRunner) VQLResponse( return nil } - path_manager, err := artifact_paths.NewArtifactPathManager( + path_manager, err := artifact_paths.NewArtifactPathManager(ctx, self.config_obj, client_id, flow_id, response.Query.Name) if err != nil { return err diff --git a/flows/housekeeping.go b/flows/housekeeping.go index b2887ac0e35..7630496fc5f 100644 --- a/flows/housekeeping.go +++ b/flows/housekeeping.go @@ -143,7 +143,7 @@ func CheckClientStatus( latest_timestamp := uint64(0) for _, hunt := range hunts { // Notify the hunt manager that we need to hunt this client. - journal.PushRowsToArtifactAsync(config_obj, + journal.PushRowsToArtifactAsync(ctx, config_obj, ordereddict.NewDict(). Set("HuntId", hunt.HuntId). Set("ClientId", client_id), diff --git a/flows/logs.go b/flows/logs.go index 53c3f60d70b..9112c25d232 100644 --- a/flows/logs.go +++ b/flows/logs.go @@ -5,6 +5,7 @@ package flows import ( + "context" "regexp" "strings" "sync" @@ -121,13 +122,13 @@ func writeLogMessages( // Flush the logs to disk. During execution the flow collects the logs // in memory and then flushes it all when done. func flushContextLogs( - config_obj *config_proto.Config, + ctx context.Context, config_obj *config_proto.Config, collection_context *CollectionContext, completion *utils.Completer) error { // Handle monitoring flow specially. if collection_context.SessionId == constants.MONITORING_WELL_KNOWN_FLOW { - return flushContextLogsMonitoring(config_obj, collection_context) + return flushContextLogsMonitoring(ctx, config_obj, collection_context) } flow_path_manager := paths.NewFlowPathManager( diff --git a/flows/monitoring.go b/flows/monitoring.go index da029d6faf2..9e4c4cc72fa 100644 --- a/flows/monitoring.go +++ b/flows/monitoring.go @@ -6,6 +6,7 @@ package flows import ( "bytes" + "context" "github.com/Velocidex/ordereddict" "github.com/prometheus/client_golang/prometheus" @@ -36,7 +37,7 @@ type jsonBatch struct { // Receive monitoring messages from the client. func MonitoringProcessMessage( - config_obj *config_proto.Config, + ctx context.Context, config_obj *config_proto.Config, collection_context *CollectionContext, message *crypto_proto.VeloMessage) error { @@ -48,7 +49,7 @@ func MonitoringProcessMessage( switch message.RequestId { case constants.TransferWellKnownFlowId: - return appendUploadDataToFile( + return appendUploadDataToFile(ctx, config_obj, collection_context, message) } @@ -81,7 +82,7 @@ func MonitoringProcessMessage( // Logs from monitoring flow need to be handled especially since they // are written with a time index. func flushContextLogsMonitoring( - config_obj *config_proto.Config, + ctx context.Context, config_obj *config_proto.Config, collection_context *CollectionContext) error { // A single packet may have multiple log messages from @@ -100,7 +101,7 @@ func flushContextLogsMonitoring( // Try to get the writer from the cache. rs_writer, pres := writers[artifact_name] if !pres { - log_path_manager, err := artifact_paths.NewArtifactLogPathManager( + log_path_manager, err := artifact_paths.NewArtifactLogPathManager(ctx, config_obj, collection_context.ClientId, collection_context.SessionId, artifact_name) if err != nil { @@ -142,7 +143,7 @@ func (self *CollectionContext) batchRows( } func flushMonitoringLogs( - config_obj *config_proto.Config, + ctx context.Context, config_obj *config_proto.Config, collection_context *CollectionContext) error { journal, err := services.GetJournal(config_obj) @@ -152,6 +153,7 @@ func flushMonitoringLogs( for query_name, jsonl_buff := range collection_context.monitoring_batch { err := journal.PushJsonlToArtifact( + ctx, config_obj, jsonl_buff.Bytes(), jsonl_buff.row_count, query_name, diff --git a/gui/velociraptor/src/components/artifacts/artifacts.jsx b/gui/velociraptor/src/components/artifacts/artifacts.jsx index 5a8e56b9c21..78dc0635136 100644 --- a/gui/velociraptor/src/components/artifacts/artifacts.jsx +++ b/gui/velociraptor/src/components/artifacts/artifacts.jsx @@ -72,6 +72,8 @@ class ArtifactInspector extends React.Component { showDeleteArtifactDialog: false, showArtifactsUploadDialog: false, current_filter: "", + + version: 0, } componentDidMount = () => { @@ -142,13 +144,20 @@ class ArtifactInspector extends React.Component { e.preventDefault(); e.stopPropagation(); + return this.getArtifactDescription(row.name); + } + + getArtifactDescription = (name) => { // Fetch the full description api.post("v1/GetArtifacts", { - names: [row.name], + names: [name], }, this.source.token).then(response=>{ let items = response.data.items; if (items.length > 0) { - this.setState({fullSelectedDescriptor: items[0]}); + this.setState({ + fullSelectedDescriptor: items[0], + version: this.state.version+1, + }); }; }); return false; @@ -235,6 +244,7 @@ class ArtifactInspector extends React.Component { // Re-apply the search in case the user updated // an artifact that should show up. this.fetchRows(this.state.current_filter); + this.getArtifactDescription(selected); this.setState({showEditedArtifactDialog: false}); }} /> @@ -341,6 +351,7 @@ class ArtifactInspector extends React.Component { { this.state.selectedDescriptor ? : diff --git a/gui/velociraptor/src/components/artifacts/reporting.jsx b/gui/velociraptor/src/components/artifacts/reporting.jsx index 78a6e755155..9d242fd0c60 100644 --- a/gui/velociraptor/src/components/artifacts/reporting.jsx +++ b/gui/velociraptor/src/components/artifacts/reporting.jsx @@ -40,6 +40,7 @@ export default class VeloReportViewer extends React.Component { data: {}, messages: [], loading: true, + version: 0, } componentDidMount() { @@ -110,8 +111,8 @@ export default class VeloReportViewer extends React.Component { messages: response.data.messages || [], data: JSON.parse(response.data.data), loading: false, + version: this.state.version += 1, }; - for (var i=0; i + ); }; diff --git a/gui/velociraptor/src/components/tools/tool-viewer.jsx b/gui/velociraptor/src/components/tools/tool-viewer.jsx index f4fd4c92f11..5ca876a14bc 100644 --- a/gui/velociraptor/src/components/tools/tool-viewer.jsx +++ b/gui/velociraptor/src/components/tools/tool-viewer.jsx @@ -12,14 +12,65 @@ import CardDeck from 'react-bootstrap/CardDeck'; import Form from 'react-bootstrap/Form'; import InputGroup from 'react-bootstrap/InputGroup'; import T from '../i8n/i8n.jsx'; +import Select from 'react-select'; +import VeloValueRenderer from '../utils/value.jsx'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import classNames from "classnames"; +class ResetToolDialog extends React.Component { + static propTypes = { + tool: PropTypes.object, + onClose: PropTypes.func.isRequired, + }; + + componentDidMount = () => { + this.source = axios.CancelToken.source(); + } + + componentWillUnmount() { + this.source.cancel("unmounted"); + } + + setToolInfo = (tool) => { + api.post("v1/SetToolInfo", tool, + this.source.token).then((response) => { + this.setState({tool: response.data}); + }).finally(() => { + this.props.onClose(); + }); + }; + + render() { + return + + {T("Tool")} { + this.props.tool && this.props.tool.name} + + +

{T("Confirm tool definition reset")}

+ {T("This will reset the tool to its original definition")} + + + + + + ; + } +} + export default class ToolViewer extends React.Component { static propTypes = { name: PropTypes.string, + version: PropTypes.number, }; componentDidMount = () => { @@ -32,17 +83,18 @@ export default class ToolViewer extends React.Component { } componentDidUpdate = (prevProps, prevState, rootNode) => { - if (this.props.name !== prevProps.name) { + if (this.props.name !== prevProps.name || + this.props.version !== prevProps.version) { this.fetchToolInfo(); } } - fetchToolInfo = () => { + fetchToolInfo = (onclose) => { api.get("v1/GetToolInfo", {name: this.props.name}, this.source.token).then((response) => { this.setState({tool: response.data}); - }); + }).then(onclose); } state = { @@ -276,8 +328,21 @@ export default class ToolViewer extends React.Component { ); } + let tool_version_options = _.map(tool.versions, x=>{ + return {value: x.artifact, + tool: x, + label: x.artifact, + isFixed: true}; + }); return ( <> + { this.state.showUpdateDialog && + this.fetchToolInfo( + x=>this.setState({showUpdateDialog: false}))}> + + }
+ { tool.versions && + <> +
{T("Other Definitions")}
+
+