Add secret support to Splunk and Elastic uploads. (Velocidex#3379)

artifacts/testdata/windows/evtx.in.yaml

@@ -1,4 +1,5 @@
 Queries:
+  # The followings are here to test the Message extraction on a live Windows system.
   - SELECT * FROM parse_evtx(filename=srcDir + '/artifacts/testdata/files/Security_1_record.evtx')
 
   - SELECT UserData FROM parse_evtx(filename=srcDir + '/artifacts/testdata/files/Security_1_record.evtx') WHERE System.EventId.Value = 1102

constants/constants.go

@@ -155,6 +155,8 @@ const (
 	AWS_S3_CREDS    = "AWS S3 Creds"
 	SSH_PRIVATE_KEY = "SSH PrivateKey"
 	HTTP_SECRETS    = "HTTP Secrets"
+	SPLUNK_CREDS    = "Splunk Creds"
+	ELASTIC_CREDS   = "Elastic Creds"
 )
 
 type key int

go.mod

@@ -99,7 +99,7 @@ require (
 	www.velocidex.com/golang/go-prefetch v0.0.0-20220801101854-338dbe61982a
 	www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe
 	www.velocidex.com/golang/regparser v0.0.0-20221020153526-bbc758cbd18b
-	www.velocidex.com/golang/vfilter v0.0.0-20240325172436-0d47688bd789
+	www.velocidex.com/golang/vfilter v0.0.0-20240328043914-09d99b52b86f
 )
 
 require (

go.sum

@@ -1339,7 +1339,7 @@ www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe h1:o9jQWSwK
 www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe/go.mod h1:R7IisRzDO7q5LVRJsCQf1xA50LrIavsPWzAjVE4THyY=
 www.velocidex.com/golang/regparser v0.0.0-20221020153526-bbc758cbd18b h1:NrnjFXwjUi7vdLEDKgSxu6cs304UJLZE/H7pSXXakVA=
 www.velocidex.com/golang/regparser v0.0.0-20221020153526-bbc758cbd18b/go.mod h1:pxSECT5mWM3goJ4sxB4HCJNKnKqiAlpyT8XnvBwkLGU=
-www.velocidex.com/golang/vfilter v0.0.0-20240325172436-0d47688bd789 h1:7SNUY6RMlg+aBIEDLSeaDQORiHje6WjYHTKLSej+xHI=
-www.velocidex.com/golang/vfilter v0.0.0-20240325172436-0d47688bd789/go.mod h1:P50KPQr2LpWVAu7ilGH8CBLBASGtOJ2971yA9YhR8rY=
+www.velocidex.com/golang/vfilter v0.0.0-20240328043914-09d99b52b86f h1:v6CyvYEK1YLTzUi23ae1Bs2fWxjzM8JBulLcpfpyH0k=
+www.velocidex.com/golang/vfilter v0.0.0-20240328043914-09d99b52b86f/go.mod h1:P50KPQr2LpWVAu7ilGH8CBLBASGtOJ2971yA9YhR8rY=
 www.velocidex.com/golang/vtypes v0.0.0-20240123105603-069d4a7f435c h1:rL/It+Ig+mvIhmy9vl5gg5b6CX2J12x0v2SXIT2RoWE=
 www.velocidex.com/golang/vtypes v0.0.0-20240123105603-069d4a7f435c/go.mod h1:tjaJNlBWbvH4cEMrEu678CFR2hrtcdyPINIpRxrOh4U=

gui/velociraptor/src/components/events/utils.jsx

@@ -75,6 +75,10 @@ function proto2tables(table, cb) {
         all_artifacts = all_artifacts.concat(event_table.artifacts);
     });
 
+    if (_.isEmpty(all_artifacts)) {
+        return {};
+    }
+
     // Now lookup all the artifacts for their definitions and replace
     // in client_event_table.
     api.post("v1/GetArtifacts", {names: all_artifacts}).then(response=>{

gui/velociraptor/src/components/flows/new-collection.jsx

@@ -238,6 +238,10 @@ class NewCollectionSelectArtifacts extends React.Component {
         let artifacts = [];
         _.each(spec, x=>artifacts.push(x.artifact));
 
+        if(_.isEmpty(artifacts)) {
+            return;
+        }
+
         api.post("v1/GetArtifacts", {
             type: this.props.artifactType,
             names: artifacts}, this.source.token).then((response) => {
@@ -852,6 +856,12 @@ class NewCollectionWizard extends React.Component {
             initialized_from_parent: true,
         });
 
+        if (_.isEmpty(request.artifacts)) {
+            this.setState({artifacts:[]});
+            return;
+        }
+
+
         // Resolve the artifacts from the request into a list of descriptors.
         api.post("v1/GetArtifacts",
                 {names: request.artifacts}, this.source.token).then(response=>{

gui/velociraptor/src/components/hunts/new-hunt.jsx

@@ -299,6 +299,12 @@ export default class NewHuntWizard extends React.Component {
             state.hunt_parameters.expires = expiry;
             state.hunt_parameters.org_ids = hunt.org_ids || [];
 
+            if (_.isEmpty(request.artifacts)) {
+                this.setState({artifacts:[]});
+                return state;
+            }
+
+
             // Resolve the artifacts from the request into a list of descriptors.
             api.post("v1/GetArtifacts", {
                 names: request.artifacts,

services/sanity/secrets.go

@@ -40,11 +40,44 @@ var built_in_definitions = []string{`{
      "url": "",
      "method": "",
      "user_agent": "",
+     "root_ca": "",
+     "skip_verify": "FALSE",
      "extra_params": "# Add extra parameters as YAML strings\n#Foo: Value\n#Baz:Value2\n",
      "extra_headers": "# Add extra headers as YAML strings\n#Authorization: Value\n",
      "cookies": "# Add cookies as YAML strings\n#Cookie1: Value\n#Cookie2: Value2\n"
   },
   "verifier": "x=>x.url"
+}`, `{
+  "typeName":"Splunk Creds",
+  "description": "Credentials to be used in upload_splunk() calls.",
+  "template": {
+     "url": "",
+     "token": "",
+     "index": "",
+     "source": "",
+     "root_ca": "",
+     "hostname": "",
+     "hostname_field": "",
+     "skip_verify": "FALSE"
+  },
+  "verifier": "x=>x.index AND x.url"
+}`, `{
+  "typeName":"Elastic Creds",
+  "description": "Credentials to be used in upload_elastic() calls.",
+  "template": {
+     "index": "",
+     "type": "",
+     "addresses": "# Add URLs one per line\n# http://www.example.com/\n",
+     "username": "",
+     "password": "",
+     "cloud_id": "",
+     "api_key": "",
+     "pipeline": "",
+     "root_ca": "",
+     "action": "",
+     "skip_verify": "FALSE"
+  },
+  "verifier": "x=>x.addresses"
 }`,
 }
 

vql/common/cache.go

@@ -56,6 +56,7 @@ func (self *_CacheObj) Materialize() {
 	defer self.mu.Unlock()
 
 	self.materialize()
+	self.expires = time.Now().Add(self.period)
 }
 
 func (self *_CacheObj) materialize() {
@@ -111,7 +112,6 @@ func (self _CacheAssociative) Associative(
 
 	if time.Now().After(cache_obj.expires) {
 		cache_obj.Materialize()
-		cache_obj.expires = time.Now().Add(cache_obj.period)
 	}
 
 	res, pres := cache_obj.cache[key]

vql/networking/http_client.go

@@ -151,6 +151,14 @@ func (self *HTTPClientCache) mergeSecretToRequest(
 		}
 	}
 
+	get_bool := func(field string, target *bool) {
+		res := vql_subsystem.GetStringFromRow(
+			scope, secret_record.Data, field)
+		if res != "" {
+			*target = vql_subsystem.GetBoolFromString(res)
+		}
+	}
+
 	// We expect Dict parameters to be a YAML formatted object.
 	get_dict := func(field string, target *ordereddict.Dict) {
 		res := vql_subsystem.GetStringFromRow(
@@ -186,6 +194,8 @@ func (self *HTTPClientCache) mergeSecretToRequest(
 	get("url", &arg.real_url)
 	get("method", &arg.Method)
 	get("user_agent", &arg.UserAgent)
+	get("root_ca", &arg.RootCerts)
+	get_bool("skip_verify", &arg.SkipVerify)
 	get_dict("extra_params", arg.Params)
 	get_dict("extra_headers", arg.Headers)
 	get_dict("cookies", arg.CookieJar)

vql/server/elastic.go

@@ -53,7 +53,9 @@ import (
 	"www.velocidex.com/golang/velociraptor/acls"
 	"www.velocidex.com/golang/velociraptor/artifacts"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+	"www.velocidex.com/golang/velociraptor/constants"
 	"www.velocidex.com/golang/velociraptor/json"
+	"www.velocidex.com/golang/velociraptor/services"
 	"www.velocidex.com/golang/velociraptor/utils"
 	"www.velocidex.com/golang/velociraptor/vql"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
@@ -80,6 +82,7 @@ type _ElasticPluginArgs struct {
 	RootCerts          string              `vfilter:"optional,field=root_ca,doc=As a better alternative to disable_ssl_security, allows root ca certs to be added here."`
 	MaxMemoryBuffer    uint64              `vfilter:"optional,field=max_memory_buffer,doc=How large we allow the memory buffer to grow to while we are trying to contact the Elastic server (default 100mb)."`
 	Action             string              `vfilter:"optional,field=action,doc=Either index or create. For data streams this must be create."`
+	Secret             string              `vfilter:"optional,field=secret,doc=Alternatively use a secret from the secrets service. Secret must be of type 'AWS S3 Creds'"`
 }
 
 type _ElasticPlugin struct{}
@@ -98,13 +101,21 @@ func (self _ElasticPlugin) Call(ctx context.Context,
 			return
 		}
 
-		arg := _ElasticPluginArgs{}
-		err = arg_parser.ExtractArgsWithContext(ctx, scope, args, &arg)
+		arg := &_ElasticPluginArgs{}
+		err = arg_parser.ExtractArgsWithContext(ctx, scope, args, arg)
 		if err != nil {
 			scope.Log("elastic: %v", err)
 			return
 		}
 
+		if arg.Secret != "" {
+			err := mergeSecretElastic(ctx, scope, arg)
+			if err != nil {
+				scope.Log("elastic_upload: %v", err)
+				return
+			}
+		}
+
 		if arg.Action == "" {
 			arg.Action = "index"
 		}
@@ -137,7 +148,7 @@ func (self _ElasticPlugin) Call(ctx context.Context,
 
 			// Start an uploader on a thread.
 			go upload_rows(ctx, config_obj, scope, output_chan,
-				row_chan, id, arg.Action, &wg, &arg)
+				row_chan, id, arg.Action, &wg, arg)
 		}
 
 		wg.Wait()
@@ -333,6 +344,64 @@ func sanitize_index(name string) string {
 		strings.ToLower(name), "_")
 }
 
+func mergeSecretElastic(ctx context.Context, scope vfilter.Scope, arg *_ElasticPluginArgs) error {
+	config_obj, ok := vql_subsystem.GetServerConfig(scope)
+	if !ok {
+		return errors.New("elastic_upload: Secrets may only be used on the server")
+	}
+
+	secrets_service, err := services.GetSecretsService(config_obj)
+	if err != nil {
+		return err
+	}
+
+	principal := vql_subsystem.GetPrincipal(scope)
+
+	secret_record, err := secrets_service.GetSecret(ctx, principal,
+		constants.ELASTIC_CREDS, arg.Secret)
+	if err != nil {
+		return err
+	}
+
+	get := func(field string) string {
+		return vql_subsystem.GetStringFromRow(
+			scope, secret_record.Data, field)
+	}
+
+	get_bool := func(field string) bool {
+		return vql_subsystem.GetBoolFromString(vql_subsystem.GetStringFromRow(
+			scope, secret_record.Data, field))
+	}
+
+	addresses := vql_subsystem.GetStringFromRow(
+		scope, secret_record.Data, "addresses")
+	arg.Addresses = nil
+	for _, line := range strings.Split(addresses, "\n") {
+		line = strings.TrimSpace(line)
+		if strings.HasPrefix(line, "#") {
+			continue
+		}
+		arg.Addresses = append(arg.Addresses, line)
+	}
+
+	if arg.Addresses == nil {
+		return errors.New("No addresses present in elastic secret!")
+	}
+
+	arg.Index = get("index")
+	arg.Type = get("type")
+	arg.Username = get("username")
+	arg.Password = get("password")
+	arg.CloudID = get("cloud_id")
+	arg.APIKey = get("api_key")
+	arg.PipeLine = get("pipeline")
+	arg.SkipVerify = get_bool("skip_verify")
+	arg.RootCerts = get("root_ca")
+	arg.Action = get("action")
+
+	return nil
+}
+
 func (self _ElasticPlugin) Info(
 	scope vfilter.Scope,
 	type_map *vfilter.TypeMap) *vfilter.PluginInfo {

vql/server/splunk.go

@@ -23,6 +23,7 @@ package server
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"sync"
 	"time"
@@ -32,6 +33,8 @@ import (
 	"www.velocidex.com/golang/velociraptor/acls"
 	"www.velocidex.com/golang/velociraptor/artifacts"
 	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+	"www.velocidex.com/golang/velociraptor/constants"
+	"www.velocidex.com/golang/velociraptor/services"
 	"www.velocidex.com/golang/velociraptor/vql"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/velociraptor/vql/functions"
@@ -43,7 +46,7 @@ import (
 type _SplunkPluginArgs struct {
 	Query          vfilter.StoredQuery `vfilter:"required,field=query,doc=Source for rows to upload."`
 	Threads        int64               `vfilter:"optional,field=threads,doc=How many threads to use."`
-	URL            string              `vfilter:"optional,field=url,doc=The Splunk Event Collector URL."`
+	URL            string              `vfilter:"required,field=url,doc=The Splunk Event Collector URL."`
 	Token          string              `vfilter:"optional,field=token,doc=Splunk HEC Token."`
 	Index          string              `vfilter:"required,field=index,doc=The name of the index to upload to."`
 	Source         string              `vfilter:"optional,field=source,doc=The source field for splunk. If not specified this will be 'velociraptor'."`
@@ -55,6 +58,7 @@ type _SplunkPluginArgs struct {
 	Hostname       string              `vfilter:"optional,field=hostname,doc=Hostname for Splunk Events. Defaults to server hostname."`
 	TimestampField string              `vfilter:"optional,field=timestamp_field,doc=Field to use as event timestamp."`
 	HostnameField  string              `vfilter:"optional,field=hostname_field,doc=Field to use as event hostname. Overrides hostname parameter."`
+	Secret         string              `vfilter:"optional,field=secret,doc=Alternatively use a secret from the secrets service. Secret must be of type 'AWS S3 Creds'"`
 }
 
 type _SplunkPlugin struct{}
@@ -72,12 +76,20 @@ func (self _SplunkPlugin) Call(ctx context.Context,
 			return
 		}
 
-		arg := _SplunkPluginArgs{}
-		err = arg_parser.ExtractArgsWithContext(ctx, scope, args, &arg)
+		arg := &_SplunkPluginArgs{}
+		err = arg_parser.ExtractArgsWithContext(ctx, scope, args, arg)
 		if err != nil {
 			return
 		}
 
+		if arg.Secret != "" {
+			err := mergeSecretSplunk(ctx, scope, arg)
+			if err != nil {
+				scope.Log("splunk_upload: %v", err)
+				return
+			}
+		}
+
 		if arg.Threads == 0 {
 			arg.Threads = 1
 		}
@@ -107,7 +119,7 @@ func (self _SplunkPlugin) Call(ctx context.Context,
 
 			// Start an uploader on a thread.
 			go _upload_rows(ctx, scope, config_obj, output_chan,
-				row_chan, &wg, &arg)
+				row_chan, &wg, arg)
 		}
 
 		wg.Wait()
@@ -268,6 +280,47 @@ func send_to_splunk(
 	}
 }
 
+func mergeSecretSplunk(ctx context.Context, scope vfilter.Scope, arg *_SplunkPluginArgs) error {
+	config_obj, ok := vql_subsystem.GetServerConfig(scope)
+	if !ok {
+		return errors.New("splunk_upload: Secrets may only be used on the server")
+	}
+
+	secrets_service, err := services.GetSecretsService(config_obj)
+	if err != nil {
+		return err
+	}
+
+	principal := vql_subsystem.GetPrincipal(scope)
+
+	secret_record, err := secrets_service.GetSecret(ctx, principal,
+		constants.SPLUNK_CREDS, arg.Secret)
+	if err != nil {
+		return err
+	}
+
+	get := func(field string) string {
+		return vql_subsystem.GetStringFromRow(
+			scope, secret_record.Data, field)
+	}
+
+	get_bool := func(field string) bool {
+		return vql_subsystem.GetBoolFromString(vql_subsystem.GetStringFromRow(
+			scope, secret_record.Data, field))
+	}
+
+	arg.URL = get("url")
+	arg.Token = get("token")
+	arg.Index = get("index")
+	arg.Source = get("source")
+	arg.RootCerts = get("root_ca")
+	arg.Hostname = get("hostname")
+	arg.HostnameField = get("hostname_field")
+	arg.SkipVerify = get_bool("skip_verify")
+
+	return nil
+}
+
 func (self _SplunkPlugin) Info(
 	scope vfilter.Scope,
 	type_map *vfilter.TypeMap) *vfilter.PluginInfo {