diff --git a/accessors/raw_registry/raw_registry.go b/accessors/raw_registry/raw_registry.go
index 9c373eff4bc..e8167eac3fb 100644
--- a/accessors/raw_registry/raw_registry.go
+++ b/accessors/raw_registry/raw_registry.go
@@ -45,6 +45,7 @@ import (
"www.velocidex.com/golang/velociraptor/accessors"
"www.velocidex.com/golang/velociraptor/constants"
"www.velocidex.com/golang/velociraptor/json"
+ "www.velocidex.com/golang/velociraptor/utils"
vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
"www.velocidex.com/golang/velociraptor/vql/readers"
"www.velocidex.com/golang/vfilter"
@@ -75,8 +76,9 @@ var (
)
type RawRegKeyInfo struct {
- key *regparser.CM_KEY_NODE
_full_path *accessors.OSPath
+ _data *ordereddict.Dict
+ _modtime time.Time
}
func (self *RawRegKeyInfo) IsDir() bool {
@@ -84,7 +86,10 @@ func (self *RawRegKeyInfo) IsDir() bool {
}
func (self *RawRegKeyInfo) Data() *ordereddict.Dict {
- return ordereddict.NewDict().Set("type", "Key")
+ if self._data == nil {
+ self._data = ordereddict.NewDict().Set("type", "Key")
+ }
+ return self._data
}
func (self *RawRegKeyInfo) Size() int64 {
@@ -104,11 +109,11 @@ func (self *RawRegKeyInfo) Mode() os.FileMode {
}
func (self *RawRegKeyInfo) Name() string {
- return self.key.Name()
+ return self._full_path.Basename()
}
func (self *RawRegKeyInfo) ModTime() time.Time {
- return self.key.LastWriteTime().Time
+ return self._modtime
}
func (self *RawRegKeyInfo) Mtime() time.Time {
@@ -143,35 +148,41 @@ func (self *RawRegKeyInfo) UnmarshalJSON(data []byte) error {
type RawRegValueInfo struct {
// Containing key
*RawRegKeyInfo
- value *regparser.CM_KEY_VALUE
- // The windows registry can store a value inside a reg key. This
- // makes the key act both as a directory and as a file
- // (i.e. ReadDir() will list the key) but Open() will read the
- // value.
- is_default_value bool
+ // Hold a reference so value can be decoded lazily.
+ _value *regparser.CM_KEY_VALUE
+ // Once value is decoded once it will be cached here.
_data *ordereddict.Dict
-}
-
-func (self *RawRegValueInfo) Name() string {
- return self.value.ValueName()
+ _size int64
+}
+
+func (self *RawRegValueInfo) Copy() *RawRegValueInfo {
+ return &RawRegValueInfo{
+ RawRegKeyInfo: &RawRegKeyInfo{
+ _full_path: self._full_path,
+ _modtime: self._modtime,
+ },
+ _value: self._value,
+ _data: self._data,
+ _size: self._size,
+ }
}
func (self *RawRegValueInfo) IsDir() bool {
- // We are also a key so act as a directory.
- return self.is_default_value
+ return false
}
func (self *RawRegValueInfo) Mode() os.FileMode {
- if self.is_default_value {
- return 0755
- }
return 0644
}
func (self *RawRegValueInfo) Size() int64 {
- return int64(self.value.DataSize())
+ if self._size > 0 {
+ return self._size
+ }
+ self._size = int64(self._value.DataSize())
+ return self._size
}
func (self *RawRegValueInfo) Data() *ordereddict.Dict {
@@ -180,11 +191,8 @@ func (self *RawRegValueInfo) Data() *ordereddict.Dict {
}
metricsReadValue.Inc()
- value_data := self.value.ValueData()
- value_type := self.value.TypeString()
- if self.is_default_value {
- value_type += "/Key"
- }
+ value_data := self._value.ValueData()
+ value_type := self._value.TypeString()
result := ordereddict.NewDict().
Set("type", value_type).
Set("data_len", len(value_data.Data))
@@ -209,6 +217,7 @@ func (self *RawRegValueInfo) Data() *ordereddict.Dict {
type RawValueBuffer struct {
*bytes.Reader
+
info *RawRegValueInfo
}
@@ -218,7 +227,7 @@ func (self *RawValueBuffer) Close() error {
func NewRawValueBuffer(buf string, stat *RawRegValueInfo) *RawValueBuffer {
return &RawValueBuffer{
- bytes.NewReader(stat.value.ValueData().Data),
+ bytes.NewReader(stat._value.ValueData().Data),
stat,
}
}
@@ -370,9 +379,85 @@ func (self *RawRegFileSystemAccessor) ReadDir(key_path string) (
return self.ReadDirWithOSPath(full_path)
}
+// Get the default value of a Registry Key if possible.
+func (self *RawRegFileSystemAccessor) getDefaultValue(
+ full_path *accessors.OSPath) (result *RawRegValueInfo, err error) {
+
+ // A Key has a default value if its parent directory contains a
+ // value with the same name as the key.
+ basename := full_path.Basename()
+ contents, err := self._readDirWithOSPath(full_path.Dirname())
+ if err != nil {
+ return nil, err
+ }
+
+ for _, item := range contents {
+ value_item, ok := item.(*RawRegValueInfo)
+ if !ok {
+ continue
+ }
+
+ if item.Name() == basename {
+ item_copy := value_item.Copy()
+ item_copy._full_path = item_copy._full_path.Append("@")
+ return item_copy, nil
+ }
+ }
+
+ return nil, utils.NotFoundError
+}
+
func (self *RawRegFileSystemAccessor) ReadDirWithOSPath(
full_path *accessors.OSPath) (result []accessors.FileInfo, err error) {
+ // Add the default value if the key has one
+ default_value, err := self.getDefaultValue(full_path)
+ if err == nil {
+ result = append(result, default_value)
+ }
+
+ contents, err := self._readDirWithOSPath(full_path)
+ if err != nil {
+ return nil, err
+ }
+
+ seen := make(map[string]bool)
+
+ for _, item := range contents {
+ basename := item.Name()
+
+ // Does this value have the same name as one of the keys? We
+ // special case it as a subdirectory with a file called @ in
+ // it:
+ // Subkeys: A, B, C
+ // Values: B -> Means Subkey B has default values.
+ //
+ // This will end up being:
+ // A/ -> Directory
+ // B/ -> Directory
+ // C/ -> Directory
+ // B/@ -> File
+ //
+ // Therefore skip such values at this level - a Glob will
+ // fetch them at the next level down.
+ _, pres := seen[basename]
+ if pres {
+ continue
+ }
+
+ seen[basename] = true
+
+ result = append(result, item)
+ }
+
+ return result, nil
+}
+
+// Return all the contents in the directory including all keys and all
+// values, even if some keys have a default value.
+func (self *RawRegFileSystemAccessor) _readDirWithOSPath(
+ full_path *accessors.OSPath) (result []accessors.FileInfo, err error) {
+
cache_key := full_path.String()
cached, err := self.readdir_lru.Get(cache_key)
if err == nil {
@@ -399,41 +484,31 @@ func (self *RawRegFileSystemAccessor) ReadDirWithOSPath(
key := OpenKeyComponents(hive, full_path.Components)
if key == nil {
- return nil, errors.New("Key not found")
+ return nil, nil
}
- seen := make(map[string]int)
- for idx, subkey := range key.Subkeys() {
+ for _, subkey := range key.Subkeys() {
basename := subkey.Name()
subkey := &RawRegKeyInfo{
- key: subkey,
_full_path: full_path.Append(basename),
+ _modtime: subkey.LastWriteTime().Time,
}
- seen[basename] = idx
result = append(result, subkey)
}
+ // All Values carry their mode time as the parent key
+ key_mod_time := key.LastWriteTime().Time
for _, value := range key.Values() {
basename := value.ValueName()
value_obj := &RawRegValueInfo{
RawRegKeyInfo: &RawRegKeyInfo{
- key: key,
_full_path: full_path.Append(basename),
+ _modtime: key_mod_time,
},
- value: value,
- }
-
- // Does this value have the same name as one of the keys?
- idx, pres := seen[basename]
- if pres {
- // Replace the old object with the value object
- value_obj.is_default_value = true
- result[idx] = value_obj
- } else {
- result = append(result, value_obj)
+ _value: value,
}
+ result = append(result, value_obj)
}
-
return result, nil
}
@@ -447,7 +522,7 @@ func (self *RawRegFileSystemAccessor) Open(path string) (
value_info, ok := stat.(*RawRegValueInfo)
if ok {
return NewValueBuffer(
- value_info.value.ValueData().Data, stat), nil
+ value_info._value.ValueData().Data, stat), nil
}
// Keys do not have any data.
@@ -465,7 +540,7 @@ func (self *RawRegFileSystemAccessor) OpenWithOSPath(path *accessors.OSPath) (
value_info, ok := stat.(*RawRegValueInfo)
if ok {
return NewValueBuffer(
- value_info.value.ValueData().Data, stat), nil
+ value_info._value.ValueData().Data, stat), nil
}
// Keys do not have any data.
@@ -494,12 +569,20 @@ func (self *RawRegFileSystemAccessor) LstatWithOSPath(
}, nil
}
- children, err := self.ReadDirWithOSPath(full_path.Dirname())
+ name := full_path.Basename()
+ container := full_path.Dirname()
+
+ // If the full_path refers to the default value of the key, return
+ // it.
+ if name == "@" {
+ return self.getDefaultValue(container)
+ }
+
+ children, err := self.ReadDirWithOSPath(container)
if err != nil {
return nil, err
}
- name := full_path.Basename()
for _, child := range children {
if child.Name() == name {
return child, nil
diff --git a/api/datastore_test.go b/api/datastore_test.go
index a86e88f4709..b5426d68f81 100644
--- a/api/datastore_test.go
+++ b/api/datastore_test.go
@@ -44,7 +44,7 @@ func (self *DatastoreAPITest) SetupTest() {
// Wait for the server to come up.
vtesting.WaitUntil(2*time.Second, self.T(), func() bool {
conn, closer, err := grpc_client.Factory.GetAPIClient(
- self.Sm.Ctx, self.ConfigObj)
+ self.Sm.Ctx, grpc_client.SuperUser, self.ConfigObj)
assert.NoError(self.T(), err)
defer closer()
@@ -64,7 +64,7 @@ func (self *DatastoreAPITest) TestDatastore() {
// Make some RPC calls
conn, closer, err := grpc_client.Factory.GetAPIClient(
- self.Sm.Ctx, self.ConfigObj)
+ self.Sm.Ctx, grpc_client.SuperUser, self.ConfigObj)
assert.NoError(self.T(), err)
defer closer()
diff --git a/api/docs.go b/api/docs.go
new file mode 100644
index 00000000000..4d1cb84b357
--- /dev/null
+++ b/api/docs.go
@@ -0,0 +1,71 @@
+package api
+
+/*
+
+# How does the Velociraptor server work?
+
+The Velociraptor server presents a GRPC API for manipulating and
+presenting data. We chose gRPC because:
+
+1. It has mutual two way authentication - the server presents a
+ certificate to identify itself and the caller must also present a
+ properly signed certificate.
+
+2. Communication is encrypted using TLS
+
+3. Each RPC call can include a complex well defined API with protocol
+ buffers as inputs and outputs.
+
+4. GRPC has a streaming mode which is useful for real time
+ communications (e.g. via the Query API point).
+
+The server's API surface is well defined in api/proto/api.proto and
+implemented in this "api" module.
+
+By tightening down the server api surface it is easier to ensure that
+ACLs are properly enforced.
+
+## ACLs and permissions
+
+The API endpoints enforce the permission model based on the identity
+of the caller. In the gRPC API the caller's identity is found by
+examining the Common Name in the certificate that the user presented.
+
+The user identity is recovered using the users service
+users.GetUserFromContext(ctx)
+
+## How is the GUI implemented?
+
+The GUI is a simple react app which communicates with the server using
+AJAX calls, such as GET or POST. As such the GUI can not make direct
+gRPC calls to the API server.
+
+To translate between REST calls to gRPC we use the grpc gateway
+proxy. This proxy service exposes HTTP handlers on /api/ URLs. When a
+HTTP connection occurs, the gateway proxy will bundle the data into
+protocol buffers and make a proper gRPC call into the API.
+
+The gateway's gRPC connections are made using the gateway identity
+(certificates generated in GUI.gw_certificate and
+GUI.gw_private_key. The real identity of the calling user is injected
+in the gRPC metadata channel under the "USER" parameter.
+
+From the API server's perspective, the user identity is:
+
+1. If the identity is not utils.GetGatewayName() then the identity is
+ fetched from the caller's X509 certificates. (After verifying the
+ certificates are issued by the internal CA)
+
+2. If the caller is really the Gateway, then the real identity of the
+ user is retrieved from the gRPC metadata "USER" variable (passed in
+ the context)
+
+This logic is implemented in services/users/grpc.go:GetGRPCUserInfo()
+
+NOTE: The gateway's certificates are critical to protect - if an actor
+ makes an API connection using these certificate they can just claim
+ to be anyone by injecting any username into the "USER" gRPC
+ metadata.
+
+
+*/
diff --git a/api/proxy.go b/api/proxy.go
index 7dabc5ca46e..765d83ce611 100644
--- a/api/proxy.go
+++ b/api/proxy.go
@@ -217,7 +217,9 @@ func PrepareGUIMux(
}
// An api handler which connects to the gRPC service (i.e. it is a
-// gRPC client).
+// gRPC client). This is used by the gRPC gateway to relay REST calls
+// to the gRPC API. This connection must be identified as the gateway
+// identity.
func GetAPIHandler(
ctx context.Context,
config_obj *config_proto.Config) (http.Handler, error) {
@@ -285,6 +287,8 @@ func GetAPIHandler(
return nil, errors.New("GUI gRPC proxy Certificate is not correct")
}
+ // The API server's TLS address is pinned to the frontend's
+ // certificate. We must only connect to the real API server.
creds := credentials.NewTLS(&tls.Config{
Certificates: []tls.Certificate{cert},
RootCAs: CA_Pool,
diff --git a/artifacts/definitions/Windows/Forensics/Shellbags.yaml b/artifacts/definitions/Windows/Forensics/Shellbags.yaml
index eb85e8e69d4..60482917795 100644
--- a/artifacts/definitions/Windows/Forensics/Shellbags.yaml
+++ b/artifacts/definitions/Windows/Forensics/Shellbags.yaml
@@ -19,7 +19,6 @@ parameters:
C:/Users/*/NTUSER.dat,\Software\Microsoft\Windows\Shell\BagMRU\**
C:/Users/*/AppData/Local/Microsoft/Windows/UsrClass.dat,\Local Settings\Software\Microsoft\Windows\Shell\BagMRU\**
-precondition: SELECT OS From info() where OS = 'windows'
imports:
# Link files use the same internal format as shellbags so we import
@@ -42,11 +41,11 @@ sources:
root=pathspec(DelegatePath=HivePath),
globs=KeyGlob,
accessor="raw_reg")
- WHERE Data.type =~ "BINARY" AND OSPath.Path =~ "[0-9]$"
+ WHERE Data.type =~ "BINARY" AND OSPath.Path =~ "[0-9]\\\\@$"
})
LET ParsedValues = SELECT
- OSPath AS KeyPath,
+ OSPath.Dirname AS KeyPath,
parse_binary(profile=Profile, filename=Data.value,
accessor="data", struct="ItemIDList") as _Parsed,
base64encode(string=Data.value) AS _RawData, ModTime
diff --git a/artifacts/testdata/server/testcases/raw_registry.in.yaml b/artifacts/testdata/server/testcases/raw_registry.in.yaml
index 78372d22814..5983f3233f2 100644
--- a/artifacts/testdata/server/testcases/raw_registry.in.yaml
+++ b/artifacts/testdata/server/testcases/raw_registry.in.yaml
@@ -2,6 +2,36 @@ Queries:
- SELECT mock(plugin='info', results=[dict(OS='windows'), dict(OS='windows')] )
FROM scope()
+ # Test semantics around listing registry keys with default
+ # values. The 0 key actually has a default value. In the glob model
+ # it is a directory (since it is a Key) as well as a file (because
+ # it also is a value). Velociraptor's glob model can not deal with
+ # directories which are also files (most artifacts test for IsDir to
+ # avoid reading directories.).
+
+ # Therefore we separate the value out into a default value called
+ # "@" within the key/directory. Globbing the parent directory will
+ # only show keys, while Globbing the keys will show the default
+ # value of the key as @.
+
+ # Just a regular key
+ - SELECT OSPath.Path AS Key, Data FROM glob(globs="*", accessor="raw_reg", root=pathspec(
+ Path="/Local Settings/Software/Microsoft/Windows/Shell/BagMRU",
+ DelegatePath=srcDir+"/artifacts/testdata/files/UsrClass.dat"))
+ WHERE OSPath.Basename =~ "0"
+
+ # A value with name @
+ - SELECT OSPath.Path AS Key, Data FROM glob(globs="*", accessor="raw_reg", root=pathspec(
+ Path="/Local Settings/Software/Microsoft/Windows/Shell/BagMRU/0",
+ DelegatePath=srcDir+"/artifacts/testdata/files/UsrClass.dat"))
+ WHERE OSPath.Basename =~ "@"
+
+ # Now test read_file on values
+ - SELECT format(format="%02x", args=read_file(accessor='raw_reg', filename=pathspec(
+ Path="/Local Settings/Software/Microsoft/Windows/Shell/BagMRU/0/@",
+ DelegatePath=srcDir+"/artifacts/testdata/files/UsrClass.dat"))) AS ValueContent
+ FROM scope()
+
# This artifact uses the raw registry parser.
- SELECT LastModified, Binary, Name, Size, ProductName, Publisher, BinFileVersion
FROM Artifact.Windows.System.Amcache(
diff --git a/artifacts/testdata/server/testcases/raw_registry.out.yaml b/artifacts/testdata/server/testcases/raw_registry.out.yaml
index 921ae150615..49a7c32f4e4 100644
--- a/artifacts/testdata/server/testcases/raw_registry.out.yaml
+++ b/artifacts/testdata/server/testcases/raw_registry.out.yaml
@@ -2,6 +2,26 @@ SELECT mock(plugin='info', results=[dict(OS='windows'), dict(OS='windows')] ) FR
{
"mock(plugin='info', results=[dict(OS='windows'), dict(OS='windows')])": null
}
+]SELECT OSPath.Path AS Key, Data FROM glob(globs="*", accessor="raw_reg", root=pathspec( Path="/Local Settings/Software/Microsoft/Windows/Shell/BagMRU", DelegatePath=srcDir+"/artifacts/testdata/files/UsrClass.dat")) WHERE OSPath.Basename =~ "0"[
+ {
+ "Key": "\\Local Settings\\Software\\Microsoft\\Windows\\Shell\\BagMRU\\0",
+ "Data": {
+ "type": "Key"
+ }
+ }
+]SELECT OSPath.Path AS Key, Data FROM glob(globs="*", accessor="raw_reg", root=pathspec( Path="/Local Settings/Software/Microsoft/Windows/Shell/BagMRU/0", DelegatePath=srcDir+"/artifacts/testdata/files/UsrClass.dat")) WHERE OSPath.Basename =~ "@"[
+ {
+ "Key": "\\Local Settings\\Software\\Microsoft\\Windows\\Shell\\BagMRU\\0\\@",
+ "Data": {
+ "type": "REG_BINARY",
+ "data_len": 22,
+ "value": "FAAfgMuFn2cgAoBAsptVQMwFqrYAAA=="
+ }
+ }
+]SELECT format(format="%02x", args=read_file(accessor='raw_reg', filename=pathspec( Path="/Local Settings/Software/Microsoft/Windows/Shell/BagMRU/0/@", DelegatePath=srcDir+"/artifacts/testdata/files/UsrClass.dat"))) AS ValueContent FROM scope()[
+ {
+ "ValueContent": "14001f80cb859f6720028040b29b5540cc05aab60000"
+ }
]SELECT LastModified, Binary, Name, Size, ProductName, Publisher, BinFileVersion FROM Artifact.Windows.System.Amcache( source="InventoryApplicationFile", amCacheGlob=srcDir+"/artifacts/testdata/files/Amcache.hve") LIMIT 5[
{
"LastModified": "2019-03-02T08:21:12Z",
diff --git a/bin/config.go b/bin/config.go
index fab5256861a..e3d17c777ef 100644
--- a/bin/config.go
+++ b/bin/config.go
@@ -39,6 +39,7 @@ import (
"www.velocidex.com/golang/velociraptor/json"
"www.velocidex.com/golang/velociraptor/logging"
"www.velocidex.com/golang/velociraptor/services"
+ "www.velocidex.com/golang/velociraptor/services/users"
"www.velocidex.com/golang/velociraptor/startup"
"www.velocidex.com/golang/velociraptor/utils"
)
@@ -231,7 +232,7 @@ func generateNewKeys(config_obj *config_proto.Config) error {
// Generate gRPC gateway certificate.
gw_certificate, err := crypto.GenerateServerCert(
- config_obj, config_obj.API.PinnedGwName)
+ config_obj, utils.GetGatewayName(config_obj))
if err != nil {
return fmt.Errorf("Unable to create Frontend cert: %w", err)
}
@@ -302,7 +303,7 @@ func doRotateKeyConfig() error {
// Generate gRPC gateway certificate.
gw_certificate, err := crypto.GenerateServerCert(
- config_obj, config_obj.API.PinnedGwName)
+ config_obj, utils.GetGatewayName(config_obj))
if err != nil {
return err
}
@@ -423,9 +424,9 @@ func doDumpApiClientConfig() error {
return err
}
- if *config_api_client_common_name == utils.GetSuperuserName(config_obj) {
- return errors.New("Name reserved! You may not name your " +
- "api keys with this name.")
+ err = users.ValidateUsername(config_obj, *config_api_client_common_name)
+ if err != nil {
+ return err
}
if config_obj.Client == nil {
diff --git a/bin/fuse_unix.go b/bin/fuse_unix.go
new file mode 100644
index 00000000000..6c40a23e3b7
--- /dev/null
+++ b/bin/fuse_unix.go
@@ -0,0 +1,142 @@
+//go:build !windows
+// +build !windows
+
+package main
+
+import (
+ "fmt"
+ "log"
+ "time"
+
+ "github.com/Velocidex/ordereddict"
+ kingpin "github.com/alecthomas/kingpin/v2"
+ "github.com/hanwen/go-fuse/v2/fs"
+ "www.velocidex.com/golang/velociraptor/accessors"
+ config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+ "www.velocidex.com/golang/velociraptor/services"
+ "www.velocidex.com/golang/velociraptor/startup"
+ "www.velocidex.com/golang/velociraptor/tools/fuse"
+ "www.velocidex.com/golang/velociraptor/vql/acl_managers"
+)
+
+var (
+ fuse_command = app.Command("fuse", "Use fuse mounts")
+
+ fuse_zip_command = fuse_command.Command("container", "Mount ZIP containers over fuse")
+
+ fuse_directory = fuse_zip_command.Arg("directory", "A directory to mount on").
+ Required().String()
+
+ fuse_tmp_dir = fuse_zip_command.Flag("tmpdir",
+ "A temporary directory to use (if not specified we use our own tempdir)").
+ String()
+
+ fuse_zip_accessor = fuse_command.Flag("accessor", "The accessor to use (default container)").
+ Default("collector").String()
+
+ fuse_zip_prefix = fuse_command.Flag("prefix", "Export all files below this directory in the zip file").
+ Default("/").String()
+
+ fuse_files = fuse_zip_command.Arg("files", "list of zip files to mount").
+ Required().Strings()
+)
+
+func doFuseZip() error {
+ server_config_obj, err := makeDefaultConfigLoader().
+ WithNullLoader().LoadAndValidate()
+ if err != nil {
+ return fmt.Errorf("Unable to load config file: %w", err)
+ }
+
+ ctx, cancel := install_sig_handler()
+ defer cancel()
+
+ config_obj := &config_proto.Config{}
+ config_obj.Frontend = server_config_obj.Frontend
+
+ config_obj.Services = services.GenericToolServices()
+ sm, err := startup.StartToolServices(ctx, config_obj)
+ defer sm.Close()
+
+ if err != nil {
+ return err
+ }
+
+ logger := &LogWriter{config_obj: sm.Config}
+ builder := services.ScopeBuilder{
+ Config: sm.Config,
+ ACLManager: acl_managers.NewRoleACLManager(sm.Config, "administrator"),
+ Logger: log.New(logger, "", 0),
+ Env: ordereddict.NewDict(),
+ }
+
+ manager, err := services.GetRepositoryManager(builder.Config)
+ if err != nil {
+ return err
+ }
+
+ scope := manager.BuildScope(builder)
+ defer scope.Close()
+
+ accessor, err := accessors.GetAccessor(*fuse_zip_accessor, scope)
+ if err != nil {
+ return err
+ }
+
+ paths := make([]*accessors.OSPath, 0, len(*fuse_files))
+
+ for _, filename := range *fuse_files {
+ ospath, err := accessor.ParsePath("")
+ if err != nil {
+ return fmt.Errorf("Parsing %v with accessor %v: %v",
+ filename, *fuse_zip_accessor, err)
+ }
+ ospath.SetPathSpec(
+ &accessors.PathSpec{
+ DelegatePath: filename,
+ Path: *fuse_zip_prefix,
+ })
+
+ paths = append(paths, ospath)
+ }
+
+ accessor_fs, err := fuse.NewAccessorFuseFS(
+ ctx, config_obj, accessor, paths)
+ if err != nil {
+ return err
+ }
+ defer accessor_fs.Close()
+
+ ttl := time.Duration(5 * time.Second)
+
+ opts := &fs.Options{
+ AttrTimeout: &ttl,
+ EntryTimeout: &ttl,
+ }
+
+ server, err := fs.Mount(*fuse_directory, accessor_fs, opts)
+ kingpin.FatalIfError(err, "Mounting fuse")
+
+ go func() {
+ defer cancel()
+
+ server.Wait()
+ }()
+
+ <-ctx.Done()
+
+ return nil
+}
+
+func init() {
+ command_handlers = append(command_handlers, func(command string) bool {
+ switch command {
+ case fuse_zip_command.FullCommand():
+ FatalIfError(fuse_zip_command, doFuseZip)
+
+ default:
+ return false
+ }
+ return true
+ })
+}
diff --git a/bin/query.go b/bin/query.go
index 39a0d4d0f32..89f2634d8ec 100644
--- a/bin/query.go
+++ b/bin/query.go
@@ -161,7 +161,10 @@ func doRemoteQuery(
ctx, cancel := install_sig_handler()
defer cancel()
- client, closer, err := grpc_client.Factory.GetAPIClient(ctx, config_obj)
+ // Make a remote query using the API - we better have user API
+ // credentials in the config file.
+ client, closer, err := grpc_client.Factory.GetAPIClient(
+ ctx, grpc_client.API_User, config_obj)
if err != nil {
return err
}
diff --git a/constants/constants.go b/constants/constants.go
index 993f74dbc8d..8ef85efb32c 100644
--- a/constants/constants.go
+++ b/constants/constants.go
@@ -138,7 +138,10 @@ const (
TZ = "TZ"
PinnedServerName = "VelociraptorServer"
- PinnedGwName = "GRPC_GW"
+
+ // Default gateway identity. This is only used when creating the
+ // gateway certificates.
+ PinnedGwName = "GRPC_GW"
CLIENT_API_VERSION = uint32(4)
diff --git a/datastore/remote.go b/datastore/remote.go
index 1056af62cd0..a14b55783f6 100644
--- a/datastore/remote.go
+++ b/datastore/remote.go
@@ -59,6 +59,7 @@ func Retry(ctx context.Context,
select {
case <-ctx.Done():
return timeoutError
+
case <-time.After(time.Duration(RPC_BACKOFF) * time.Second):
}
@@ -96,7 +97,9 @@ func (self *RemoteDataStore) _GetSubject(
time.Duration(RPC_TIMEOUT)*time.Second)
defer cancel()
- conn, closer, err := grpc_client.Factory.GetAPIClient(ctx, config_obj)
+ // Make the call as the superuser
+ conn, closer, err := grpc_client.Factory.GetAPIClient(ctx,
+ grpc_client.SuperUser, config_obj)
if err != nil {
return err
}
@@ -191,7 +194,9 @@ func (self *RemoteDataStore) _SetSubjectWithCompletion(
time.Duration(RPC_TIMEOUT)*time.Second)
defer cancel()
- conn, closer, err := grpc_client.Factory.GetAPIClient(ctx, config_obj)
+ // Make the call as the superuser
+ conn, closer, err := grpc_client.Factory.GetAPIClient(
+ ctx, grpc_client.SuperUser, config_obj)
defer closer()
_, err = conn.SetSubject(ctx, &api_proto.DataRequest{
@@ -225,7 +230,8 @@ func (self *RemoteDataStore) _DeleteSubjectWithCompletion(
time.Duration(RPC_TIMEOUT)*time.Second)
defer cancel()
- conn, closer, err := grpc_client.Factory.GetAPIClient(ctx, config_obj)
+ conn, closer, err := grpc_client.Factory.GetAPIClient(
+ ctx, grpc_client.SuperUser, config_obj)
defer closer()
_, err = conn.DeleteSubject(ctx, &api_proto.DataRequest{
@@ -262,7 +268,8 @@ func (self *RemoteDataStore) _DeleteSubject(
time.Duration(RPC_TIMEOUT)*time.Second)
defer cancel()
- conn, closer, err := grpc_client.Factory.GetAPIClient(ctx, config_obj)
+ conn, closer, err := grpc_client.Factory.GetAPIClient(
+ ctx, grpc_client.SuperUser, config_obj)
defer closer()
_, err = conn.DeleteSubject(ctx, &api_proto.DataRequest{
@@ -302,7 +309,8 @@ func (self *RemoteDataStore) _ListChildren(
time.Duration(RPC_TIMEOUT)*time.Second)
defer cancel()
- conn, closer, err := grpc_client.Factory.GetAPIClient(ctx, config_obj)
+ conn, closer, err := grpc_client.Factory.GetAPIClient(
+ ctx, grpc_client.SuperUser, config_obj)
defer closer()
result, err := conn.ListChildren(ctx, &api_proto.DataRequest{
diff --git a/datastore/remote_test.go b/datastore/remote_test.go
index c85543f53a6..8efe5cfbce9 100644
--- a/datastore/remote_test.go
+++ b/datastore/remote_test.go
@@ -54,7 +54,8 @@ func (self *RemoteTestSuite) SetupTest() {
self.TestSuite.SetupTest()
- grpc_client.EnsureInit(self.Ctx, self.ConfigObj, true)
+ // Reset the api clients
+ grpc_client.Factory = &grpc_client.DummyGRPCAPIClient{}
}
func (self *RemoteTestSuite) startAPIServer() {
@@ -95,6 +96,7 @@ func (self *RemoteTestSuite) TestRemoteDataStoreMissing() {
}
datastore.RPC_BACKOFF = 0
+ datastore.RPC_RETRY = 2
logging.ClearMemoryLogs()
db := datastore.NewRemoteDataStore(self.Ctx)
@@ -118,8 +120,8 @@ func (self *RemoteTestSuite) TestRemoteDataStoreMissing() {
}
}
- // We had at least 10 retries
- assert.True(self.T(), len(matches) > 10)
+ // We had at least 5 retries to the various calls
+ assert.True(self.T(), len(matches) > 5)
}
func TestRemoteTestSuite(t *testing.T) {
diff --git a/glob/fileinfo.go b/glob/fileinfo.go
index cbbbbbb84ae..6894c2831fb 100644
--- a/glob/fileinfo.go
+++ b/glob/fileinfo.go
@@ -4,6 +4,7 @@ import (
"sort"
"www.velocidex.com/golang/velociraptor/accessors"
+ "www.velocidex.com/golang/velociraptor/utils"
)
// A FileInfo that reports the globs that matched.
@@ -15,7 +16,15 @@ type GlobHit struct {
// Report all matching globs
func (self GlobHit) Globs() []string {
- return self.globs
+ ret := make([]string, 0, len(self.globs))
+
+ // Should be short so O(1) is OK
+ for _, i := range self.globs {
+ if !utils.InString(ret, i) {
+ ret = append(ret, i)
+ }
+ }
+ return ret
}
func NewGlobHit(base accessors.FileInfo, globs []string) *GlobHit {
diff --git a/go.mod b/go.mod
index 2132f6ed556..33fac9f499f 100644
--- a/go.mod
+++ b/go.mod
@@ -99,7 +99,7 @@ require (
www.velocidex.com/golang/go-prefetch v0.0.0-20220801101854-338dbe61982a
www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe
www.velocidex.com/golang/regparser v0.0.0-20221020153526-bbc758cbd18b
- www.velocidex.com/golang/vfilter v0.0.0-20240328043914-09d99b52b86f
+ www.velocidex.com/golang/vfilter v0.0.0-20240331180259-417052f5aba4
)
require (
@@ -129,6 +129,7 @@ require (
github.com/golang-jwt/jwt/v4 v4.4.3
github.com/golang/protobuf v1.5.4
github.com/gorilla/websocket v1.5.2-0.20240215025916-695e9095ce87
+ github.com/hanwen/go-fuse/v2 v2.5.1
github.com/hashicorp/go-retryablehttp v0.7.2
github.com/hillu/go-archive-zip-crypto v0.0.0-20200712202847-bd5cf365dd44
github.com/hirochachacha/go-smb2 v1.1.0
diff --git a/go.sum b/go.sum
index ae0177a2672..7ece5ef9617 100644
--- a/go.sum
+++ b/go.sum
@@ -451,6 +451,8 @@ github.com/gregjones/httpcache v0.0.0-20170920190843-316c5e0ff04e/go.mod h1:Fecb
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 h1:RtRsiaGvWxcwd8y3BiRZxsylPT8hLWZ5SPcfI+3IDNk=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0/go.mod h1:TzP6duP4Py2pHLVPPQp42aoYI92+PCrVotyR5e8Vqlk=
+github.com/hanwen/go-fuse/v2 v2.5.1 h1:OQBE8zVemSocRxA4OaFJbjJ5hlpCmIWbGr7r0M4uoQQ=
+github.com/hanwen/go-fuse/v2 v2.5.1/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -532,6 +534,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8=
github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is=
@@ -585,6 +588,8 @@ github.com/mitchellh/panicwrap v1.0.0/go.mod h1:pKvZHwWrZowLUzftuFq7coarnxbBXU4a
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=
+github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
@@ -1341,5 +1346,7 @@ www.velocidex.com/golang/regparser v0.0.0-20221020153526-bbc758cbd18b h1:NrnjFXw
www.velocidex.com/golang/regparser v0.0.0-20221020153526-bbc758cbd18b/go.mod h1:pxSECT5mWM3goJ4sxB4HCJNKnKqiAlpyT8XnvBwkLGU=
www.velocidex.com/golang/vfilter v0.0.0-20240328043914-09d99b52b86f h1:v6CyvYEK1YLTzUi23ae1Bs2fWxjzM8JBulLcpfpyH0k=
www.velocidex.com/golang/vfilter v0.0.0-20240328043914-09d99b52b86f/go.mod h1:P50KPQr2LpWVAu7ilGH8CBLBASGtOJ2971yA9YhR8rY=
+www.velocidex.com/golang/vfilter v0.0.0-20240331180259-417052f5aba4 h1:MPELjrZrnW7N0nINpVj7mBhUd4ktZASNuw1eRYDmHM4=
+www.velocidex.com/golang/vfilter v0.0.0-20240331180259-417052f5aba4/go.mod h1:P50KPQr2LpWVAu7ilGH8CBLBASGtOJ2971yA9YhR8rY=
www.velocidex.com/golang/vtypes v0.0.0-20240123105603-069d4a7f435c h1:rL/It+Ig+mvIhmy9vl5gg5b6CX2J12x0v2SXIT2RoWE=
www.velocidex.com/golang/vtypes v0.0.0-20240123105603-069d4a7f435c/go.mod h1:tjaJNlBWbvH4cEMrEu678CFR2hrtcdyPINIpRxrOh4U=
diff --git a/grpc_client/dummy.go b/grpc_client/dummy.go
new file mode 100644
index 00000000000..1f0eb3beef0
--- /dev/null
+++ b/grpc_client/dummy.go
@@ -0,0 +1,34 @@
+package grpc_client
+
+import (
+ "context"
+ "sync"
+
+ api_proto "www.velocidex.com/golang/velociraptor/api/proto"
+ config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+)
+
+type DummyGRPCAPIClient struct {
+ mu sync.Mutex
+ *GRPCAPIClient
+}
+
+func (self *DummyGRPCAPIClient) GetAPIClient(
+ ctx context.Context,
+ identity CallerIdentity,
+ config_obj *config_proto.Config) (
+ api_proto.APIClient, func() error, error) {
+ self.mu.Lock()
+ client := self.GRPCAPIClient
+ if client == nil {
+ new_client, err := NewGRPCAPIClient(config_obj)
+ if err != nil {
+ return nil, nil, err
+ }
+ self.GRPCAPIClient = new_client
+ client = new_client
+ }
+ self.mu.Unlock()
+
+ return client.GetAPIClient(ctx, identity, config_obj)
+}
diff --git a/grpc_client/grpc.go b/grpc_client/grpc.go
index 2ce2e9b8377..a1a6e2cd19d 100644
--- a/grpc_client/grpc.go
+++ b/grpc_client/grpc.go
@@ -37,15 +37,26 @@ import (
"www.velocidex.com/golang/velociraptor/utils"
)
-var (
- // Cache the creds for internal gRPC connections.
- mu sync.Mutex
- creds credentials.TransportCredentials
-
- pool *grpcpool.Pool
- address string
+// The different types of caller supported. This affects the user
+// identity we use when making the API call.
+type CallerIdentity int
+
+const (
+ // Used by the gateway proxy to call the API on behalf of the
+ // GUI. This identity causes the real user identity to be
+ // recovered from the embedded metadata.
+ GRPC_GW CallerIdentity = iota + 1
+
+ // Used by user API calls
+ API_User
+
+ // Used by the Velociraptor Server to make minion to master API
+ // calls. Implicitely trusted.
+ SuperUser
+)
- Factory APIClientFactory = GRPCAPIClient{}
+var (
+ Factory APIClientFactory = &DummyGRPCAPIClient{}
grpcCallCounter = promauto.NewCounter(prometheus.CounterOpts{
Name: "grpc_client_calls",
@@ -63,63 +74,159 @@ var (
})
)
-func getCreds(config_obj *config_proto.Config) (credentials.TransportCredentials, error) {
- if creds == nil {
- var certificate, private_key, ca_certificate, server_name string
+type gRPCPool struct {
+ // Cache the creds for internal gRPC connections.
+ mu sync.Mutex
+ creds credentials.TransportCredentials
+
+ pool *grpcpool.Pool
+ address string
+
+ config_obj *config_proto.Config
+}
+// There are three types of pools reserved for different caller
+// identities. Keeping the pools separated ensures we can never call
+// the API with the wrong identities. The server assigns different
+// permissions to the different identities.
+func NewGRPCPool(config_obj *config_proto.Config,
+ identity CallerIdentity) (*gRPCPool, error) {
+ var certificate, private_key, ca_certificate string
+
+ // Expect the server present the correct server certificate. This
+ // pins the acceptable server certificate to ensure we can not
+ // connect to the wrong server.
+ server_name := utils.GetSuperuserName(config_obj)
+
+ // Configure the credentials based on which identity is required.
+ switch identity {
+ case SuperUser:
if config_obj.Frontend != nil && config_obj.Client != nil {
+ // Present the frontend certificate as our identity. This
+ // will be implicitely trusted for every ACL.
certificate = config_obj.Frontend.Certificate
private_key = config_obj.Frontend.PrivateKey
ca_certificate = config_obj.Client.CaCertificate
- server_name = utils.GetSuperuserName(config_obj)
}
+
+ case API_User:
if config_obj.ApiConfig != nil &&
config_obj.ApiConfig.ClientCert != "" {
+ // For an API connection, present the API certificate to
+ // connect with.
certificate = config_obj.ApiConfig.ClientCert
private_key = config_obj.ApiConfig.ClientPrivateKey
ca_certificate = config_obj.ApiConfig.CaCertificate
- server_name = utils.GetSuperuserGWName(config_obj)
- }
-
- if certificate == "" {
- return nil, errors.New("Unable to load api certificate")
}
- // We use the Frontend's certificate because this connection
- // represents an internal connection.
- cert, err := tls.X509KeyPair(
- []byte(certificate),
- []byte(private_key))
- if err != nil {
- return nil, err
+ case GRPC_GW:
+ if config_obj.GUI != nil &&
+ config_obj.GUI.GwCertificate != "" &&
+ config_obj.Client != nil {
+ // For an API connection, present the API certificate to
+ // connect with.
+ certificate = config_obj.GUI.GwCertificate
+ private_key = config_obj.GUI.GwPrivateKey
+ ca_certificate = config_obj.Client.CaCertificate
}
+ }
- // The server cert must be signed by our CA.
- CA_Pool := x509.NewCertPool()
- CA_Pool.AppendCertsFromPEM([]byte(ca_certificate))
+ // Identity not configured - This is not really an error but we
+ // wont be able to make calls using this identity.
+ if certificate == "" {
+ return nil, nil
+ }
- creds = credentials.NewTLS(&tls.Config{
- Certificates: []tls.Certificate{cert},
- RootCAs: CA_Pool,
- ServerName: server_name,
- })
+ // We use the Frontend's certificate because this connection
+ // represents an internal connection.
+ cert, err := tls.X509KeyPair(
+ []byte(certificate),
+ []byte(private_key))
+ if err != nil {
+ // This is a critical error - the certs are broken
+ return nil, err
}
- return creds, nil
+ // The server cert must be signed by our CA.
+ CA_Pool := x509.NewCertPool()
+ CA_Pool.AppendCertsFromPEM([]byte(ca_certificate))
+
+ creds := credentials.NewTLS(&tls.Config{
+ Certificates: []tls.Certificate{cert},
+ RootCAs: CA_Pool,
+ ServerName: server_name,
+ })
+
+ // Do not create the pool until first call
+ return &gRPCPool{
+ creds: creds,
+ address: GetAPIConnectionString(config_obj),
+ config_obj: config_obj,
+ }, nil
}
type APIClientFactory interface {
- GetAPIClient(ctx context.Context,
- config_obj *config_proto.Config) (api_proto.APIClient, func() error, error)
+ GetAPIClient(
+ ctx context.Context,
+ identity CallerIdentity,
+ config_obj *config_proto.Config) (
+ api_proto.APIClient, func() error, error)
+}
+
+// Maintain separate pools for different identities.
+type GRPCAPIClient struct {
+ GRPC_GW, API_User, SuperUser *gRPCPool
}
-type GRPCAPIClient struct{}
+func NewGRPCAPIClient(config_obj *config_proto.Config) (*GRPCAPIClient, error) {
+ res := &GRPCAPIClient{}
+ var err error
+
+ res.GRPC_GW, err = NewGRPCPool(config_obj, GRPC_GW)
+ if err != nil {
+ return nil, err
+ }
+ res.API_User, err = NewGRPCPool(config_obj, API_User)
+ if err != nil {
+ return nil, err
+ }
+
+ res.SuperUser, err = NewGRPCPool(config_obj, SuperUser)
+ if err != nil {
+ return nil, err
+ }
+
+ return res, nil
+}
func (self GRPCAPIClient) GetAPIClient(
- ctx context.Context,
+ ctx context.Context, identity CallerIdentity,
config_obj *config_proto.Config) (
api_proto.APIClient, func() error, error) {
- channel, err := getChannel(ctx, config_obj)
+
+ var pool *gRPCPool
+ switch identity {
+ case GRPC_GW:
+ pool = self.GRPC_GW
+ if pool == nil {
+ return nil, nil, errors.New(
+ "No gateway identity configured (GUI.gw_certificate)")
+ }
+ case API_User:
+ pool = self.API_User
+ if pool == nil {
+ return nil, nil, errors.New(
+ "No API user identity configured (API.client_cert)")
+ }
+ case SuperUser:
+ pool = self.SuperUser
+ if pool == nil {
+ return nil, nil, errors.New(
+ "No server identity configured (Frontend.certificate)")
+ }
+ }
+
+ channel, err := pool.getChannel(ctx)
if err != nil {
return nil, nil, err
}
@@ -129,9 +236,7 @@ func (self GRPCAPIClient) GetAPIClient(
return api_proto.NewAPIClient(channel.ClientConn), channel.Close, err
}
-func getChannel(
- ctx context.Context,
- config_obj *config_proto.Config) (*grpcpool.ClientConn, error) {
+func (self *gRPCPool) getChannel(ctx context.Context) (*grpcpool.ClientConn, error) {
// Collect number of callers waiting for a channel - this
// indicates backpressure from the grpc pool.
@@ -139,20 +244,20 @@ func getChannel(
defer grpcPoolWaiters.Dec()
// Make sure pool is initialized.
- err := EnsureInit(ctx, config_obj, false /* recreate */)
+ err := self.EnsureInit(ctx, false /* recreate */)
if err != nil {
return nil, err
}
for {
- conn, err := pool.Get(ctx)
+ conn, err := self.pool.Get(ctx)
if err == grpcpool.ErrTimeout {
grpcTimeoutCounter.Inc()
time.Sleep(time.Second)
// Try to force a new connection pool in case the master
// changed it's DNS mapping.
- err := EnsureInit(ctx, config_obj, true /* recreate */)
+ err := self.EnsureInit(ctx, true /* recreate */)
if err != nil {
return nil, err
}
@@ -163,6 +268,7 @@ func getChannel(
}
}
+// Figure out the correct API connection string from the config
func GetAPIConnectionString(config_obj *config_proto.Config) string {
if config_obj.ApiConfig != nil && config_obj.ApiConfig.ApiConnectionString != "" {
return config_obj.ApiConfig.ApiConnectionString
@@ -190,46 +296,40 @@ func GetAPIConnectionString(config_obj *config_proto.Config) string {
panic("Unknown API.BindScheme")
}
-func EnsureInit(
- ctx context.Context,
- config_obj *config_proto.Config,
- recreate bool) error {
+// Make sure the pool is established and running.
+func (self *gRPCPool) EnsureInit(
+ ctx context.Context, recreate bool) (err error) {
- mu.Lock()
- defer mu.Unlock()
+ self.mu.Lock()
+ defer self.mu.Unlock()
- if !recreate && pool != nil {
+ if !recreate && self.pool != nil {
return nil
}
- address = GetAPIConnectionString(config_obj)
- creds, err := getCreds(config_obj)
- if err != nil {
- return err
- }
-
+ // Build a new pool.
factory := func(ctx context.Context) (*grpc.ClientConn, error) {
- return grpc.DialContext(ctx, address,
- grpc.WithTransportCredentials(creds))
+ return grpc.DialContext(ctx, self.address,
+ grpc.WithTransportCredentials(self.creds))
}
max_size := 100
max_wait := 60
- if config_obj.Frontend != nil {
- if config_obj.Frontend.GRPCPoolMaxSize > 0 {
- max_size = int(config_obj.Frontend.GRPCPoolMaxSize)
+ if self.config_obj.Frontend != nil {
+ if self.config_obj.Frontend.GRPCPoolMaxSize > 0 {
+ max_size = int(self.config_obj.Frontend.GRPCPoolMaxSize)
}
- if config_obj.Frontend.GRPCPoolMaxWait > 0 {
- max_wait = int(config_obj.Frontend.GRPCPoolMaxWait)
+ if self.config_obj.Frontend.GRPCPoolMaxWait > 0 {
+ max_wait = int(self.config_obj.Frontend.GRPCPoolMaxWait)
}
}
- pool, err = grpcpool.NewWithContext(ctx,
+ self.pool, err = grpcpool.NewWithContext(ctx,
factory, 1, max_size, time.Duration(max_wait)*time.Second)
if err != nil {
return fmt.Errorf(
- "Unable to connect to gRPC server: %v: %v", address, err)
+ "Unable to connect to gRPC server: %v: %v", self.address, err)
}
return nil
}
diff --git a/services/frontend/frontend.go b/services/frontend/frontend.go
index 54fe7fc3ae8..d0f2e7251bb 100644
--- a/services/frontend/frontend.go
+++ b/services/frontend/frontend.go
@@ -378,7 +378,9 @@ func (self MinionFrontendManager) IsMaster() bool {
// The minion frontend replicates to the master node.
func (self MinionFrontendManager) GetMasterAPIClient(ctx context.Context) (
api_proto.APIClient, func() error, error) {
- client, closer, err := grpc_client.Factory.GetAPIClient(ctx, self.config_obj)
+ // Connect as the SuperUser
+ client, closer, err := grpc_client.Factory.GetAPIClient(
+ ctx, grpc_client.SuperUser, self.config_obj)
if err != nil {
return nil, nil, err
}
diff --git a/services/repository/plugin.go b/services/repository/plugin.go
index 8efd7a380c9..f337cadb3f0 100644
--- a/services/repository/plugin.go
+++ b/services/repository/plugin.go
@@ -263,6 +263,10 @@ func (self *ArtifactRepositoryPlugin) copyScope(
scope vfilter.Scope, my_name string) (
vfilter.Scope, error) {
env := ordereddict.NewDict()
+
+ // TODO: Move most of these to the scope context as they dont
+ // change with subscopes so it should be faster to get them from
+ // the context.
for _, field := range []string{
vql_subsystem.ACL_MANAGER_VAR,
constants.SCOPE_MOCK,
@@ -300,15 +304,10 @@ func (self *ArtifactRepositoryPlugin) copyScope(
result.ClearContext()
result.AppendVars(env)
- // Copy the scope cache from the caller
- cache, pres := scope.GetContext(vql_subsystem.CACHE_VAR)
- if pres {
- result.SetContext(vql_subsystem.CACHE_VAR, cache)
- }
-
// Copy critical context variables
for _, field := range []string{
constants.SCOPE_RESPONDER_CONTEXT,
+ vql_subsystem.CACHE_VAR,
} {
value, pres := scope.GetContext(field)
if pres {
diff --git a/services/repository/scope.go b/services/repository/scope.go
index 503b7c839f4..d8ae9116e7d 100644
--- a/services/repository/scope.go
+++ b/services/repository/scope.go
@@ -51,7 +51,7 @@ func _build(self services.ScopeBuilder, from_scratch bool) vfilter.Scope {
scope.SetLogger(self.Logger)
- // Make a new fresh context.
+ // Make a new fresh cache context.
scope.SetContext(vql_subsystem.CACHE_VAR, vql_subsystem.NewScopeCache())
device_manager := accessors.GetDefaultDeviceManager(
diff --git a/services/sanity/api.go b/services/sanity/api.go
new file mode 100644
index 00000000000..ea4b48d511a
--- /dev/null
+++ b/services/sanity/api.go
@@ -0,0 +1,29 @@
+package sanity
+
+import (
+ "fmt"
+
+ config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+ crypto_utils "www.velocidex.com/golang/velociraptor/crypto/utils"
+)
+
+func (self *SanityChecks) CheckAPISettings(
+ config_obj *config_proto.Config) error {
+
+ // Make sure to fill in the pinned gateway name from the gateway's
+ // certificate.
+ if config_obj.GUI != nil &&
+ config_obj.GUI.GwCertificate != "" {
+ cert, err := crypto_utils.ParseX509CertFromPemStr([]byte(config_obj.GUI.GwCertificate))
+ if err != nil {
+ return fmt.Errorf("CheckAPISettings: While parsing GUI.gw_certificate: %w", err)
+ }
+
+ if config_obj.API == nil {
+ config_obj.API = &config_proto.APIConfig{}
+ }
+
+ config_obj.API.PinnedGwName = crypto_utils.GetSubjectName(cert)
+ }
+ return nil
+}
diff --git a/services/sanity/sanity.go b/services/sanity/sanity.go
index d8268580ec1..6683c6e7f6b 100644
--- a/services/sanity/sanity.go
+++ b/services/sanity/sanity.go
@@ -71,6 +71,11 @@ func (self *SanityChecks) CheckRootOrg(
return err
}
+ err = self.CheckAPISettings(config_obj)
+ if err != nil {
+ return err
+ }
+
// Make sure our internal VelociraptorServer service account is
// properly created. Default accounts are created with org admin
// so they can add new orgs as required.
diff --git a/services/users/add_org.go b/services/users/add_org.go
index 79b275cdd7d..5845690a864 100644
--- a/services/users/add_org.go
+++ b/services/users/add_org.go
@@ -26,7 +26,7 @@ func (self *UserManager) AddUserToOrg(
principal, username string,
orgs []string, policy *acl_proto.ApiClientACL) error {
- err := validateUsername(self.config_obj, username)
+ err := ValidateUsername(self.config_obj, username)
if err != nil {
return err
}
diff --git a/services/users/delete.go b/services/users/delete.go
index 48e0ffbd151..4d81420b512 100644
--- a/services/users/delete.go
+++ b/services/users/delete.go
@@ -16,7 +16,7 @@ func (self *UserManager) DeleteUser(
principal, username string,
orgs []string) error {
- err := validateUsername(self.config_obj, username)
+ err := ValidateUsername(self.config_obj, username)
if err != nil {
return err
}
diff --git a/services/users/grpc.go b/services/users/grpc.go
index 3d8f2e604da..5a1e7ccc647 100644
--- a/services/users/grpc.go
+++ b/services/users/grpc.go
@@ -23,10 +23,8 @@ func (self UserManager) GetUserFromContext(ctx context.Context) (
grpc_user_info := GetGRPCUserInfo(self.config_obj, ctx, self.ca_pool)
- // This is not a real user but represents the grpc gateway
- // connection - it is always allowed.
- if grpc_user_info.Name == utils.GetSuperuserName(org_config_obj) ||
- grpc_user_info.Name == utils.GetSuperuserGWName(org_config_obj) {
+ // If the call comes from the super user we allow it.
+ if grpc_user_info.Name == utils.GetSuperuserName(org_config_obj) {
user_record = &api_proto.VelociraptorUser{
Name: grpc_user_info.Name,
}
@@ -110,7 +108,7 @@ func GetGRPCUserInfo(
// convert web side authentication to a valid
// user name which it may pass in the call
// context.
- if result.Name == config_obj.API.PinnedGwName {
+ if result.Name == utils.GetGatewayName(config_obj) {
md, ok := metadata.FromIncomingContext(ctx)
if ok {
userinfo := md.Get("USER")
diff --git a/services/users/set_user.go b/services/users/set_user.go
index 8a97021697b..bb8877427a3 100644
--- a/services/users/set_user.go
+++ b/services/users/set_user.go
@@ -30,7 +30,7 @@ func (self *UserManager) SetUserPassword(
principal, username string,
password, current_org string) error {
- err := validateUsername(config_obj, username)
+ err := ValidateUsername(config_obj, username)
if err != nil {
return err
}
diff --git a/services/users/storage.go b/services/users/storage.go
index 772aef588ef..418899ef2c8 100644
--- a/services/users/storage.go
+++ b/services/users/storage.go
@@ -129,7 +129,7 @@ func (self *UserStorageManager) GetUserWithHashes(ctx context.Context, username
cache = &_CachedUserObject{}
}
- err = validateUsername(self.config_obj, username)
+ err = ValidateUsername(self.config_obj, username)
if err != nil {
return nil, err
}
@@ -172,7 +172,7 @@ func (self *UserStorageManager) SetUser(
return errors.New("Must set a username")
}
- err := validateUsername(self.config_obj, user_record.Name)
+ err := ValidateUsername(self.config_obj, user_record.Name)
if err != nil {
return err
}
diff --git a/services/users/users.go b/services/users/users.go
index 5ada640abdf..0dfcc6ad1bc 100644
--- a/services/users/users.go
+++ b/services/users/users.go
@@ -98,7 +98,9 @@ type UserManager struct {
storage IUserStorageManager
}
-func validateUsername(config_obj *config_proto.Config, name string) error {
+// Prevent certificates from being minted for critical privileged
+// accounts.
+func ValidateUsername(config_obj *config_proto.Config, name string) error {
if !validUsernameRegEx.MatchString(name) {
return fmt.Errorf("Unacceptable username %v", name)
}
@@ -112,8 +114,8 @@ func validateUsername(config_obj *config_proto.Config, name string) error {
return fmt.Errorf("Username is reserved: %v", name)
}
- if name == utils.GetSuperuserGWName(config_obj) {
- return fmt.Errorf("Username is reserved: %v", name)
+ if name == utils.GetGatewayName(config_obj) {
+ return fmt.Errorf("Username is reserved for the gateway: %v", name)
}
return nil
@@ -121,7 +123,7 @@ func validateUsername(config_obj *config_proto.Config, name string) error {
func NewUserRecord(config_obj *config_proto.Config,
name string) (*api_proto.VelociraptorUser, error) {
- err := validateUsername(config_obj, name)
+ err := ValidateUsername(config_obj, name)
if err != nil {
return nil, err
}
diff --git a/tools/fuse/accessors.go b/tools/fuse/accessors.go
new file mode 100644
index 00000000000..893770b14dd
--- /dev/null
+++ b/tools/fuse/accessors.go
@@ -0,0 +1,107 @@
+//go:build !windows
+// +build !windows
+
+package fuse
+
+import (
+ "context"
+
+ "github.com/hanwen/go-fuse/v2/fs"
+ "github.com/hanwen/go-fuse/v2/fuse"
+ "www.velocidex.com/golang/velociraptor/accessors"
+ config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+ "www.velocidex.com/golang/velociraptor/logging"
+ "www.velocidex.com/golang/vfilter"
+)
+
+type AccessorFuseFS struct {
+ fs.Inode
+
+ config_obj *config_proto.Config
+
+ scope vfilter.Scope
+ accessor accessors.FileSystemAccessor
+ containers []*accessors.OSPath
+
+ file_count int
+}
+
+func (self *AccessorFuseFS) Close() {
+ logger := logging.GetLogger(self.config_obj, &logging.ToolComponent)
+ logger.Info("Fuse: Existing! Dont forget to unmount the filesystem")
+}
+
+func (self *AccessorFuseFS) add(
+ ctx context.Context,
+ accessor accessors.FileSystemAccessor,
+ ospath *accessors.OSPath,
+ node *fs.Inode) error {
+
+ children, err := accessor.ReadDirWithOSPath(ospath)
+ if err != nil {
+ return err
+ }
+
+ for _, child := range children {
+ child_ospath := child.OSPath()
+ basename := child_ospath.Basename()
+ if child.IsDir() {
+ // Check if there is a directory node already
+ child_node := node.GetChild(basename)
+ if child_node == nil {
+ child_node = node.NewPersistentInode(ctx, &fs.Inode{},
+ fs.StableAttr{Mode: fuse.S_IFDIR})
+ node.AddChild(basename, child_node, true)
+ }
+
+ err := self.add(ctx, accessor, child.OSPath(), child_node)
+ if err != nil {
+ return err
+ }
+
+ } else {
+ child_node := node.NewPersistentInode(
+ ctx, &FileNode{
+ accessor: accessor,
+ ospath: child_ospath,
+ }, fs.StableAttr{})
+ node.AddChild(basename, child_node, true)
+ self.file_count++
+ }
+ }
+ return nil
+}
+
+// Initialize the filesystem by scanning all the containers.
+func (self *AccessorFuseFS) OnAdd(ctx context.Context) {
+ logger := logging.GetLogger(self.config_obj, &logging.ToolComponent)
+
+ for _, filename := range self.containers {
+ start := self.file_count
+ err := self.add(ctx, self.accessor, filename, &self.Inode)
+ if err != nil {
+ logger.Error("Fuse: Unable to load from %v: %v",
+ filename.DelegatePath(), err)
+ } else {
+ logger.Info("Fuse: Loaded %v files from %v",
+ self.file_count-start, filename.DelegatePath())
+ }
+ }
+
+}
+
+func NewAccessorFuseFS(
+ ctx context.Context,
+ config_obj *config_proto.Config,
+ accessor accessors.FileSystemAccessor,
+ files []*accessors.OSPath) (*AccessorFuseFS, error) {
+
+ fs := &AccessorFuseFS{
+ containers: files,
+ accessor: accessor,
+ config_obj: config_obj,
+ }
+ return fs, nil
+}
+
+var _ = (fs.NodeOnAdder)((*AccessorFuseFS)(nil))
diff --git a/tools/fuse/nodes.go b/tools/fuse/nodes.go
new file mode 100644
index 00000000000..d5f20b28f53
--- /dev/null
+++ b/tools/fuse/nodes.go
@@ -0,0 +1,84 @@
+//go:build !windows
+// +build !windows
+
+package fuse
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "os"
+ "syscall"
+
+ "github.com/hanwen/go-fuse/v2/fs"
+ "github.com/hanwen/go-fuse/v2/fuse"
+ "www.velocidex.com/golang/velociraptor/accessors"
+)
+
+// Build the directory tree
+type FileNode struct {
+ fs.Inode
+
+ accessor accessors.FileSystemAccessor
+ ospath *accessors.OSPath
+}
+
+func (self FileNode) Getattr(ctx context.Context,
+ f fs.FileHandle, out *fuse.AttrOut) syscall.Errno {
+
+ stat, err := self.accessor.LstatWithOSPath(self.ospath)
+ if err != nil {
+ return syscall.EIO
+ }
+
+ out.Mode = uint32(0644)
+ out.Nlink = 1
+ out.Mtime = uint64(stat.Mtime().Unix())
+ out.Atime = out.Mtime
+ out.Ctime = out.Mtime
+ out.Size = uint64(stat.Size())
+
+ const bs = 512
+ out.Blksize = bs
+ out.Blocks = (out.Size + bs - 1) / bs
+
+ return 0
+}
+
+func (self *FileNode) Open(
+ ctx context.Context, flags uint32) (
+ fs.FileHandle, uint32, syscall.Errno) {
+
+ // We don't return a filehandle since we don't really need
+ // one. The file content is immutable, so hint the kernel to
+ // cache the data.
+ return nil, fuse.FOPEN_KEEP_CACHE, 0
+}
+
+func (self *FileNode) Read(ctx context.Context,
+ f fs.FileHandle, dest []byte, off int64) (
+ fuse.ReadResult, syscall.Errno) {
+
+ fd, err := self.accessor.OpenWithOSPath(self.ospath)
+ if err != nil {
+ return nil, syscall.EIO
+ }
+ defer fd.Close()
+
+ _, err = fd.Seek(off, os.SEEK_SET)
+ if err != nil {
+ return nil, syscall.EIO
+ }
+
+ n, err := fd.Read(dest)
+ if err != nil && err != io.EOF {
+ fmt.Printf("ERROR: While opening %v: %v\n",
+ self.ospath.String(), err)
+ return nil, syscall.EIO
+ }
+
+ return fuse.ReadResultData(dest[:n]), 0
+}
+
+var _ = (fs.NodeOpener)((*FileNode)(nil))
+var _ = (fs.NodeGetattrer)((*FileNode)(nil))
diff --git a/utils/errors.go b/utils/errors.go
index 8279c4f98a3..d0ecc8f91c9 100644
--- a/utils/errors.go
+++ b/utils/errors.go
@@ -15,6 +15,7 @@ var (
TypeError = errors.New("TypeError")
NotImplementedError = errors.New("Not implemented")
InvalidConfigError = errors.New("InvalidConfigError")
+ NotFoundError = errors.New("NotFoundError")
)
// This is a custom error type that wraps an inner error but does not
diff --git a/utils/users.go b/utils/users.go
index 806c2e5ebaa..32907e988b9 100644
--- a/utils/users.go
+++ b/utils/users.go
@@ -16,7 +16,9 @@ func GetSuperuserName(
return config_obj.Client.PinnedServerName
}
-func GetSuperuserGWName(
+// The name of the gateway certificate. This is specified in the
+// GUI.gw_certificate and is populated by the sanity service.
+func GetGatewayName(
config_obj *config_proto.Config) string {
if config_obj == nil ||
config_obj.API == nil ||
diff --git a/vql/filesystem/filesystem.go b/vql/filesystem/filesystem.go
index 7f9c205b104..838fedf67ed 100644
--- a/vql/filesystem/filesystem.go
+++ b/vql/filesystem/filesystem.go
@@ -1,19 +1,19 @@
/*
- Velociraptor - Dig Deeper
- Copyright (C) 2019-2024 Rapid7 Inc.
+Velociraptor - Dig Deeper
+Copyright (C) 2019-2024 Rapid7 Inc.
- This program is free software: you can redistribute it and/or modify
- it under the terms of the GNU Affero General Public License as published
- by the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published
+by the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Affero General Public License for more details.
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
- You should have received a copy of the GNU Affero General Public License
- along with this program. If not, see .
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see .
*/
package filesystem
@@ -437,6 +437,53 @@ func (self StatPlugin) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) *vfi
}
}
+type StatFunction struct{}
+
+func (self *StatFunction) Call(
+ ctx context.Context,
+ scope vfilter.Scope,
+ args *ordereddict.Dict) vfilter.Any {
+
+ arg := &StatArgs{}
+ err := arg_parser.ExtractArgsWithContext(ctx, scope, args, arg)
+ if err != nil {
+ scope.Log("stat: %s", err.Error())
+ return vfilter.Null{}
+ }
+
+ err = vql_subsystem.CheckFilesystemAccess(scope, arg.Accessor)
+ if err != nil {
+ scope.Log("stat: %s", err.Error())
+ return vfilter.Null{}
+ }
+
+ accessor, err := accessors.GetAccessor(arg.Accessor, scope)
+ if err != nil {
+ scope.Log("stat: %s", err.Error())
+ return vfilter.Null{}
+ }
+
+ f, err := accessor.LstatWithOSPath(arg.Filename)
+ if err != nil {
+ return vfilter.Null{}
+ }
+
+ return f
+}
+
+func (self StatFunction) Name() string {
+ return "stat"
+}
+
+func (self StatFunction) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.FunctionInfo {
+ return &vfilter.FunctionInfo{
+ Name: "stat",
+ Doc: "Get file information. Unlike glob() this does not support wildcards.",
+ ArgType: type_map.AddType(scope, &StatArgs{}),
+ Metadata: vql.VQLMetadata().Permissions(acls.FILESYSTEM_READ).Build(),
+ }
+}
+
func init() {
vql_subsystem.RegisterPlugin(&GlobPlugin{})
vql_subsystem.RegisterPlugin(&ReadFilePlugin{})
@@ -459,4 +506,5 @@ func init() {
})
vql_subsystem.RegisterPlugin(&StatPlugin{})
vql_subsystem.RegisterFunction(&ReadFileFunction{})
+ vql_subsystem.RegisterFunction(&StatFunction{})
}
diff --git a/vql/filesystem/raw_registry.go b/vql/filesystem/raw_registry.go
index 7517d140a87..5ef184efda7 100644
--- a/vql/filesystem/raw_registry.go
+++ b/vql/filesystem/raw_registry.go
@@ -69,7 +69,11 @@ func (self ReadKeyValues) Call(
if value_data != nil {
value, pres := value_data.Get("value")
if pres {
- result.Set(item.Name(), value)
+ name := item.Name()
+ if name == "" {
+ name = "@"
+ }
+ result.Set(name, value)
}
}
}
diff --git a/vql/functions/time.go b/vql/functions/time.go
index 6743f6aa10c..df9c994f3f2 100644
--- a/vql/functions/time.go
+++ b/vql/functions/time.go
@@ -160,15 +160,15 @@ func (self _Timestamp) Call(ctx context.Context, scope vfilter.Scope,
}
if arg.CocoaTime > 0 {
- return time.Unix((arg.CocoaTime + 978307200), 0)
+ return time.Unix((arg.CocoaTime + 978307200), 0).UTC()
}
if arg.MacTime > 0 {
- return time.Unix((arg.MacTime - 2082844800), 0)
+ return time.Unix((arg.MacTime - 2082844800), 0).UTC()
}
if arg.WinFileTime > 0 {
- return time.Unix((arg.WinFileTime/10000000)-11644473600, 0)
+ return time.Unix((arg.WinFileTime/10000000)-11644473600, 0).UTC()
}
if arg.String != "" {
@@ -183,7 +183,7 @@ func (self _Timestamp) Call(ctx context.Context, scope vfilter.Scope,
if err != nil {
return vfilter.Null{}
}
- return result
+ return result.UTC()
}
}
@@ -192,7 +192,7 @@ func (self _Timestamp) Call(ctx context.Context, scope vfilter.Scope,
return vfilter.Null{}
}
- return result
+ return result.UTC()
}
func TimeFromAny(ctx context.Context,
diff --git a/vql/grouper/mergegrouper.go b/vql/grouper/mergegrouper.go
index 642ca4d501d..e731430eec6 100644
--- a/vql/grouper/mergegrouper.go
+++ b/vql/grouper/mergegrouper.go
@@ -40,16 +40,15 @@ type AggregateContext struct {
}
/*
- This is a memory efficient grouper with a contrained upper bound on
- memory consumption.
-
- 1. Grouped by rows are grouped into bins with a constant group key
- 2. When the number of bins exceeds the chunk size, we:
- 3. Sort the bins by the group key and then serialized the bins into a tmp file.
- 4. When the query is finished we perform a merge-sort on the resulting files:
- 5. Reading the bins from various files by order of the group key, we
- can group duplicate bins from each file.
-
+This is a memory efficient grouper with a contrained upper bound on
+memory consumption.
+
+ 1. Grouped by rows are grouped into bins with a constant group key
+ 2. When the number of bins exceeds the chunk size, we:
+ 3. Sort the bins by the group key and then serialized the bins into a tmp file.
+ 4. When the query is finished we perform a merge-sort on the resulting files:
+ 5. Reading the bins from various files by order of the group key, we
+ can group duplicate bins from each file.
*/
type MergeSortGrouper struct {
ChunkSize int
@@ -173,7 +172,8 @@ func (self *MergeSortGrouper) transformRow(
// mask original row (from plugin).
new_scope.AppendVars(row)
new_scope.AppendVars(transformed_row)
- new_scope.SetContextDict(context)
+ new_scope.SetContext(
+ types.AGGREGATOR_CONTEXT_TAG, context)
return actor.MaterializeRow(ctx, transformed_row, new_scope)
}
@@ -206,7 +206,9 @@ func (self *MergeSortGrouper) Group(
// The transform function receives its own unique context
// for the specific aggregate group.
- new_scope.SetContextDict(aggregate_ctx.context)
+ new_scope.SetContext(
+ types.AGGREGATOR_CONTEXT_TAG,
+ aggregate_ctx.context)
// Update the row with the transformed columns. Note we
// must materialize these rows because evaluating the row
diff --git a/vql/sigma/evaluator/modifiers/vql.go b/vql/sigma/evaluator/modifiers/vql.go
index a02103603de..07a261befd3 100644
--- a/vql/sigma/evaluator/modifiers/vql.go
+++ b/vql/sigma/evaluator/modifiers/vql.go
@@ -3,6 +3,7 @@ package modifiers
import (
"context"
"errors"
+ "sync"
"www.velocidex.com/golang/velociraptor/utils"
vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
@@ -14,7 +15,31 @@ const (
SIGMA_VQL_TAG = "_SIGMA_VQL"
)
-type LambdaCache map[string]*vfilter.Lambda
+type LambdaCache struct {
+ mu sync.Mutex
+ cache map[string]*vfilter.Lambda
+}
+
+func NewLambdaCache() *LambdaCache {
+ return &LambdaCache{
+ cache: make(map[string]*vfilter.Lambda),
+ }
+}
+
+func (self *LambdaCache) Get(in string) (*vfilter.Lambda, bool) {
+ self.mu.Lock()
+ defer self.mu.Unlock()
+
+ res, ok := self.cache[in]
+ return res, ok
+}
+
+func (self *LambdaCache) Set(in string, l *vfilter.Lambda) {
+ self.mu.Lock()
+ defer self.mu.Unlock()
+
+ self.cache[in] = l
+}
type vql struct{}
@@ -30,22 +55,22 @@ func (self vql) Matches(
lambda_cache_any := vql_subsystem.CacheGet(scope, SIGMA_VQL_TAG)
if utils.IsNil(lambda_cache_any) {
- lambda_cache_any = make(LambdaCache)
+ lambda_cache_any = NewLambdaCache()
vql_subsystem.CacheSet(scope, SIGMA_VQL_TAG, lambda_cache_any)
}
- lambda_cache, ok := lambda_cache_any.(LambdaCache)
+ lambda_cache, ok := lambda_cache_any.(*LambdaCache)
if !ok {
return false, errors.New("LambdaCache is incorrect")
}
- lambda, pres := lambda_cache[expected_str]
+ lambda, pres := lambda_cache.Get(expected_str)
if !pres {
lambda, err = vfilter.ParseLambda(expected_str)
if err != nil {
return false, err
}
- lambda_cache[expected_str] = lambda
+ lambda_cache.Set(expected_str, lambda)
}
return scope.Bool(
lambda.Reduce(ctx, scope, []types.Any{actual})), nil