diff --git a/accessors/api.go b/accessors/api.go index c76e2d1887d..44f917629c3 100644 --- a/accessors/api.go +++ b/accessors/api.go @@ -9,6 +9,7 @@ import ( "time" "github.com/Velocidex/ordereddict" + errors "github.com/go-errors/errors" "www.velocidex.com/golang/velociraptor/json" "www.velocidex.com/golang/velociraptor/utils" "www.velocidex.com/golang/vfilter" @@ -320,3 +321,31 @@ type FileSystemAccessor interface { LstatWithOSPath(path *OSPath) (FileInfo, error) New(scope vfilter.Scope) (FileSystemAccessor, error) } + +// Some filesystems can attempt to retrieve the underlying file. If +// this interface exists on the accessor **and** the +// GetUnderlyingAPIFilename() call succeeds, then it should be +// possible to directly access the returned filename using the OS +// APIs. +type RawFileAPIAccessor interface { + GetUnderlyingAPIFilename(path *OSPath) (string, error) +} + +var ( + NotRawFileSystem = errors.New("NotRawFileSystem") +) + +func GetUnderlyingAPIFilename(accessor string, + scope vfilter.Scope, path *OSPath) (string, error) { + accessor_obj, err := GetAccessor(accessor, scope) + if err != nil { + return "", err + } + + raw_accessor, ok := accessor_obj.(RawFileAPIAccessor) + if !ok { + return "", NotRawFileSystem + } + + return raw_accessor.GetUnderlyingAPIFilename(path) +} diff --git a/accessors/file/accessor_common.go b/accessors/file/accessor_common.go index 66fb4e41e1f..70b77d1c331 100644 --- a/accessors/file/accessor_common.go +++ b/accessors/file/accessor_common.go @@ -293,6 +293,11 @@ func (self OSFileSystemAccessor) ReadDir(dir string) ([]accessors.FileInfo, erro return self.ReadDirWithOSPath(full_path) } +func (self *OSFileSystemAccessor) GetUnderlyingAPIFilename( + full_path *accessors.OSPath) (string, error) { + return full_path.PathSpec().Path, nil +} + func (self OSFileSystemAccessor) ReadDirWithOSPath( full_path *accessors.OSPath) ([]accessors.FileInfo, error) { dir := full_path.PathSpec().Path diff --git a/accessors/file/auto_windows.go b/accessors/file/auto_windows.go index 65d32ba7c91..4eb77cb3582 100644 --- a/accessors/file/auto_windows.go +++ b/accessors/file/auto_windows.go @@ -135,6 +135,11 @@ func (self AutoFilesystemAccessor) New(scope vfilter.Scope) (accessors.FileSyste }, nil } +func (self *AutoFilesystemAccessor) GetUnderlyingAPIFilename( + full_path *accessors.OSPath) (string, error) { + return full_path.PathSpec().Path, nil +} + func (self *AutoFilesystemAccessor) ReadDirWithOSPath( path *accessors.OSPath) ([]accessors.FileInfo, error) { result, err := self.file_delegate.ReadDirWithOSPath(path) diff --git a/accessors/file/os_windows.go b/accessors/file/os_windows.go index 09f4990bc5c..28563042e35 100644 --- a/accessors/file/os_windows.go +++ b/accessors/file/os_windows.go @@ -151,6 +151,11 @@ func (self OSFileSystemAccessor) New(scope vfilter.Scope) ( return result, nil } +func (self *OSFileSystemAccessor) GetUnderlyingAPIFilename( + full_path *accessors.OSPath) (string, error) { + return full_path.PathSpec().Path, nil +} + func discoverDriveLetters() ([]accessors.FileInfo, error) { result := []accessors.FileInfo{} diff --git a/artifacts/testdata/server/testcases/remapping.out.yaml b/artifacts/testdata/server/testcases/remapping.out.yaml index 64ca334d4c1..7ac25297e4d 100644 --- a/artifacts/testdata/server/testcases/remapping.out.yaml +++ b/artifacts/testdata/server/testcases/remapping.out.yaml @@ -171,6 +171,7 @@ LET _ <= remap(config=format(format=RemappingTemplate, args=[ srcDir+'/artifacts ]SELECT * FROM parse_ntfs_i30( accessor='ntfs', device='c:/$MFT', inode="41-144-1")[ { "MFTId": "45", + "SequenceNumber": 0, "Mtime": "2018-09-24T07:55:44.4592119Z", "Atime": "2022-03-18T04:09:07.2885951Z", "Ctime": "2018-09-24T07:55:20.6489276Z", diff --git a/go.mod b/go.mod index 908e40e48b7..997eb833043 100644 --- a/go.mod +++ b/go.mod @@ -97,7 +97,7 @@ require ( howett.net/plist v1.0.0 www.velocidex.com/golang/evtx v0.2.1-0.20220404133451-1fdf8be7325e www.velocidex.com/golang/go-ese v0.1.1-0.20220107095505-c38622559671 - www.velocidex.com/golang/go-ntfs v0.1.2-0.20230728152253-4d399c766ed6 + www.velocidex.com/golang/go-ntfs v0.1.2-0.20230815140127-6a3dd72bfbf1 www.velocidex.com/golang/go-pe v0.1.1-0.20230228112150-ef2eadf34bc3 www.velocidex.com/golang/go-prefetch v0.0.0-20220801101854-338dbe61982a www.velocidex.com/golang/oleparse v0.0.0-20230217092320-383a0121aafe diff --git a/go.sum b/go.sum index 7dc5d8f4887..62bb8a6d44f 100644 --- a/go.sum +++ b/go.sum @@ -1231,8 +1231,8 @@ www.velocidex.com/golang/evtx v0.2.1-0.20220404133451-1fdf8be7325e h1:AhcXPgNKhJ www.velocidex.com/golang/evtx v0.2.1-0.20220404133451-1fdf8be7325e/go.mod h1:ykEQ7AUF9AL+mfCefDmLwmZOnU2So6wM3qKM8xdsHhU= www.velocidex.com/golang/go-ese v0.1.1-0.20220107095505-c38622559671 h1:pfvo7NFo0eJj6Zr7d+4vMx/Zr2JriMMPEWRHUf1YjUw= www.velocidex.com/golang/go-ese v0.1.1-0.20220107095505-c38622559671/go.mod h1:qnzHyB9yD2khtYO+wf3ck9FQxX3wFhXeJHFBnuUIZcc= -www.velocidex.com/golang/go-ntfs v0.1.2-0.20230728152253-4d399c766ed6 h1:CQTXpiMZ01PJIvpelSzpWJlZEUoQM831YgHEVdaZic4= -www.velocidex.com/golang/go-ntfs v0.1.2-0.20230728152253-4d399c766ed6/go.mod h1:itvbHQcnLdTVIDY6fI3lR0zeBwXwBYBdUFtswE0x1vc= +www.velocidex.com/golang/go-ntfs v0.1.2-0.20230815140127-6a3dd72bfbf1 h1:6NMITYv1pi4tzmDcqB/enNUXKmS8dnTb72HBghqhnAM= +www.velocidex.com/golang/go-ntfs v0.1.2-0.20230815140127-6a3dd72bfbf1/go.mod h1:itvbHQcnLdTVIDY6fI3lR0zeBwXwBYBdUFtswE0x1vc= www.velocidex.com/golang/go-pe v0.1.1-0.20220107093716-e91743c801de/go.mod h1:j9Xy8Z9wxzY2SCB8CqDkkoSzy+eUwevnOrRm/XM2q/A= www.velocidex.com/golang/go-pe v0.1.1-0.20230228112150-ef2eadf34bc3 h1:W394TEIFuHFxHY8mzTJPHI5v+M+NLKEHmHn7KY/VpEM= www.velocidex.com/golang/go-pe v0.1.1-0.20230228112150-ef2eadf34bc3/go.mod h1:agYwYzeeytVtdwkRrvxZAjgIA8SCeM/Tg7Ym2/jBwmA= diff --git a/vql/common/yara.go b/vql/common/yara.go index 437216774a2..7aa35c0bc94 100644 --- a/vql/common/yara.go +++ b/vql/common/yara.go @@ -146,23 +146,32 @@ func (self YaraScanPlugin) Call( } matcher.filename = filename - // If accessor is not specified we call yara's - // ScanFile API which mmaps the entire file - // into memory avoiding the need for - // buffering. - if arg.Accessor == "" || arg.Accessor == "file" { - err := matcher.scanFile(ctx, output_chan) + accessor, err := accessors.GetAccessor(arg.Accessor, scope) + if err != nil { + scope.Log("yara: %v", err) + return + } + + // As an optimization, we try to call yara's ScanFile API + // which mmaps the entire file into memory avoiding the + // need for buffering. + raw_accessor, ok := accessor.(accessors.RawFileAPIAccessor) + if ok { + underlying_file, err := raw_accessor.GetUnderlyingAPIFilename(filename) if err == nil { - continue - } else { - scope.Log("Directly scanning file %v failed, will use accessor", - filename.String()) + err := matcher.scanFile(ctx, underlying_file, output_chan) + if err == nil { + continue + } else { + scope.Log("Directly scanning file %v failed, will use accessor", + filename.String()) + } } } // If scanning with the file api failed above // we fall back to accessor scanning. - matcher.scanFileByAccessor(ctx, arg.Accessor, + matcher.scanFileByAccessor(ctx, arg.Accessor, accessor, arg.Blocksize, arg.Start, arg.End, output_chan) } }() @@ -234,18 +243,13 @@ func getYaraRules(key, namespace, rules string, func (self *scanReporter) scanFileByAccessor( ctx context.Context, accessor_name string, + accessor accessors.FileSystemAccessor, blocksize uint64, start, end uint64, output_chan chan vfilter.Row) { defer utils.CheckForPanic("Panic in scanFileByAccessor") - accessor, err := accessors.GetAccessor(accessor_name, self.scope) - if err != nil { - self.scope.Log("yara: %v", err) - return - } - // Open the file with the accessor f, err := accessor.OpenWithOSPath(self.filename) if err != nil { @@ -354,9 +358,11 @@ func (self *scanReporter) scanRange(start, end uint64, f accessors.ReadSeekClose // filename to libyara directly for faster scanning using mmap. This // also ensures that all yara features (like the PE plugin) work. func (self *scanReporter) scanFile( - ctx context.Context, output_chan chan vfilter.Row) error { + ctx context.Context, + underlying_file string, + output_chan chan vfilter.Row) error { - fd, err := os.Open(self.filename.String()) + fd, err := os.Open(underlying_file) if err != nil { return err } @@ -377,7 +383,7 @@ func (self *scanReporter) scanFile( err = scanner.SetCallback(self). SetTimeout(10 * time.Second). SetFlags(self.yara_flag). - ScanFile(self.filename.String()) + ScanFile(underlying_file) if err != nil { return err } diff --git a/vql/parsers/csv/csv.go b/vql/parsers/csv/csv.go index fdda780c495..f9e83fd8133 100644 --- a/vql/parsers/csv/csv.go +++ b/vql/parsers/csv/csv.go @@ -224,7 +224,7 @@ func (self _WatchCSVPlugin) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) } type WriteCSVPluginArgs struct { - Filename string `vfilter:"required,field=filename,doc=CSV files to open"` + Filename *accessors.OSPath `vfilter:"required,field=filename,doc=CSV files to open"` Accessor string `vfilter:"optional,field=accessor,doc=The accessor to use"` Query vfilter.StoredQuery `vfilter:"required,field=query,doc=query to write into the file."` } @@ -257,7 +257,14 @@ func (self WriteCSVPlugin) Call( return } - file, err := os.OpenFile(arg.Filename, + underlying_file, err := accessors.GetUnderlyingAPIFilename( + arg.Accessor, scope, arg.Filename) + if err != nil { + scope.Log("write_csv: %s", err) + return + } + + file, err := os.OpenFile(underlying_file, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0700) if err != nil { scope.Log("write_csv: Unable to open file %s: %s", diff --git a/vql/parsers/json.go b/vql/parsers/json.go index 422deb65983..8102a91416d 100644 --- a/vql/parsers/json.go +++ b/vql/parsers/json.go @@ -485,7 +485,7 @@ func (self _IndexAssociativeProtocol) GetMembers( } type WriteJSONPluginArgs struct { - Filename string `vfilter:"required,field=filename,doc=CSV files to open"` + Filename *accessors.OSPath `vfilter:"required,field=filename,doc=CSV files to open"` Accessor string `vfilter:"optional,field=accessor,doc=The accessor to use"` Query vfilter.StoredQuery `vfilter:"required,field=query,doc=query to write into the file."` } @@ -518,7 +518,14 @@ func (self WriteJSONPlugin) Call( return } - file, err := os.OpenFile(arg.Filename, + underlying_file, err := accessors.GetUnderlyingAPIFilename( + arg.Accessor, scope, arg.Filename) + if err != nil { + scope.Log("write_csv: %s", err) + return + } + + file, err := os.OpenFile(underlying_file, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0700) if err != nil { scope.Log("write_jsonl: Unable to open file %s: %s", diff --git a/vql/parsers/leveldb.go b/vql/parsers/leveldb.go index 530022330a2..bff52b6a99e 100644 --- a/vql/parsers/leveldb.go +++ b/vql/parsers/leveldb.go @@ -32,6 +32,7 @@ func (self LevelDBPlugin) Call( scope vfilter.Scope, args *ordereddict.Dict) <-chan vfilter.Row { output_chan := make(chan vfilter.Row) + go func() { defer close(output_chan) defer utils.RecoverVQL(scope) @@ -53,54 +54,9 @@ func (self LevelDBPlugin) Call( return } - var db *leveldb.DB - switch arg.Accessor { - case "", "auto", "file": - db, err = leveldb.OpenFile(arg.Filename.String(), &opt.Options{ - ReadOnly: true, - Strict: opt.NoStrict, - }) - if err != nil { - if !retriableError(err) { - scope.Log("leveldb: %v", err) - return - } - scope.Log("DEBUG:leveldb: Directly opening file faild with %v, retrying on a local copy", err) - local_path, err1 := maybeMakeLocalCopy(ctx, scope, arg) - if err1 != nil { - scope.Log("leveldb: %v", err) - scope.Log("leveldb: %v", err1) - return - } - - // Try again with the copy - db, err = leveldb.OpenFile(local_path, &opt.Options{ - ReadOnly: true, - Strict: opt.NoStrict, - }) - if err != nil { - scope.Log("leveldb: %v", err) - return - } - } - - // For other accessors we just always make a copy. - default: - local_path, err := maybeMakeLocalCopy(ctx, scope, arg) - if err != nil { - scope.Log("leveldb: %v", err) - return - } - - // Try again with the copy - db, err = leveldb.OpenFile(local_path, &opt.Options{ - ReadOnly: true, - Strict: opt.NoStrict, - }) - if err != nil { - scope.Log("leveldb: %v", err) - return - } + db, err := getLevelDBHandle(ctx, scope, arg.Accessor, arg.Filename) + if err != nil { + return } defer db.Close() @@ -134,25 +90,61 @@ func (self LevelDBPlugin) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) * } } +func getLevelDBHandle( + ctx context.Context, scope vfilter.Scope, + accessor string, filename *accessors.OSPath) ( + db *leveldb.DB, err error) { + + underlying_file, err := accessors.GetUnderlyingAPIFilename( + accessor, scope, filename) + if err == nil { + // Try to open the underlying_file + db, err = leveldb.OpenFile(underlying_file, &opt.Options{ + ReadOnly: true, + Strict: opt.NoStrict, + }) + if err == nil { + // Ok it worked, lets use it. + return db, nil + } + scope.Log("DEBUG:leveldb: Directly opening file faild with %v, retrying on a local copy", err) + } + + local_path, err := makeLocalCopy(ctx, scope, accessor, filename) + if err != nil { + scope.Log("leveldb: %v", err) + return + } + + // Try again with the copy + return leveldb.OpenFile(local_path, &opt.Options{ + ReadOnly: true, + Strict: opt.NoStrict, + }) +} + // Maybe make a local copy of the database files. -func maybeMakeLocalCopy( +func makeLocalCopy( ctx context.Context, scope vfilter.Scope, - arg *LevelDBPluginArgs) (string, error) { - accessor, err := accessors.GetAccessor(arg.Accessor, scope) + accessor_name string, + filename *accessors.OSPath) (string, error) { + + accessor, err := accessors.GetAccessor(accessor_name, scope) if err != nil { return "", err } - files, err := accessor.ReadDirWithOSPath(arg.Filename) + files, err := accessor.ReadDirWithOSPath(filename) if err != nil { return "", err } // Create a temp directory to contain all the files. - tmpdir_any := (&filesystem.TempdirFunction{}).Call(ctx, scope, ordereddict.NewDict()) + tmpdir_any := (&filesystem.TempdirFunction{}).Call( + ctx, scope, ordereddict.NewDict()) tmpdir, ok := tmpdir_any.(string) if !ok { - return "", errors.New("Unable to create tempdir") + return "", errors.New("leveldb: Unable to create tempdir") } total_bytes := 0 @@ -178,7 +170,7 @@ func maybeMakeLocalCopy( scope.Log("INFO:leveldb: Copied db %v with accessor %v to local "+ "tmp directory %v (Copied %v files, %v bytes)\n", - arg.Filename, arg.Accessor, tmpdir, len(files), total_bytes) + filename.String(), accessor_name, tmpdir, len(files), total_bytes) return tmpdir, nil } diff --git a/vql/parsers/sql.go b/vql/parsers/sql.go index 83e35050150..217b2ee0c2c 100644 --- a/vql/parsers/sql.go +++ b/vql/parsers/sql.go @@ -12,6 +12,7 @@ import ( _ "github.com/go-sql-driver/mysql" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" + "www.velocidex.com/golang/velociraptor/accessors" "www.velocidex.com/golang/velociraptor/acls" utils "www.velocidex.com/golang/velociraptor/utils" "www.velocidex.com/golang/velociraptor/vql" @@ -25,12 +26,12 @@ var ( ) type SQLPluginArgs struct { - Driver string `vfilter:"required,field=driver, doc=sqlite, mysql,or postgres"` - ConnString string `vfilter:"optional,field=connstring, doc=SQL Connection String"` - Filename string `vfilter:"optional,field=file, doc=Required if using sqlite driver"` - Accessor string `vfilter:"optional,field=accessor,doc=The accessor to use if using sqlite"` - Query string `vfilter:"required,field=query"` - Args vfilter.Any `vfilter:"optional,field=args"` + Driver string `vfilter:"required,field=driver, doc=sqlite, mysql,or postgres"` + ConnString string `vfilter:"optional,field=connstring, doc=SQL Connection String"` + Filename *accessors.OSPath `vfilter:"optional,field=file, doc=Required if using sqlite driver"` + Accessor string `vfilter:"optional,field=accessor,doc=The accessor to use if using sqlite"` + Query string `vfilter:"required,field=query"` + Args vfilter.Any `vfilter:"optional,field=args"` } type SQLPlugin struct{} diff --git a/vql/parsers/sqlite.go b/vql/parsers/sqlite.go index bc16fd6d51b..e9477685579 100644 --- a/vql/parsers/sqlite.go +++ b/vql/parsers/sqlite.go @@ -42,10 +42,10 @@ import ( ) type SQLiteArgs struct { - Filename string `vfilter:"required,field=file"` - Accessor string `vfilter:"optional,field=accessor,doc=The accessor to use."` - Query string `vfilter:"required,field=query"` - Args vfilter.Any `vfilter:"optional,field=args"` + Filename *accessors.OSPath `vfilter:"required,field=file"` + Accessor string `vfilter:"optional,field=accessor,doc=The accessor to use."` + Query string `vfilter:"required,field=query"` + Args vfilter.Any `vfilter:"optional,field=args"` } type SQLitePlugin struct{} @@ -71,13 +71,10 @@ func VFSPathToFilesystemPath(path string) string { // Check the file header - ignore if this is not really an sqlite // file. -func checkSQLiteHeader(scope vfilter.Scope, accessor, filename string) (bool, error) { - fs, err := accessors.GetAccessor(accessor, scope) - if err != nil { - return false, err - } - - file, err := fs.Open(filename) +func checkSQLiteHeader(scope vfilter.Scope, + accessor accessors.FileSystemAccessor, + filename *accessors.OSPath) (bool, error) { + file, err := accessor.OpenWithOSPath(filename) if err != nil { return false, err } @@ -95,9 +92,13 @@ func checkSQLiteHeader(scope vfilter.Scope, accessor, filename string) (bool, er func GetHandleSqlite(ctx context.Context, arg *SQLPluginArgs, scope vfilter.Scope) ( handle *sqlx.DB, err error) { - filename := VFSPathToFilesystemPath(arg.Filename) + accessor, err := accessors.GetAccessor(arg.Accessor, scope) + if err != nil { + return nil, err + } - if filename == "" { + filename := arg.Filename.String() + if arg.Filename == nil || filename == "" { return nil, errors.New("file parameter required for sqlite driver!") } @@ -107,14 +108,25 @@ func GetHandleSqlite(ctx context.Context, // Check the header quickly to ensure that we dont copy the // file needlessly. If the file does not exist, we allow a // connection because this will create a new file. - header_ok, err := checkSQLiteHeader(scope, arg.Accessor, filename) + header_ok, err := checkSQLiteHeader(scope, accessor, arg.Filename) if !errors.Is(err, os.ErrNotExist) && !header_ok { return nil, notValidDatabase } - should_make_copy := vql_subsystem.GetBoolFromRow(scope, scope, constants.SQLITE_ALWAYS_MAKE_TEMPFILE) - if arg.Accessor != "file" && arg.Accessor != "" { - should_make_copy = true + should_make_copy := vql_subsystem.GetBoolFromRow( + scope, scope, constants.SQLITE_ALWAYS_MAKE_TEMPFILE) + + if !should_make_copy { + // We need raw file access to use the sqlite library directly. + raw_accessor, ok := accessor.(accessors.RawFileAPIAccessor) + if !ok { + should_make_copy = true + } else { + filename, err = raw_accessor.GetUnderlyingAPIFilename(arg.Filename) + if err != nil { + should_make_copy = true + } + } } if !should_make_copy { @@ -147,10 +159,10 @@ func GetHandleSqlite(ctx context.Context, parts := strings.Split(filename, "?") filename, err = _MakeTempfile(ctx, arg, parts[0], scope) if err != nil { - scope.Log("Unable to create temp file: %v", err) + scope.Log("sqlite: Unable to create temp file: %v", err) return nil, err } - scope.Log("Using local copy %v", filename) + scope.Log("sqlite: Using local copy %v", filename) } // All other accessors, make a copy and @@ -160,7 +172,7 @@ func GetHandleSqlite(ctx context.Context, if err != nil { return nil, err } - scope.Log("Using local copy %v", filename) + scope.Log("sqlite: Using local copy %v", filename) } // Try once again to connect to the new file @@ -179,7 +191,7 @@ func GetHandleSqlite(ctx context.Context, handle.Close() }) if err != nil { - scope.Log("Unable to set destructor for %v", filename) + scope.Log("sqlite: Unable to set destructor for %v", filename) handle.Close() return nil, err } @@ -193,7 +205,7 @@ func _MakeTempfile(ctx context.Context, string, error) { if arg.Accessor != "data" { - scope.Log("Will try to copy %v to temp file", filename) + scope.Log("sqlite: Will try to copy %v to temp file", filename) } tmpfile, err := ioutil.TempFile("", "tmp*.sqlite")