{ this.state.currently_editing && selected &&
diff --git a/gui/velociraptor/src/components/notebooks/notebook-uploads.jsx b/gui/velociraptor/src/components/notebooks/notebook-uploads.jsx
index a5e14dfd7a0..d69ca944db7 100644
--- a/gui/velociraptor/src/components/notebooks/notebook-uploads.jsx
+++ b/gui/velociraptor/src/components/notebooks/notebook-uploads.jsx
@@ -18,6 +18,7 @@ const POLL_TIME = 5000;
export default class NotebookUploads extends Component {
static propTypes = {
notebook: PropTypes.object,
+ cell: PropTypes.object,
closeDialog: PropTypes.func.isRequired,
}
@@ -75,16 +76,6 @@ export default class NotebookUploads extends Component {
;
}
- getDeleteLink = (cell, row) =>{
- var stats = row.stats || {};
- return ;
- }
-
render() {
let files = this.state.notebook &&
this.state.notebook.available_uploads &&
@@ -92,7 +83,6 @@ export default class NotebookUploads extends Component {
files = files || [];
let columns = formatColumns([
- {dataField: "name_", text: "", formatter: this.getDeleteLink},
{dataField: "name", text: T("Name"),
sort: true, filtered: true, formatter: this.getDownloadLink},
{dataField: "size", text: T("Size")},
diff --git a/gui/velociraptor/src/components/notebooks/notebook.jsx b/gui/velociraptor/src/components/notebooks/notebook.jsx
index 84e6013f228..8fc47cbd7ef 100644
--- a/gui/velociraptor/src/components/notebooks/notebook.jsx
+++ b/gui/velociraptor/src/components/notebooks/notebook.jsx
@@ -99,7 +99,7 @@ class Notebooks extends React.Component {
api.get("v1/GetNotebooks", {
notebook_id: notebook.notebook_id,
- include_uploads: true,
+ include_uploads: false,
}, this.source.token).then(response=>{
if (response.cancel) return;
diff --git a/paths/notebooks.go b/paths/notebooks.go
index 84007ad9e0a..1b1d7cdada9 100644
--- a/paths/notebooks.go
+++ b/paths/notebooks.go
@@ -21,14 +21,22 @@ func NotebookDir() api.DSPathSpec {
return NOTEBOOK_ROOT
}
-// Where to store attachments? In the notebook path.
+// Attachments are not the same as uploads - they are usually uploaded
+// by pasting in the cell eg an image. We want the attachment to
+// remain whenever the cell is updated to a new version.
+// Example workflow:
+// - User uploads an attachment into a cell
+// - Cell is updated with a link to the attachment
+// - User continues to edit the cell in newer versions but the link
+// remains valid because the attachment is stored in the notebook
+// and not the cell.
func (self *NotebookPathManager) Attachment(name string) api.FSPathSpec {
- return self.root.AddUnsafeChild(self.notebook_id, "uploads", "attach/"+name).
+ return self.root.AddUnsafeChild(self.notebook_id, "attach", name).
AsFilestorePath().SetType(api.PATH_TYPE_FILESTORE_ANY)
}
func (self *NotebookPathManager) AttachmentDirectory() api.FSPathSpec {
- return self.root.AddChild(self.notebook_id, "uploads").
+ return self.root.AddChild(self.notebook_id, "attach").
AsFilestorePath().SetType(api.PATH_TYPE_FILESTORE_ANY)
}
@@ -67,10 +75,13 @@ func (self *NotebookPathManager) DSDirectory() api.DSPathSpec {
return self.root.AddChild(self.notebook_id)
}
-func (self *NotebookPathManager) HtmlExport() api.FSPathSpec {
- return DOWNLOADS_ROOT.AddChild("notebooks", self.notebook_id,
- fmt.Sprintf("%s-%s", self.notebook_id,
- self.Clock.Now().UTC().Format("20060102150405Z"))).
+func (self *NotebookPathManager) HtmlExport(prefered_name string) api.FSPathSpec {
+ if prefered_name == "" {
+ prefered_name = fmt.Sprintf("%s-%s", self.notebook_id,
+ self.Clock.Now().UTC().Format("20060102150405Z"))
+ }
+ return DOWNLOADS_ROOT.AddChild(
+ "notebooks", self.notebook_id, prefered_name).
SetType(api.PATH_TYPE_FILESTORE_DOWNLOAD_REPORT)
}
@@ -201,16 +212,15 @@ func (self *NotebookCellPathManager) QueryStorage(id int64) *NotebookCellQuery {
// Uploads are stored at the network level.
func (self *NotebookCellPathManager) UploadsDir() api.FSPathSpec {
return self.root.AsFilestorePath().
- AddUnsafeChild(self.notebook_id, "uploads").
+ AddUnsafeChild(self.notebook_id, self.cell_id, "uploads").
SetType(api.PATH_TYPE_FILESTORE_ANY)
}
func (self *NotebookCellPathManager) GetUploadsFile(filename string) api.FSPathSpec {
- // Cell id and filename are combined so we can read all
- // attachments in a single ListDir
+ // Uploads exist inside each cell so when cells are reaped we
+ // remove the uploads.
return self.root.AsFilestorePath().
- AddUnsafeChild(self.notebook_id,
- "uploads", fmt.Sprintf("%s/%s", self.cell_id, filename)).
+ AddUnsafeChild(self.notebook_id, self.cell_id, "uploads", filename).
SetType(api.PATH_TYPE_FILESTORE_ANY)
}
diff --git a/paths/notebooks_test.go b/paths/notebooks_test.go
index dcf7a91e24f..f9fcaefe73c 100644
--- a/paths/notebooks_test.go
+++ b/paths/notebooks_test.go
@@ -15,12 +15,12 @@ func (self *PathManagerTestSuite) TestNotebookPathManager() {
assert.Equal(self.T(), "/ds/notebooks/N.123.json.db",
self.getDatastorePath(manager.Path()))
- assert.Equal(self.T(), "/fs/notebooks/N.123/uploads/attach%2FNA.123%2Fimage.png",
+ assert.Equal(self.T(), "/fs/notebooks/N.123/attach/NA.123%2Fimage.png",
self.getFilestorePath(manager.Attachment("NA.123/image.png")))
// Exports are available in the (authenticated) downloads directory.
assert.Equal(self.T(), "/fs/downloads/notebooks/N.123/N.123-20010909014640Z.html",
- self.getFilestorePath(manager.HtmlExport()))
+ self.getFilestorePath(manager.HtmlExport("")))
assert.Equal(self.T(), "/fs/downloads/notebooks/N.123/N.123-20010909014640Z.zip",
self.getFilestorePath(manager.ZipExport()))
diff --git a/reporting/gui.go b/reporting/gui.go
index 191bf20645e..fa098115fb9 100644
--- a/reporting/gui.go
+++ b/reporting/gui.go
@@ -26,6 +26,7 @@ import (
actions_proto "www.velocidex.com/golang/velociraptor/actions/proto"
artifacts_proto "www.velocidex.com/golang/velociraptor/artifacts/proto"
config_proto "www.velocidex.com/golang/velociraptor/config/proto"
+ "www.velocidex.com/golang/velociraptor/constants"
"www.velocidex.com/golang/velociraptor/file_store"
"www.velocidex.com/golang/velociraptor/json"
"www.velocidex.com/golang/velociraptor/paths"
@@ -56,7 +57,7 @@ type GuiTemplateEngine struct {
*BaseTemplateEngine
tmpl *template.Template
ctx context.Context
- log_writer *notebooCellLogger
+ log_writer *notebookCellLogger
path_manager *paths.NotebookCellPathManager
Data map[string]*actions_proto.VQLResponse
Progress utils.ProgressReporter
@@ -92,7 +93,7 @@ func parseOptions(values []interface{}) (*ordereddict.Dict, []interface{}) {
// When rendering a table the user can introduce options into the
// scope.
func (self *GuiTemplateEngine) getTableOptions() (*ordereddict.Dict, error) {
- column_types, pres := self.Scope.Resolve("ColumnTypes")
+ column_types, pres := self.Scope.Resolve(constants.COLUMN_TYPES)
if !pres {
return nil, errors.New("Not found")
}
@@ -547,6 +548,7 @@ func (self *GuiTemplateEngine) MoreMessages() bool {
func (self *GuiTemplateEngine) Close() {
if self.log_writer != nil {
self.log_writer.Flush()
+ self.log_writer.Close()
}
self.BaseTemplateEngine.Close()
}
diff --git a/reporting/logging.go b/reporting/logging.go
index 84186201fec..c4183cd234e 100644
--- a/reporting/logging.go
+++ b/reporting/logging.go
@@ -2,7 +2,6 @@ package reporting
import (
"sync"
- "time"
"github.com/Velocidex/ordereddict"
config_proto "www.velocidex.com/golang/velociraptor/config/proto"
@@ -14,7 +13,7 @@ import (
"www.velocidex.com/golang/velociraptor/utils"
)
-type notebooCellLogger struct {
+type notebookCellLogger struct {
mu sync.Mutex
messages []string
@@ -26,7 +25,7 @@ type notebooCellLogger struct {
func newNotebookCellLogger(
config_obj *config_proto.Config, log_path api.FSPathSpec) (
- *notebooCellLogger, error) {
+ *notebookCellLogger, error) {
file_store_factory := file_store.GetFileStore(config_obj)
// Create a new result set to write the logs
@@ -37,15 +36,15 @@ func newNotebookCellLogger(
return nil, err
}
- return ¬ebooCellLogger{
+ return ¬ebookCellLogger{
rs_writer: rs_writer,
}, nil
}
-func (self *notebooCellLogger) Write(b []byte) (int, error) {
+func (self *notebookCellLogger) Write(b []byte) (int, error) {
level, msg := logging.SplitIntoLevelAndLog(b)
self.rs_writer.Write(ordereddict.NewDict().
- Set("Timestamp", time.Now().UTC().UnixNano()/1000).
+ Set("Timestamp", utils.GetTime().Now().UTC().UnixNano()/1000).
Set("Level", level).
Set("message", msg))
@@ -74,7 +73,7 @@ func (self *notebooCellLogger) Write(b []byte) (int, error) {
return len(b), nil
}
-func (self *notebooCellLogger) Messages() []string {
+func (self *notebookCellLogger) Messages() []string {
self.mu.Lock()
defer self.mu.Unlock()
@@ -82,13 +81,17 @@ func (self *notebooCellLogger) Messages() []string {
}
// Are there additional messages?
-func (self *notebooCellLogger) MoreMessages() bool {
+func (self *notebookCellLogger) MoreMessages() bool {
self.mu.Lock()
defer self.mu.Unlock()
return self.more_messages
}
-func (self *notebooCellLogger) Flush() {
+func (self *notebookCellLogger) Flush() {
self.rs_writer.Flush()
}
+
+func (self *notebookCellLogger) Close() {
+ self.rs_writer.Close()
+}
diff --git a/reporting/paths.go b/reporting/paths.go
index 76c030a415d..dc1b3033323 100644
--- a/reporting/paths.go
+++ b/reporting/paths.go
@@ -14,13 +14,8 @@ func (self *NotebookExportPathManager) CellMetadata(
return self.root.Append(self.notebook_id, cell_id+".db")
}
-func (self *NotebookExportPathManager) UploadRoot() *accessors.OSPath {
- return self.root.Append(self.notebook_id, "files")
-}
-
-func (self *NotebookExportPathManager) CellUploadRoot(
- cell_id string) *accessors.OSPath {
- return self.root.Append(self.notebook_id, cell_id, "uploads")
+func (self *NotebookExportPathManager) AttachmentRoot() *accessors.OSPath {
+ return self.root.Append(self.notebook_id, "attach")
}
func (self *NotebookExportPathManager) CellDirectory(
diff --git a/services/notebook.go b/services/notebook.go
index 03717d51f32..35e9543058d 100644
--- a/services/notebook.go
+++ b/services/notebook.go
@@ -7,6 +7,11 @@ import (
config_proto "www.velocidex.com/golang/velociraptor/config/proto"
)
+const (
+ DO_NOT_INCLUDE_UPLOADS = false
+ INCLUDE_UPLOADS = true
+)
+
func GetNotebookManager(config_obj *config_proto.Config) (NotebookManager, error) {
org_manager, err := GetOrgManager()
if err != nil {
diff --git a/services/notebook/calculate.go b/services/notebook/calculate.go
index a33f622e0b1..6750a673c34 100644
--- a/services/notebook/calculate.go
+++ b/services/notebook/calculate.go
@@ -54,6 +54,15 @@ func (self *NotebookManager) UpdateNotebookCell(
if err != nil {
return nil, err
}
+
+ // Preserve the cell type
+ if in.Type == "" {
+ in.Type = cell_metadata.Type
+ }
+ }
+
+ if in.Type == "" {
+ in.Type = "markdown"
}
// Write the cell record as calculating while we attempt to
@@ -70,6 +79,16 @@ func (self *NotebookManager) UpdateNotebookCell(
AvailableVersions: in.AvailableVersions,
}
+ // If the output field is specified, we just set it as is without
+ // actually calculating it.
+ if in.Output != "" {
+ notebook_cell.Calculating = false
+ notebook_cell.CurrentlyEditing = false
+ notebook_cell.Output = in.Output
+ return notebook_cell, self.Store.SetNotebookCell(
+ notebook_metadata.NotebookId, notebook_cell)
+ }
+
err := self.Store.SetNotebookCell(
notebook_metadata.NotebookId, notebook_cell)
if err != nil {
diff --git a/services/notebook/downloads.go b/services/notebook/downloads.go
index 5f371745351..24157737ce2 100644
--- a/services/notebook/downloads.go
+++ b/services/notebook/downloads.go
@@ -3,13 +3,14 @@ package notebook
import (
"context"
"errors"
- "strings"
+ "os"
"time"
api_proto "www.velocidex.com/golang/velociraptor/api/proto"
"www.velocidex.com/golang/velociraptor/datastore"
"www.velocidex.com/golang/velociraptor/file_store"
"www.velocidex.com/golang/velociraptor/file_store/api"
+ "www.velocidex.com/golang/velociraptor/file_store/path_specs"
"www.velocidex.com/golang/velociraptor/paths"
"www.velocidex.com/golang/velociraptor/reporting"
)
@@ -35,7 +36,7 @@ func (self *NotebookStoreImpl) GetAvailableDownloadFiles(
notebook_id string) (*api_proto.AvailableDownloads, error) {
download_path := paths.NewNotebookPathManager(notebook_id).
- HtmlExport().Dir()
+ HtmlExport("X").Dir()
return reporting.GetAvailableDownloadFiles(self.config_obj, download_path)
}
@@ -46,30 +47,37 @@ func (self *NotebookStoreImpl) GetAvailableUploadFiles(notebook_id string) (
notebook_path_manager := paths.NewNotebookPathManager(notebook_id)
file_store_factory := file_store.GetFileStore(self.config_obj)
- files, err := file_store_factory.ListDirectory(
- notebook_path_manager.AttachmentDirectory())
+
+ notebook, err := self.GetNotebook(notebook_id)
if err != nil {
return nil, err
}
-
- for _, item := range files {
- ps := item.PathSpec()
- parts := strings.SplitN(ps.Base(), "/", 2)
- if len(parts) < 2 {
- continue
+ for _, cell_metadata := range notebook.CellMetadata {
+ cell_manager := notebook_path_manager.Cell(
+ cell_metadata.CellId, cell_metadata.CurrentVersion)
+
+ err := api.Walk(file_store_factory, cell_manager.UploadsDir(),
+ func(ps api.FSPathSpec, info os.FileInfo) error {
+ stat, err := file_store_factory.StatFile(ps)
+ if err != nil {
+ return nil
+ }
+
+ result.Files = append(result.Files, &api_proto.AvailableDownloadFile{
+ Name: ps.Base(),
+ Size: uint64(stat.Size()),
+ Date: stat.ModTime().UTC().Format(time.RFC3339),
+ Type: api.GetExtensionForFilestore(ps),
+ Stats: &api_proto.ContainerStats{
+ Components: ps.Components(),
+ },
+ })
+ return nil
+ })
+ if err != nil {
+ return nil, err
}
-
- result.Files = append(result.Files, &api_proto.AvailableDownloadFile{
- Name: parts[1],
- Size: uint64(item.Size()),
- Date: item.ModTime().UTC().Format(time.RFC3339),
- Type: api.GetExtensionForFilestore(ps),
- Stats: &api_proto.ContainerStats{
- Components: ps.Components(),
- },
- })
}
-
return result, nil
}
@@ -86,8 +94,12 @@ func (self *NotebookStoreImpl) RemoveAttachment(ctx context.Context,
}
notebook_path_manager := paths.NewNotebookPathManager(notebook_id)
- attachment_path := notebook_path_manager.AttachmentDirectory().
- AddUnsafeChild(components[len(components)-1])
+ attachment_path := path_specs.NewUnsafeFilestorePath(components...)
+ if !path_specs.IsSubPath(
+ notebook_path_manager.AttachmentDirectory(),
+ attachment_path) {
+ return errors.New("Attachment must be within the notebook directory")
+ }
file_store_factory := file_store.GetFileStore(self.config_obj)
return file_store_factory.Delete(attachment_path)
diff --git a/services/notebook/initial.go b/services/notebook/initial.go
index 2bbbce49cc5..abe41c12d25 100644
--- a/services/notebook/initial.go
+++ b/services/notebook/initial.go
@@ -77,6 +77,7 @@ func (self *NotebookManager) NewNotebookCell(
// Create the new cell with fresh content.
new_cell_request := &api_proto.NotebookCellRequest{
Input: in.Input,
+ Output: in.Output,
NotebookId: in.NotebookId,
CellId: notebook.LatestCellId,
Version: new_version,
diff --git a/services/notebook/notebook.go b/services/notebook/notebook.go
index b5ca415fad4..6d80d16c647 100644
--- a/services/notebook/notebook.go
+++ b/services/notebook/notebook.go
@@ -18,6 +18,10 @@ import (
"www.velocidex.com/golang/vfilter/reformat"
)
+var (
+ invalidNotebookId = errors.New("Invalid notebook id")
+)
+
type NotebookManager struct {
config_obj *config_proto.Config
Store NotebookStore
@@ -27,6 +31,11 @@ func (self *NotebookManager) GetNotebook(
ctx context.Context, notebook_id string, include_uploads bool) (
*api_proto.NotebookMetadata, error) {
+ err := verifyNotebookId(notebook_id)
+ if err != nil {
+ return nil, err
+ }
+
notebook, err := self.Store.GetNotebook(notebook_id)
if err != nil {
return nil, err
@@ -79,6 +88,10 @@ func (self *NotebookManager) NewNotebook(
func (self *NotebookManager) UpdateNotebook(
ctx context.Context, in *api_proto.NotebookMetadata) error {
+ err := verifyNotebookId(in.NotebookId)
+ if err != nil {
+ return err
+ }
in.ModifiedTime = utils.GetTime().Now().Unix()
return self.Store.SetNotebook(in)
@@ -87,6 +100,11 @@ func (self *NotebookManager) UpdateNotebook(
func (self *NotebookManager) GetNotebookCell(ctx context.Context,
notebook_id, cell_id, version string) (*api_proto.NotebookCell, error) {
+ err := verifyNotebookId(notebook_id)
+ if err != nil {
+ return nil, err
+ }
+
notebook_cell, err := self.Store.GetNotebookCell(notebook_id, cell_id, version)
// Cell does not exist, make it a default cell.
@@ -112,6 +130,11 @@ func (self *NotebookManager) GetNotebookCell(ctx context.Context,
func (self *NotebookManager) CancelNotebookCell(
ctx context.Context, notebook_id, cell_id, version string) error {
+ err := verifyNotebookId(notebook_id)
+ if err != nil {
+ return err
+ }
+
// Unset the calculating bit in the notebook in case the
// renderer is not actually running (e.g. server restart).
notebook_cell, err := self.Store.GetNotebookCell(notebook_id, cell_id, version)
@@ -147,7 +170,12 @@ func (self *NotebookManager) UploadNotebookAttachment(
return nil, err
}
- filename := NewNotebookAttachmentId() + in.Filename
+ err = verifyNotebookId(in.NotebookId)
+ if err != nil {
+ return nil, err
+ }
+
+ filename := NewNotebookAttachmentId() + "-" + in.Filename
full_path, err := self.Store.StoreAttachment(
in.NotebookId, filename, decoded)
@@ -158,7 +186,9 @@ func (self *NotebookManager) UploadNotebookAttachment(
result := &api_proto.NotebookFileUploadResponse{
Url: full_path.AsClientPath() + "?org_id=" +
url.QueryEscape(utils.NormalizedOrgId(self.config_obj.OrgId)),
+ Filename: filename,
}
+
return result, nil
}
@@ -207,3 +237,10 @@ func (self *NotebookManager) ReformatVQL(
return strings.Join(trimmed, "\n"), nil
}
+
+func verifyNotebookId(notebook_id string) error {
+ if !strings.HasPrefix(notebook_id, "N.") {
+ return invalidNotebookId
+ }
+ return nil
+}
diff --git a/services/notebook/storage.go b/services/notebook/storage.go
index bde96172d5c..cc2ed120d50 100644
--- a/services/notebook/storage.go
+++ b/services/notebook/storage.go
@@ -26,18 +26,23 @@ type NotebookStore interface {
SetNotebook(in *api_proto.NotebookMetadata) error
GetNotebook(notebook_id string) (*api_proto.NotebookMetadata, error)
SetNotebookCell(notebook_id string, in *api_proto.NotebookCell) error
- GetNotebookCell(notebook_id, cell_id, version string) (*api_proto.NotebookCell, error)
+ GetNotebookCell(notebook_id, cell_id, version string) (
+ *api_proto.NotebookCell, error)
// progress_chan receives information about deletion. It may be
// nil if callers dont care about it.
RemoveNotebookCell(
ctx context.Context, config_obj *config_proto.Config,
- notebook_id, cell_id, version string, progress_chan chan *ordereddict.Dict) error
+ notebook_id, cell_id, version string,
+ progress_chan chan *ordereddict.Dict) error
- StoreAttachment(notebook_id, filename string, data []byte) (api.FSPathSpec, error)
- RemoveAttachment(ctx context.Context, notebook_id string, components []string) error
+ StoreAttachment(notebook_id,
+ filename string, data []byte) (api.FSPathSpec, error)
+ RemoveAttachment(ctx context.Context,
+ notebook_id string, components []string) error
- GetAvailableDownloadFiles(notebook_id string) (*api_proto.AvailableDownloads, error)
+ GetAvailableDownloadFiles(notebook_id string) (
+ *api_proto.AvailableDownloads, error)
GetAvailableTimelines(notebook_id string) []string
GetAvailableUploadFiles(notebook_id string) (
*api_proto.AvailableDownloads, error)
@@ -257,7 +262,8 @@ func (self *NotebookStoreImpl) GetNotebookCell(
return notebook_cell, err
}
-func (self *NotebookStoreImpl) StoreAttachment(notebook_id, filename string, data []byte) (api.FSPathSpec, error) {
+func (self *NotebookStoreImpl) StoreAttachment(
+ notebook_id, filename string, data []byte) (api.FSPathSpec, error) {
full_path := paths.NewNotebookPathManager(notebook_id).
Attachment(filename)
file_store_factory := file_store.GetFileStore(self.config_obj)
diff --git a/services/notebook/worker.go b/services/notebook/worker.go
index d6c55bbfa31..e41765fdbce 100644
--- a/services/notebook/worker.go
+++ b/services/notebook/worker.go
@@ -100,6 +100,8 @@ func (self *NotebookWorker) ProcessUpdateRequest(
notebook_path_manager.Cell(in.CellId, in.Version),
"Server.Internal.ArtifactDescription")
if err != nil {
+ logger.Debug("NotebookWorker: While evaluating template: %v", err)
+
return nil, err
}
defer tmpl.Close()
@@ -359,6 +361,7 @@ func (self *NotebookWorker) Start(
config_obj *config_proto.Config,
name string,
scheduler services.Scheduler) {
+
for {
err := self.RegisterWorker(ctx, config_obj, name, scheduler)
if err != nil {
@@ -508,8 +511,8 @@ func (self *NotebookManager) Start(
wg *sync.WaitGroup) error {
// Only start this once for all orgs. Otherwise we would have as
- // many workers as orgs. Orgs will switch into the correct org for
- // processing
+ // many workers as orgs. Workers will switch into the correct org
+ // for processing
if !utils.IsRootOrg(config_obj.OrgId) {
return nil
}
diff --git a/services/scheduler/fixtures/TestNotebookMinionScheduler.golden b/services/scheduler/fixtures/TestNotebookMinionScheduler.golden
index 64272046316..095bd3c55e9 100644
--- a/services/scheduler/fixtures/TestNotebookMinionScheduler.golden
+++ b/services/scheduler/fixtures/TestNotebookMinionScheduler.golden
@@ -7,7 +7,6 @@
"messages": [
"DEBUG:Query Stats: {\"RowsScanned\":1,\"PluginsCalled\":1,\"FunctionsCalled\":1,\"ProtocolSearch\":0,\"ScopeC ..."
],
- "timestamp": 10,
"type": "vql",
"current_version": "XXX",
"available_versions": [
diff --git a/services/scheduler/minion_test.go b/services/scheduler/minion_test.go
index fda3692c4a5..b163aaa2cd2 100644
--- a/services/scheduler/minion_test.go
+++ b/services/scheduler/minion_test.go
@@ -41,6 +41,7 @@ func (self *MinionSchedulerTestSuite) SetupTest() {
// Do not start local workers to force us to go through the remote
// one.
self.ConfigObj.Defaults.NotebookNumberOfLocalWorkers = -1
+ self.ConfigObj.Defaults.NotebookWaitTimeForWorkerMs = -1
self.ConfigObj.API.BindPort = 8345
// Mock out cell ID generation for tests
@@ -111,6 +112,7 @@ func (self *MinionSchedulerTestSuite) TestNotebookMinionScheduler() {
})
assert.NoError(self.T(), err)
+ cell.Timestamp = 0
golden := ordereddict.NewDict().
Set("Updated Cell", cell)
diff --git a/services/scheduler/scheduler.go b/services/scheduler/scheduler.go
index 8a6dc2b3144..a2214b7444d 100644
--- a/services/scheduler/scheduler.go
+++ b/services/scheduler/scheduler.go
@@ -69,6 +69,8 @@ type Scheduler struct {
mu sync.Mutex
queues map[string][]*Worker
+
+ config_obj *config_proto.Config
}
func (self *Scheduler) RegisterWorker(
@@ -133,23 +135,52 @@ func (self *Scheduler) WriteProfile(ctx context.Context,
func (self *Scheduler) Schedule(ctx context.Context,
job services.SchedulerJob) (chan services.JobResponse, error) {
+
+ var wait_time time.Duration
+ if self.config_obj.Defaults != nil {
+ config_wait_time := self.config_obj.Defaults.NotebookWaitTimeForWorkerMs
+ if config_wait_time > 0 {
+ wait_time = time.Millisecond * time.Duration(config_wait_time)
+ } else if config_wait_time == 0 {
+ wait_time = 10 * time.Second
+ } else if config_wait_time < 0 {
+ wait_time = 0
+ }
+ }
+
for {
// The following does not block so we can do it all under lock
- self.mu.Lock()
-
var available_workers []*Worker
- // Find a ready worker
- workers, _ := self.queues[job.Queue]
- for _, w := range workers {
- if !w.IsBusy() {
- available_workers = append(available_workers, w)
+ // Retry a few times to get a worker from the queue.
+ start := utils.GetTime().Now()
+ for {
+ self.mu.Lock()
+
+ // Find a ready worker
+ workers, _ := self.queues[job.Queue]
+ for _, w := range workers {
+ if !w.IsBusy() {
+ available_workers = append(available_workers, w)
+ }
+ }
+
+ // Yes we got some workers.
+ if len(available_workers) > 0 {
+ // Hold the lock on break
+ break
}
- }
- if len(available_workers) == 0 {
+ // Do not wait with the lock held
self.mu.Unlock()
- return nil, fmt.Errorf("No workers available on queue %v!", job.Queue)
+
+ // Give up after 10 seconds.
+ if utils.GetTime().Now().Sub(start) > wait_time {
+ return nil, fmt.Errorf("No workers available on queue %v!", job.Queue)
+ }
+
+ // Try again soon
+ utils.GetTime().Sleep(100 * time.Millisecond)
}
result_chan := make(chan services.JobResponse)
@@ -246,7 +277,8 @@ func StartSchedulerService(
logger.Info("Starting Server Scheduler Service for %v", services.GetOrgName(config_obj))
scheduler := &Scheduler{
- queues: make(map[string][]*Worker),
+ queues: make(map[string][]*Worker),
+ config_obj: config_obj,
}
services.RegisterScheduler(scheduler)
diff --git a/utils/errors.go b/utils/errors.go
index 0b1e168f071..1ec0effcbc9 100644
--- a/utils/errors.go
+++ b/utils/errors.go
@@ -9,8 +9,9 @@ var (
// Error relayed when the error details are added inline. The GUI
// API will strip this error as the details are included in the
// response already.
- InlineError = errors.New("InlineError")
- TimeoutError = errors.New("Timeout")
+ InlineError = errors.New("InlineError")
+ TimeoutError = errors.New("Timeout")
+ InvalidStatus = errors.New("InvalidStatus")
)
// This is a custom error type that wraps an inner error but does not
diff --git a/vql/functions/time.go b/vql/functions/time.go
index f3631a052a5..6743f6aa10c 100644
--- a/vql/functions/time.go
+++ b/vql/functions/time.go
@@ -12,6 +12,7 @@ import (
"github.com/Velocidex/ordereddict"
"github.com/araddon/dateparse"
+ "www.velocidex.com/golang/velociraptor/constants"
"www.velocidex.com/golang/velociraptor/third_party/cache"
"www.velocidex.com/golang/velociraptor/utils"
vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
@@ -79,7 +80,7 @@ func getTimezone(scope types.Scope) (*time.Location, string) {
// Otherwise the user may have specified a global timezone (which
// also affects output).
- tz, pres = scope.Resolve("TZ")
+ tz, pres = scope.Resolve(constants.TZ)
if pres {
tz_str, ok := tz.(string)
if ok {
diff --git a/vql/json.go b/vql/json.go
index eb7ac54bad3..0b9e5746e88 100644
--- a/vql/json.go
+++ b/vql/json.go
@@ -5,6 +5,7 @@ import (
"time"
"github.com/Velocidex/json"
+ "www.velocidex.com/golang/velociraptor/constants"
vjson "www.velocidex.com/golang/velociraptor/json"
"www.velocidex.com/golang/vfilter"
)
@@ -26,7 +27,7 @@ func EncOptsFromScope(scope vfilter.Scope) *json.EncOpts {
// If the scope contains a TZ variable, then we will use that
// instead.
- location_name, pres := scope.Resolve("TZ")
+ location_name, pres := scope.Resolve(constants.TZ)
if pres {
location_str, ok := location_name.(string)
if ok {
diff --git a/vql/server/notebooks/create.go b/vql/server/notebooks/create.go
new file mode 100644
index 00000000000..3d295f469cd
--- /dev/null
+++ b/vql/server/notebooks/create.go
@@ -0,0 +1,100 @@
+package notebooks
+
+import (
+ "context"
+
+ "github.com/Velocidex/ordereddict"
+ "www.velocidex.com/golang/velociraptor/acls"
+ api_proto "www.velocidex.com/golang/velociraptor/api/proto"
+ "www.velocidex.com/golang/velociraptor/services"
+ "www.velocidex.com/golang/velociraptor/vql"
+ vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
+ "www.velocidex.com/golang/vfilter"
+ "www.velocidex.com/golang/vfilter/arg_parser"
+)
+
+type CreateNotebookFunctionArg struct {
+ Name string `vfilter:"optional,field=name,doc=The name of the notebook"`
+ Description string `vfilter:"optional,field=description,doc=The description of the notebook"`
+ Collaborators []string `vfilter:"optional,field=collaborators,doc=A list of users to share the notebook with."`
+ Public bool `vfilter:"optional,field=public,doc=If set the notebook will be public."`
+ Artifacts []string `vfilter:"optional,field=artifacts,doc=A list of NOTEBOOK artifacts to create the notebook with (Notebooks.Default)"`
+ Env *ordereddict.Dict `vfilter:"optional,field=env,doc=An environment to initialize the notebook with"`
+}
+
+type CreateNotebookFunction struct{}
+
+func (self *CreateNotebookFunction) Call(ctx context.Context,
+ scope vfilter.Scope,
+ args *ordereddict.Dict) vfilter.Any {
+
+ err := vql_subsystem.CheckAccess(scope, acls.COLLECT_SERVER)
+ if err != nil {
+ scope.Log("notebook_create: %v", err)
+ return vfilter.Null{}
+ }
+
+ arg := &CreateNotebookFunctionArg{}
+ err = arg_parser.ExtractArgsWithContext(ctx, scope, args, arg)
+ if err != nil {
+ scope.Log("notebook_create: %v", err)
+ return vfilter.Null{}
+ }
+
+ principal := vql_subsystem.GetPrincipal(scope)
+ new_notebook := &api_proto.NotebookMetadata{
+ Name: arg.Name,
+ Description: arg.Description,
+ Creator: principal,
+ Collaborators: arg.Collaborators,
+ Artifacts: arg.Artifacts,
+ Public: arg.Public,
+ }
+
+ config_obj, pres := vql_subsystem.GetServerConfig(scope)
+ if !pres {
+ scope.Log("notebook_create: must be running on the server")
+ return vfilter.Null{}
+ }
+
+ notebook_manager, err := services.GetNotebookManager(config_obj)
+ if err != nil {
+ scope.Log("notebook_create: %v", err)
+ return vfilter.Null{}
+ }
+
+ new_notebook, err = notebook_manager.NewNotebook(ctx,
+ principal, new_notebook)
+ if err != nil {
+ scope.Log("notebook_create: %v", err)
+ return vfilter.Null{}
+ }
+
+ services.LogAudit(ctx,
+ config_obj, principal, "CreateNotebook",
+ ordereddict.NewDict().
+ Set("notebook_id", new_notebook.NotebookId).
+ Set("details", vfilter.RowToDict(ctx, scope, arg)))
+
+ err = fillNotebookCells(ctx, config_obj, new_notebook)
+ if err != nil {
+ scope.Log("notebook_create: %v", err)
+ return vfilter.Null{}
+ }
+
+ return new_notebook
+}
+
+func (self CreateNotebookFunction) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.FunctionInfo {
+ return &vfilter.FunctionInfo{
+ Name: "notebook_create",
+ Doc: "Create a new notebook.",
+ ArgType: type_map.AddType(scope, &CreateNotebookFunctionArg{}),
+ Metadata: vql.VQLMetadata().Permissions(
+ acls.COLLECT_SERVER).Build(),
+ }
+}
+
+func init() {
+ vql_subsystem.RegisterFunction(&CreateNotebookFunction{})
+}
diff --git a/vql/server/notebooks/download.go b/vql/server/notebooks/download.go
index b2e62cc1649..9f91ac1b2bc 100644
--- a/vql/server/notebooks/download.go
+++ b/vql/server/notebooks/download.go
@@ -7,8 +7,6 @@ import (
"github.com/Velocidex/ordereddict"
"www.velocidex.com/golang/velociraptor/acls"
"www.velocidex.com/golang/velociraptor/file_store"
- "www.velocidex.com/golang/velociraptor/paths"
- "www.velocidex.com/golang/velociraptor/reporting"
"www.velocidex.com/golang/velociraptor/vql"
vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
"www.velocidex.com/golang/vfilter"
@@ -17,6 +15,7 @@ import (
type CreateNotebookDownloadArgs struct {
NotebookId string `vfilter:"required,field=notebook_id,doc=Notebook ID to export."`
+ Filename string `vfilter:"optional,field=filename,doc=The name of the export. If not set this will be named according to the notebook id and timestamp"`
}
type CreateNotebookDownload struct{}
@@ -44,10 +43,10 @@ func (self *CreateNotebookDownload) Call(ctx context.Context,
return vfilter.Null{}
}
- notebook_path_manager := paths.NewNotebookPathManager(arg.NotebookId)
wg := &sync.WaitGroup{}
-
- err = reporting.ExportNotebookToZip(ctx, config_obj, wg, notebook_path_manager)
+ principal := vql_subsystem.GetPrincipal(scope)
+ path, err := ExportNotebookToZip(ctx,
+ config_obj, wg, arg.NotebookId, principal, arg.Filename)
if err != nil {
scope.Log("create_notebook_download: %s", err)
return vfilter.Null{}
@@ -57,7 +56,7 @@ func (self *CreateNotebookDownload) Call(ctx context.Context,
wg.Wait()
file_store.FlushFilestore(config_obj)
- return notebook_path_manager.ZipExport()
+ return path
}
func (self CreateNotebookDownload) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.FunctionInfo {
diff --git a/reporting/notebooks.go b/vql/server/notebooks/export.go
similarity index 54%
rename from reporting/notebooks.go
rename to vql/server/notebooks/export.go
index 32aa7b7c19c..4c9ab96eb86 100644
--- a/reporting/notebooks.go
+++ b/vql/server/notebooks/export.go
@@ -1,12 +1,13 @@
-package reporting
+package notebooks
import (
"bytes"
"context"
+ "crypto/sha256"
"encoding/base64"
+ "encoding/hex"
"fmt"
"html"
- "io"
"io/ioutil"
"net/url"
"os"
@@ -14,29 +15,126 @@ import (
"sync"
"time"
+ "github.com/Velocidex/ordereddict"
"github.com/Velocidex/yaml/v2"
"github.com/go-errors/errors"
"www.velocidex.com/golang/velociraptor/accessors"
+ "www.velocidex.com/golang/velociraptor/acls"
actions_proto "www.velocidex.com/golang/velociraptor/actions/proto"
api_proto "www.velocidex.com/golang/velociraptor/api/proto"
config_proto "www.velocidex.com/golang/velociraptor/config/proto"
"www.velocidex.com/golang/velociraptor/datastore"
"www.velocidex.com/golang/velociraptor/file_store"
"www.velocidex.com/golang/velociraptor/file_store/api"
+ "www.velocidex.com/golang/velociraptor/file_store/path_specs"
"www.velocidex.com/golang/velociraptor/json"
"www.velocidex.com/golang/velociraptor/logging"
"www.velocidex.com/golang/velociraptor/paths"
+ "www.velocidex.com/golang/velociraptor/reporting"
"www.velocidex.com/golang/velociraptor/result_sets"
+ "www.velocidex.com/golang/velociraptor/services"
"www.velocidex.com/golang/velociraptor/utils"
+ "www.velocidex.com/golang/velociraptor/vql"
+ vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
+ "www.velocidex.com/golang/vfilter"
+ "www.velocidex.com/golang/vfilter/arg_parser"
)
+var ()
+
+type ExportNotebookArg struct {
+ NotebookId string `vfilter:"required,field=notebook_id,doc=The id of the notebook to export"`
+ Filename string `vfilter:"optional,field=filename,doc=The name of the export. If not set this will be named according to the notebook id and timestamp"`
+ Type string `vfilter:"optional,field=type,doc=Set the type of the export (html or zip)."`
+}
+
+type ExportNotebookFunction struct{}
+
+func (self ExportNotebookFunction) Call(ctx context.Context,
+ scope vfilter.Scope,
+ args *ordereddict.Dict) vfilter.Any {
+
+ arg := &ExportNotebookArg{}
+ err := arg_parser.ExtractArgsWithContext(ctx, scope, args, arg)
+ if err != nil {
+ scope.Log("notebook_export: %v", err)
+ return vfilter.Null{}
+ }
+
+ err = vql_subsystem.CheckAccess(scope, acls.PREPARE_RESULTS)
+ if err != nil {
+ scope.Log("notebook_export: %v", err)
+ return vfilter.Null{}
+ }
+
+ config_obj, ok := vql_subsystem.GetServerConfig(scope)
+ if !ok {
+ scope.Log("notebook_export: Command can only run on the server")
+ return vfilter.Null{}
+ }
+
+ wg := &sync.WaitGroup{}
+ defer func() {
+ // Wait here until the export is done.
+ wg.Wait()
+
+ // Make sure the data is flushed to disk so the VQL can get
+ // it.
+ file_store.FlushFilestore(config_obj)
+ }()
+
+ principal := vql_subsystem.GetPrincipal(scope)
+
+ switch arg.Type {
+ case "", "zip":
+ result, err := ExportNotebookToZip(ctx,
+ config_obj, wg, arg.NotebookId,
+ principal, arg.Filename)
+ if err != nil {
+ scope.Log("notebook_export: %v", err)
+ return vfilter.Null{}
+ }
+ return result
+
+ case "html":
+ result, err := ExportNotebookToHTML(
+ config_obj, wg, arg.NotebookId,
+ principal, arg.Filename)
+ if err != nil {
+ scope.Log("notebook_export: %v", err)
+ return vfilter.Null{}
+ }
+ return result
+
+ default:
+ scope.Log("notebook_export: unsupported export type %v", arg.Type)
+ return vfilter.Null{}
+ }
+}
+
+func (self ExportNotebookFunction) Info(scope vfilter.Scope, type_map *vfilter.TypeMap) *vfilter.FunctionInfo {
+ return &vfilter.FunctionInfo{
+ Name: "notebook_export",
+ Doc: "Exports a notebook to a zip file or HTML.",
+ ArgType: type_map.AddType(scope, &ExportNotebookArg{}),
+ Metadata: vql.VQLMetadata().Permissions(acls.PREPARE_RESULTS).Build(),
+ }
+}
+
+func init() {
+ vql_subsystem.RegisterFunction(&ExportNotebookFunction{})
+}
+
var (
// Must match the output emitted by GuiTemplateEngine.Table
csvViewerRegexp = regexp.MustCompile(
`![](\"/notebooks/(?P<NotebookId)
N.[^/]+)/(?PNA.[^.]+.png)\" (?P[^>]*)>`)
+ `/attach/(?P<Attachment>NA\.[^\?)
[^>]*)>`)
+
+ hrefRegex = regexp.MustCompile(
+ `[^>]*)>(?P[^<]+) `)
)
const (
@@ -53,6 +151,32 @@ const (
+
+
+