Post Zeek Logs (brimdata#594)

* test: Add sample logs to test file detection

* fix: Prevent toolbar button text from wrapping

* feat: Allow the picker to accept multiple files

* feat: Add the default types.json

* feat: Add post logs api call to zealot client

* feat: Module to gather ingest params from a list of files

* feat: a way to run multiple steps in an transactional way

* feat: Refactor and extend openPacket

The code from openPacket was refactored and extended to handle
posting zeek logs and json logs. It uses the transaction function
which makes sure all steps succeed, and rollsback if they do not.

* refactor: Moved lib/FileType to brim/ingest

* fix: Normalize paths for windows tests

* fix: Removed openPacket and moved pcap tests

* fix: Removed unused sample files

* fix: Removed duplicate file and moved test

* fix: Only update space once you've got a snapshot_count of >0

* fix: Tests for error case

src/css/_toolbar-button.scss

@@ -28,6 +28,7 @@
     line-height: 16px;
     height: 16px;
     padding: 0 4px;
+    white-space: nowrap;
   }
 
   .dropdown-arrow {

src/js/lib/fileType.js → src/js/brim/ingest/detectFileType.js

@@ -1,5 +1,4 @@
 /* @flow */
-
 import fs from "fs"
 import readline from "readline"
 
@@ -8,13 +7,15 @@ const PCAP_2_HEX = "a1b2c3d4"
 const PCAPNG_HEX = "0a0d0d0a"
 const PCAP_HEXES = [PCAP_1_HEX, PCAP_2_HEX, PCAPNG_HEX]
 
-type IngestableType = "pcap" | "zeek" | "unknown"
+export type IngestFileType = "pcap" | "json" | "zeek" | "unknown"
 
-export default async function fileType(path: string): Promise<IngestableType> {
+export default async function(path: string): Promise<IngestFileType> {
   if (await isPcap(path)) {
     return "pcap"
-  } else if ((await isZeekAscii(path)) || (await isZeekJson(path))) {
+  } else if (await isZeekAscii(path)) {
     return "zeek"
+  } else if (await isZeekJson(path)) {
+    return "json"
   } else {
     return "unknown"
   }

src/js/brim/ingest/detectFileTypes.js

@@ -0,0 +1,12 @@
+/* @flow */
+import type {FileListData} from "./fileList"
+import detectFileType from "./detectFileType"
+
+export default function(paths: string[]): Promise<FileListData> {
+  return Promise.all(
+    paths.map(async (path) => {
+      let type = await detectFileType(path)
+      return {type, path}
+    })
+  )
+}

src/js/brim/ingest/detectFileTypes.test.js

@@ -0,0 +1,23 @@
+/* @flow */
+
+import ingest from "./"
+import itestFile from "../../test/itestFile"
+
+const json = itestFile("sample.ndjson")
+const pcap = itestFile("sample.pcap")
+const pcapng = itestFile("sample.pcapng")
+const unknown = itestFile("setup.js")
+const zeek = itestFile("sample.tsv")
+
+test("add file types", async () => {
+  let paths = [pcap, pcapng, zeek, json, unknown]
+  let types = await ingest.detectFileTypes(paths)
+
+  expect(types).toEqual([
+    {type: "pcap", path: pcap},
+    {type: "pcap", path: pcapng},
+    {type: "zeek", path: zeek},
+    {type: "json", path: json},
+    {type: "unknown", path: unknown}
+  ])
+})

src/js/brim/ingest/fileList.js

@@ -0,0 +1,48 @@
+/* @flow */
+
+import type {IngestFileType} from "./detectFileType"
+import lib from "../../lib"
+
+export type FileListData = {type: IngestFileType, path: string}[]
+
+export default function fileList(files: FileListData) {
+  return {
+    first() {
+      return files[0]
+    },
+
+    oneFile() {
+      return files.length === 1
+    },
+
+    multiple() {
+      return files.length > 1
+    },
+
+    paths(): string[] {
+      return files.map((f) => f.path)
+    },
+
+    allPcap() {
+      return files.every((f) => f.type === "pcap")
+    },
+
+    allZeek() {
+      return files.every((f) => f.type === "zeek")
+    },
+
+    mixed() {
+      return !files.every((f) => f.type === files[0].type)
+    },
+
+    inSameDir() {
+      return files.every(
+        (f) => lib.file(f.path).dirName() === lib.file(files[0].path).dirName()
+      )
+    },
+
+    dirName() {
+      return lib.file(files[0].path).dirName()
+    }
+  }
+}

src/js/brim/ingest/getParams.js

@@ -0,0 +1,55 @@
+/* @flow */
+import os from "os"
+import path from "path"
+
+import type {IngestFileType} from "./detectFileType"
+import fileList, {type FileListData} from "./fileList"
+import time from "../time"
+
+export type IngestParams = {
+  dataDir: string,
+  endpoint: IngestFileType,
+  paths: string[]
+}
+
+export type IngestParamsError = {
+  error: string
+}
+
+export default function getParams(
+  data: FileListData,
+  home?: string = os.homedir(),
+  now?: Date = new Date()
+): IngestParams | IngestParamsError {
+  let files = fileList(data)
+
+  if (files.multiple() && files.allPcap()) {
+    return {
+      error: "Only one pcap can be opened at a time."
+    }
+  }
+
+  if (files.multiple() && files.mixed()) {
+    return {
+      error: "Only files of a single type (zeek or pcap) can be opened."
+    }
+  }
+
+  function getDataDir() {
+    if (files.oneFile()) return path.normalize(files.first().path)
+
+    let dirName = files.inSameDir() ? files.dirName() : generateDirName(now)
+
+    return path.join(home, ".brim", dirName)
+  }
+
+  return {
+    dataDir: getDataDir() + ".brim",
+    endpoint: files.first().type,
+    paths: files.paths()
+  }
+}
+
+function generateDirName(now) {
+  return "zeek_" + time(now).format("YYYY-MM-DD_HH:mm:ss")
+}

src/js/brim/ingest/index.js

@@ -0,0 +1,8 @@
+/* @flow */
+import detectFileTypes from "./detectFileTypes"
+import getParams from "./getParams"
+
+export default {
+  getParams,
+  detectFileTypes
+}

src/js/brim/ingest/test.js

@@ -0,0 +1,79 @@
+/* @flow */
+import path from "path"
+
+import ingest from "./"
+
+test("one pcap", () => {
+  let data = ingest.getParams([{type: "pcap", path: "/work/my.pcap"}])
+
+  expect(data).toEqual({
+    dataDir: path.join("/work", "my.pcap.brim"),
+    endpoint: "pcap",
+    paths: ["/work/my.pcap"]
+  })
+})
+
+test("one zeek log", () => {
+  let data = ingest.getParams([{type: "zeek", path: "/work/zeek.log"}])
+
+  expect(data).toEqual({
+    dataDir: path.join("/work", "zeek.log.brim"),
+    endpoint: "zeek",
+    paths: ["/work/zeek.log"]
+  })
+})
+
+test("two zeek logs in same dir", () => {
+  let data = ingest.getParams(
+    [
+      {type: "zeek", path: "/work/zeek-1.log"},
+      {type: "zeek", path: "/work/zeek-2.log"}
+    ],
+    "/home"
+  )
+
+  expect(data).toEqual({
+    dataDir: path.join("/home", ".brim", "work.brim"),
+    endpoint: "zeek",
+    paths: ["/work/zeek-1.log", "/work/zeek-2.log"]
+  })
+})
+
+test("two zeek logs in different dir", () => {
+  let data = ingest.getParams(
+    [
+      {type: "zeek", path: "/work/day-1/zeek.log"},
+      {type: "zeek", path: "/work/day-2/zeek.log"}
+    ],
+    "/home",
+    new Date(0)
+  )
+
+  expect(data).toEqual({
+    dataDir: path.join("/home", ".brim", "zeek_1969-12-31_16:00:00.brim"),
+    endpoint: "zeek",
+    paths: ["/work/day-1/zeek.log", "/work/day-2/zeek.log"]
+  })
+})
+
+test("two pcaps", () => {
+  let data = ingest.getParams([
+    {type: "pcap", path: "/pcap-1"},
+    {type: "pcap", path: "/pcap-2"}
+  ])
+
+  expect(data).toEqual({
+    error: "Only one pcap can be opened at a time."
+  })
+})
+
+test("1 pcap and 1 zeek", () => {
+  let data = ingest.getParams([
+    {type: "pcap", path: "/pcap-1"},
+    {type: "zeek", path: "/zeek-1"}
+  ])
+
+  expect(data).toEqual({
+    error: "Only files of a single type (zeek or pcap) can be opened."
+  })
+})

src/js/components/PcapFileInput.js

@@ -21,6 +21,7 @@ export default function PcapFileInput({onChange}: Props) {
       <ArrowOrangeSvg className="upload-arrow" />
       <input
         type="file"
+        multiple
         title=""
         onChange={_onChange}
         {...reactElementProps("pcapsFileInput")}

src/js/components/TabWelcome.js

@@ -8,7 +8,7 @@ import SavedSpacesList from "./SavedSpacesList"
 import SpaceDeletedNotice from "./SpaceDeletedNotice"
 import Spaces from "../state/Spaces"
 import Tab from "../state/Tab"
-import openPacket from "../flows/openPacket"
+import ingestFiles from "../flows/ingestFiles"
 import refreshSpaceNames from "../flows/refreshSpaceNames"
 
 export default function TabWelcome() {
@@ -21,9 +21,9 @@ export default function TabWelcome() {
     dispatch(refreshSpaceNames())
   }, [])
 
-  function onChange(_e, [file]) {
-    if (!file) return
-    dispatch(openPacket(file))
+  function onChange(_e, files) {
+    if (!files.length) return
+    dispatch(ingestFiles(files))
   }
 
   return (

src/js/errors/index.js

@@ -1,6 +1,8 @@
 /* @flow */
+import logsIngest from "./logsIngest"
 import pcapIngest from "./pcapIngest"
 
 export default {
-  pcapIngest
+  pcapIngest,
+  logsIngest
 }

src/js/errors/logsIngest.js

@@ -0,0 +1,18 @@
+/* @flow */
+import type {BrimError} from "./types"
+
+export default function(err: string): BrimError {
+  return {
+    type: "LogsIngestError",
+    message: "Unable to load these logs",
+    details: getDetails(err)
+  }
+}
+
+function getDetails(err) {
+  let details = [`Detail: ${err}`]
+  if (/sort limit/.test(err)) {
+    details.push("Reached internal line count limit")
+  }
+  return details
+}

src/js/errors/types.js

@@ -1,6 +1,9 @@
 /* @flow */
 
-export type BrimErrorType = "PCAPIngestError" | "NetworkError"
+export type BrimErrorType =
+  | "PCAPIngestError"
+  | "NetworkError"
+  | "LogsIngestError"
 
 export type BrimError = {
   type: BrimErrorType,