Super Support

apps/zui/package.json

@@ -74,7 +74,6 @@
     "acorn": "^7.4.1",
     "ajv": "^6.9.1",
     "animejs": "^3.2.0",
-    "brimcap": "brimdata/brimcap#v1.18.0",
     "bullet": "^0.0.7",
     "chalk": "^4.1.0",
     "chrono-node": "^2.5.0",
@@ -152,14 +151,14 @@
     "set-tz": "^0.2.0",
     "sprintf-js": "^1.1.2",
     "styled-components": "^5.3.5",
+    "super": "github:brimdata/super#main",
     "tmp": "^0.1.0",
     "tree-model": "^1.0.7",
     "use-resize-observer": "^8.0.0",
     "utopia-core-scss": "^1.0.1",
     "web-file-polyfill": "^1.0.4",
     "web-streams-polyfill": "^3.2.0",
     "when-clause": "^0.0.4",
-    "zed": "brimdata/zed#f62dfb623defc702d8e2fe62adf21d1184792b19",
     "zui-test-data": "workspace:*"
   },
   "peerDependencies": {

apps/zui/scripts/download-zdeps/index.js

@@ -5,26 +5,23 @@ const fs = require("fs-extra")
 const got = require("got")
 const path = require("path")
 const tmp = require("tmp")
-const zuiPackage = require("../../package.json")
+const package = require("../../package.json")
 const decompress = require("decompress")
-const zdepsPath = path.resolve("zdeps")
+const depsDir = path.resolve("zdeps")
 
 const platformDefs = {
   darwin: {
-    zqBin: "zq",
-    zedBin: "zed",
+    superBin: "super",
     osarch: "darwin-amd64",
     ext: "tar.gz",
   },
   linux: {
-    zqBin: "zq",
-    zedBin: "zed",
+    superBin: "super",
     osarch: "linux-amd64",
     ext: "tar.gz",
   },
   win32: {
-    zqBin: "zq.exe",
-    zedBin: "zed.exe",
+    superBin: "super.exe",
     osarch: "windows-amd64",
     ext: "zip",
   },
@@ -52,11 +49,11 @@ async function unzipTo(zipfile, dir) {
   await decompress(zipfile, dir)
 }
 
-function zedArtifactPaths(version) {
+function artifactPaths(version) {
   const plat = platformDefs[process.platform]
 
-  const artifactFile = `zed-${version}.${plat.osarch}.${plat.ext}`
-  const artifactUrl = `https://github.com/brimdata/zed/releases/download/${version}/${artifactFile}`
+  const artifactFile = `super-${version}.${plat.osarch}.${plat.ext}`
+  const artifactUrl = `https://github.com/brimdata/super/releases/download/${version}/${artifactFile}`
 
   return {
     artifactFile,
@@ -66,9 +63,9 @@ function zedArtifactPaths(version) {
 
 // Download and extract the zed binary for this platform to the specified
 // directory. Returns the absolute path of the zed binary file.
-async function zedArtifactsDownload(version, destPath) {
+async function artifactsDownload(version, destPath) {
   const plat = platformDefs[process.platform]
-  const paths = zedArtifactPaths(version)
+  const paths = artifactPaths(version)
 
   const tmpdir = tmp.dirSync({unsafeCleanup: true})
   try {
@@ -78,7 +75,7 @@ async function zedArtifactsDownload(version, destPath) {
     console.log("Download and unzip success")
     fs.mkdirpSync(destPath)
 
-    for (let f of [plat.zqBin, plat.zedBin]) {
+    for (let f of [plat.superBin]) {
       fs.moveSync(path.join(tmpdir.name, f), path.join(destPath, f), {
         overwrite: true,
       })
@@ -88,74 +85,51 @@ async function zedArtifactsDownload(version, destPath) {
   }
 }
 
-async function zedDevBuild(destPath) {
+async function devBuild(destPath) {
   if (!(process.platform in platformDefs)) {
     throw new Error("unsupported platform")
   }
   const plat = platformDefs[process.platform]
 
-  const zedPackageDir = path.join(
+  const packageDir = path.join(
     __dirname,
     "..",
     "..",
     "..",
     "..",
     "node_modules",
-    "zed"
+    "super"
   )
 
   fs.mkdirpSync(destPath)
 
-  for (let f of [plat.zqBin, plat.zedBin]) {
-    fs.copyFileSync(path.join(zedPackageDir, "dist", f), path.join(destPath, f))
-  }
-}
-
-// Suricata rules are dropped from the Windows build to fix a false positive
-// malware flagging. See https://github.com/brimdata/zui/issues/2857.
-const filterBrimcapZdeps = (src, dest) => {
-  if (process.platform == "win32" &&
-      (/suricata\.rules$/.test(src) || /emerging\.rules\.tar\.gz$/.test(src)) &&
-      fs.statSync(src).isFile()) {
-    return false
-  } else {
-    return true
+  for (let f of [plat.superBin]) {
+    fs.copyFileSync(path.join(packageDir, "dist", f), path.join(destPath, f))
   }
 }
 
 async function main() {
   try {
-    fs.copySync(
-      path.resolve("..", "..", "node_modules", "brimcap", "build", "dist"),
-      zdepsPath,
-      { filter: filterBrimcapZdeps }
-    )
-    const brimcapVersion = child_process
-      .execSync(path.join(zdepsPath, "brimcap") + " -version")
-      .toString()
-      .trim()
-    console.log("copied brimcap artifacts " + brimcapVersion)
-
     // The Zed dependency should be a git tag or commit. Any tag that
     // begins with "v*" is expected to be a released artifact, and will
     // be downloaded from the Zed repository. Otherwise, copy Zed
     // artifacts from node_modules via zedDevBuild.
-    const zedVersion = zuiPackage.devDependencies.zed.split("#")[1]
-    if (zedVersion.startsWith("v")) {
-      await zedArtifactsDownload(zedVersion, zdepsPath)
-      console.log("downloaded Zed artifacts version " + zedVersion)
+    const version = package.devDependencies.super.split("#")[1]
+    if (version.startsWith("v")) {
+      await artifactsDownload(version, depsDir)
+      console.log("downloaded artifacts version " + version)
     } else {
-      await zedDevBuild(zdepsPath)
+      await devBuild(depsDir)
       // Print the version inside zq derived during prepack as
       // opposed to what's in package.json.
-      let realZqVersion = child_process
-        .execSync(path.join(zdepsPath, "zq") + " -version")
+      let realVersion = child_process
+        .execSync(path.join(depsDir, "super") + " -version")
         .toString()
         .trim()
-      console.log("copied Zed artifacts " + realZqVersion)
+      console.log("copied artifacts " + realVersion)
     }
   } catch (err) {
-    console.error("zdeps setup: ", err)
+    console.error("setup: ", err)
     process.exit(1)
   }
 }

apps/zui/src/config/links.ts

@@ -1,18 +1,18 @@
 import pkg from "../../package.json"
 
-const currentZedTag = pkg.devDependencies.zed.split("#")[1] || "main"
-const zedDocsTag = currentZedTag.startsWith("v")
-  ? currentZedTag.replace(/\.\d+$/, ".0")
+const gitRef = pkg.devDependencies.super.split("#")[1] || "main"
+const docsGitRef = gitRef.startsWith("v")
+  ? gitRef.replace(/\.\d+$/, ".0")
   : "v1.18.0" // Change back to "next" when we make the zed->super transition
 
 export default {
-  ZED_DOCS_ROOT: `https://zed.brimdata.io/docs/${zedDocsTag}/commands/zed`,
-  ZED_DOCS_LANGUAGE: `https://zed.brimdata.io/docs/${zedDocsTag}/language`,
-  ZED_DOCS_FORMATS_ZJSON: `https://zed.brimdata.io/docs/${zedDocsTag}/formats/zjson`,
-  ZED_DOCS_FORMATS_ZNG: `https://zed.brimdata.io/docs/${zedDocsTag}/formats/zng`,
-  ZED_DOCS_FORMATS_ZSON: `https://zed.brimdata.io/docs/${zedDocsTag}/formats/zson`,
-  ZED_DOCS_FORMATS_VNG: `https://zed.brimdata.io/docs/${zedDocsTag}/formats/vng`,
-  ZUI_DOCS_ROOT: `https://zui.brimdata.io/docs`,
-  ZUI_DOCS_CONNNECTION_TROUBLESHOOTING: `https://zui.brimdata.io/docs/support/Troubleshooting#zui-shows-an-error-the-service-could-not-be-reached`,
-  ZUI_DOWNLOAD: `https://www.brimdata.io/download/`,
+  DOCS_ROOT: `https://zed.brimdata.io/docs/${docsGitRef}/commands/zed`,
+  DOCS_LANGUAGE: `https://zed.brimdata.io/docs/${docsGitRef}/language`,
+  DOCS_FORMATS_ZJSON: `https://zed.brimdata.io/docs/${docsGitRef}/formats/zjson`,
+  DOCS_FORMATS_ZNG: `https://zed.brimdata.io/docs/${docsGitRef}/formats/zng`,
+  DOCS_FORMATS_ZSON: `https://zed.brimdata.io/docs/${docsGitRef}/formats/zson`,
+  DOCS_FORMATS_VNG: `https://zed.brimdata.io/docs/${docsGitRef}/formats/vng`,
+  DESKTOP_DOCS_ROOT: `https://zui.brimdata.io/docs`,
+  DESKTOP_DOCS_CONNNECTION_TROUBLESHOOTING: `https://zui.brimdata.io/docs/support/Troubleshooting#zui-shows-an-error-the-service-could-not-be-reached`,
+  DESKTOP_DOWNLOAD: `https://www.brimdata.io/download/`,
 }

apps/zui/src/core/main/main-object.ts

@@ -89,7 +89,7 @@ export class MainObject {
       addr: DefaultLake.listenAddr,
       port: this.args.lakePort,
       logs: this.args.lakeLogs,
-      bin: zdeps.zed,
+      bin: zdeps.superdb,
       corsOrigins: ["*"],
     })
   }

apps/zui/src/core/zq.ts

@@ -2,9 +2,9 @@ import * as zed from "@brimdata/zed-node"
 import * as zdeps from "../electron/zdeps"
 
 export const zq = ((opts) => {
-  return zed.zq({...opts, bin: zdeps.zq})
+  return zed.zq({...opts, bin: zdeps.superdb})
 }) as typeof zed.zq
 
 export const createReadableStream = ((opts) => {
-  return zed.createReadableStream({...opts, bin: zdeps.zq})
+  return zed.createReadableStream({...opts, bin: zdeps.superdb})
 }) as typeof zed.createReadableStream

apps/zui/src/domain/loads/file-loader.ts

@@ -33,7 +33,7 @@ export class FileLoader implements Loader {
       res = await client.load(body, {
         pool: ctx.poolId,
         branch: ctx.branch,
-        format: "zng",
+        format: "bsup",
         message: {
           author: ctx.author,
           body: ctx.body,

apps/zui/src/domain/loads/handlers/preview-load-files.ts

@@ -2,31 +2,22 @@ import {createHandler} from "src/core/handlers"
 import Current from "src/js/state/Current"
 import LoadDataForm from "src/js/state/LoadDataForm"
 import Pools from "src/js/state/Pools"
-import {quickLoadFiles} from "./quick-load-files"
 import Modal from "src/js/state/Modal"
 
 export const previewLoadFiles = createHandler(
   "loads.previewLoadFiles",
-  async (
-    {dispatch, invoke, select},
-    opts: {files: string[]; poolId?: string}
-  ) => {
-    const files = await invoke("loads.getFileTypes", opts.files)
+  async ({dispatch, select}, opts: {files: string[]; poolId?: string}) => {
     const lakeId = select(Current.getLakeId)
     const pool = select(Pools.get(lakeId, opts.poolId))
     const poolId = pool ? pool.id : null
 
-    if (files.length === 1 && files[0].type === "pcap") {
-      quickLoadFiles({files: files.map((f) => f.path), poolId})
+    if (select(LoadDataForm.getShow)) {
+      // The preview load is already opened
+      dispatch(LoadDataForm.addFiles(opts.files))
     } else {
-      if (select(LoadDataForm.getShow)) {
-        // The preview load is already opened
-        dispatch(LoadDataForm.addFiles(opts.files))
-      } else {
-        dispatch(LoadDataForm.setFiles(opts.files))
-        dispatch(LoadDataForm.setPoolId(poolId))
-        dispatch(Modal.show("preview-load"))
-      }
+      dispatch(LoadDataForm.setFiles(opts.files))
+      dispatch(LoadDataForm.setPoolId(poolId))
+      dispatch(Modal.show("preview-load"))
     }
   }
 )

apps/zui/src/domain/loads/messages.ts

@@ -19,7 +19,6 @@ export type LoadFormData = {
 export type LoadsOperations = {
   "loads.create": typeof ops.create
   "loads.preview": typeof ops.preview
-  "loads.getFileTypes": typeof ops.getFileTypes
   "loads.abortPreview": typeof ops.abortPreview
   "loads.abort": typeof ops.abort
   "loads.paste": typeof ops.paste

apps/zui/src/domain/loads/operations/create.ts

@@ -1,7 +1,6 @@
 import {createOperation} from "src/core/operations"
 import * as zui from "src/zui"
 import {Pool} from "src/models/pool"
-import {ZedScript} from "src/models/zed-script"
 import {LoadFormData} from "../messages"
 import {errorToString} from "src/util/error-to-string"
 import {deriveName} from "src/domain/pools/utils"
@@ -13,7 +12,6 @@ export const create = createOperation(
   "loads.create",
   async (ctx, data: LoadFormData) => {
     const pool = await createPool(data)
-    const script = new ZedScript(data.shaper || "")
     // Async so that we can return this and subscribe to updates on the load.
     // Do not wait for the load to finish in this operation.
     zui.pools
@@ -25,7 +23,7 @@ export const create = createOperation(
         branch: "main",
         query: data.query,
         files: data.files,
-        shaper: script.isEmpty() ? "*" : data.shaper,
+        shaper: data.shaper,
         author: data.author,
         body: data.body,
       })

apps/zui/src/domain/loads/operations/index.ts

@@ -1,4 +1,3 @@
-export * from "./get-file-types"
 export * from "./preview"
 export * from "./create"
 export * from "./abort"

apps/zui/src/domain/updates/linux-updater.ts

@@ -30,7 +30,7 @@ export class LinuxUpdater implements Updater {
     if (env.isInsiders) {
       return pkg.repository + "/releases/latest"
     } else {
-      return links.ZUI_DOWNLOAD
+      return links.DESKTOP_DOWNLOAD
     }
   }
 }

apps/zui/src/electron/windows/search/app-menu.ts

@@ -263,13 +263,13 @@ export function compileTemplate(
       {
         label: "Zui Documentation",
         click() {
-          shell.openExternal(links.ZUI_DOCS_ROOT)
+          shell.openExternal(links.DESKTOP_DOCS_ROOT)
         },
       },
       {
         label: "Zed Language Documentation",
         click() {
-          shell.openExternal(links.ZED_DOCS_LANGUAGE)
+          shell.openExternal(links.DOCS_LANGUAGE)
         },
       },
 

apps/zui/src/electron/zdeps.ts

@@ -12,5 +12,4 @@ function getBin(name) {
   return path.join(getDirectory(), name)
 }
 
-export const zed = getBin("zed")
-export const zq = getBin("zq")
+export const superdb = getBin("super")

apps/zui/src/js/components/ConnectionError.tsx

@@ -205,7 +205,7 @@ const ConnectionError = (props: {onRetry: () => void | Promise<void>}) => {
           <SubmitButton onClick={onClick}>Retry</SubmitButton>
         </p>
         <p>
-          <Link href={links.ZUI_DOCS_CONNNECTION_TROUBLESHOOTING}>
+          <Link href={links.DESKTOP_DOCS_CONNNECTION_TROUBLESHOOTING}>
             View Troubleshooting Docs
           </Link>
         </p>