diff --git a/.github/workflows/clojure.yml b/.github/workflows/clojure.yml index d4aa9209f..b43b5583d 100644 --- a/.github/workflows/clojure.yml +++ b/.github/workflows/clojure.yml @@ -29,7 +29,7 @@ jobs: echo "did_change=False" >> $GITHUB_OUTPUT for file in ${changeArr[*]} do - if echo $file | grep -E '^server/'; then + if echo $file | grep -E '^server/' | grep -E -v '^server/refinery/'; then echo "did_change=True" >> $GITHUB_OUTPUT break fi @@ -246,19 +246,69 @@ jobs: sed -i "s|IMAGE_REPLACE_ME|597134865416.dkr.ecr.us-east-1.amazonaws.com/instant-prod-ecr:${IMAGE_TAG}|g" docker-compose.yml cat docker-compose.yml - - name: Update logdna.sh - working-directory: server - env: - SHA: ${{ github.sha }} - run: | - sed -i "s|SHA_REPLACE_ME|${SHA}|g" .platform/hooks/prebuild/logdna.sh - cat .platform/hooks/prebuild/logdna.sh - - name: Create eb application version uses: "./.github/actions/elastic-beanstalk" with: working-directory: "server" - files: '["docker-compose.yml", ".platform/hooks/prebuild/logdna.sh", "refinery/config.yaml", "refinery/config-redis.yaml", "refinery/rules.yaml", "refinery/.env"]' + files: '["docker-compose.yml", ".ebextensions/resources.config"]' aws-region: "us-east-1" bucket: "elasticbeanstalk-us-east-1-597134865416" application-name: "instant-docker-prod" + + detect_refinery: + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' + outputs: + did_change: ${{ steps.detect-change.outputs.did_change }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }} + - name: Get changed files + id: changed-files + run: | + if ${{ github.event_name == 'pull_request' || github.event.before == '0000000000000000000000000000000000000000' }}; then + echo "changed_files=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> $GITHUB_OUTPUT + else + echo "changed_files=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }} | xargs)" >> $GITHUB_OUTPUT + fi + - name: Detect refinery changes + id: detect-change + run: | + changeArr=($(echo ${{ steps.changed-files.outputs.changed_files }})) + echo "did_change=False" >> $GITHUB_OUTPUT + for file in ${changeArr[*]} + do + if echo $file | grep -E '^server/refinery/'; then + echo "did_change=True" >> $GITHUB_OUTPUT + break + fi + done + + publish-refinery-eb: + runs-on: ubuntu-latest + if: needs.detect_refinery.outputs.did_change == 'True' + needs: + - detect_refinery + permissions: + id-token: write + contents: read + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::597134865416:role/aws-credentials-for-github-actions-Role-x0lUbtwJNb7G + aws-region: us-east-1 + + - name: Create eb application version + uses: "./.github/actions/elastic-beanstalk" + with: + working-directory: "server/refinery/" + files: '["docker-compose.yml", "config.yaml", "config-redis.yaml", "rules.yaml", "refinery.env", ".ebextensions/resources.config"]' + aws-region: "us-east-1" + bucket: "elasticbeanstalk-us-east-1-597134865416" + application-name: "refinery" diff --git a/client/packages/admin/package.json b/client/packages/admin/package.json index ceaa1ab35..08179ad05 100644 --- a/client/packages/admin/package.json +++ b/client/packages/admin/package.json @@ -1,6 +1,6 @@ { "name": "@instantdb/admin", - "version": "v0.15.2", + "version": "v0.15.7", "description": "Admin SDK for Instant DB", "main": "dist/index.js", "module": "dist/module/index.js", diff --git a/client/packages/admin/src/index.ts b/client/packages/admin/src/index.ts index 5198f84c3..75e31c616 100644 --- a/client/packages/admin/src/index.ts +++ b/client/packages/admin/src/index.ts @@ -42,6 +42,8 @@ import { type ValueTypes, type InstantSchemaDef, type InstantUnknownSchema, + type InstaQLEntity, + type InstaQLResult, } from "@instantdb/core"; import version from "./version"; @@ -757,7 +759,7 @@ class InstantAdminDatabase> { headers: authorizedHeaders(this.config, this.impersonationOpts), body: JSON.stringify({ query: query, - "inference?": true, + "inference?": !!this.config.schema, }), }); }; @@ -793,7 +795,10 @@ class InstantAdminDatabase> { return jsonFetch(`${this.config.apiURI}/admin/transact`, { method: "POST", headers: authorizedHeaders(this.config, this.impersonationOpts), - body: JSON.stringify({ steps: steps }), + body: JSON.stringify({ + steps: steps, + "throw-on-missing-attrs?": !!this.config.schema, + }), }); }; @@ -923,4 +928,6 @@ export { type ValueTypes, type InstantSchemaDef, type InstantUnknownSchema, + type InstaQLEntity, + type InstaQLResult, }; diff --git a/client/packages/admin/src/version.js b/client/packages/admin/src/version.js index 042dd5158..7ef8752ed 100644 --- a/client/packages/admin/src/version.js +++ b/client/packages/admin/src/version.js @@ -1,4 +1,4 @@ // Autogenerated by publish_packages.clj -const version = "v0.15.2-dev"; +const version = "v0.15.7-dev"; export default version; diff --git a/client/packages/cli/index.js b/client/packages/cli/index.js index 7fe93f231..e6694fd3a 100644 --- a/client/packages/cli/index.js +++ b/client/packages/cli/index.js @@ -6,14 +6,24 @@ import { join } from "path"; import { randomUUID } from "crypto"; import dotenv from "dotenv"; import chalk from "chalk"; -import { program } from "commander"; -import { input, confirm } from "@inquirer/prompts"; +import { program, Option } from "commander"; +import { input, confirm, select } from "@inquirer/prompts"; import envPaths from "env-paths"; import { loadConfig } from "unconfig"; import { packageDirectory } from "pkg-dir"; import openInBrowser from "open"; import ora from "ora"; import terminalLink from "terminal-link"; +import { exec } from "child_process"; +import { promisify } from "util"; +import { + detectPackageManager, + getInstallCommand, +} from "./src/util/packageManager.js"; +import { pathExists, readJsonFile } from "./src/util/fs.js"; +import prettier from 'prettier'; + +const execAsync = promisify(exec); // config dotenv.config(); @@ -21,8 +31,35 @@ dotenv.config(); const dev = Boolean(process.env.INSTANT_CLI_DEV); const verbose = Boolean(process.env.INSTANT_CLI_VERBOSE); +// logs + +function warn(firstArg, ...rest) { + console.warn(chalk.yellow("[warning]") + " " + firstArg, ...rest); +} + +function error(firstArg, ...rest) { + console.error(chalk.red("[error]") + " " + firstArg, ...rest); +} + // consts +const potentialEnvs = { + catchall: "INSTANT_APP_ID", + next: "NEXT_PUBLIC_INSTANT_APP_ID", + svelte: "PUBLIC_INSTANT_APP_ID", + vite: "VITE_INSTANT_APP_ID", +}; + +const noAppIdErrorMessage = ` +Couldn't find an app ID. + +You can either: +a. Set ${chalk.green("`INSTANT_APP_ID`")} in your .env file. [1] +b. Or provide an app ID via the CLI: ${chalk.green("`instant-cli push|pull -a `")}. + +[1] Alternatively, If you have ${chalk.green("`NEXT_PUBLIC_INSTANT_APP_ID`")}, ${chalk.green("`VITE_INSTANT_APP_ID`")}, we can detect those too! +`.trim(); + const instantDashOrigin = dev ? "http://localhost:3000" : "https://instantdb.com"; @@ -31,110 +68,367 @@ const instantBackendOrigin = process.env.INSTANT_CLI_API_URI || (dev ? "http://localhost:8888" : "https://api.instantdb.com"); -const instantCLIDescription = ` -${chalk.magenta(`Instant CLI`)} -Docs: ${chalk.underline(`https://www.instantdb.com/docs/cli`)} -Dash: ${chalk.underline(`https://www.instantdb.com/dash`)} -Discord: ${chalk.underline(`https://discord.com/invite/VU53p7uQcE`)}`.trim(); +const PUSH_PULL_OPTIONS = new Set(["schema", "perms", "all"]); + +function convertArgToBagWithErrorLogging(arg) { + if (!arg) { + return { ok: true, bag: "all" }; + } else if (PUSH_PULL_OPTIONS.has(arg.trim().lowercase())) { + return { ok: true, bag: arg }; + } else { + error( + `${chalk.red(arg)} must be one of ${chalk.green(Array.from(PUSH_PULL_OPTIONS).join(", "))}`, + ); + return { ok: false }; + } +} + +// Note: Nov 20, 2024 +// We can eventually deprecate this +// once we're confident that users no longer +// provide app ID as their first argument +function convertPushPullToCurrentFormat(cmdName, arg, opts) { + if (arg && !PUSH_PULL_OPTIONS.has(arg) && !opts.app) { + warnDeprecation(`${cmdName} ${arg}`, `${cmdName} --app ${arg}`); + return { ok: true, bag: "all", opts: { ...opts, app: arg } }; + } + const { ok, bag } = convertArgToBagWithErrorLogging(arg); + if (!ok) return { ok: false }; + return { ok: true, bag, opts }; +} + +async function packageDirectoryWithErrorLogging() { + const pkgDir = await packageDirectory(); + if (!pkgDir) { + error("Couldn't find your root directory. Is there a package.json file?"); + return; + } + return pkgDir; +} // cli +// Header -- this shows up in every command +const logoChalk = chalk.bold("instant-cli"); +const versionChalk = chalk.dim(`${version.trim()}`); +const headerChalk = `${logoChalk} ${versionChalk} ` + "\n"; + +// Help Footer -- this only shows up in help commands +const helpFooterChalk = + "\n" + + chalk.dim.bold("Want to learn more?") + + "\n" + + `Check out the docs: ${chalk.blueBright.underline("https://instantdb.com/docs")} +Join the Discord: ${chalk.blueBright.underline("https://discord.com/invite/VU53p7uQcE")} +`.trim(); + +program.addHelpText("after", helpFooterChalk); + +program.addHelpText("beforeAll", headerChalk); + +function getLocalAndGlobalOptions(cmd, helper) { + const mixOfLocalAndGlobal = helper.visibleOptions(cmd); + const localOptionsFromMix = mixOfLocalAndGlobal.filter( + (option) => !option.__global, + ); + const globalOptionsFromMix = mixOfLocalAndGlobal.filter( + (option) => option.__global, + ); + const globalOptions = helper.visibleGlobalOptions(cmd); + + return [localOptionsFromMix, globalOptionsFromMix.concat(globalOptions)]; +} + +// custom `formatHelp` +// original: https://github.com/tj/commander.js/blob/master/lib/help.js +function formatHelp(cmd, helper) { + const termWidth = helper.padWidth(cmd, helper); + const helpWidth = helper.helpWidth || 80; + const itemIndentWidth = 2; + const itemSeparatorWidth = 2; // between term and description + function formatItem(term, description) { + if (description) { + const fullText = `${term.padEnd(termWidth + itemSeparatorWidth)}${description}`; + return helper.wrap( + fullText, + helpWidth - itemIndentWidth, + termWidth + itemSeparatorWidth, + ); + } + return term; + } + function formatList(textArray) { + return textArray.join("\n").replace(/^/gm, " ".repeat(itemIndentWidth)); + } + + // Usage + let output = [`${helper.commandUsage(cmd)}`, ""]; + + // Description + const commandDescription = helper.commandDescription(cmd); + if (commandDescription.length > 0) { + output = output.concat([helper.wrap(commandDescription, helpWidth, 0), ""]); + } + + // Arguments + const argumentList = helper.visibleArguments(cmd).map((argument) => { + return formatItem( + helper.argumentTerm(argument), + helper.argumentDescription(argument), + ); + }); + if (argumentList.length > 0) { + output = output.concat([ + chalk.dim.bold("Arguments"), + formatList(argumentList), + "", + ]); + } + const [visibleOptions, visibleGlobalOptions] = getLocalAndGlobalOptions( + cmd, + helper, + ); + + // Options + const optionList = visibleOptions.map((option) => { + return formatItem( + helper.optionTerm(option), + helper.optionDescription(option), + ); + }); + if (optionList.length > 0) { + output = output.concat([ + chalk.dim.bold("Options"), + formatList(optionList), + "", + ]); + } + // Commands + const commandList = helper.visibleCommands(cmd).map((cmd) => { + return formatItem( + helper.subcommandTerm(cmd), + helper.subcommandDescription(cmd), + ); + }); + if (commandList.length > 0) { + output = output.concat([ + chalk.dim.bold("Commands"), + formatList(commandList), + "", + ]); + } + + if (this.showGlobalOptions) { + const globalOptionList = visibleGlobalOptions.map((option) => { + return formatItem( + helper.optionTerm(option), + helper.optionDescription(option), + ); + }); + if (globalOptionList.length > 0) { + output = output.concat([ + chalk.dim.bold("Global Options"), + formatList(globalOptionList), + "", + ]); + } + } + + return output.join("\n"); +} + +program.configureHelp({ + showGlobalOptions: true, + formatHelp, +}); + +function globalOption(flags, description, argParser) { + const opt = new Option(flags, description); + if (argParser) { + opt.argParser(argParser); + } + // @ts-ignore + // __global does not exist on `Option`, + // but we use it in `getLocalAndGlobalOptions`, to produce + // our own custom list of local and global options. + // For more info, see the original PR: + // https://github.com/instantdb/instant/pull/505 + opt.__global = true; + return opt; +} + +function warnDeprecation(oldCmd, newCmd) { + warn( + chalk.yellow("`instant-cli " + oldCmd + "` is deprecated.") + + " Use " + + chalk.green("`instant-cli " + newCmd + "`") + + " instead." + + "\n", + ); +} + program .name("instant-cli") - .description(instantCLIDescription) - .option("-t --token ", "auth token override") - .option("-y", "skip confirmation prompt") - .option("-v --version", "output the version number", () => { - console.log(version); - process.exit(0); - }); + .addOption(globalOption("-t --token ", "Auth token override")) + .addOption(globalOption("-y --yes", "Answer 'yes' to all prompts")) + .addOption( + globalOption("-v --version", "Print the version number", () => { + console.log(version); + process.exit(0); + }), + ) + .addHelpOption(globalOption("-h --help", "Print the help text for a command")) + .usage(` ${chalk.dim("[options] [args]")}`); program .command("login") - .description("Authenticates with Instant") - .option("-p --print", "print auth token") + .description("Log into your account") + .option("-p --print", "Prints the auth token into the console.") .action(login); program .command("init") - .description("Creates a new app with configuration files") - .action(init); + .description("Set up a new project.") + .option( + "-a --app ", + "If you have an existing app ID, we can pull schema and perms from there.", + ) + .action(async function (opts) { + await handlePull("all", opts); + }); +// Note: Nov 20, 2024 +// We can eventually delete this, +// once we know most people use the new pull and push commands program - .command("push-schema") - .argument("[ID]") - .description("Pushes local instant.schema definition to production.") + .command("push-schema", { hidden: true }) + .argument("[app-id]") + .description("Push schema to production.") .option( "--skip-check-types", "Don't check types on the server when pushing schema", ) - .action((id, opts) => { - pushSchema(id, opts); + .action(async (appIdOrName, opts) => { + warnDeprecation("push-schema", "push schema"); + await handlePush("schema", { app: appIdOrName, ...opts }); }); +// Note: Nov 20, 2024 +// We can eventually delete this, +// once we know most people use the new pull and push commands program - .command("push-perms") - .argument("[ID]") - .description("Pushes local instant.perms rules to production.") - .action(() => { - pushPerms(); + .command("push-perms", { hidden: true }) + .argument("[app-id]") + .description("Push perms to production.") + .action(async (appIdOrName) => { + warnDeprecation("push-perms", "push perms"); + await handlePush("perms", { app: appIdOrName }); }); program .command("push") - .argument("[ID]") + .argument( + "[schema|perms|all]", + "Which configuration to push. Defaults to `all`", + ) + .option( + "-a --app ", + "App ID to push too. Defaults to *_INSTANT_APP_ID in .env", + ) .option( "--skip-check-types", "Don't check types on the server when pushing schema", ) - .description( - "Pushes local instant.schema and instant.perms rules to production.", - ) - .action(pushAll); + .description("Push schema and perm files to production.") + .action(async function (arg, inputOpts) { + const ret = convertPushPullToCurrentFormat("push", arg, inputOpts); + if (!ret.ok) return; + const { bag, opts } = ret; + await handlePush(bag, opts); + }); +// Note: Nov 20, 2024 +// We can eventually delete this, +// once we know most people use the new pull and push commands program - .command("pull-schema") - .argument("[ID]") - .description( - "Generates an initial instant.schema definition from production state.", - ) - .action((appIdOrName) => { - pullSchema(appIdOrName); + .command("pull-schema", { hidden: true }) + .argument("[app-id]") + .description("Generate instant.schema.ts from production") + .action(async (appIdOrName) => { + warnDeprecation("pull-schema", "pull schema"); + await handlePull("schema", { app: appIdOrName }); }); +// Note: Nov 20, 2024 +// We can eventually delete this, +// once we know most people use the new pull and push commands program - .command("pull-perms") - .argument("[ID]") - .description( - "Generates an initial instant.perms definition from production rules.", - ) - .action((appIdOrName) => { - pullPerms(appIdOrName); + .command("pull-perms", { hidden: true }) + .argument("[app-id]") + .description("Generate instant.perms.ts from production.") + .action(async (appIdOrName) => { + warnDeprecation("pull-perms", "pull perms"); + await handlePull("perms", { app: appIdOrName }); }); program .command("pull") - .argument("[ID]") - .description( - "Generates initial instant.schema and instant.perms definition from production state.", + .argument( + "[schema|perms|all]", + "Which configuration to push. Defaults to `all`", ) - .action(pullAll); + .option( + "-a --app ", + "App ID to push to. Defaults to *_INSTANT_APP_ID in .env", + ) + .description("Pull schema and perm files from production.") + .action(async function (arg, inputOpts) { + const ret = convertPushPullToCurrentFormat("pull", arg, inputOpts); + if (!ret.ok) return; + const { bag, opts } = ret; + await handlePull(bag, opts); + }); program.parse(process.argv); // command actions - -async function pushAll(appIdOrName, opts) { - const ok = await pushSchema(appIdOrName, opts); +async function handlePush(bag, opts) { + const { ok, appId } = await detectOrCreateAppWithErrorLogging(opts); if (!ok) return; + await push(bag, appId, opts); +} - await pushPerms(appIdOrName); +async function push(bag, appId, opts) { + if (bag === "schema" || bag === "all") { + const ok = await pushSchema(appId, opts); + if (!ok) return; + } + if (bag === "perms" || bag === "all") { + await pushPerms(appId); + } } -async function pullAll(appIdOrName) { - const ok = await pullSchema(appIdOrName); +async function handlePull(bag, opts) { + const pkgAndAuthInfo = await resolvePackageAndAuthInfoWithErrorLogging(); + if (!pkgAndAuthInfo) return; + const { ok, appId, appTitle, isCreated } = + await detectOrCreateAppWithErrorLogging(opts); if (!ok) return; - await pullPerms(appIdOrName); + if (isCreated) { + await handleCreatedApp(pkgAndAuthInfo, appId, appTitle); + } else { + await pull(bag, appId, pkgAndAuthInfo); + } +} + +async function pull(bag, appId, pkgAndAuthInfo) { + if (bag === "schema" || bag === "all") { + const ok = await pullSchema(appId, pkgAndAuthInfo); + if (!ok) return; + } + if (bag === "perms" || bag === "all") { + await pullPerms(appId, pkgAndAuthInfo); + } } async function login(options) { @@ -149,7 +443,7 @@ async function login(options) { if (!registerRes.ok) return; const { secret, ticket } = registerRes.data; - + console.log("Let's log you in!"); const ok = await promptOk( `This will open instantdb.com in your browser, OK to proceed?`, ); @@ -174,62 +468,176 @@ async function login(options) { } } -async function init() { - const pkgDir = await packageDirectory(); - if (!pkgDir) { - console.error("Failed to locate app root dir."); +async function getOrInstallInstantModuleWithErrorLogging(pkgDir) { + const pkgJson = await getPackageJSONWithErrorLogging(pkgDir); + if (!pkgJson) { return; } + console.log("Checking for an Instant SDK..."); + const instantModuleName = await getInstantModuleName(pkgJson); + if (instantModuleName) { + console.log( + `Found ${chalk.green(instantModuleName)} in your package.json.`, + ); + return instantModuleName; + } + console.log( + "Couldn't find an Instant SDK in your package.json, let's install one!", + ); + const moduleName = await select({ + message: "Which package would you like to use?", + choices: [ + { name: "@instantdb/react", value: "@instantdb/react" }, + { name: "@instantdb/react-native", value: "@instantdb/react-native" }, + { name: "@instantdb/core", value: "@instantdb/core" }, + { name: "@instantdb/admin", value: "@instantdb/admin" }, + ], + }); - const instantModuleName = await getInstantModuleName(pkgDir); - const schema = await readLocalSchemaFile(); - const { perms } = await readLocalPermsFile(); - const authToken = await readConfigAuthToken(); - if (!authToken) { - console.error("Unauthenticated. Please log in with `instant-cli login`!"); + const packageManager = await detectPackageManager(pkgDir); + const installCommand = getInstallCommand(packageManager, moduleName); + + const spinner = ora( + `Installing ${moduleName} using ${packageManager}...`, + ).start(); + + try { + await execAsync(installCommand, pkgDir); + spinner.succeed(`Installed ${moduleName} using ${packageManager}.`); + } catch (e) { + spinner.fail(`Failed to install ${moduleName} using ${packageManager}.`); + error(e.message); return; } + return moduleName; +} + +async function promptCreateApp() { const id = randomUUID(); const token = randomUUID(); - - const title = await input({ - message: "Enter a name for your app", + const _title = await input({ + message: "What would you like to call it?", + default: "My cool app", required: true, }).catch(() => null); + const title = _title?.trim(); + if (!title) { - console.error("No name provided. Exiting."); - return; + error("No name provided. Exiting."); + return { ok: false }; } - + const app = { id, title, admin_token: token }; const appRes = await fetchJson({ method: "POST", path: "/dash/apps", debugName: "App create", errorMessage: "Failed to create app.", - body: { id, title, admin_token: token }, + body: app, + }); + + if (!appRes.ok) return { ok: false }; + return { + ok: true, + appId: id, + appTitle: title, + isCreated: true, + }; +} + +async function promptImportAppOrCreateApp() { + const res = await fetchJson({ + debugName: "Fetching apps", + method: "GET", + path: "/dash", + errorMessage: "Failed to fetch apps.", + }); + if (!res.ok) { + return { ok: false }; + } + const { apps } = res.data; + if (!apps.length) { + const ok = await promptOk( + "You don't have any apps. Want to create a new one?", + ); + if (!ok) return { ok: false }; + return await promptCreateApp(); + } + const choice = await select({ + message: "Which app would you like to import?", + choices: res.data.apps.map((app) => { + return { name: `${app.title} (${app.id})`, value: app.id }; + }), + }).catch(() => null); + if (!choice) return { ok: false }; + return { ok: true, appId: choice }; +} + +async function detectOrCreateAppWithErrorLogging(opts) { + const fromOpts = await detectAppIdFromOptsWithErrorLogging(opts); + if (!fromOpts.ok) return fromOpts; + if (fromOpts.appId) { + return { ok: true, appId: fromOpts.appId }; + } + + const fromEnv = detectAppIdFromEnvWithErrorLogging(); + if (!fromEnv.ok) return fromEnv; + if (fromEnv.found) { + const { envName, value } = fromEnv.found; + console.log(`Found ${chalk.green(envName)}: ${value}`); + return { ok: true, appId: value }; + } + + const action = await select({ + message: "What would you like to do?", + choices: [ + { name: "Create a new app", value: "create" }, + { name: "Import an existing app", value: "import" }, + ], + }).catch(() => null); + + if (action === "create") { + return await promptCreateApp(); + } + + return await promptImportAppOrCreateApp(); +} + +async function writeTypescript(path, content, encoding) { + const prettierConfig = await prettier.resolveConfig(path); + const formattedCode = await prettier.format(content, { + ...prettierConfig, + parser: 'typescript', }); + return await writeFile(path, formattedCode, encoding); +} - if (!appRes.ok) return; +async function handleCreatedApp( + { pkgDir, instantModuleName }, + appId, + appTitle, +) { + const schema = await readLocalSchemaFile(); + const { perms } = await readLocalPermsFile(); - console.log(chalk.green(`Successfully created your Instant app "${title}"`)); + console.log(chalk.green(`Successfully created your Instant app "${appId}"`)); console.log(`Please add your app ID to your .env config:`); - console.log(chalk.magenta(`INSTANT_APP_ID=${id}`)); - console.log(chalk.underline(appDashUrl(id))); + console.log(chalk.magenta(`INSTANT_APP_ID=${appId}`)); + console.log(terminalLink("Dashboard", appDashUrl(appId))); if (!schema) { const schemaPath = join(pkgDir, "instant.schema.ts"); - await writeFile( + await writeTypescript( schemaPath, - instantSchemaTmpl(title, id, instantModuleName), + instantSchemaTmpl(appTitle, appId, instantModuleName), "utf-8", ); console.log("Start building your schema: " + schemaPath); } if (!perms) { - await writeFile( + await writeTypescript( join(pkgDir, "instant.perms.ts"), examplePermsTmpl, "utf-8", @@ -237,39 +645,44 @@ async function init() { } } -async function getInstantModuleName(pkgDir) { - const pkgJson = await readJsonFile(join(pkgDir, "package.json")); - const instantModuleName = pkgJson?.dependencies?.["@instantdb/react"] - ? "@instantdb/react" - : pkgJson?.dependencies?.["@instantdb/core"] - ? "@instantdb/core" - : null; +async function getInstantModuleName(pkgJson) { + const deps = pkgJson.dependencies || {}; + const instantModuleName = [ + "@instantdb/react", + "@instantdb/react-native", + "@instantdb/core", + "@instantdb/admin", + ].find((name) => deps[name]); return instantModuleName; } -async function pullSchema(appIdOrName) { - const pkgDir = await packageDirectory(); - if (!pkgDir) { - console.error("Failed to locate app root dir."); +async function getPackageJSONWithErrorLogging(pkgDir) { + const pkgJson = await readJsonFile(join(pkgDir, "package.json")); + if (!pkgJson) { + error(`Couldn't find a packge.json file in: ${pkgDir}. Please add one.`); return; } + return pkgJson; +} - const appId = await getAppIdWithErrorLogging(appIdOrName); - if (!appId) return; - - const instantModuleName = await getInstantModuleName(pkgDir); +async function resolvePackageAndAuthInfoWithErrorLogging() { + const pkgDir = await packageDirectoryWithErrorLogging(); + if (!pkgDir) { + return; + } + const instantModuleName = + await getOrInstallInstantModuleWithErrorLogging(pkgDir); if (!instantModuleName) { - console.warn( - "Missing Instant dependency in package.json. Please install `@instantdb/react` or `@instantdb/core`.", - ); + return; } - - const authToken = await readConfigAuthToken(); + const authToken = await readConfigAuthTokenWithErrorLogging(); if (!authToken) { - console.error("Unauthenticated. Please log in with `login`!"); return; } + return { pkgDir, instantModuleName, authToken }; +} +async function pullSchema(appId, { pkgDir, instantModuleName }) { console.log("Pulling schema..."); const pullRes = await fetchJson({ @@ -302,7 +715,7 @@ async function pullSchema(appIdOrName) { !countEntities(pullRes.data.schema.refs) && !countEntities(pullRes.data.schema.blobs) ) { - console.log("Schema is empty. Skipping."); + console.log("Schema is empty. Skipping."); return; } @@ -316,7 +729,7 @@ async function pullSchema(appIdOrName) { } const schemaPath = join(pkgDir, "instant.schema.ts"); - await writeFile( + await writeTypescript( schemaPath, generateSchemaTypescriptFile( appId, @@ -332,24 +745,9 @@ async function pullSchema(appIdOrName) { return true; } -async function pullPerms(appIdOrName) { +async function pullPerms(appId, { pkgDir }) { console.log("Pulling perms..."); - const appId = await getAppIdWithErrorLogging(appIdOrName); - if (!appId) return; - - const pkgDir = await packageDirectory(); - if (!pkgDir) { - console.error("Failed to locate app root dir."); - return; - } - - const authToken = await readConfigAuthToken(); - if (!authToken) { - console.error("Unauthenticated. Please log in with `login`!"); - return; - } - const pullRes = await fetchJson({ path: `/dash/apps/${appId}/perms/pull`, debugName: "Perms pull", @@ -372,7 +770,7 @@ async function pullPerms(appIdOrName) { } const permsPath = join(pkgDir, "instant.perms.ts"); - await writeFile( + await writeTypescript( permsPath, `export default ${JSON.stringify(pullRes.data.perms, null, " ")};`, "utf-8", @@ -580,10 +978,7 @@ async function waitForIndexingJobsToFinish(appId, data) { } } -async function pushSchema(appIdOrName, opts) { - const appId = await getAppIdWithErrorLogging(appIdOrName); - if (!appId) return; - +async function pushSchema(appId, opts) { const schema = await readLocalSchemaFileWithErrorLogging(); if (!schema) return; @@ -700,13 +1095,9 @@ async function pushSchema(appIdOrName, opts) { return true; } -async function pushPerms(appIdOrName) { - const appId = await getAppIdWithErrorLogging(appIdOrName); - if (!appId) return; - - const { perms } = await readLocalPermsFile(); +async function pushPerms(appId) { + const perms = await readLocalPermsFileWithErrorLogging(); if (!perms) { - console.error("Missing instant.perms file!"); return; } @@ -735,25 +1126,26 @@ async function pushPerms(appIdOrName) { async function waitForAuthToken({ secret }) { for (let i = 1; i <= 120; i++) { await sleep(1000); - - try { - const authCheckRes = await fetchJson({ - method: "POST", - debugName: "Auth check", - errorMessage: "Failed to check auth status.", - path: "/dash/cli/auth/check", - body: { secret }, - noAuth: true, - noLogError: true, - }); - - if (authCheckRes.ok) { - return authCheckRes.data; - } - } catch (error) {} + const authCheckRes = await fetchJson({ + method: "POST", + debugName: "Auth check", + errorMessage: "Failed to check auth status.", + path: "/dash/cli/auth/check", + body: { secret }, + noAuth: true, + noLogError: true, + }); + if (authCheckRes.ok) { + return authCheckRes.data; + } + if (authCheckRes.data?.hint.errors?.[0]?.issue === "waiting-for-user") { + continue; + } + error('Failed to authenticate '); + prettyPrintJSONErr(authCheckRes.data); + return; } - - console.error("Login timed out."); + error("Timed out waiting for authentication"); return null; } @@ -785,9 +1177,8 @@ async function fetchJson({ const withErrorLogging = !noLogError; let authToken = null; if (withAuth) { - authToken = await readConfigAuthToken(); + authToken = await readConfigAuthTokenWithErrorLogging(); if (!authToken) { - console.error("Unauthenticated. Please log in with `instant-cli login`"); return { ok: false, data: undefined }; } } @@ -815,23 +1206,13 @@ async function fetchJson({ } catch { data = null; } - + if (verbose && data) { + console.log(debugName, "json:", JSON.stringify(data)); + } if (!res.ok) { if (withErrorLogging) { - console.error(errorMessage); - if (data?.message) { - console.error(data.message); - } - if (Array.isArray(data?.hint?.errors)) { - for (const error of data.hint.errors) { - console.error( - `${error.in ? error.in.join("->") + ": " : ""}${error.message}`, - ); - } - } - if (!data) { - console.error("Failed to parse error response"); - } + error(errorMessage); + prettyPrintJSONErr(data); } return { ok: false, data }; } @@ -844,17 +1225,31 @@ async function fetchJson({ } catch (err) { if (withErrorLogging) { if (err.name === "AbortError") { - console.error( - `Timeout: It took more than ${timeoutMs / 60000} minutes to get the result!`, + error( + `Timeout: It took more than ${timeoutMs / 60000} minutes to get the result.`, ); } else { - console.error(`Error: type: ${err.name}, message: ${err.message}`); + error(`Error: type: ${err.name}, message: ${err.message}`); } } return { ok: false, data: null }; } } +function prettyPrintJSONErr(data) { + if (data?.message) { + error(data.message); + } + if (Array.isArray(data?.hint?.errors)) { + for (const err of data.hint.errors) { + error(`${err.in ? err.in.join("->") + ": " : ""}${err.message}`); + } + } + if (!data) { + error("Failed to parse error response"); + } +} + async function promptOk(message) { const options = program.opts(); @@ -866,15 +1261,6 @@ async function promptOk(message) { }).catch(() => false); } -async function pathExists(f) { - try { - await stat(f); - return true; - } catch { - return false; - } -} - async function readLocalPermsFile() { const { config, sources } = await loadConfig({ sources: [ @@ -895,6 +1281,16 @@ async function readLocalPermsFile() { }; } +async function readLocalPermsFileWithErrorLogging() { + const { perms } = await readLocalPermsFile(); + if (!perms) { + error( + `We couldn't find your ${chalk.yellow("`instant.perms.ts`")} file. Make sure it's in the root directory.`, + ); + } + return perms; +} + async function readLocalSchemaFile() { return ( await loadConfig({ @@ -933,26 +1329,15 @@ async function readLocalSchemaFileWithErrorLogging() { const schema = await readLocalSchemaFile(); if (!schema) { - console.error("Missing instant.schema file!"); + error( + `We couldn't find your ${chalk.yellow("`instant.schema.ts`")} file. Make sure it's in the root directory.`, + ); return; } return schema; } -async function readJsonFile(path) { - if (!pathExists(path)) { - return null; - } - - try { - const data = await readFile(path, "utf-8"); - return JSON.parse(data); - } catch (error) {} - - return null; -} - async function readConfigAuthToken() { const options = program.opts(); if (options.token) { @@ -970,6 +1355,15 @@ async function readConfigAuthToken() { return authToken; } +async function readConfigAuthTokenWithErrorLogging() { + const token = await readConfigAuthToken(); + if (!token) { + error( + `Looks like you are not logged in. Please log in with ${chalk.green("`instant-cli login`")}`, + ); + } + return token; +} async function saveConfigAuthToken(authToken) { const authPaths = getAuthPaths(); @@ -1057,48 +1451,61 @@ function isUUID(uuid) { return uuidRegex.test(uuid); } -async function getAppIdWithErrorLogging(defaultAppIdOrName) { - if (defaultAppIdOrName) { - const config = await readInstantConfigFile(); - - const nameMatch = config?.apps?.[defaultAppIdOrName]; - const namedAppId = nameMatch?.id && isUUID(nameMatch.id) ? nameMatch : null; - const uuidAppId = - defaultAppIdOrName && isUUID(defaultAppIdOrName) - ? defaultAppIdOrName - : null; - - if (nameMatch && !namedAppId) { - console.error( - `App ID for \`${defaultAppIdOrName}\` is not a valid UUID.`, - ); - } else if (!namedAppId && !uuidAppId) { - console.error(`The provided app ID is not a valid UUID.`); - } +async function detectAppIdFromOptsWithErrorLogging(opts) { + if (!opts.app) return { ok: true }; + const appId = opts.app; + const config = await readInstantConfigFile(); + const nameMatch = config?.apps?.[appId]; + const namedAppId = nameMatch?.id && isUUID(nameMatch.id) ? nameMatch : null; + const uuidAppId = appId && isUUID(appId) ? appId : null; + + if (nameMatch && !namedAppId) { + error(`Expected \`${appId}\` to point to a UUID, but got ${nameMatch.id}.`); + return { ok: false }; + } + if (!namedAppId && !uuidAppId) { + error(`Expected App ID to be a UUID, but got: ${chalk.red(appId)}`); + return { ok: false }; + } + return { ok: true, appId: namedAppId || uuidAppId }; +} - return ( - // first, check for a config and whether the provided arg - // matched a named ID - namedAppId || - // next, check whether there's a provided arg at all - uuidAppId +function detectAppIdFromEnvWithErrorLogging() { + const found = Object.keys(potentialEnvs) + .map((type) => { + const envName = potentialEnvs[type]; + const value = process.env[envName]; + return { type, envName, value }; + }) + .find(({ value }) => !!value); + if (found && !isUUID(found.value)) { + error( + `Found ${chalk.green("`" + found.envName + "`")} but it's not a valid UUID.`, ); + return { ok: false, found }; } + return { ok: true, found }; +} - const appId = - // finally, check .env - process.env.INSTANT_APP_ID || - process.env.NEXT_PUBLIC_INSTANT_APP_ID || - process.env.PUBLIC_INSTANT_APP_ID || // for Svelte - process.env.VITE_INSTANT_APP_ID || - null; - - // otherwise, instruct the user to set one of these up - if (!appId) { - console.error(noAppIdErrorMessage); +async function getAppIdWithErrorLogging(arg) { + const fromArg = await detectAppIdFromOptsWithErrorLogging({ + app: arg, + }); + if (!fromArg.ok) return; + if (fromArg.appId) { + return fromArg.appId; } + const fromEnv = detectAppIdFromEnvWithErrorLogging(); + if (!fromEnv.ok) return; + if (fromEnv.found) { + const { envName, value } = fromEnv.found; + console.log(`Found ${chalk.green(envName)}: ${value}`); + return value; + } + // otherwise, instruct the user to set one of these up + error(noAppIdErrorMessage); - return appId; + return; } function appDashUrl(id) { @@ -1264,10 +1671,3 @@ ${indentLines(JSON.stringify(linksEntriesCode, null, " "), 1)} export default graph; `; } - -const noAppIdErrorMessage = ` -No app ID found. -Add \`INSTANT_APP_ID=\` to your .env file. -(Or \`NEXT_PUBLIC_INSTANT_APP_ID\`, \`VITE_INSTANT_APP_ID\`) -Or provide an app ID via the CLI \`instant-cli pull-schema \`. -`.trim(); diff --git a/client/packages/cli/package.json b/client/packages/cli/package.json index 53c52f0de..947519b50 100644 --- a/client/packages/cli/package.json +++ b/client/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "instant-cli", "type": "module", - "version": "v0.15.2", + "version": "v0.15.7", "description": "Instant's CLI", "scripts": { "publish-package": "npm publish --access public" @@ -19,6 +19,7 @@ "open": "^10.1.0", "ora": "^8.1.1", "pkg-dir": "^8.0.0", + "prettier": "^3.3.3", "terminal-link": "^3.0.0", "unconfig": "^0.5.5" } diff --git a/client/packages/cli/src/util/fs.js b/client/packages/cli/src/util/fs.js new file mode 100644 index 000000000..d439bc3fc --- /dev/null +++ b/client/packages/cli/src/util/fs.js @@ -0,0 +1,23 @@ +import { readFile, stat } from "fs/promises"; + +export async function pathExists(f) { + try { + await stat(f); + return true; + } catch { + return false; + } +} + +export async function readJsonFile(path) { + if (!pathExists(path)) { + return null; + } + + try { + const data = await readFile(path, "utf-8"); + return JSON.parse(data); + } catch (error) {} + + return null; +} diff --git a/client/packages/cli/src/util/packageManager.js b/client/packages/cli/src/util/packageManager.js new file mode 100644 index 000000000..370d131d2 --- /dev/null +++ b/client/packages/cli/src/util/packageManager.js @@ -0,0 +1,77 @@ +// Note: +// Extracted the main logic for `detectPackageManager` from: +// https://github.com/vercel/vercel/blob/eb7fe8a9266563cfeaf275cd77cd9fad3f17c92b/packages/build-utils/src/fs/run-user-scripts.ts + +import { pathExists, readJsonFile } from "./fs.js"; +import path from "path"; + +async function detectPackageManager(destPath) { + const lockfileNames = { + "yarn.lock": "yarn", + "package-lock.json": "npm", + "pnpm-lock.yaml": "pnpm", + "bun.lockb": "bun", + }; + + for (const dir of traverseUpDirectories(destPath)) { + for (const [lockfileName, cliType] of Object.entries(lockfileNames)) { + const lockfilePath = path.join(dir, lockfileName); + if (await pathExists(lockfilePath)) { + return cliType; + } + } + + const packageJsonPath = path.join(dir, "package.json"); + if (await pathExists(packageJsonPath)) { + const packageJson = await readJsonFile(packageJsonPath); + if (packageJson.packageManager) { + const corepackPackageManager = parsePackageManagerField( + packageJson.packageManager, + ); + if (corepackPackageManager) { + return corepackPackageManager.packageName; + } + } + } + + if (dir === path.parse(dir).root) { + break; + } + } + + return "npm"; +} + +function* traverseUpDirectories(start) { + let current = path.resolve(start); + while (true) { + yield current; + const parent = path.dirname(current); + if (parent === current) { + break; + } + current = parent; + } +} + +function parsePackageManagerField(packageManager) { + if (!packageManager) return null; + const atIndex = packageManager.lastIndexOf("@"); + if (atIndex <= 0) return null; // '@' at position 0 is invalid + const packageName = packageManager.slice(0, atIndex); + const packageVersion = packageManager.slice(atIndex + 1); + if (!packageName || !packageVersion) { + return null; + } + return { packageName, packageVersion }; +} + +function getInstallCommand(packageManager, moduleName) { + if (packageManager === "npm") { + return `npm install ${moduleName}`; + } else { + return `${packageManager} add ${moduleName}`; + } +} + +export { detectPackageManager, getInstallCommand }; diff --git a/client/packages/cli/src/version.js b/client/packages/cli/src/version.js index 042dd5158..7ef8752ed 100644 --- a/client/packages/cli/src/version.js +++ b/client/packages/cli/src/version.js @@ -1,4 +1,4 @@ // Autogenerated by publish_packages.clj -const version = "v0.15.2-dev"; +const version = "v0.15.7-dev"; export default version; diff --git a/client/packages/core/__tests__/src/Reactor.test.js b/client/packages/core/__tests__/src/Reactor.test.js index c1beceaa2..389bf6ea1 100644 --- a/client/packages/core/__tests__/src/Reactor.test.js +++ b/client/packages/core/__tests__/src/Reactor.test.js @@ -112,7 +112,7 @@ test("rewrite mutations", () => { ]; // create transactions without any attributes - const optimisticSteps = instaml.transform({}, ops); + const optimisticSteps = instaml.transform({ attrs: {} }, ops); const mutations = new Map([["k", { "tx-steps": optimisticSteps }]]); @@ -128,7 +128,7 @@ test("rewrite mutations", () => { ._rewriteMutations(zenecaIdToAttr, mutations) .get("k")["tx-steps"]; - const serverSteps = instaml.transform(zenecaIdToAttr, ops); + const serverSteps = instaml.transform({ attrs: zenecaIdToAttr }, ops); expect(rewrittenSteps).toEqual(serverSteps); }); @@ -159,7 +159,7 @@ test("rewrite mutations works with multiple transactions", () => { for (const k of keys) { const attrs = reactor.optimisticAttrs(); - const steps = instaml.transform(attrs, ops); + const steps = instaml.transform({ attrs }, ops); const mut = { op: "transact", "tx-steps": steps, @@ -176,7 +176,7 @@ test("rewrite mutations works with multiple transactions", () => { reactor.pendingMutations.currentValue, ); - const serverSteps = instaml.transform(zenecaIdToAttr, ops); + const serverSteps = instaml.transform({ attrs: zenecaIdToAttr }, ops); for (const k of keys) { expect(rewrittenMutations.get(k)["tx-steps"]).toEqual(serverSteps); } diff --git a/client/packages/core/__tests__/src/instaml.test.js b/client/packages/core/__tests__/src/instaml.test.js index 113aa5867..080cdc339 100644 --- a/client/packages/core/__tests__/src/instaml.test.js +++ b/client/packages/core/__tests__/src/instaml.test.js @@ -3,6 +3,7 @@ import * as instaml from "../../src/instaml"; import * as instatx from "../../src/instatx"; import zenecaAttrs from "./data/zeneca/attrs.json"; import uuid from "../../src/utils/uuid"; +import { i } from "../../src/index"; const zenecaAttrToId = zenecaAttrs.reduce((res, x) => { res[`${x["forward-identity"][1]}/${x["forward-identity"][2]}`] = x.id; @@ -13,7 +14,7 @@ test("simple update transform", () => { const testId = uuid(); const ops = instatx.tx.books[testId].update({ title: "New Title" }); - const result = instaml.transform(zenecaAttrs, ops); + const result = instaml.transform({ attrs: zenecaAttrs }, ops); const expected = [ ["add-triple", testId, zenecaAttrToId["books/title"], "New Title"], @@ -29,8 +30,11 @@ test("simple update transform", () => { test("ignores id attrs", () => { const testId = uuid(); - const ops = instatx.tx.books[testId].update({ title: "New Title", id: 'ploop' }); - const result = instaml.transform(zenecaAttrs, ops); + const ops = instatx.tx.books[testId].update({ + title: "New Title", + id: "ploop", + }); + const result = instaml.transform({ attrs: zenecaAttrs }, ops); const expected = [ ["add-triple", testId, zenecaAttrToId["books/title"], "New Title"], @@ -47,7 +51,7 @@ test("optimistically adds attrs if they don't exist", () => { const ops = instatx.tx.books[testId].update({ newAttr: "New Title" }); - const result = instaml.transform(zenecaAttrs, ops); + const result = instaml.transform({ attrs: zenecaAttrs }, ops); const expected = [ [ @@ -81,7 +85,7 @@ test("lookup resolves attr ids", () => { const stopaLookup = [zenecaAttrToId["users/email"], "stopa@instantdb.com"]; - const result = instaml.transform(zenecaAttrs, ops); + const result = instaml.transform({ attrs: zenecaAttrs }, ops); const expected = [ ["add-triple", stopaLookup, zenecaAttrToId["users/handle"], "stopa"], @@ -107,7 +111,7 @@ test("lookup creates unique attrs for custom lookups", () => { "newAttrValue", ]; - const result = instaml.transform(zenecaAttrs, ops); + const result = instaml.transform({ attrs: zenecaAttrs }, ops); const expected = [ [ "add-attr", @@ -137,7 +141,7 @@ test("lookup creates unique attrs for lookups in link values", () => { .update({}) .link({ posts: instatx.lookup("slug", "life-is-good") }); - const result = instaml.transform({}, ops); + const result = instaml.transform({ attrs: {} }, ops); expect(result).toEqual([ [ @@ -196,7 +200,7 @@ test("lookup creates unique attrs for lookups in link values with arrays", () => ], }); - const result = instaml.transform({}, ops); + const result = instaml.transform({ attrs: {} }, ops); const expected = [ [ @@ -274,7 +278,10 @@ test("lookup creates unique attrs for lookups in link values when fwd-ident exis "index?": true, }; - const result = instaml.transform({ attrId: existingRefAttr }, ops); + const result = instaml.transform( + { attrs: { [attrId]: existingRefAttr } }, + ops, + ); expect(result).toEqual([ [ @@ -328,7 +335,10 @@ test("lookup creates unique attrs for lookups in link values when rev-ident exis "index?": true, }; - const result = instaml.transform({ attrId: existingRefAttr }, ops); + const result = instaml.transform( + { attrs: { [attrId]: existingRefAttr } }, + ops, + ); expect(result).toEqual([ [ @@ -403,7 +413,7 @@ test("lookup doesn't override attrs for lookups in link values", () => { }, }; - const result = instaml.transform(attrs, ops); + const result = instaml.transform({ attrs }, ops); expect(result).toEqual([ ["add-triple", uid, userIdAttrId, uid], @@ -448,8 +458,7 @@ test("lookup doesn't override attrs for lookups in self links", () => { .update({}) .link({ parent: instatx.lookup("slug", "life-is-good") }); - - const result1 = instaml.transform(attrs, ops1); + const result1 = instaml.transform({ attrs }, ops1); expect(result1.filter((x) => x[0] !== "add-triple")).toEqual([]); @@ -457,8 +466,7 @@ test("lookup doesn't override attrs for lookups in self links", () => { .update({}) .link({ child: instatx.lookup("slug", "life-is-good") }); - - const result2 = instaml.transform(attrs, ops2); + const result2 = instaml.transform({ attrs }, ops2); expect(result2.filter((x) => x[0] !== "add-triple")).toEqual([]); }); @@ -476,7 +484,7 @@ test("lookup creates unique ref attrs for ref lookup", () => { uid, ]; - const result = instaml.transform({}, ops); + const result = instaml.transform({ attrs: {} }, ops); const expected = [ [ "add-attr", @@ -539,7 +547,7 @@ test("lookup creates unique ref attrs for ref lookup in link value", () => { uid, ]; - const result = instaml.transform({}, ops); + const result = instaml.transform({ attrs: {} }, ops); const expected = [ [ @@ -580,7 +588,7 @@ test("lookup creates unique ref attrs for ref lookup in link value", () => { test("it throws if you use an invalid link attr", () => { expect(() => instaml.transform( - {}, + { attrs: {} }, instatx.tx.users[ instatx.lookup("user_pref.email", "test@example.com") ].update({ @@ -623,7 +631,7 @@ test("it doesn't throw if you have a period in your attr", () => { expect( instaml.transform( - attrs, + { attrs }, instatx.tx.users[instatx.lookup("attr.with.dot", "value")].update({ a: 1, }), @@ -642,7 +650,7 @@ test("it doesn't create duplicate ref attrs", () => { instatx.tx.nsB[bid].update({}).link({ nsA: aid }), ]; - const result = instaml.transform({}, ops); + const result = instaml.transform({ attrs: {} }, ops); const expected = [ [ @@ -694,3 +702,675 @@ test("it doesn't create duplicate ref attrs", () => { } }); +test("Schema: uses info in `attrs` and `links`", () => { + const schema = i.graph( + { + comments: i.entity({ + slug: i.string().unique().indexed(), + }), + books: i.entity({}), + }, + { + commentBooks: { + forward: { + on: "comments", + has: "one", + label: "book", + }, + reverse: { + on: "books", + has: "many", + label: "comments", + }, + }, + }, + ); + + const commentId = uuid(); + const bookId = uuid(); + const ops = instatx.tx.comments[commentId] + .update({ + slug: "test-slug", + }) + .link({ + book: bookId, + }); + + const result = instaml.transform( + { + attrs: zenecaAttrs, + schema: schema, + }, + ops, + ); + + const expected = [ + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "comments", "slug"], + "value-type": "blob", + cardinality: "one", + "unique?": true, + "index?": true, + "checked-data-type": "string", + isUnsynced: true, + }, + ], + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "comments", "id"], + "value-type": "blob", + cardinality: "one", + "unique?": true, + "index?": false, + isUnsynced: true, + }, + ], + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "comments", "book"], + "reverse-identity": [expect.any(String), "books", "comments"], + "value-type": "ref", + cardinality: "one", + "unique?": false, + "index?": false, + isUnsynced: true, + }, + ], + ["add-triple", commentId, expect.any(String), commentId], + ["add-triple", commentId, expect.any(String), "test-slug"], + ["add-triple", commentId, expect.any(String), bookId], + ]; + expect(result).toHaveLength(expected.length); + for (const item of expected) { + expect(result).toContainEqual(item); + } +}); + +test("Schema: doesn't create duplicate ref attrs", () => { + const schema = i.graph( + { + comments: i.entity({}), + books: i.entity({}), + }, + { + commentBooks: { + forward: { + on: "comments", + has: "one", + label: "book", + }, + reverse: { + on: "books", + has: "many", + label: "comments", + }, + }, + }, + ); + + const commentId = uuid(); + const bookId = uuid(); + const ops = [ + instatx.tx.comments[commentId].update({}).link({ book: bookId }), + instatx.tx.books[bookId].update({}).link({ comments: commentId }), + ]; + + const result = instaml.transform({ attrs: zenecaAttrs, schema }, ops); + + const expected = [ + [ + "add-attr", + { + cardinality: "one", + "forward-identity": [expect.any(String), "comments", "id"], + id: expect.any(String), + "index?": false, + isUnsynced: true, + "unique?": true, + "value-type": "blob", + }, + ], + [ + "add-attr", + { + cardinality: "one", + "forward-identity": [expect.any(String), "comments", "book"], + id: expect.any(String), + "index?": false, + isUnsynced: true, + "reverse-identity": [expect.any(String), "books", "comments"], + "unique?": false, + "value-type": "ref", + }, + ], + ["add-triple", commentId, expect.any(String), commentId], + ["add-triple", commentId, expect.any(String), bookId], + ["add-triple", bookId, expect.any(String), bookId], + ["add-triple", commentId, expect.any(String), bookId], + ]; + expect(result).toHaveLength(expected.length); + for (const item of expected) { + expect(result).toContainEqual(item); + } +}); + +test("Schema: lookup creates unique attrs for custom lookups", () => { + const schema = i.graph( + { + users: i.entity({ + nickname: i.string().unique().indexed(), + }), + }, + {}, + ); + + const ops = instatx.tx.users[instatx.lookup("nickname", "stopanator")].update( + { + handle: "stopa", + }, + ); + + const lookup = [ + // The attr is going to be created, so we don't know its value yet + expect.any(String), + "stopanator", + ]; + + const result = instaml.transform({ attrs: zenecaAttrs, schema }, ops); + const expected = [ + [ + "add-attr", + { + cardinality: "one", + "forward-identity": [expect.any(String), "users", "nickname"], + id: expect.any(String), + "index?": true, + isUnsynced: true, + "unique?": true, + "checked-data-type": "string", + "value-type": "blob", + }, + ], + ["add-triple", lookup, zenecaAttrToId["users/handle"], "stopa"], + ["add-triple", lookup, zenecaAttrToId["users/id"], lookup], + ]; + + expect(result).toHaveLength(expected.length); + for (const item of expected) { + expect(result).toContainEqual(item); + } +}); + +test("Schema: lookup creates unique attrs for lookups in link values", () => { + const schema = i.graph( + { + posts: i.entity({ + slug: i.string().unique().indexed(), + }), + users: i.entity({}), + }, + { + postUsers: { + forward: { + on: "users", + has: "many", + label: "authoredPosts", + }, + reverse: { + on: "posts", + has: "one", + label: "author", + }, + }, + }, + ); + + const uid = uuid(); + const ops = instatx.tx.users[uid] + .update({}) + .link({ authoredPosts: instatx.lookup("slug", "life-is-good") }); + + const result = instaml.transform({ attrs: {}, schema }, ops); + + expect(result).toEqual([ + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "users", "authoredPosts"], + "reverse-identity": [expect.any(String), "posts", "author"], + "value-type": "ref", + // TODO: should this be one? + cardinality: "one", + "unique?": true, + "index?": true, + isUnsynced: true, + }, + ], + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "posts", "slug"], + "value-type": "blob", + cardinality: "one", + "unique?": true, + "index?": true, + "checked-data-type": "string", + isUnsynced: true, + }, + ], + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "users", "id"], + "value-type": "blob", + cardinality: "one", + "unique?": true, + "index?": false, + isUnsynced: true, + }, + ], + ["add-triple", uid, expect.any(String), uid], + [ + "add-triple", + uid, + expect.any(String), + [expect.any(String), "life-is-good"], + ], + ]); +}); + +test("Schema: lookup creates unique attrs for lookups in link values with arrays", () => { + const schema = i.graph( + { + posts: i.entity({ + slug: i.string().unique().indexed(), + }), + users: i.entity({}), + }, + { + postUsers: { + forward: { + on: "users", + has: "many", + label: "authoredPosts", + }, + reverse: { + on: "posts", + has: "one", + label: "author", + }, + }, + }, + ); + + const uid = uuid(); + const ops = instatx.tx.users[uid].update({}).link({ + authoredPosts: [ + instatx.lookup("slug", "life-is-good"), + instatx.lookup("slug", "check-this-out"), + ], + }); + + const result = instaml.transform({ attrs: {}, schema }, ops); + + const expected = [ + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "users", "authoredPosts"], + "reverse-identity": [expect.any(String), "posts", "author"], + "value-type": "ref", + cardinality: "one", + "unique?": true, + "index?": true, + isUnsynced: true, + }, + ], + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "posts", "slug"], + "value-type": "blob", + cardinality: "one", + "unique?": true, + "index?": true, + "checked-data-type": "string", + isUnsynced: true, + }, + ], + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "users", "id"], + "value-type": "blob", + cardinality: "one", + "unique?": true, + "index?": false, + isUnsynced: true, + }, + ], + ["add-triple", uid, expect.any(String), uid], + [ + "add-triple", + uid, + expect.any(String), + [expect.any(String), "life-is-good"], + ], + [ + "add-triple", + uid, + expect.any(String), + [expect.any(String), "check-this-out"], + ], + ]; + + expect(result).toHaveLength(expected.length); + for (const item of expected) { + expect(result).toContainEqual(item); + } +}); + +test("Schema: lookup creates unique ref attrs for ref lookup", () => { + const schema = i.graph( + { + users: i.entity({}), + user_prefs: i.entity({}), + }, + { + user_user_prefs: { + forward: { + on: "user_prefs", + has: "one", + label: "user", + }, + reverse: { + on: "users", + has: "one", + label: "user_pref", + }, + }, + }, + ); + + const uid = uuid(); + const ops = [ + instatx.tx.users[uid].update({}), + instatx.tx.user_prefs[instatx.lookup("user.id", uid)].update({}), + ]; + + const lookup = [ + // The attr is going to be created, so we don't know its value yet + expect.any(String), + uid, + ]; + + const result = instaml.transform({ attrs: {}, schema }, ops); + const expected = [ + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "users", "id"], + "value-type": "blob", + cardinality: "one", + "unique?": true, + "index?": false, + isUnsynced: true, + }, + ], + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "user_prefs", "id"], + "value-type": "blob", + cardinality: "one", + "unique?": true, + "index?": false, + isUnsynced: true, + }, + ], + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "user_prefs", "user"], + "reverse-identity": [expect.any(String), "users", "user_pref"], + "value-type": "ref", + cardinality: "one", + "unique?": true, + "index?": true, + isUnsynced: true, + }, + ], + ["add-triple", uid, expect.any(String), uid], + ["add-triple", lookup, expect.any(String), lookup], + ]; + + expect(result).toHaveLength(expected.length); + for (const item of expected) { + expect(result).toContainEqual(item); + } +}); + +test("Schema: lookup creates unique ref attrs for ref lookup in link value", () => { + const schema = i.graph( + { + users: i.entity({}), + user_prefs: i.entity({}), + }, + { + user_user_prefs: { + forward: { + on: "users", + has: "one", + label: "user_pref", + }, + reverse: { + on: "user_prefs", + has: "one", + label: "user", + }, + }, + }, + ); + const uid = uuid(); + const ops = [ + instatx.tx.users[uid] + .update({}) + .link({ user_pref: instatx.lookup("user.id", uid) }), + ]; + + const lookup = [ + // The attr is going to be created, so we don't know its value yet + expect.any(String), + uid, + ]; + + const result = instaml.transform({ attrs: {}, schema }, ops); + + const expected = [ + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "users", "user_pref"], + "reverse-identity": [expect.any(String), "user_prefs", "user"], + "value-type": "ref", + cardinality: "one", + "unique?": true, + "index?": true, + isUnsynced: true, + }, + ], + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "users", "id"], + "value-type": "blob", + cardinality: "one", + "unique?": true, + "index?": false, + isUnsynced: true, + }, + ], + ["add-triple", uid, expect.any(String), uid], + ["add-triple", uid, expect.any(String), lookup], + ]; + + expect(result).toHaveLength(expected.length); + for (const item of expected) { + expect(result).toContainEqual(item); + } +}); + +test("Schema: populates checked-data-type", () => { + const schema = i.graph( + { + comments: i.entity({ + s: i.string(), + n: i.number(), + d: i.date(), + b: i.boolean(), + a: i.any(), + j: i.json(), + }), + }, + {}, + ); + + const commentId = uuid(); + const ops = instatx.tx.comments[commentId].update({ + s: "str", + n: "num", + d: "date", + b: "bool", + a: "any", + j: "json", + }); + + const result = instaml.transform( + { + attrs: zenecaAttrs, + schema: schema, + }, + ops, + ); + + const expected = [ + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "comments", "s"], + "value-type": "blob", + cardinality: "one", + "unique?": false, + "index?": false, + isUnsynced: true, + "checked-data-type": "string", + }, + ], + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "comments", "n"], + "value-type": "blob", + cardinality: "one", + "unique?": false, + "index?": false, + isUnsynced: true, + "checked-data-type": "number", + }, + ], + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "comments", "d"], + "value-type": "blob", + cardinality: "one", + "unique?": false, + "index?": false, + isUnsynced: true, + "checked-data-type": "date", + }, + ], + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "comments", "b"], + "value-type": "blob", + cardinality: "one", + "unique?": false, + "index?": false, + isUnsynced: true, + "checked-data-type": "boolean", + }, + ], + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "comments", "a"], + "value-type": "blob", + cardinality: "one", + "unique?": false, + "index?": false, + isUnsynced: true, + }, + ], + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "comments", "j"], + "value-type": "blob", + cardinality: "one", + "unique?": false, + "index?": false, + isUnsynced: true, + }, + ], + [ + "add-attr", + { + id: expect.any(String), + "forward-identity": [expect.any(String), "comments", "id"], + "value-type": "blob", + cardinality: "one", + "unique?": true, + "index?": false, + isUnsynced: true, + }, + ], + ["add-triple", commentId, expect.any(String), commentId], + ["add-triple", commentId, expect.any(String), "str"], + ["add-triple", commentId, expect.any(String), "num"], + ["add-triple", commentId, expect.any(String), "date"], + ["add-triple", commentId, expect.any(String), "bool"], + ["add-triple", commentId, expect.any(String), "any"], + ["add-triple", commentId, expect.any(String), "json"], + ]; + + expect(result).toHaveLength(expected.length); + for (const item of expected) { + expect(result).toContainEqual(item); + } +}); diff --git a/client/packages/core/__tests__/src/instaql.test.js b/client/packages/core/__tests__/src/instaql.test.js index 45ab3174d..86b2fe9b0 100644 --- a/client/packages/core/__tests__/src/instaql.test.js +++ b/client/packages/core/__tests__/src/instaql.test.js @@ -5,6 +5,7 @@ import zenecaTriples from "./data/zeneca/triples.json"; import { createStore, transact } from "../../src/store"; import query from "../../src/instaql"; import { tx } from "../../src/instatx"; +import { i } from "../../src/index"; import * as instaml from "../../src/instaml"; import { randomUUID } from "crypto"; @@ -541,6 +542,81 @@ test("multiple connections", () => { ]); }); +test("query forward references work with and without id", () => { + const bookshelf = query( + { store }, + { + bookshelves: { + $: { where: { "users.handle": "stopa" } }, + }, + }, + ).data.bookshelves[0]; + + const usersByBookshelfId = query( + { store }, + { + users: { + $: { where: { "bookshelves.id": bookshelf.id } }, + }, + }, + ).data.users.map((x) => x.handle); + + const usersByBookshelfLinkFIeld = query( + { store }, + { + users: { + $: { where: { bookshelves: bookshelf.id } }, + }, + }, + ).data.users.map((x) => x.handle); + + expect(usersByBookshelfId).toEqual(["stopa"]); + expect(usersByBookshelfLinkFIeld).toEqual(["stopa"]); +}); + +test("query reverse references work with and without id", () => { + const stopa = query( + { store }, + { + users: { + $: { where: { handle: "stopa" } }, + }, + }, + ).data.users[0]; + + const stopaBookshelvesByHandle = query( + { store }, + { + bookshelves: { + $: { where: { "users.handle": "stopa" } }, + }, + }, + ).data.bookshelves; + + const stopaBookshelvesById = query( + { store }, + { + bookshelves: { + $: { where: { "users.id": stopa.id } }, + }, + }, + ).data.bookshelves; + + const stopaBookshelvesByLinkField = query( + { store }, + { + bookshelves: { + $: { where: { users: stopa.id } }, + }, + }, + ).data.bookshelves; + + expect(stopaBookshelvesByHandle.length).toBe(16); + + expect(stopaBookshelvesByHandle).toEqual(stopaBookshelvesById); + expect(stopaBookshelvesByHandle).toEqual(stopaBookshelvesByLinkField); +}); + test("objects are created by etype", () => { const stopa = query( { store }, @@ -554,7 +630,7 @@ test("objects are created by etype", () => { const chunk = tx.user[stopa.id].update({ email: "this-should-not-change-users-stopa@gmail.com", }); - const txSteps = instaml.transform(store.attrs, chunk); + const txSteps = instaml.transform({ attrs: store.attrs }, chunk); const newStore = transact(store, txSteps); const newStopa = query( { store: newStore }, @@ -581,7 +657,7 @@ test("object values", () => { jsonField: { hello: "world" }, otherJsonField: { world: "hello" }, }); - const txSteps = instaml.transform(store.attrs, chunk); + const txSteps = instaml.transform({ attrs: store.attrs }, chunk); const newStore = transact(store, txSteps); const newStopa = query( { store: newStore }, @@ -745,7 +821,7 @@ test("$isNull", () => { tx.books[randomUUID()].update({ title: null }), tx.books[randomUUID()].update({ pageCount: 20 }), ]; - const txSteps = instaml.transform(store.attrs, chunks); + const txSteps = instaml.transform({ attrs: store.attrs }, chunks); const newStore = transact(store, txSteps); expect(query({ store: newStore }, q).data.books.map((x) => x.title)).toEqual([ null, @@ -757,7 +833,7 @@ test("$isNull with relations", () => { const q = { users: { $: { where: { bookshelves: { $isNull: true } } } } }; expect(query({ store }, q).data.users.length).toEqual(0); const chunks = [tx.users[randomUUID()].update({ handle: "dww" })]; - const txSteps = instaml.transform(store.attrs, chunks); + const txSteps = instaml.transform({ attrs: store.attrs }, chunks); const newStore = transact(store, txSteps); expect(query({ store: newStore }, q).data.users.map((x) => x.handle)).toEqual( ["dww"], @@ -781,7 +857,7 @@ test("$isNull with relations", () => { const storeWithNullTitle = transact( newStore, - instaml.transform(newStore.attrs, [ + instaml.transform({ attrs: newStore.attrs }, [ tx.books[bookId].update({ title: null }), ]), ); @@ -800,6 +876,22 @@ test("$isNull with relations", () => { expect(usersWithNullTitle).toEqual([...usersWithBook, "dww"]); }); +test("$isNull with reverse relations", () => { + const q = { + bookshelves: { $: { where: { "users.id": { $isNull: true } } }, users: {} }, + }; + expect(query({ store }, q).data.bookshelves.length).toBe(0); + + const chunks = [ + tx.bookshelves[randomUUID()].update({ name: "Lonely shelf" }), + ]; + const txSteps = instaml.transform({ attrs: store.attrs }, chunks); + const newStore = transact(store, txSteps); + expect( + query({ store: newStore }, q).data.bookshelves.map((x) => x.name), + ).toEqual(["Lonely shelf"]); +}); + test("$not", () => { const q = { tests: { $: { where: { val: { $not: "a" } } } } }; expect(query({ store }, q).data.tests.length).toEqual(0); @@ -810,7 +902,7 @@ test("$not", () => { tx.tests[randomUUID()].update({ val: null }), tx.tests[randomUUID()].update({ undefinedVal: "d" }), ]; - const txSteps = instaml.transform(store.attrs, chunks); + const txSteps = instaml.transform({ attrs: store.attrs }, chunks); const newStore = transact(store, txSteps); expect(query({ store: newStore }, q).data.tests.map((x) => x.val)).toEqual([ "b", @@ -819,3 +911,80 @@ test("$not", () => { undefined, ]); }); + +test("comparators", () => { + const schema = i.graph( + { + tests: i.entity({ + string: i.string().indexed(), + number: i.number().indexed(), + date: i.date().indexed(), + boolean: i.boolean().indexed(), + }), + }, + {}, + ); + + const txSteps = []; + for (let i = 0; i < 5; i++) { + txSteps.push( + tx.tests[randomUUID()].update({ + string: `${i}`, + number: i, + date: i, + boolean: i % 2 === 0, + }), + ); + } + + const newStore = transact( + store, + instaml.transform({ attrs: store.attrs, schema: schema }, txSteps), + ); + + function runQuery(dataType, op, value) { + const res = query( + { store: newStore }, + { + tests: { + $: { where: { [dataType]: { [op]: value } } }, + }, + }, + ); + return res.data.tests.map((x) => x[dataType]); + } + + expect(runQuery("string", "$gt", "2")).toEqual(["3", "4"]); + expect(runQuery("string", "$gte", "2")).toEqual(["2", "3", "4"]); + expect(runQuery("string", "$lt", "2")).toEqual(["0", "1"]); + expect(runQuery("string", "$lte", "2")).toEqual(["0", "1", "2"]); + + expect(runQuery("number", "$gt", 2)).toEqual([3, 4]); + expect(runQuery("number", "$gte", 2)).toEqual([2, 3, 4]); + expect(runQuery("number", "$lt", 2)).toEqual([0, 1]); + expect(runQuery("number", "$lte", 2)).toEqual([0, 1, 2]); + + expect(runQuery("date", "$gt", 2)).toEqual([3, 4]); + expect(runQuery("date", "$gte", 2)).toEqual([2, 3, 4]); + expect(runQuery("date", "$lt", 2)).toEqual([0, 1]); + expect(runQuery("date", "$lte", 2)).toEqual([0, 1, 2]); + + // Accepts string dates + expect( + runQuery("date", "$lt", JSON.parse(JSON.stringify(new Date()))), + ).toEqual([0, 1, 2, 3, 4]); + expect( + runQuery("date", "$gt", JSON.parse(JSON.stringify(new Date()))), + ).toEqual([]); + + expect(runQuery("boolean", "$gt", true)).toEqual([]); + expect(runQuery("boolean", "$gte", true)).toEqual([true, true, true]); + expect(runQuery("boolean", "$lt", true)).toEqual([false, false]); + expect(runQuery("boolean", "$lte", true)).toEqual([ + true, + false, + true, + false, + true, + ]); +}); diff --git a/client/packages/core/__tests__/src/store.test.js b/client/packages/core/__tests__/src/store.test.js index 85df10563..a16c3c582 100644 --- a/client/packages/core/__tests__/src/store.test.js +++ b/client/packages/core/__tests__/src/store.test.js @@ -1,7 +1,13 @@ import { test, expect } from "vitest"; import zenecaAttrs from "./data/zeneca/attrs.json"; import zenecaTriples from "./data/zeneca/triples.json"; -import { createStore, transact, allMapValues, toJSON, fromJSON } from "../../src/store"; +import { + createStore, + transact, + allMapValues, + toJSON, + fromJSON, +} from "../../src/store"; import query from "../../src/instaql"; import uuid from "../../src/utils/uuid"; import { tx } from "../../src/instatx"; @@ -60,7 +66,7 @@ function checkIndexIntegrity(store) { test("simple add", () => { const id = uuid(); const chunk = tx.users[id].update({ handle: "bobby" }); - const txSteps = instaml.transform(store.attrs, chunk); + const txSteps = instaml.transform({ attrs: store.attrs }, chunk); const newStore = transact(store, txSteps); expect( query({ store: newStore }, { users: {} }).data.users.map((x) => x.handle), @@ -74,7 +80,7 @@ test("cardinality-one add", () => { const chunk = tx.users[id] .update({ handle: "bobby" }) .update({ handle: "bob" }); - const txSteps = instaml.transform(store.attrs, chunk); + const txSteps = instaml.transform({ attrs: store.attrs }, chunk); const newStore = transact(store, txSteps); const ret = datalog .query(newStore, { @@ -96,7 +102,10 @@ test("link/unlink", () => { const bookshelfChunk = tx.bookshelves[bookshelfId].update({ name: "my books", }); - const txSteps = instaml.transform(store.attrs, [userChunk, bookshelfChunk]); + const txSteps = instaml.transform({ attrs: store.attrs }, [ + userChunk, + bookshelfChunk, + ]); const newStore = transact(store, txSteps); expect( query( @@ -120,7 +129,7 @@ test("link/unlink", () => { bookshelves: bookshelfId, }) .link({ bookshelves: secondBookshelfId }); - const secondTxSteps = instaml.transform(newStore.attrs, [ + const secondTxSteps = instaml.transform({ attrs: newStore.attrs }, [ unlinkFirstChunk, secondBookshelfChunk, ]); @@ -153,7 +162,7 @@ test("link/unlink multi", () => { const bookshelf2Chunk = tx.bookshelves[bookshelfId2].update({ name: "my books 2", }); - const txSteps = instaml.transform(store.attrs, [ + const txSteps = instaml.transform({ attrs: store.attrs }, [ userChunk, bookshelf1Chunk, bookshelf2Chunk, @@ -182,7 +191,7 @@ test("link/unlink multi", () => { bookshelves: [bookshelfId1, bookshelfId2], }) .link({ bookshelves: bookshelfId3 }); - const secondTxSteps = instaml.transform(newStore.attrs, [ + const secondTxSteps = instaml.transform({ attrs: newStore.attrs }, [ unlinkChunk, bookshelf3Chunk, ]); @@ -210,7 +219,10 @@ test("delete entity", () => { const bookshelfChunk = tx.bookshelves[bookshelfId].update({ name: "my books", }); - const txSteps = instaml.transform(store.attrs, [userChunk, bookshelfChunk]); + const txSteps = instaml.transform({ attrs: store.attrs }, [ + userChunk, + bookshelfChunk, + ]); const newStore = transact(store, txSteps); checkIndexIntegrity(newStore); @@ -230,7 +242,7 @@ test("delete entity", () => { expect(retTwo).contains(userId); const txStepsTwo = instaml.transform( - newStore.attrs, + { attrs: newStore.attrs }, tx.bookshelves[bookshelfId].delete(), ); const newStoreTwo = transact(newStore, txStepsTwo); @@ -259,7 +271,10 @@ test("new attrs", () => { .update({ handle: "bobby" }) .link({ colors: colorId }); const colorChunk = tx.colors[colorId].update({ name: "red" }); - const txSteps = instaml.transform(store.attrs, [userChunk, colorChunk]); + const txSteps = instaml.transform({ attrs: store.attrs }, [ + userChunk, + colorChunk, + ]); const newStore = transact(store, txSteps); expect( query( diff --git a/client/packages/core/package.json b/client/packages/core/package.json index 7fb1d0fd2..591f2c182 100644 --- a/client/packages/core/package.json +++ b/client/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@instantdb/core", - "version": "v0.15.2", + "version": "v0.15.7", "description": "Instant's core local abstraction", "main": "dist/index.js", "module": "dist/module/index.js", diff --git a/client/packages/core/src/Reactor.js b/client/packages/core/src/Reactor.js index f1e346ee2..f7d3e1f4c 100644 --- a/client/packages/core/src/Reactor.js +++ b/client/packages/core/src/Reactor.js @@ -899,7 +899,10 @@ export default class Reactor { /** Applies transactions locally and sends transact message to server */ pushTx = (chunks) => { try { - const txSteps = instaml.transform(this.optimisticAttrs(), chunks); + const txSteps = instaml.transform( + { attrs: this.optimisticAttrs(), schema: this.config.schema }, + chunks, + ); return this.pushOps(txSteps); } catch (e) { return this.pushOps([], e); diff --git a/client/packages/core/src/datalog.js b/client/packages/core/src/datalog.js index 9e6c74fd8..4ae4d7f50 100644 --- a/client/packages/core/src/datalog.js +++ b/client/packages/core/src/datalog.js @@ -17,39 +17,43 @@ function matchExact(patternPart, triplePart, context) { return patternPart === triplePart ? context : null; } -function matchWithArgMap(patternPart, triplePart, context) { - const { in: inList, $in: $inList } = patternPart; - if ( - (inList && inList.includes(triplePart)) || - ($inList && $inList.includes(triplePart)) - ) { - return context; - } - - if ( - patternPart.hasOwnProperty("$not") || - patternPart.hasOwnProperty("$isNull") || - patternPart.hasOwnProperty("$like") - ) { - // We've already done the filtering in `getTriples` - return context; - } - return null; -} - function matcherForPatternPart(patternPart) { switch (typeof patternPart) { case "string": return patternPart.startsWith("?") ? matchVariable : matchExact; - case "object": - return matchWithArgMap; default: return matchExact; } } +const validArgMapProps = [ + "in", + "$in", + "$not", + "$isNull", + "$comparator", // covers all of $gt, $lt, etc. +]; + +// Checks if an object is an args map +function isArgsMap(patternPart) { + for (const prop of validArgMapProps) { + if (patternPart.hasOwnProperty(prop)) { + return true; + } + } + return false; +} + function matchPart(patternPart, triplePart, context) { if (!context) return null; + if (typeof patternPart === "object") { + // This is an args map, so we'll have already fitered the triples + // in `getRelevantTriples` + if (isArgsMap(patternPart)) { + return context; + } + return null; + } const matcher = matcherForPatternPart(patternPart); return matcher(patternPart, triplePart, context); } diff --git a/client/packages/core/src/instaml.js b/client/packages/core/src/instaml.js index f96a1d269..5fdf113c1 100644 --- a/client/packages/core/src/instaml.js +++ b/client/packages/core/src/instaml.js @@ -207,14 +207,14 @@ function expandDeepMerge(attrs, [etype, eid, obj]) { // id first so that we don't clobber updates on the lookup field return [idTuple].concat(attrTuples); } -function removeIdFromArgs(step) { +function removeIdFromArgs(step) { const [op, etype, eid, obj] = step; if (!obj) { return step; } - const newObj = {...obj}; - delete newObj.id - return [op, etype, eid, newObj] + const newObj = { ...obj }; + delete newObj.id; + return [op, etype, eid, newObj]; } function toTxSteps(attrs, step) { @@ -238,7 +238,38 @@ function toTxSteps(attrs, step) { // --------- // transform -function createObjectAttr(etype, label, props) { +function checkedDataTypeOfValueType(valueType) { + switch (valueType) { + case "string": + case "date": + case "boolean": + case "number": + return valueType; + default: + return undefined; + } +} + +function objectPropsFromSchema(schema, etype, label) { + const attr = schema.entities[etype]?.attrs?.[label]; + if (label === "id") return null; + if (!attr) { + throw new Error(`${etype}.${label} does not exist in your schema`); + } + const { unique, indexed } = attr?.config; + const checkedDataType = checkedDataTypeOfValueType(attr?.valueType); + + return { + "index?": indexed, + "unique?": unique, + "checked-data-type": checkedDataType, + }; +} + +function createObjectAttr(schema, etype, label, props) { + const schemaObjectProps = schema + ? objectPropsFromSchema(schema, etype, label) + : null; const attrId = uuid(); const fwdIdentId = uuid(); const fwdIdent = [fwdIdentId, etype, label]; @@ -250,16 +281,42 @@ function createObjectAttr(etype, label, props) { "unique?": false, "index?": false, isUnsynced: true, + ...(schemaObjectProps || {}), ...(props || {}), }; } -function createRefAttr(etype, label, props) { +function findSchemaLink(schema, etype, label) { + const found = Object.values(schema.links).find((x) => { + return ( + (x.forward.on === etype && x.forward.label === label) || + (x.reverse.on === etype && x.reverse.label === label) + ); + }); + return found; +} + +function refPropsFromSchema(schema, etype, label) { + const found = findSchemaLink(schema, etype, label); + if (!found) { + throw new Error(`Couldn't find the link ${etype}.${label} in your schema`); + } + const { forward, reverse } = found; + return { + "forward-identity": [uuid(), forward.on, forward.label], + "reverse-identity": [uuid(), reverse.on, reverse.label], + cardinality: forward.has === "one" ? "one" : "many", + "unique?": reverse.has === "one", + }; +} + +function createRefAttr(schema, etype, label, props) { + const schemaRefProps = schema + ? refPropsFromSchema(schema, etype, label) + : null; const attrId = uuid(); - const fwdIdentId = uuid(); - const revIdentId = uuid(); - const fwdIdent = [fwdIdentId, etype, label]; - const revIdent = [revIdentId, label, etype]; + const fwdIdent = [uuid(), etype, label]; + const revIdent = [uuid(), label, etype]; return { id: attrId, "forward-identity": fwdIdent, @@ -269,6 +326,7 @@ function createRefAttr(etype, label, props) { "unique?": false, "index?": false, isUnsynced: true, + ...(schemaRefProps || {}), ...(props || {}), }; } @@ -317,7 +375,7 @@ function lookupPairsOfOp(op) { return res; } -function createMissingAttrs(existingAttrs, ops) { +function createMissingAttrs({ attrs: existingAttrs, schema }, ops) { const [addedIds, attrs, addOps] = [new Set(), { ...existingAttrs }, []]; function addAttr(attr) { attrs[attr.id] = attr; @@ -338,7 +396,7 @@ function createMissingAttrs(existingAttrs, ops) { addUnsynced(fwdAttr); addUnsynced(revAttr); if (!fwdAttr && !revAttr) { - addAttr(createRefAttr(etype, label, refLookupProps)); + addAttr(createRefAttr(schema, etype, label, refLookupProps)); } } @@ -369,7 +427,9 @@ function createMissingAttrs(existingAttrs, ops) { } else { const attr = getAttrByFwdIdentName(attrs, linkEtype, identName); if (!attr) { - addAttr(createObjectAttr(linkEtype, identName, lookupProps)); + addAttr( + createObjectAttr(schema, linkEtype, identName, lookupProps), + ); } addUnsynced(attr); } @@ -378,7 +438,7 @@ function createMissingAttrs(existingAttrs, ops) { } else { const attr = getAttrByFwdIdentName(attrs, etype, identName); if (!attr) { - addAttr(createObjectAttr(etype, identName, lookupProps)); + addAttr(createObjectAttr(schema, etype, identName, lookupProps)); } addUnsynced(attr); } @@ -398,6 +458,7 @@ function createMissingAttrs(existingAttrs, ops) { if (!fwdAttr) { addAttr( createObjectAttr( + schema, etype, label, label === "id" ? { "unique?": true } : null, @@ -408,7 +469,7 @@ function createMissingAttrs(existingAttrs, ops) { if (REF_ACTIONS.has(action)) { const revAttr = getAttrByReverseIdentName(attrs, etype, label); if (!fwdAttr && !revAttr) { - addAttr(createRefAttr(etype, label)); + addAttr(createRefAttr(schema, etype, label)); } addUnsynced(revAttr); } @@ -418,10 +479,10 @@ function createMissingAttrs(existingAttrs, ops) { return [attrs, addOps]; } -export function transform(attrs, inputChunks) { +export function transform(ctx, inputChunks) { const chunks = Array.isArray(inputChunks) ? inputChunks : [inputChunks]; const ops = chunks.flatMap((tx) => getOps(tx)); - const [newAttrs, addAttrTxSteps] = createMissingAttrs(attrs, ops); + const [newAttrs, addAttrTxSteps] = createMissingAttrs(ctx, ops); const txSteps = ops.flatMap((op) => toTxSteps(newAttrs, op)); return [...addAttrTxSteps, ...txSteps]; } diff --git a/client/packages/core/src/instaql.js b/client/packages/core/src/instaql.js index 17ade131f..6976ada3a 100644 --- a/client/packages/core/src/instaql.js +++ b/client/packages/core/src/instaql.js @@ -88,12 +88,78 @@ function refAttrPat(makeVar, store, etype, level, label) { return [nextEtype, nextLevel, attrPat, attr, isForward]; } +function parseValue(attr, v) { + if ( + typeof v !== "object" || + v.hasOwnProperty("$in") || + v.hasOwnProperty("in") + ) { + return v; + } + + const isDate = attr["checked-data-type"] === "date"; + + if (v.hasOwnProperty("$gt")) { + return { + $comparator: true, + $op: isDate + ? function gtDate(triple) { + return new Date(triple[2]) > new Date(v.$gt); + } + : function gt(triple) { + return triple[2] > v.$gt; + }, + }; + } + if (v.hasOwnProperty("$gte")) { + return { + $comparator: true, + $op: isDate + ? function gteDate(triple) { + return new Date(triple[2]) >= new Date(v.$gte); + } + : function gte(triple) { + return triple[2] >= v.$gte; + }, + }; + } + + if (v.hasOwnProperty("$lt")) { + return { + $comparator: true, + $op: isDate + ? function ltDate(triple) { + return new Date(triple[2]) < new Date(v.$lt); + } + : function lt(triple) { + return triple[2] < v.$lt; + }, + }; + } + if (v.hasOwnProperty("$lte")) { + return { + $comparator: true, + $op: isDate + ? function lteDate(triple) { + return new Date(triple[2]) <= new Date(v.$lte); + } + : function lte(triple) { + return triple[2] <= v.$lte; + }, + }; + } + + return v; +} + function valueAttrPat(makeVar, store, valueEtype, valueLevel, valueLabel, v) { - const attr = s.getAttrByFwdIdentName(store, valueEtype, valueLabel); + const fwdAttr = s.getAttrByFwdIdentName(store, valueEtype, valueLabel); + const revAttr = s.getAttrByReverseIdentName(store, valueEtype, valueLabel); + const attr = fwdAttr || revAttr; if (!attr) { throw new AttrNotFoundError( - `No attr for etype = ${valueEtype} label = ${valueLabel} value-label`, + `No attr for etype = ${valueEtype} label = ${valueLabel}`, ); } @@ -101,28 +167,27 @@ function valueAttrPat(makeVar, store, valueEtype, valueLevel, valueLabel, v) { const idAttr = s.getAttrByFwdIdentName(store, valueEtype, "id"); if (!idAttr) { throw new AttrNotFoundError( - `No attr for etype = ${valueEtype} label = id value-label`, + `No attr for etype = ${valueEtype} label = id`, ); } + return [ makeVar(valueEtype, valueLevel), idAttr.id, - { $isNull: { attrId: attr.id, isNull: v.$isNull } }, + { $isNull: { attrId: attr.id, isNull: v.$isNull, reverse: !fwdAttr } }, wildcard("time"), ]; } - if (v?.hasOwnProperty("$like")) { + if (fwdAttr) { return [ makeVar(valueEtype, valueLevel), attr.id, - { $like: v.$like }, + parseValue(attr, v), wildcard("time"), ]; } - - - return [makeVar(valueEtype, valueLevel), attr.id, v, wildcard("time")]; + return [v, attr.id, makeVar(valueEtype, valueLevel), wildcard("time")]; } function refAttrPats(makeVar, store, etype, level, refsPath) { diff --git a/client/packages/core/src/queryTypes.ts b/client/packages/core/src/queryTypes.ts index f04796abb..5a31d3e2d 100644 --- a/client/packages/core/src/queryTypes.ts +++ b/client/packages/core/src/queryTypes.ts @@ -10,6 +10,12 @@ import type { ResolveEntityAttrs, } from "./schemaTypes"; +type Expand = T extends object + ? T extends infer O + ? { [K in keyof O]: Expand } + : never + : T; + // NonEmpty disallows {}, so that you must provide at least one field type NonEmpty = { [K in keyof T]-?: Required>; @@ -21,6 +27,10 @@ type WhereArgs = { $in?: (string | number | boolean)[]; $not?: string | number | boolean; $isNull?: boolean; + $gt?: string | number | boolean; + $lt?: string | number | boolean; + $gte?: string | number | boolean; + $lte?: string | number | boolean; }; type WhereClauseValue = string | number | boolean | NonEmpty; @@ -145,12 +155,10 @@ type Exactly = Parent & { // ========== // InstaQL helpers -type InstaQLSubqueryResult< +type InstaQLEntitySubqueryResult< Schema extends IContainEntitiesAndLinks, EntityName extends keyof Schema["entities"], - Query extends { - [LinkAttrName in keyof Schema["entities"][EntityName]["links"]]?: any; - }, + Query extends InstaQLEntitySubquery = {}, > = { [QueryPropName in keyof Query]: Schema["entities"][EntityName]["links"][QueryPropName] extends LinkAttrDef< infer Cardinality, @@ -159,17 +167,9 @@ type InstaQLSubqueryResult< ? LinkedEntityName extends keyof Schema["entities"] ? Cardinality extends "one" ? - | InstaQLEntity< - Schema, - LinkedEntityName, - Query[QueryPropName] - > + | InstaQLEntity | undefined - : InstaQLEntity< - Schema, - LinkedEntityName, - Query[QueryPropName] - >[] + : InstaQLEntity[] : never : never; }; @@ -216,11 +216,11 @@ type InstaQLQueryEntityLinksResult< type InstaQLEntity< Schema extends IContainEntitiesAndLinks, EntityName extends keyof Schema["entities"], - Subquery extends { - [QueryPropName in keyof Schema["entities"][EntityName]["links"]]?: any; - } = {}, -> = { id: string } & ResolveEntityAttrs & - InstaQLSubqueryResult; + Subquery extends InstaQLEntitySubquery = {}, +> = Expand< + { id: string } & ResolveEntityAttrs & + InstaQLEntitySubqueryResult +>; type InstaQLQueryEntityResult< Entities extends EntitiesDef, @@ -254,11 +254,21 @@ type InstaQLQueryResult< type InstaQLResult< Schema extends IContainEntitiesAndLinks, - Query, -> = { + Query extends InstaQLParams, +> = Expand<{ [QueryPropName in keyof Query]: QueryPropName extends keyof Schema["entities"] ? InstaQLEntity[] : never; +}>; + +type InstaQLEntitySubquery< + Schema extends IContainEntitiesAndLinks, + EntityName extends keyof Schema["entities"], +> = { + [QueryPropName in keyof Schema["entities"][EntityName]["links"]]?: InstaQLEntitySubquery< + Schema, + Schema["entities"][EntityName]["links"][QueryPropName]["entityName"] + >; }; type InstaQLQuerySubqueryParams< @@ -283,14 +293,15 @@ type InstaQLParams> = { /** * @deprecated * `InstaQLQueryParams` is deprecated. Use `InstaQLParams` instead. - * + * * @example * // Before - * const myQuery = {...} satisfies InstaQLQueryParams + * const myQuery = {...} satisfies InstaQLQueryParams * // After * const myQuery = {...} satisfies InstaQLParams */ -type InstaQLQueryParams> = InstaQLParams; +type InstaQLQueryParams> = + InstaQLParams; export { Query, diff --git a/client/packages/core/src/schemaTypes.ts b/client/packages/core/src/schemaTypes.ts index aa5a967df..4cef43a7e 100644 --- a/client/packages/core/src/schemaTypes.ts +++ b/client/packages/core/src/schemaTypes.ts @@ -371,7 +371,6 @@ type EntityDefFromShape = EntityDef< >; /** - * @deprecated * If you were using the old `schema` types, you can use this to help you * migrate. * diff --git a/client/packages/core/src/store.js b/client/packages/core/src/store.js index c013e197d..a7623d091 100644 --- a/client/packages/core/src/store.js +++ b/client/packages/core/src/store.js @@ -460,7 +460,7 @@ function matchesLikePattern(value, pattern) { function triplesByValue(store, m, v) { const res = []; - if (v?.hasOwnProperty('$not')) { + if (v?.hasOwnProperty("$not")) { for (const candidate of m.keys()) { if (v.$not !== candidate) { res.push(m.get(candidate)); @@ -469,31 +469,37 @@ function triplesByValue(store, m, v) { return res; } - if (v?.hasOwnProperty('$isNull')) { - const { attrId, isNull } = v.$isNull; + if (v?.hasOwnProperty("$isNull")) { + const { attrId, isNull, reverse } = v.$isNull; - const aMap = store.aev.get(attrId); - for (const candidate of m.keys()) { - const isValNull = - !aMap || aMap.get(candidate)?.get(null) || !aMap.get(candidate); - if (isNull ? isValNull : !isValNull) { - res.push(m.get(candidate)); + if (reverse) { + for (const candidate of m.keys()) { + const vMap = store.vae.get(candidate); + const isValNull = + !vMap || vMap.get(attrId)?.get(null) || !vMap.get(attrId); + if (isNull ? isValNull : !isValNull) { + res.push(m.get(candidate)); + } + } + } else { + const aMap = store.aev.get(attrId); + for (const candidate of m.keys()) { + const isValNull = + !aMap || aMap.get(candidate)?.get(null) || !aMap.get(candidate); + if (isNull ? isValNull : !isValNull) { + res.push(m.get(candidate)); + } } } return res; } - if (v?.hasOwnProperty('$like')) { - const pattern = v.$like; - for (const candidate of m.keys()) { - if (matchesLikePattern(candidate, pattern)) { - res.push(m.get(candidate)); - } - } - return res; + if (v?.$comparator) { + // TODO: A sorted index would be nice here + return allMapValues(m, 1).filter(v.$op); } - const values = v.in || v.$in ? (v.in || v.$in) : [v]; + const values = v.in || v.$in || [v]; for (const value of values) { const triple = m.get(value); @@ -501,6 +507,7 @@ function triplesByValue(store, m, v) { res.push(triple); } } + return res; } @@ -606,7 +613,7 @@ export function getPrimaryKeyAttr(store, etype) { if (fromPrimary) { return fromPrimary; } - return store.attrIndexes.forwardIdents.get(etype)?.get('id'); + return store.attrIndexes.forwardIdents.get(etype)?.get("id"); } export function transact(store, txSteps) { diff --git a/client/packages/core/src/version.js b/client/packages/core/src/version.js index 042dd5158..7ef8752ed 100644 --- a/client/packages/core/src/version.js +++ b/client/packages/core/src/version.js @@ -1,4 +1,4 @@ // Autogenerated by publish_packages.clj -const version = "v0.15.2-dev"; +const version = "v0.15.7-dev"; export default version; diff --git a/client/packages/react-native/package.json b/client/packages/react-native/package.json index bdd3b288d..91f2aa094 100644 --- a/client/packages/react-native/package.json +++ b/client/packages/react-native/package.json @@ -1,6 +1,6 @@ { "name": "@instantdb/react-native", - "version": "v0.15.2", + "version": "v0.15.7", "description": "Instant DB for React Native", "main": "dist/index.js", "module": "dist/module/index.js", diff --git a/client/packages/react-native/src/version.js b/client/packages/react-native/src/version.js index 042dd5158..7ef8752ed 100644 --- a/client/packages/react-native/src/version.js +++ b/client/packages/react-native/src/version.js @@ -1,4 +1,4 @@ // Autogenerated by publish_packages.clj -const version = "v0.15.2-dev"; +const version = "v0.15.7-dev"; export default version; diff --git a/client/packages/react/package.json b/client/packages/react/package.json index ece268342..fb213ac3f 100644 --- a/client/packages/react/package.json +++ b/client/packages/react/package.json @@ -1,6 +1,6 @@ { "name": "@instantdb/react", - "version": "v0.15.2", + "version": "v0.15.7", "description": "Instant DB for React", "main": "dist/index.js", "module": "dist/module/index.js", diff --git a/client/packages/react/src/version.js b/client/packages/react/src/version.js index 042dd5158..7ef8752ed 100644 --- a/client/packages/react/src/version.js +++ b/client/packages/react/src/version.js @@ -1,4 +1,4 @@ // Autogenerated by publish_packages.clj -const version = "v0.15.2-dev"; +const version = "v0.15.7-dev"; export default version; diff --git a/client/pnpm-lock.yaml b/client/pnpm-lock.yaml index a1d42c493..cd3c23e8b 100644 --- a/client/pnpm-lock.yaml +++ b/client/pnpm-lock.yaml @@ -69,6 +69,9 @@ importers: pkg-dir: specifier: ^8.0.0 version: 8.0.0 + prettier: + specifier: ^3.3.3 + version: 3.3.3 terminal-link: specifier: ^3.0.0 version: 3.0.0 @@ -14236,7 +14239,6 @@ packages: resolution: {integrity: sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==} engines: {node: '>=14'} hasBin: true - dev: true /pretty-bytes@5.6.0: resolution: {integrity: sha512-FFw039TmrBqFK8ma/7OL3sDz/VytdtJr044/QUJtH0wK9lb9jLq9tJyIxUwtQJHwar2BqtiA4iCWSwo9JLkzFg==} diff --git a/client/sandbox/react-nextjs/pages/play/missing-attrs.tsx b/client/sandbox/react-nextjs/pages/play/missing-attrs.tsx new file mode 100644 index 000000000..ab3af30af --- /dev/null +++ b/client/sandbox/react-nextjs/pages/play/missing-attrs.tsx @@ -0,0 +1,205 @@ +import { useEffect, useState } from "react"; +import config from "../../config"; +import { init, init_experimental, tx, id, i } from "@instantdb/react"; +import { useRouter } from "next/router"; + +const schema = i.graph( + { + comments: i.entity({ + slug: i.string().unique().indexed(), + }), + $users: i.entity({ + email: i.string().unique().indexed(), + }), + }, + { + commentAuthors: { + forward: { + on: "comments", + has: "one", + label: "author", + }, + reverse: { + on: "$users", + has: "many", + label: "authoredComments", + }, + }, + }, +); + +function Example({ appId, useSchema }: { appId: string; useSchema: boolean }) { + const myConfig = { ...config, appId }; + const db = useSchema + ? init_experimental({ ...myConfig, schema }) + : (init(myConfig) as any); + const q = db.useQuery({ comments: {} }); + const [attrs, setAttrs] = useState(); + useEffect(() => { + const unsub = db._core._reactor.subscribeAttrs((res: any) => { + setAttrs(res); + }); + return unsub; + }); + + return ( +
+
+ + +
+
+
+
Using Schema? = {JSON.stringify(useSchema)}
+
Attrs:
+
+          {JSON.stringify(
+            Object.values(attrs || {}).filter(
+              (x: any) => x.catalog !== "system",
+            ),
+            null,
+            2,
+          )}
+          {JSON.stringify(q, null, 2)}
+        
+
+
+ ); +} + +async function provisionEphemeralApp() { + const r = await fetch(`${config.apiURI}/dash/apps/ephemeral`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + title: "Pagination example", + // Uncomment and start a new app to test rules + /* rules: { + code: { + goals: { + allow: { + // data.number % 2 == 0 gives me a typecasting error + // so does int(data.number) % 2 == 0 + view: "data.number == 2 || data.number == 4 || data.number == 6 || data.number == 8 || data.number == 10", + }, + }, + }, + }, */ + }), + }); + + return r.json(); +} + +async function verifyEphemeralApp({ appId }: { appId: string }) { + const r = await fetch(`${config.apiURI}/dash/apps/ephemeral/${appId}`, { + headers: { + "Content-Type": "application/json", + }, + }); + + return r.json(); +} + +function App({ + urlAppId, + useSchema, +}: { + urlAppId: string | undefined; + useSchema: boolean; +}) { + const router = useRouter(); + const [appId, setAppId] = useState(); + + const [error, setError] = useState(null); + + useEffect(() => { + if (appId) { + return; + } + if (urlAppId) { + verifyEphemeralApp({ appId: urlAppId }).then((res): any => { + if (res.app) { + setAppId(res.app.id); + } else { + provisionEphemeralApp().then((res) => { + if (res.app) { + router.replace({ + pathname: router.pathname, + query: { ...router.query, app: res.app.id }, + }); + + setAppId(res.app.id); + } else { + console.log(res); + setError("Could not create app."); + } + }); + } + }); + } else { + provisionEphemeralApp().then((res) => { + if (res.app) { + router.replace({ + pathname: router.pathname, + query: { ...router.query, app: res.app.id }, + }); + + setAppId(res.app.id); + } else { + console.log(res); + setError("Could not create app."); + } + }); + } + }, []); + + if (error) { + return ( +
+

There was an error

+

{error}

+
+ ); + } + + if (!appId) { + return
Loading...
; + } + + return ; +} + +function Page() { + const router = useRouter(); + if (router.isReady) { + return ( + + ); + } else { + return
Loading...
; + } +} + +export default Page; diff --git a/client/sandbox/react-nextjs/pages/play/operators.tsx b/client/sandbox/react-nextjs/pages/play/operators.tsx new file mode 100644 index 000000000..94937d818 --- /dev/null +++ b/client/sandbox/react-nextjs/pages/play/operators.tsx @@ -0,0 +1,211 @@ +import { useEffect, useState } from "react"; +import config from "../../config"; +import { init_experimental, tx, id, i } from "@instantdb/react"; +import { useRouter } from "next/router"; + +const schema = i.graph( + { + comments: i.entity({ + slug: i.string().unique().indexed(), + someString: i.string().indexed(), + date: i.date().indexed(), + order: i.number().indexed(), + bool: i.boolean().indexed(), + }), + $users: i.entity({ + email: i.string().unique().indexed(), + }), + }, + { + commentAuthors: { + forward: { + on: "comments", + has: "one", + label: "author", + }, + reverse: { + on: "$users", + has: "many", + label: "authoredComments", + }, + }, + }, +); + +function randInt(max: number) { + return Math.floor(Math.random() * max); +} + +const d = new Date(); + +function Example({ appId }: { appId: string }) { + const router = useRouter(); + const myConfig = { ...config, appId, schema }; + const db = init_experimental(myConfig); + + const { data } = db.useQuery({ + comments: { + $: { where: { order: { $gt: 50 } } }, + }, + }); + + return ( +
+
+ + + +
+
+
+
+
+ All items ({data?.comments?.length || 0}): + + {data?.comments?.map((item) => ( +
+ {" "} + order = {item.order} +
+ ))} +
+
+
+
+ ); +} + +async function provisionEphemeralApp() { + const r = await fetch(`${config.apiURI}/dash/apps/ephemeral`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + title: "Comparisons example", + }), + }); + + return r.json(); +} + +async function verifyEphemeralApp({ appId }: { appId: string }) { + const r = await fetch(`${config.apiURI}/dash/apps/ephemeral/${appId}`, { + headers: { + "Content-Type": "application/json", + }, + }); + + return r.json(); +} + +function App({ urlAppId }: { urlAppId: string | undefined }) { + const router = useRouter(); + const [appId, setAppId] = useState(); + + const [error, setError] = useState(null); + + useEffect(() => { + if (appId) { + return; + } + if (urlAppId) { + verifyEphemeralApp({ appId: urlAppId }).then((res): any => { + if (res.app) { + setAppId(res.app.id); + } else { + provisionEphemeralApp().then((res) => { + if (res.app) { + router.replace({ + pathname: router.pathname, + query: { ...router.query, app: res.app.id }, + }); + + setAppId(res.app.id); + } else { + console.log(res); + setError("Could not create app."); + } + }); + } + }); + } else { + provisionEphemeralApp().then((res) => { + if (res.app) { + router.replace({ + pathname: router.pathname, + query: { ...router.query, app: res.app.id }, + }); + + setAppId(res.app.id); + } else { + console.log(res); + setError("Could not create app."); + } + }); + } + }, []); + + if (error) { + return ( +
+

There was an error

+

{error}

+
+ ); + } + + if (!appId) { + return
Loading...
; + } + return ; +} + +function Page() { + const router = useRouter(); + if (router.isReady) { + return ; + } else { + return
Loading...
; + } +} + +export default Page; diff --git a/client/sandbox/strong-init-vite/instant.schema.ts b/client/sandbox/strong-init-vite/instant.schema.ts index aaab2b822..83d223d6c 100644 --- a/client/sandbox/strong-init-vite/instant.schema.ts +++ b/client/sandbox/strong-init-vite/instant.schema.ts @@ -23,6 +23,7 @@ const _graph = i.graph( }, }, }, + ); type _Graph = typeof _graph; diff --git a/client/sandbox/strong-init-vite/src/typescript_tests_experimental_v2.tsx b/client/sandbox/strong-init-vite/src/typescript_tests_experimental_v2.tsx index b66958bd2..87a04464f 100644 --- a/client/sandbox/strong-init-vite/src/typescript_tests_experimental_v2.tsx +++ b/client/sandbox/strong-init-vite/src/typescript_tests_experimental_v2.tsx @@ -163,10 +163,7 @@ let coreMessageWithCreator: CoreMessageWithCreator = 1 as any; coreMessageWithCreator.content; coreMessageWithCreator.creator?.email; -type MessageCreatorResult = InstaQLResult< - AppSchema, - InstaQLParams ->; +type MessageCreatorResult = InstaQLResult>; function subMessagesWithCreator( resultCB: (data: MessageCreatorResult) => void, ) { @@ -177,6 +174,61 @@ function subMessagesWithCreator( }); } +// Test that the `Q` bit is typed +type DeeplyNestedQueryWorks = InstaQLEntity< + AppSchema, + "messages", + { creator: { createdMessages: { creator: {} } } } +>; +let deeplyNestedQuery: DeeplyNestedQueryWorks = 1 as any; +deeplyNestedQuery.creator?.createdMessages[0].creator?.email; + +type DeeplyNestedQueryWillFailsBadInput = InstaQLEntity< + AppSchema, + "messages", + // Type '{ foo: {}; }' has no properties in common with type 'InstaQLSubqueryParams' + // @ts-expect-error + { creator: { createdMessages: { foo: {} } } } +>; +let deeplyNestedQueryFailed: DeeplyNestedQueryWillFailsBadInput = 1 as any; + +type DeeplyNestedResultWorks = InstaQLResult< + AppSchema, + { + messages: { + creator: { + createdMessages: { + creator: {}; + }; + }; + }; + } +>; +let deeplyNestedResult: DeeplyNestedResultWorks = 1 as any; +deeplyNestedQuery.creator?.createdMessages[0].creator?.email; + +type DeeplyNestedResultFailsBadInput = InstaQLResult< + AppSchema, + // @ts-expect-error + { + messages: { + creator: { + createdMessages: { + // Type '{ foo: {}; }' is not assignable to type + // '$Option | ($Option & InstaQLQuerySubqueryParams) + // | undefined' + foo: {}; + }; + }; + }; + } +>; +let deeplyNestedResultFailed: DeeplyNestedResultFailsBadInput = 1 as any; + // to silence ts warnings +deeplyNestedQueryFailed; +deeplyNestedResultFailed; messagesQuery; subMessagesWithCreator; +deeplyNestedQuery; +deeplyNestedResult; diff --git a/client/sandbox/strong-init-vite/src/typescript_tests_normal_as_experimental.tsx b/client/sandbox/strong-init-vite/src/typescript_tests_normal_as_experimental.tsx index f52dc90f9..23a4e5eb1 100644 --- a/client/sandbox/strong-init-vite/src/typescript_tests_normal_as_experimental.tsx +++ b/client/sandbox/strong-init-vite/src/typescript_tests_normal_as_experimental.tsx @@ -90,7 +90,6 @@ function ReactNormalApp() { } const { messages } = data; messages[0].content; - // transactions reactDB.transact( reactDB.tx.messages[id()] diff --git a/client/version.md b/client/version.md index a240c92a4..21ddc09de 100644 --- a/client/version.md +++ b/client/version.md @@ -1 +1 @@ -v0.15.2 +v0.15.7 diff --git a/client/www/pages/docs/cli.md b/client/www/pages/docs/cli.md index dd068e04a..f201a4599 100644 --- a/client/www/pages/docs/cli.md +++ b/client/www/pages/docs/cli.md @@ -55,12 +55,12 @@ Similar to `git init`, running `instant-cli init` will generate a new app id and ### Push schema ```sh -npx instant-cli push-schema +npx instant-cli push schema ``` -`push-schema` evals your `instant.schema.ts` file and applies it your app's production database. [Read more about schema as code](/docs/schema). +`push schema` evals your `instant.schema.ts` file and applies it your app's production database. [Read more about schema as code](/docs/schema). -Note, to avoid accidental data loss, `push-schema` does not delete entities or fields you've removed from your schema. You can manually delete them in the [Explorer](https://www.instantdb.com/dash?s=main&t=explorer). +Note, to avoid accidental data loss, `push schema` does not delete entities or fields you've removed from your schema. You can manually delete them in the [Explorer](https://www.instantdb.com/dash?s=main&t=explorer). Here's an example `instant.schema.ts` file. @@ -100,10 +100,10 @@ export default graph; ### Push perms ```sh -npx instant-cli push-perms +npx instant-cli push perms ``` -`push-perms` evals your `instant.perms.ts` file and applies it your app's production database. `instant.perms.ts` should export an object implementing Instant's standard permissions CEL+JSON format. [Read more about permissions in Instant](/docs/permissions). +`push perms` evals your `instant.perms.ts` file and applies it your app's production database. `instant.perms.ts` should export an object implementing Instant's standard permissions CEL+JSON format. [Read more about permissions in Instant](/docs/permissions). Here's an example `instant.perms.ts` file. @@ -126,16 +126,16 @@ export default { ### Pull: migrating from the dashboard If you already created an app in the dashboard and created some schema and -permissions, you can run `npx instant-cli pull ` to generate an `instant.schema.ts` and `instant.perms.ts` files based on your production configuration. +permissions, you can run `npx instant-cli pull --app ` to generate an `instant.schema.ts` and `instant.perms.ts` files based on your production configuration. ```bash -npx instant-cli pull-schema -npx instant-cli pull-perms +npx instant-cli pull schema +npx instant-cli pull perms npx instant-cli pull # pulls both schema and perms ``` {% callout type="warning" %} -Note: Strongly typed attributes are under active development. For now, `pull-schema` will default all attribute types to `i.any()`. +Note: Strongly typed attributes are under active development. For now, `pull schema` will default all attribute types to `i.any()`. {% /callout %} diff --git a/client/www/pages/docs/permissions.md b/client/www/pages/docs/permissions.md index 9389738a9..81457bd95 100644 --- a/client/www/pages/docs/permissions.md +++ b/client/www/pages/docs/permissions.md @@ -26,7 +26,7 @@ You can manage permissions via configuration files or through the Instant dashbo The permissions definition file is `instant.perms.ts` -This file lives in the root of your project and will be consumed by [the Instant CLI](/docs/cli). You can immediately deploy permission changes to your database with `npx instant-cli push-perms`. +This file lives in the root of your project and will be consumed by [the Instant CLI](/docs/cli). You can immediately deploy permission changes to your database with `npx instant-cli push perms`. These changes will be reflected in the Permissions tab of the Instant dashboard. The default export of `instant.perms.ts` should be an object of rules as defined diff --git a/client/www/pages/docs/schema.md b/client/www/pages/docs/schema.md index cf5dbca8d..16ee61e97 100644 --- a/client/www/pages/docs/schema.md +++ b/client/www/pages/docs/schema.md @@ -4,7 +4,7 @@ title: Schema-as-code **The schema definition file: `instant.schema.ts`** -This file lives in the root of your project and will be consumed by [the Instant CLI](/docs/cli). You can apply your schema to the production database with `npx instant-cli push-schema`. +This file lives in the root of your project and will be consumed by [the Instant CLI](/docs/cli). You can apply your schema to the production database with `npx instant-cli push schema`. The default export of `instant.schema.ts` should always be the result of a call to `i.graph`. @@ -50,7 +50,7 @@ First we specify the expected type of the attribute: `i.string()`, `i.number()`, We can then chain modifiers: `.optional()`, `.unique()` and `.indexed()`. -When adding a type to an existing attribute, `push-schema` will kick off a job to check the existing data for the attribute before setting the type on the attribute. If you prefer not to enforce the type, you can run `push-schema` with the `--skip-check-types` flag. +When adding a type to an existing attribute, `push schema` will kick off a job to check the existing data for the attribute before setting the type on the attribute. If you prefer not to enforce the type, you can run `push schema` with the `--skip-check-types` flag. Here are some examples: diff --git a/server/.ebextensions/resources.config b/server/.ebextensions/resources.config new file mode 100644 index 000000000..74b777b25 --- /dev/null +++ b/server/.ebextensions/resources.config @@ -0,0 +1,5 @@ +Resources: + AWSEBAutoScalingGroup: + Type: AWS::AutoScaling::AutoScalingGroup + Properties: + PlacementGroup: "Instant-docker-prod-placement-group" diff --git a/server/.platform/hooks/prebuild/logdna.sh b/server/.platform/hooks/prebuild/logdna.sh deleted file mode 100644 index e2673ca50..000000000 --- a/server/.platform/hooks/prebuild/logdna.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# Generates a logdna.env file that will be picked up by docker-compose.yml and -# set up some logdna tags - -envfile="/var/app/staging/logdna.env" - -token=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 60"` - -hostname=$(curl -H "X-aws-ec2-metadata-token: $token" http://169.254.169.254/latest/meta-data/hostname) -instance_id=$(curl -H "X-aws-ec2-metadata-token: $token" http://169.254.169.254/latest/meta-data/instance-id) -instance_type=$(curl -H "X-aws-ec2-metadata-token: $token" http://169.254.169.254/latest/meta-data/instance-type) -env_name=$(/opt/elasticbeanstalk/bin/get-config container -k environment_name) -git_sha="SHA_REPLACE_ME" - -echo "MZ_HOSTNAME=${hostname}" > $envfile -echo "MZ_TAGS=${instance_id},${instance_type},${env_name},${git_sha}" >> $envfile diff --git a/server/Makefile b/server/Makefile index 84c2a4e52..94161fc2f 100644 --- a/server/Makefile +++ b/server/Makefile @@ -97,12 +97,12 @@ psql-dump: pg_dump $${DATABASE_URL} -f dump.sql tail-web: - aws logs tail /aws/elasticbeanstalk/Instant-docker-prod-env/var/log/eb-docker/containers/eb-current-app/stdouterr.log --follow + aws logs tail /aws/elasticbeanstalk/Instant-docker-prod-env-2/var/log/eb-docker/containers/eb-current-app/stdouterr.log --follow MINS?=30 errors: aws logs filter-log-events \ - --log-group-name "/aws/elasticbeanstalk/Instant-docker-prod-env/var/log/eb-docker/containers/eb-current-app/stdouterr.log" \ + --log-group-name "/aws/elasticbeanstalk/Instant-docker-prod-env-2/var/log/eb-docker/containers/eb-current-app/stdouterr.log" \ --start-time $$(expr `date -v-$(MINS)M +%s` \* 1000) \ --filter-pattern "ERROR" \ --output text diff --git a/server/docker-compose.yml b/server/docker-compose.yml index 2e6fc9589..b4560244e 100644 --- a/server/docker-compose.yml +++ b/server/docker-compose.yml @@ -1,43 +1,14 @@ services: web: image: IMAGE_REPLACE_ME - depends_on: - - refinery environment: - BEANSTALK_PORT=80 - PRODUCTION=true - - HONEYCOMB_ENDPOINT=http://localhost:8082 + # Internal load balancer + - HONEYCOMB_ENDPOINT=http://refinery.us-east-1.elasticbeanstalk.com:8082 env_file: - .env network_mode: "host" - command: sh -c 'java $${JAVA_OPTS} -agentpath:/usr/local/YourKit-JavaProfiler-2024.9/bin/linux-x86-64/libyjpagent.so=port=10001,listen=all -server -jar target/instant-standalone.jar' stop_grace_period: 1m restart: on-failure - - logdna: - image: logdna/logdna-agent:3.8 - environment: - - MZ_INGESTION_KEY=89611186ea52cb24e48a5e44d98d666c - - MZ_LOG_DIRS=/var/log - - MZ_INCLUSION_RULES=*.log,/var/log/messages - env_file: "logdna.env" - volumes: - - /var/log/eb-engine.log:/var/log/eb-engine.log:ro - - /var/log/messages:/var/log/messages:ro - - /var/log/eb-docker:/var/log/eb-docker:ro - - refinery: - image: honeycombio/refinery:2.8.4 - volumes: - - ./refinery/config.yaml:/etc/refinery/refinery.yaml - - ./refinery/config-redis.yaml:/etc/refinery/refinery-redis.yaml - - ./refinery/rules.yaml:/etc/refinery/rules.yaml - environment: - - REFINERY_HONEYCOMB_API_KEY=${REFINERY_HONEYCOMB_API_KEY} - - REFINERY_REDIS_HOST=${REFINERY_REDIS_HOST} - env_file: - - ./refinery/.env - - network_mode: "host" - restart: on-failure - stop_grace_period: 30s + command: sh -c 'java $${JAVA_OPTS} -agentpath:/usr/local/YourKit-JavaProfiler-2024.9/bin/linux-x86-64/libyjpagent.so=port=10001,listen=all -server -jar target/instant-standalone.jar' diff --git a/server/refinery/.ebextensions/resources.config b/server/refinery/.ebextensions/resources.config new file mode 100644 index 000000000..80e205ab1 --- /dev/null +++ b/server/refinery/.ebextensions/resources.config @@ -0,0 +1,102 @@ +Resources: + AWSEBAutoScalingGroup: + Type: AWS::AutoScaling::AutoScalingGroup + Metadata: + AWS::ElasticBeanstalk::Ext: + PreferredAvailabilityZones: "us-east-1b" + Properties: + AutoScalingGroupName: eb-refinery-auto-scaling-group + AvailabilityZones: + - 'us-east-1b' + AWSEBSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: "refinery auto scaling group" + SecurityGroupIngress: + - Description: "Inbound from prod" + SourceSecurityGroupId: "sg-04c436f7a3ea769cb" + FromPort: '8082' + ToPort: '8082' + IpProtocol: tcp + - Description: "Inbound from staging" + SourceSecurityGroupId: "sg-06dad1fbc6421604e" + FromPort: '8082' + ToPort: '8082' + IpProtocol: tcp + - Description: "Inbound from elb" + SourceSecurityGroupId: + Ref: "AWSEBLoadBalancerSecurityGroup" + FromPort: '8082' + ToPort: '8082' + IpProtocol: tcp + - Description: "ec2 instance connect" + SourcePrefixListId: "pl-0e4bcff02b13bef1e" + FromPort: '22' + ToPort: '22' + IpProtocol: tcp + + RefineryPeerToPeerSGIngress: + Type: 'AWS::EC2::SecurityGroupIngress' + DependsOn: AWSEBSecurityGroup + Properties: + Description: "communicate with peers" + GroupId: + Fn::GetAtt: "AWSEBSecurityGroup.GroupId" + FromPort: '8081' + ToPort: '8081' + IpProtocol: tcp + SourceSecurityGroupId: + Fn::GetAtt: "AWSEBSecurityGroup.GroupId" + + AWSEBEC2LaunchTemplate: + Type: AWS::EC2::LaunchTemplate + Properties: + LaunchTemplateName: eb-refinery-launch-template + AWSEBV2LoadBalancerListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + Port: 8082 + Protocol: TCP + AWSEBV2LoadBalancer: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Name: eb-refinery-elb + Scheme: internal + Type: network + SecurityGroups: + - Ref: "AWSEBLoadBalancerSecurityGroup" + AWSEBLoadBalancerSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: "refinery load balancer" + VpcId: "vpc-00063e3a899656167" + SecurityGroupIngress: + - Description: "Inbound from prod" + SourceSecurityGroupId: "sg-04c436f7a3ea769cb" + FromPort: '8082' + ToPort: '8082' + IpProtocol: tcp + - Description: "Inbound from staging" + SourceSecurityGroupId: "sg-06dad1fbc6421604e" + FromPort: '8082' + ToPort: '8082' + IpProtocol: tcp + SecurityGroupEgress: + - Description: "Outbound" + CidrIp: "0.0.0.0/0" + FromPort: "8081" + ToPort: "8082" + IpProtocol: "tcp" + AWSEBV2LoadBalancerTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + Name: eb-refinery-target-group + Protocol: TCP + Port: 8082 + HealthCheckEnabled: true + UnhealthyThresholdCount: 2 + HealthyThresholdCount: 5 + HealthCheckProtocol: TCP + HealthCheckTimeoutSeconds: 4 + HealthCheckIntervalSeconds: 5 + HealthCheckPort: traffic-port diff --git a/server/refinery/docker-compose.yml b/server/refinery/docker-compose.yml new file mode 100644 index 000000000..c8f6942d5 --- /dev/null +++ b/server/refinery/docker-compose.yml @@ -0,0 +1,15 @@ +services: + refinery: + image: honeycombio/refinery:2.8.4 + volumes: + - ./config.yaml:/etc/refinery/refinery.yaml + - ./config-redis.yaml:/etc/refinery/refinery-redis.yaml + - ./rules.yaml:/etc/refinery/rules.yaml + environment: + - REFINERY_HONEYCOMB_API_KEY=${REFINERY_HONEYCOMB_API_KEY} + - REFINERY_REDIS_HOST=${REFINERY_REDIS_HOST} + env_file: + - ./refinery.env + network_mode: "host" + restart: on-failure + stop_grace_period: 30s diff --git a/server/refinery/.env b/server/refinery/refinery.env similarity index 100% rename from server/refinery/.env rename to server/refinery/refinery.env diff --git a/server/refinery/rules.yaml b/server/refinery/rules.yaml index f0da4ebc1..c094daf7d 100644 --- a/server/refinery/rules.yaml +++ b/server/refinery/rules.yaml @@ -11,6 +11,7 @@ Samplers: Conditions: - Field: error Operator: exists + - Name: default rule Sampler: EMAThroughputSampler: diff --git a/server/resources/migrations/39_fix_extract_boolean.down.sql b/server/resources/migrations/39_fix_extract_boolean.down.sql new file mode 100644 index 000000000..b4eaa2125 --- /dev/null +++ b/server/resources/migrations/39_fix_extract_boolean.down.sql @@ -0,0 +1 @@ +-- no down migration diff --git a/server/resources/migrations/39_fix_extract_boolean.up.sql b/server/resources/migrations/39_fix_extract_boolean.up.sql new file mode 100644 index 000000000..49dbbb94d --- /dev/null +++ b/server/resources/migrations/39_fix_extract_boolean.up.sql @@ -0,0 +1,13 @@ +-- Had the wrong return type +drop function triples_extract_boolean_value; + +create or replace function triples_extract_boolean_value(value jsonb) +returns boolean as $$ + begin + if jsonb_typeof(value) = 'boolean' then + return (value->>0)::boolean; + else + return null; + end if; + end; +$$ language plpgsql immutable; diff --git a/server/resources/migrations/40_checked_indexes.down.sql b/server/resources/migrations/40_checked_indexes.down.sql new file mode 100644 index 000000000..46e81b0ab --- /dev/null +++ b/server/resources/migrations/40_checked_indexes.down.sql @@ -0,0 +1,10 @@ +drop index triples_string_trgm_gist_idx; + +drop index triples_number_type_idx; + +drop index triples_boolean_type_idx; + +drop index triples_date_type_idx; + +drop extension btree_gist; +drop extension pg_trgm; diff --git a/server/resources/migrations/40_checked_indexes.up.sql b/server/resources/migrations/40_checked_indexes.up.sql new file mode 100644 index 000000000..febe49543 --- /dev/null +++ b/server/resources/migrations/40_checked_indexes.up.sql @@ -0,0 +1,36 @@ +-- TODO: Run in production with `create index concurrently` before running this migration + +create extension if not exists btree_gist; +create extension if not exists pg_trgm; + +create index if not exists triples_string_trgm_gist_idx on triples using gist ( + app_id, + attr_id, + triples_extract_string_value(value) gist_trgm_ops, + entity_id + ) + where ave and checked_data_type = 'string'; + +create index if not exists triples_number_type_idx on triples ( + app_id, + attr_id, + triples_extract_number_value(value), + entity_id + ) + where ave and checked_data_type = 'number'; + +create index if not exists triples_boolean_type_idx on triples ( + app_id, + attr_id, + triples_extract_boolean_value(value), + entity_id + ) + where ave and checked_data_type = 'boolean'; + +create index if not exists triples_date_type_idx on triples ( + app_id, + attr_id, + triples_extract_date_value(value), + entity_id + ) + where ave and checked_data_type = 'date'; diff --git a/server/scripts/eb_deploy.clj b/server/scripts/eb_deploy.clj index 4e77f8e92..f372c82e4 100755 --- a/server/scripts/eb_deploy.clj +++ b/server/scripts/eb_deploy.clj @@ -16,7 +16,7 @@ {:out :string :err :string :continue true} - "aws elasticbeanstalk describe-environments --region us-east-1 --environment-name Instant-docker-prod-env" + "aws elasticbeanstalk describe-environments --region us-east-1 --environment-name Instant-docker-prod-env-2" *command-line-args*)] (when-not (string/blank? out) (-> (json/parse-string out) @@ -72,7 +72,7 @@ (println "Deploying " (get version "VersionLabel") " " (get version "Description")) (Thread/sleep 500) (apply exec - (str "eb deploy instant-docker-prod-env --version " (get version "VersionLabel")) + (str "eb deploy Instant-docker-prod-env-2 --version " (get version "VersionLabel")) *command-line-args*)) (defn main-loop [] diff --git a/server/scripts/manual_deploy.sh b/server/scripts/manual_deploy.sh index 6db5145ba..88958d667 100755 --- a/server/scripts/manual_deploy.sh +++ b/server/scripts/manual_deploy.sh @@ -27,4 +27,4 @@ aws s3api put-object --region us-east-1 --bucket "$bucket" --key "$key" --body $ aws elasticbeanstalk create-application-version --region us-east-1 --application-name instant-docker-prod --version-label "$application_version" --description "Manual deploy from $(hostname)" --source-bundle "S3Bucket=$bucket,S3Key=$key" -eb deploy instant-docker-prod-env --version "$application_version" +eb deploy Instant-docker-prod-env-2 --version "$application_version" diff --git a/server/scripts/prod_ssh.sh b/server/scripts/prod_ssh.sh index ea2b49de3..2bdd0ac21 100755 --- a/server/scripts/prod_ssh.sh +++ b/server/scripts/prod_ssh.sh @@ -9,7 +9,7 @@ done if [ -z "$instance_id" ]; then instance_id=$( aws ec2 describe-instances \ - --filter "Name=tag:elasticbeanstalk:environment-name,Values=Instant-docker-prod-env" \ + --filter "Name=tag:elasticbeanstalk:environment-name,Values=Instant-docker-prod-env-2" \ --query "Reservations[].Instances[?State.Name == 'running'].InstanceId[]" \ --output text ) diff --git a/server/scripts/prod_tunnel.sh b/server/scripts/prod_tunnel.sh index 99c39d15f..80e76f985 100755 --- a/server/scripts/prod_tunnel.sh +++ b/server/scripts/prod_tunnel.sh @@ -16,7 +16,7 @@ done if [ -z "$instance_id" ]; then instance_id=$( aws ec2 describe-instances \ - --filter "Name=tag:elasticbeanstalk:environment-name,Values=Instant-docker-prod-env" \ + --filter "Name=tag:elasticbeanstalk:environment-name,Values=Instant-docker-prod-env-2" \ --query "Reservations[].Instances[?State.Name == 'running'].InstanceId[]" \ --output text ) diff --git a/server/scripts/yourkit_tunnel.sh b/server/scripts/yourkit_tunnel.sh index 135b0a35b..1173bc6c3 100755 --- a/server/scripts/yourkit_tunnel.sh +++ b/server/scripts/yourkit_tunnel.sh @@ -16,7 +16,7 @@ done if [ -z "$instance_id" ]; then instance_id=$( aws ec2 describe-instances \ - --filter "Name=tag:elasticbeanstalk:environment-name,Values=Instant-docker-prod-env" \ + --filter "Name=tag:elasticbeanstalk:environment-name,Values=Instant-docker-prod-env-2" \ --query "Reservations[].Instances[?State.Name == 'running'].InstanceId[]" \ --output text ) diff --git a/server/src/instant/admin/model.clj b/server/src/instant/admin/model.clj index 7b789c76a..409eb8f7a 100644 --- a/server/src/instant/admin/model.clj +++ b/server/src/instant/admin/model.clj @@ -330,8 +330,20 @@ (create-lookup-attrs ops) (create-attrs-from-objs ops))) -(defn transform [attrs steps] +(defn transform [{:keys [attrs throw-on-missing-attrs?] :as _ctx} steps] (let [{attrs :attrs add-attr-tx-steps :add-ops} (create-missing-attrs attrs steps) + _ (when (and throw-on-missing-attrs? (seq add-attr-tx-steps)) + (let [ident-names (->> add-attr-tx-steps + (map (comp + #(string/join "." %1) + attr-model/ident-name + :forward-identity + second)))] + (ex/throw-validation-err! + :steps + steps + [{:message "Attributes are missing in your schema" + :hint {:attributes ident-names}}]))) tx-steps (mapcat (fn [step] (to-tx-steps attrs step)) steps)] (concat add-attr-tx-steps tx-steps))) @@ -392,7 +404,7 @@ (str (UUID/randomUUID))) (defn ->tx-steps! - [attrs steps] + [ctx steps] (let [coerced-admin-steps (<-json (->json steps) false) valid? (s/valid? ::ops coerced-admin-steps) _ (when-not valid? @@ -401,7 +413,7 @@ steps (ex/explain->validation-errors (s/explain-data ::ops steps)))) - tx-steps (transform attrs coerced-admin-steps) + tx-steps (transform ctx coerced-admin-steps) coerced (tx/coerce! tx-steps) _ (tx/validate! coerced)] coerced)) @@ -409,8 +421,8 @@ (comment (def counters-app-id #uuid "b502cabc-11ed-4534-b340-349d46548642") (def attrs (attr-model/get-by-app-id counters-app-id)) - (->tx-steps! attrs [["merge" "goals" (str-uuid) {"title" "plop"}]]) - (->tx-steps! attrs + (->tx-steps! {:attrs attrs} [["merge" "goals" (str-uuid) {"title" "plop"}]]) + (->tx-steps! {:attrs attrs} [["update" "goals" (str-uuid) {"title" "moop"}] ["link" "goals" (str-uuid) {"todos" (str-uuid)}] ["unlink" "goals" (str-uuid) {"todos" (str-uuid)}] diff --git a/server/src/instant/admin/routes.clj b/server/src/instant/admin/routes.clj index d84589834..5d1cbec51 100644 --- a/server/src/instant/admin/routes.clj +++ b/server/src/instant/admin/routes.clj @@ -140,6 +140,9 @@ (defn transact-post [req] (let [steps (ex/get-param! req [:body :steps] #(when (coll? %) %)) + throw-on-missing-attrs? (ex/get-optional-param! + req + [:body :throw-on-missing-attrs?] boolean) {:keys [app-id] :as perms} (get-perms! req) attrs (attr-model/get-by-app-id app-id) ctx (merge {:db {:conn-pool aurora/conn-pool} @@ -148,7 +151,9 @@ :datalog-query-fn d/query :rules (rule-model/get-by-app-id {:app-id app-id})} perms) - tx-steps (admin-model/->tx-steps! attrs steps) + tx-steps (admin-model/->tx-steps! {:attrs attrs + :throw-on-missing-attrs? throw-on-missing-attrs?} + steps) {tx-id :id} (permissioned-tx/transact! ctx tx-steps)] (cond :else @@ -163,6 +168,9 @@ commit-tx (-> req :body :dangerously-commit-tx) dry-run (not commit-tx) steps (ex/get-param! req [:body :steps] #(when (coll? %) %)) + throw-on-missing-attrs? (ex/get-optional-param! + req + [:body :throw-on-missing-attrs?] boolean) attrs (attr-model/get-by-app-id app-id) rules (if rules-override {:app_id app-id :code rules-override} @@ -175,7 +183,9 @@ :admin-check? true :admin-dry-run? dry-run} perms) - tx-steps (admin-model/->tx-steps! attrs steps) + tx-steps (admin-model/->tx-steps! {:attrs attrs + :throw-on-missing-attrs? throw-on-missing-attrs?} + steps) result (permissioned-tx/transact! ctx tx-steps) cleaned-result {:tx-id (:id result) :all-checks-ok? (:all-checks-ok? result) diff --git a/server/src/instant/dash/routes.clj b/server/src/instant/dash/routes.clj index 410a89bb8..72224e42d 100644 --- a/server/src/instant/dash/routes.clj +++ b/server/src/instant/dash/routes.clj @@ -1124,11 +1124,8 @@ (defn cli-auth-check-post [req] (let [secret (ex/get-param! req [:body :secret] uuid-util/coerce) - cli-auth (instant-cli-login-model/check! aurora/conn-pool {:secret secret}) + cli-auth (instant-cli-login-model/use! aurora/conn-pool {:secret secret}) user-id (:user_id cli-auth) - _ (ex/assert-valid! :cli-auth - (:id cli-auth) - (when-not user-id [{:message "Invalid CLI auth ticket"}])) refresh-token (instant-user-refresh-token-model/create! {:id (UUID/randomUUID) :user-id user-id}) token (:id refresh-token) {email :email} (instant-user-model/get-by-id! {:id user-id}) diff --git a/server/src/instant/data/bootstrap.clj b/server/src/instant/data/bootstrap.clj index c5fe97c39..158cf7267 100644 --- a/server/src/instant/data/bootstrap.clj +++ b/server/src/instant/data/bootstrap.clj @@ -16,7 +16,7 @@ (:import (java.util UUID))) -(defn extract-zeneca-txes [] +(defn extract-zeneca-txes [checked-data?] (let [imported (<-json (slurp (io/resource "sample_triples/zeneca.json"))) triples (->> imported (remove (fn [[_ a v]] @@ -77,12 +77,25 @@ :unique? true :index? true} :else - {:id uuid - :forward-identity [(java.util.UUID/randomUUID) nsp idn] - :cardinality :one - :value-type :blob - :unique? (boolean (#{"email" "handle" "isbn13"} idn)) - :index? (boolean (#{"email" "handle"} idn))})])))) + (merge + {:id uuid + :forward-identity [(java.util.UUID/randomUUID) nsp idn] + :cardinality :one + :value-type :blob + :unique? (boolean (#{"email" "handle" "isbn13"} idn)) + :index? (boolean (#{"email" "handle"} idn))} + (when-let [data-type (when checked-data? + (case idn + ("email" + "handle" + "isbn13" + "title" + "fullName" + "description") "string" + ("order") "number" + ("createdAt") "date" + nil))] + {:checked-data-type data-type})))])))) triples-to-insert (map (fn [[e a v]] @@ -97,41 +110,42 @@ (defn add-zeneca-to-app! "Bootstraps an app with zeneca data." - [app-id] - ;; Note: This is ugly code, but it works. - ;; Maybe we clean it up later, but we don't really need to right now. - ;; One idea for a cleanup, is to create an "exported app" file. - ;; We can then write a function that works on this kind of file schema. - (attr-model/delete-by-app-id! aurora/conn-pool app-id) - (let [txes (extract-zeneca-txes) - _ (tx/transact! - aurora/conn-pool - (attr-model/get-by-app-id app-id) - app-id - txes) - triples (triple-model/fetch - aurora/conn-pool - app-id) - attrs (attr-model/get-by-app-id app-id) - users (for [[_ group] (group-by first (map :triple triples)) - :when (= (attr-model/fwd-etype - (attr-model/seek-by-id (second (first group)) - attrs)) - "users") - :let [{:strs [email id]} - (entity-model/triples->map {:attrs attrs} group)]] - {:email email - :id id - :app-id app-id})] - (doseq [user users] - (app-user-model/create! user)) + ([app-id] (add-zeneca-to-app! false app-id)) + ([checked-data? app-id] + ;; Note: This is ugly code, but it works. + ;; Maybe we clean it up later, but we don't really need to right now. + ;; One idea for a cleanup, is to create an "exported app" file. + ;; We can then write a function that works on this kind of file schema. + (attr-model/delete-by-app-id! aurora/conn-pool app-id) + (let [txes (extract-zeneca-txes checked-data?) + _ (tx/transact! + aurora/conn-pool + (attr-model/get-by-app-id app-id) + app-id + txes) + triples (triple-model/fetch + aurora/conn-pool + app-id) + attrs (attr-model/get-by-app-id app-id) + users (for [[_ group] (group-by first (map :triple triples)) + :when (= (attr-model/fwd-etype + (attr-model/seek-by-id (second (first group)) + attrs)) + "users") + :let [{:strs [email id]} + (entity-model/triples->map {:attrs attrs} group)]] + {:email email + :id id + :app-id app-id})] + (doseq [user users] + (app-user-model/create! user)) - (count triples))) + (count triples)))) (defn add-zeneca-to-byop-app! "Bootstraps an app with zeneca data." [conn] - (let [txes (extract-zeneca-txes) + (let [txes (extract-zeneca-txes false) {:keys [add-triple add-attr]} (group-by first txes) attrs (map second add-attr) attrs-by-id (reduce (fn [acc attr] diff --git a/server/src/instant/db/datalog.clj b/server/src/instant/db/datalog.clj index 73d5d3c07..e6397921f 100644 --- a/server/src/instant/db/datalog.clj +++ b/server/src/instant/db/datalog.clj @@ -57,15 +57,26 @@ (s/def ::$isNull (s/keys :req-un [::attr-id ::nil?])) (s/def ::$like string?) +(s/def ::op #{:$gt :$gte :$lt :$lte :$like}) +(s/def ::data-type #{:string :number :date :boolean}) +(s/def ::value any?) +(s/def ::$comparator (s/keys :req-un [::op ::data-type ::value])) + (s/def ::value-pattern-component (s/or :constant (s/coll-of ::triple-model/value :kind set? :min-count 0) :any #{'_} :variable symbol? - :function (s/keys :req-un [(or ::$not ::$isNull ::$like)]))) + :function (s/keys :req-un [(or ::$not ::$isNull ::$comparator)]))) + +(s/def ::idx-key #{:ea :eav :av :ave :vae}) +(s/def ::data-type #{:string :number :boolean :date}) +(s/def ::index-map (s/keys :req-un [::idx-key ::data-type])) +(s/def ::index (s/or :keyword ::idx-key + :map ::index-map)) (s/def ::pattern - (s/cat :idx ::triple-model/index + (s/cat :idx ::index :e (pattern-component ::triple-model/lookup) :a (pattern-component uuid?) :v ::value-pattern-component @@ -115,8 +126,8 @@ (and (= i 2) (map? c) (or (contains? c :$not) - (contains? c :$like) - (contains? c :$isNull))) + (contains? c :$isNull) + (contains? c :$comparator))) (symbol? c) (set? c)) c @@ -144,6 +155,12 @@ (coll/pad 5 '_) ensure-set-constants))) +(defn idx-key [idx] + (let [[tag v] idx] + (case tag + :map (:idx-key v) + :keyword v))) + (defn untag-e "Removes the tag from the entity-id position, where it can be either :entity-id or :lookup-ref. We can inspect the type (uuid? vs vector?) to determine if it's @@ -206,7 +223,7 @@ "Given a named pattern, returns a mapping of symbols to their binding paths: - idx [:eav ?a ?b ?c] + pattern-idx [:eav ?a ?b ?c] ;=> @@ -214,15 +231,15 @@ {?a [[idx 0]], ?b [[idx 1]], ?c [[idx 2]]}" - [idx {:keys [e a v]}] + [pattern-idx {:keys [e a v]}] (reduce (fn [acc [x path]] (if (named-variable? x) (update acc (uspec/tagged-unwrap x) (fnil conj []) path) acc)) {} - [[e [idx 0]] - [a [idx 1]] - [v [idx 2]]])) + [[e [pattern-idx 0]] + [a [pattern-idx 1]] + [v [pattern-idx 2]]])) ;; ---- ;; join-vals @@ -331,18 +348,7 @@ '_ #{(-> v second :$isNull :attr-id)} '_]] - - ;; There's probably a more clever way to invalidate against like - ;; values but this is a simple way for now - (and (= :function (first v)) - (contains? (second v) :$like)) - [[idx - (component->topic-component symbol-values :e e) - (component->topic-component symbol-values :a a) - '_]] - - :else - [[idx + [[(idx-key idx) (component->topic-component symbol-values :e e) (component->topic-component symbol-values :a a) (component->topic-component symbol-values :v v)]])) @@ -533,7 +539,22 @@ (defn- value->jsonb [x] [:cast (->json x) :jsonb]) -(defn- constant->where-part [app-id component-type [_ v]] +(defn- in-or-eq-value [idx v-set] + (let [[tag idx-val] idx + data-type (case tag + :keyword nil + :map (:data-type idx-val))] + (if (empty? v-set) + [:= 0 1] + (if-not data-type + (in-or-eq :value (map value->jsonb v-set)) + (list* :or (map (fn [v] + [:and + [:= :checked_data_type [:cast [:inline (name data-type)] :checked_data_type]] + [:= [(kw :triples_extract_ data-type :_value) :value] v]]) + v-set)))))) + +(defn- constant->where-part [idx app-id component-type [_ v]] (condp = component-type :e (list* :or (for [lookup v] @@ -548,7 +569,7 @@ [:= :value [:cast (->json (second lookup)) :jsonb]] [:= :attr-id [:cast (first lookup) :uuid]]]}]))) :a (in-or-eq :attr-id v) - :v (in-or-eq :value (map value->jsonb v)))) + :v (in-or-eq-value idx v))) (defn- value-function-clauses [[v-tag v-value]] (case v-tag @@ -568,7 +589,18 @@ :where [:and [:= :t.entity-id :entity-id] [:= :t.attr-id (:attr-id val)] - [:not= :t.value [:cast (->json nil) :jsonb]]]}]])) + [:not= :t.value [:cast (->json nil) :jsonb]]]}]] + :$comparator (let [{:keys [op value data-type]} val] + [[(case op + :$gt :> + :$gte :>= + :$lt :< + :$lte :<=) + [(kw :triples_extract_ data-type :_value) + :value] + value] + ;; Need this check so that postgres knows it can use the index + [:= :checked_data_type [:cast [:inline (name data-type)] :checked_data_type]]]))) [])) (defn- function-clauses [named-pattern] @@ -589,11 +621,11 @@ (list* :and [:= :app-id app-id] - [:= idx :true] + [:= (idx-key idx) :true] (concat (->> named-pattern constant-components (map (fn [[component-type v]] - (constant->where-part app-id component-type v)))) + (constant->where-part idx app-id component-type v)))) (function-clauses named-pattern) additional-clauses))) @@ -789,7 +821,7 @@ (if (named-variable? x) (assoc acc pat-idx {:sym sym :ref-value? (and (= :v component) - (= :eav (:idx named-p)))}) + (= :eav (idx-key (:idx named-p))))}) acc))) {} [[:e 0] [:a 1] [:v 2]])) @@ -1709,6 +1741,24 @@ (throw-invalid-nested-patterns nested-named-patterns) (send-query-nested ctx (:conn-pool db) app-id nested-named-patterns))) +(defn explain + "Takes nested patterns and returns the explain result from running + the postgres query. Useful for testing and debugging." + [ctx patterns] + (assert (map? patterns) "explain only works with nested patterns.") + (let [nested-named-patterns (nested->named-patterns patterns)] + (throw-invalid-nested-patterns nested-named-patterns) + (let [{:keys [query]} (nested-match-query ctx + :match-0- + (:app-id ctx) + nested-named-patterns) + sql-query (update (hsql/format query) + 0 + (fn [s] + (str "explain (analyze, verbose, buffers, timing, format json) " s)))] + + (first (sql/select-string-keys (-> ctx :db :conn-pool) sql-query))))) + (defn query "Executes a Datalog(ish) query over the given aurora `conn`, Instant `app_id` and `patterns` diff --git a/server/src/instant/db/instaql.clj b/server/src/instant/db/instaql.clj index 33c89731a..18ad28289 100644 --- a/server/src/instant/db/instaql.clj +++ b/server/src/instant/db/instaql.clj @@ -14,7 +14,6 @@ [instant.data.resolvers :as resolvers] [instant.util.tracer :as tracer] [instant.util.coll :as ucoll] - [instant.util.async :as ua] [instant.model.rule :as rule-model] [instant.db.cel :as cel] [instant.util.exception :as ex] @@ -38,12 +37,26 @@ (s/def ::$not where-value-valid?) (s/def ::$isNull boolean?) +(s/def ::comparator (s/or :string string? + :number number? + :boolean boolean?)) + +(s/def ::$gt ::comparator) +(s/def ::$gte ::comparator) +(s/def ::$lt ::comparator) +(s/def ::$lte ::comparator) +(s/def ::$like ::comparator) (defn where-value-valid-keys? [m] - (every? #{:in :$in :$not :$isNull :$like} (keys m))) + (every? #{:in :$in + :$not :$isNull + :$gt :$gte :$lt :$lte + :$like} + (keys m))) (s/def ::where-args-map (s/and - (s/keys :opt-un [::in ::$in ::$not ::$isNull ::$like]) + (s/keys :opt-un [::in ::$in ::$not ::$isNull + ::$gt ::$gte ::$lt ::$lte ::$like]) where-value-valid-keys?)) (s/def ::where-v @@ -490,9 +503,7 @@ :args-map (let [[func args-map-val] (first v-value)] (case func (:$in :in) args-map-val - :$not {:$not args-map-val} - :$isNull {:$isNull args-map-val} - :$like {:$like args-map-val}))) + {func args-map-val}))) [refs-path value-label] (ucoll/split-last path) [last-etype last-level ref-attr-pats referenced-etypes] @@ -1471,12 +1482,12 @@ (reduce (fn [acc [e a]] (let [etype (-> (attr-model/seek-by-id a attrs) - :forward-identity - second) - next-acc (assoc-in acc [:eid->etype e] etype)] - (if-not (contains? (:etype->program acc) etype) - (assoc-in next-acc [:etype->program etype] (rule-model/get-program! rules etype "view")) - next-acc))) + attr-model/fwd-etype)] + (-> acc + (update-in [:etype->eids+program etype :eids] (fnil conj #{}) e) + (update-in [:etype->eids+program etype :program] (fn [p] + (or p + (rule-model/get-program! rules etype "view"))))))) acc join-rows)) acc @@ -1500,8 +1511,7 @@ "Takes the result of `query` and generates a query cache of datalog-query -> datalog-result, and maps for etype->program and eid->type." ([ctx instaql-res] - (extract-permission-helpers {:eid->etype {} - :etype->program {} + (extract-permission-helpers {:etype->eids+program {} :query-cache {}} ctx instaql-res)) @@ -1510,18 +1520,27 @@ (tracer/with-span! {:name "extract-permission-helpers"} (extract-permission-helpers* acc ctx instaql-res)))) -(defn permissioned-node [eid->check res] +(defn permissioned-node [{:keys [attrs] :as ctx} etype+eid->check res] (let [cleaned-join-rows (->> res :data :datalog-result :join-rows (filter (fn [triples] - (every? (comp :result eid->check first) triples))) + (every? (fn [[e a]] + (let [etype (-> (attr-model/seek-by-id a attrs) + attr-model/fwd-etype) + check (get etype+eid->check [etype e])] + (:result check))) + triples))) set) cleaned-page-info (when (get-in res [:data :datalog-result :page-info]) - (when-let [filtered-rows (seq (filter (comp :result eid->check first) + (when-let [filtered-rows (seq (filter (fn [[e a]] + (let [etype (-> (attr-model/seek-by-id a attrs) + attr-model/fwd-etype) + check (get etype+eid->check [etype e])] + (:result check))) (get-in res [:data :datalog-result :page-info-rows])))] @@ -1545,7 +1564,7 @@ (if (empty? cleaned-join-rows) [] (->> child-nodes - (map (partial permissioned-node eid->check)) + (map (partial permissioned-node ctx etype+eid->check)) (filter (fn [node] (seq (-> node :data :datalog-result :join-rows)))) @@ -1564,76 +1583,76 @@ (defn extract-refs "Extracts a list of refs that can be passed to cel/prefetch-data-refs. Returns: [{:etype string path-str string eids #{uuid}}]" - [user-id eid->etype etype->program] - (let [etype->eid (ucoll/map-invert-key-set eid->etype)] - (reduce (fn [acc [etype program]] - (if-let [refs (some-> program - :cel-ast - cel/collect-ref-uses - seq)] - (reduce (fn [acc {:keys [obj path]}] - (case obj - "data" (conj acc {:etype etype - :path-str path - :eids (get etype->eid etype)}) - "auth" (conj acc {:etype "$users" - :path-str path - :eids (if user-id - #{user-id} - #{})}) - - acc)) - acc - refs) - acc)) - [] - etype->program))) - -(defn preload-refs [ctx eid->etype etype->program] + [user-id etype->eids+program] + (reduce-kv (fn [acc etype {:keys [eids program]}] + (if-let [refs (some-> program + :cel-ast + cel/collect-ref-uses + seq)] + (reduce (fn [acc {:keys [obj path]}] + (case obj + "data" (conj acc {:etype etype + :path-str path + :eids eids}) + "auth" (conj acc {:etype "$users" + :path-str path + :eids (if user-id + #{user-id} + #{})}) + + acc)) + acc + refs) + acc)) + [] + etype->eids+program)) + +(defn preload-refs [ctx etype->eids+program] (let [refs (extract-refs (-> ctx :current-user :id) - eid->etype - etype->program)] + etype->eids+program)] (if (seq refs) (cel/prefetch-data-refs ctx refs) {}))) -(defn get-eid-check-result! [{:keys [current-user] :as ctx} - {:keys [eid->etype etype->program query-cache]}] +(defn get-etype+eid-check-result! [{:keys [current-user] :as ctx} + {:keys [etype->eids+program query-cache]}] (tracer/with-span! {:name "instaql/get-eid-check-result!"} (let [preloaded-refs (tracer/with-span! {:name "instaql/preload-refs"} - (let [res (preload-refs ctx eid->etype etype->program)] + (let [res (preload-refs ctx etype->eids+program)] (tracer/add-data! {:attributes {:ref-count (count res)}}) res))] - (->> eid->etype - (ua/vfuture-pmap - (fn [[eid etype]] - (let [p (etype->program etype)] - [eid (if-not p - {:result true} - {:program p - :result - (let [em (io/warn-io :instaql/entity-map - (entity-map ctx - query-cache - etype - eid)) - ctx (assoc ctx - :preloaded-refs preloaded-refs)] - (io/warn-io :instaql/eval-program - (cel/eval-program! - p - {"auth" (cel/->cel-map {:ctx ctx - :type :auth - :etype "$users"} - current-user) - "data" (cel/->cel-map {:ctx ctx - :etype etype - :type :data} - em)})))})]))) - - (into {}))))) + (reduce-kv (fn [acc etype {:keys [eids program]}] + (reduce (fn [acc eid] + (assoc acc + [etype eid] + (if-not program + {:result true} + {:program program + :result + (let [em (io/warn-io :instaql/entity-map + (entity-map ctx + query-cache + etype + eid)) + ctx (assoc ctx + :preloaded-refs preloaded-refs)] + (io/warn-io :instaql/eval-program + (cel/eval-program! + program + {"auth" (cel/->cel-map {:ctx ctx + :type :auth + :etype "$users"} + current-user) + "data" (cel/->cel-map {:ctx ctx + :etype etype + :type :data} + em)})))}))) + acc + eids)) + {} + etype->eids+program)))) (defn permissioned-query [{:keys [app-id current-user admin?] :as ctx} o] (tracer/with-span! {:name "instaql/permissioned-query" @@ -1650,9 +1669,9 @@ (extract-permission-helpers {:attrs (:attrs ctx) :rules rules} res) - eid->check (get-eid-check-result! ctx perm-helpers) + etype+eid->check (get-etype+eid-check-result! ctx perm-helpers) res' (tracer/with-span! {:name "instaql/map-permissioned-node"} - (mapv (partial permissioned-node eid->check) res))] + (mapv (partial permissioned-node ctx etype+eid->check) res))] res'))))) (defn permissioned-query-check [{:keys [app-id] :as ctx} o rules-override] @@ -1663,22 +1682,22 @@ (extract-permission-helpers {:attrs (:attrs ctx) :rules rules} res) - eid->check (get-eid-check-result! ctx perm-helpers) + etype+eid->check (get-etype+eid-check-result! ctx perm-helpers) check-results (map - (fn [[id {:keys [result program]}]] + (fn [[[etype id] {:keys [result program]}]] {:id id - :entity (get (:eid->etype perm-helpers) id) + :entity etype :record (entity-map ctx (:query-cache perm-helpers) - (get (:eid->etype perm-helpers) id) + etype id) :program (select-keys program [:code :display-code :etype :action]) :check result}) - eid->check) - nodes (mapv (partial permissioned-node eid->check) res)] + etype+eid->check) + nodes (mapv (partial permissioned-node ctx etype+eid->check) res)] {:nodes nodes :check-results check-results})) ;; ---- diff --git a/server/src/instant/db/model/attr.clj b/server/src/instant/db/model/attr.clj index 5bb5f7d96..dec43d7c1 100644 --- a/server/src/instant/db/model/attr.clj +++ b/server/src/instant/db/model/attr.clj @@ -138,6 +138,11 @@ "Given an attr, return it's reverse etype or nil" (comp second :reverse-identity)) +(defn fwd-friendly-name + "Given an attr, returns `etype.label`" + [attr] + (format "%s.%s" (fwd-etype attr) (fwd-label attr))) + ;; ------- ;; caching diff --git a/server/src/instant/db/model/attr_pat.clj b/server/src/instant/db/model/attr_pat.clj index c77da860d..0e324c6ef 100644 --- a/server/src/instant/db/model/attr_pat.clj +++ b/server/src/instant/db/model/attr_pat.clj @@ -1,13 +1,14 @@ (ns instant.db.model.attr-pat - (:require [instant.db.datalog :as d] + (:require [clojure.spec.alpha :as s] + [instant.db.datalog :as d] [instant.db.model.attr :as attr-model] [instant.util.exception :as ex] [instant.util.json :as json] [instant.util.uuid :as uuid-util] [instant.jdbc.aurora :as aurora] [instant.data.constants :refer [zeneca-app-id]] - [clojure.spec.alpha :as s] - [instant.db.model.triple :as triple-model])) + [instant.db.model.triple :as triple-model]) + (:import (java.time Instant))) (s/def ::attr-pat (s/cat :e (d/pattern-component uuid?) @@ -24,10 +25,22 @@ [post-owner-attr true] => :vae if ?owner is actualized [posts-owner-attr false] => :eav if ?owner isn't actualized" - [{:keys [value-type index? indexing? unique? setting-unique?]} v-actualized?] + [{:keys [value-type + index? + indexing? + unique? + setting-unique? + checked-data-type + checking-data-type?]} + v-actualized?] (let [ref? (= value-type :ref) e-idx (if ref? :eav :ea) v-idx (cond + (and index? + (not indexing?) + checked-data-type + (not checking-data-type?)) {:idx-key :ave + :data-type checked-data-type} (and unique? (not setting-unique?)) :av (and index? (not indexing?)) :ave ref? :vae @@ -166,6 +179,155 @@ :else (uuid-util/coerce v))) +(defn assert-checked-attr-data-type! [state attr] + (cond (:checking-data-type? attr) + (ex/throw-validation-err! + :query + (:root state) + [{:expected? 'checked-data-type? + :in (:in state) + :message (format "The `%s` attribute is still in the process of checking its data type. It must finish before using comparison operators." + (attr-model/fwd-friendly-name attr))}]) + + (:indexing? attr) + (ex/throw-validation-err! + :query + (:root state) + [{:expected? 'indexed? + :in (:in state) + :message (format "The `%s` attribute is still in the process of indexing. It must finish before using comparison operators." + (attr-model/fwd-friendly-name attr))}]) + + (not (:index? attr)) + (ex/throw-validation-err! + :query + (:root state) + [{:expected? 'indexed? + :in (:in state) + :message (format "The `%s` attribute must be indexed to use comparison operators." + (attr-model/fwd-friendly-name attr))}]) + + (not (:checked-data-type attr)) + (ex/throw-validation-err! + :query + (:root state) + [{:expected? 'checked-data-type? + :in (:in state) + :message (format "The `%s` attribute must have an enforced type to use comparison operators." + (attr-model/fwd-friendly-name attr))}]) + + :else (:checked-data-type attr))) + +(defn throw-invalid-timestamp! [state attr value] + (ex/throw-validation-err! + :query + (:root state) + [{:expected? 'timestamp? + :in (:in state) + :message (format "The data type of `%s` is `date`, but the query got value `%s` of type `%s`." + (attr-model/fwd-friendly-name attr) + (json/->json value) + (json/json-type-of-clj value))}])) + +(defn throw-invalid-date-string! [state attr value] + (ex/throw-validation-err! + :query + (:root state) + [{:expected? 'date-string? + :in (:in state) + :message (format "The data type of `%s` is `date`, but the query got value `%s` of type `%s`." + (attr-model/fwd-friendly-name attr) + (json/->json value) + (json/json-type-of-clj value))}])) + +(defn throw-invalid-data-value! + [state attr data-type value] + (ex/throw-validation-err! + :query + (:root state) + [{:expected? (symbol (format "%s?" (name data-type))) + :in (:in state) + :message (format "The data type of `%s` is `%s`, but the query got the value `%s` of type `%s`." + (attr-model/fwd-friendly-name attr) + (name data-type) + (json/->json value) + (json/json-type-of-clj value))}])) + +(defn throw-on-invalid-value-data-value! + "Validates an individual value" + [state attr data-type v] + (case data-type + :string (when-not (string? v) + (throw-invalid-data-value! state attr data-type v)) + :number (when-not (number? v) + (throw-invalid-data-value! state attr data-type v)) + :boolean (when-not (boolean? v) + (throw-invalid-data-value! state attr data-type v)) + :date (cond (number? v) + (try + (Instant/ofEpochMilli v) + (catch Exception _e + (throw-invalid-timestamp! state attr v))) + + (string? v) + (try + (Instant/parse v) + (catch Exception _e + (throw-invalid-date-string! state attr v))) + + :else + (throw-invalid-data-value! state attr data-type v)) + nil)) + +(defn coerced-type-comparison-value! [state attr attr-data-type tag value] + (case attr-data-type + :date (case tag + :number (try + (Instant/ofEpochMilli value) + (catch Exception _e + (throw-invalid-timestamp! state attr value))) + :string (try + (Instant/parse value) + (catch Exception _e + (throw-invalid-date-string! state attr value)))) + (if-not (= tag attr-data-type) + (throw-invalid-data-value! state attr attr-data-type value) + value))) + +(defn validate-value-type! [state attr data-type v] + (doseq [v (if (set? v) v [v]) + :let [v (if (and (map? v) + (contains? v :$not)) + (:$not v) + v)]] + (throw-on-invalid-value-data-value! state attr data-type v)) + v) + +(defn coerce-value-for-typed-comparison! + "Coerces the value for a typed comparison, throwing a validation error + if the attr doesn't support the comparison." + [state attr v] + (cond (symbol? v) v + + (and (map? v) + (= (count v) 1) + (contains? #{:$gt :$gte :$lt :$lte} (ffirst v))) + (let [[op [tag value]] (first v) + attr-data-type (assert-checked-attr-data-type! state attr) + state (update state :in conj op)] + {:$comparator + {:op op + :value (coerced-type-comparison-value! state attr attr-data-type tag value) + :data-type attr-data-type}}) + + + :else + (if (and (:checked-data-type attr) + (not (:checking-data-type? attr))) + (validate-value-type! state attr (:checked-data-type attr) v) + v))) + + (defn ->value-attr-pat "Take the where-cond: [\"users\" \"bookshelves\" \"books\" \"title\"] \"Foo\" @@ -183,7 +345,10 @@ :attr {:args [value-etype value-label]}) v-coerced (if (not= :ref value-type) - v + (let [state (update state :in conj :$ :where value-label)] + (coerce-value-for-typed-comparison! state + attr + v)) (if (set? v) (set (map (fn [vv] (if-let [v-coerced (coerce-value-uuid vv)] @@ -192,7 +357,7 @@ :query (:root state) [{:expected 'uuid? - :in (conj (:in state) [:$ :where value-label]) + :in (conj (:in state) :$ :where value-label) :message (format "Expected %s to match on a uuid, found %s in %s" value-label (json/->json vv) @@ -205,7 +370,7 @@ :query (:root state) [{:expected 'uuid? - :in (conj (:in state) [:$ :where value-label]) + :in (conj (:in state) :$ :where value-label) :message (format "Expected %s to be a uuid, got %s" value-label (json/->json v))}]))))] diff --git a/server/src/instant/fixtures.clj b/server/src/instant/fixtures.clj index 10e888c77..501407192 100644 --- a/server/src/instant/fixtures.clj +++ b/server/src/instant/fixtures.clj @@ -51,6 +51,13 @@ r (resolvers/make-zeneca-resolver id)] (f app r))))) +(defn with-zeneca-checked-data-app [f] + (with-empty-app + (fn [{:keys [id] :as app}] + (let [_ (bootstrap/add-zeneca-to-app! true id) + r (resolvers/make-zeneca-resolver id)] + (f app r))))) + (defn with-zeneca-byop [f] (with-empty-app (fn [{:keys [id] :as app}] diff --git a/server/src/instant/model/instant_cli_login.clj b/server/src/instant/model/instant_cli_login.clj index 6ff234e0e..b58c04c1c 100644 --- a/server/src/instant/model/instant_cli_login.clj +++ b/server/src/instant/model/instant_cli_login.clj @@ -1,7 +1,11 @@ (ns instant.model.instant-cli-login (:require [instant.jdbc.aurora :as aurora] [instant.jdbc.sql :as sql] - [instant.util.crypt :as crypt-util])) + [instant.util.crypt :as crypt-util] + [instant.util.exception :as ex]) + (:import + (java.time Instant) + (java.time.temporal ChronoUnit))) (defn create! ([params] (create! aurora/conn-pool params)) @@ -28,23 +32,54 @@ id = ?" user-id ticket]))) -(defn check! - ([params] (check! aurora/conn-pool params)) +(defn expired? + ([magic-code] (expired? (Instant/now) magic-code)) + ([now {created-at :created_at}] + (> (.between ChronoUnit/MINUTES (.toInstant created-at) now) 2))) + +(defn voided? + [{used? :used user-id :user_id :as _login}] + (and used? (not user-id))) + +(defn use! + ([params] (use! aurora/conn-pool params)) ([conn {:keys [secret]}] - (sql/execute-one! - conn - ["UPDATE - instant_cli_logins - SET - used = true - WHERE - secret = ? - AND created_at > now() - interval '2 minutes' - AND user_id IS NOT NULL - AND used = false - RETURNING - user_id" - (crypt-util/uuid->sha256 secret)]))) + (let [{user-id :user_id id :id :as login} + (sql/select-one conn + ["SELECT * FROM instant_cli_logins WHERE secret = ?" + (crypt-util/uuid->sha256 secret)]) + + _ (ex/assert-record! login :instant-cli-login {}) + + _ (when (expired? login) + (ex/throw-expiration-err! :instant-cli-login {:args [id]})) + + _ (when (voided? login) + (ex/throw-validation-err! :instant-cli-login id [{:issue :user-voided-request + :message "This request has been denied"}])) + _ (when-not user-id + (ex/throw-validation-err! :instant-cli-login id [{:issue :waiting-for-user + :message "Waiting for a user to accept this request"}])) + + claimed (sql/execute-one! + conn + ["UPDATE + instant_cli_logins + SET + used = true + WHERE + secret = ? AND + user_id IS NOT NULL AND + used = false + RETURNING *" + (crypt-util/uuid->sha256 secret)]) + + _ (when-not claimed + (ex/throw-validation-err! :instant-cli-login + :id + [{:issue :user-already-claimed + :message "This request has already been claimed"}]))] + claimed))) (defn void! ([params] (void! aurora/conn-pool params)) @@ -57,4 +92,4 @@ used = true WHERE id = ?::uuid" - ticket]))) \ No newline at end of file + ticket]))) diff --git a/server/src/instant/reactive/session.clj b/server/src/instant/reactive/session.clj index e27c340f0..2edbd916d 100644 --- a/server/src/instant/reactive/session.clj +++ b/server/src/instant/reactive/session.clj @@ -165,7 +165,7 @@ :instaql-result instaql-result :result-changed? result-changed?})) -(defn- handle-refresh! [store-conn sess-id _event] +(defn- handle-refresh! [store-conn sess-id _event debug-info] (let [auth (get-auth! store-conn sess-id) app-id (-> auth :app :id) current-user (-> auth :user) @@ -180,6 +180,8 @@ :table-info table-info :admin? admin?} processed-tx-id (rs/get-processed-tx-id @store-conn app-id) + _ (reset! debug-info {:processed-tx-id processed-tx-id + :instaql-queries (map :instaql-query/query stale-queries)}) recompute-results (->> stale-queries (ua/vfuture-pmap (partial recompute-instaql-query! opts))) {computations true spam false} (group-by :result-changed? recompute-results) @@ -223,6 +225,25 @@ :tx-id tx-id :client-event-id client-event-id}))) +;; ----- +;; error + +(defn handle-error! [store-conn sess-id {:keys [status + client-event-id + original-event + type + message + hint]}] + (rs/send-event! store-conn + sess-id + {:op :error + :status status + :client-event-id client-event-id + :original-event original-event + :type type + :message message + :hint hint})) + ;; ------ ;; worker @@ -333,7 +354,7 @@ :topic topic :data data}))) -(defn handle-event [store-conn eph-store-atom session event] +(defn handle-event [store-conn eph-store-atom session event debug-info] (tracer/with-span! {:name "receive-worker/handle-event"} (let [{:keys [op]} event {:keys [session/socket]} session @@ -343,8 +364,9 @@ :init (handle-init! store-conn id event) :add-query (handle-add-query! store-conn id event) :remove-query (handle-remove-query! store-conn id event) - :refresh (handle-refresh! store-conn id event) + :refresh (handle-refresh! store-conn id event debug-info) :transact (handle-transact! store-conn id event) + :error (handle-error! store-conn id event) ;; ----- ;; EPH events :join-room (handle-join-room! store-conn eph-store-atom id event) @@ -357,63 +379,72 @@ ;; -------------- ;; Receive Workers -(defn- handle-instant-exception [store-conn session original-event instant-ex] +(defn- handle-instant-exception [session original-event instant-ex debug-info] (let [sess-id (:session/id session) + q (:receive-q (:session/socket session)) {:keys [client-event-id]} original-event {:keys [::ex/type ::ex/message ::ex/hint] :as err-data} (ex-data instant-ex)] (tracer/add-data! {:attributes {:err-data (pr-str err-data)}}) - (condp contains? type - #{::ex/record-not-found - ::ex/record-expired - ::ex/record-not-unique - ::ex/record-foreign-key-invalid - ::ex/record-check-violation - ::ex/sql-raise - - ::ex/permission-denied - ::ex/permission-evaluation-failed - - ::ex/param-missing - ::ex/param-malformed - - ::ex/validation-failed} - (rs/try-send-event! store-conn sess-id - {:op :error - :status 400 - :client-event-id client-event-id - :original-event original-event - :type (keyword (name type)) - :message message - :hint hint}) - - #{::ex/session-missing - ::ex/socket-missing - ::ex/socket-error} + (case type + (::ex/record-not-found + ::ex/record-expired + ::ex/record-not-unique + ::ex/record-foreign-key-invalid + ::ex/record-check-violation + ::ex/sql-raise + + ::ex/permission-denied + ::ex/permission-evaluation-failed + + ::ex/param-missing + ::ex/param-malformed + + ::ex/validation-failed) + (receive-queue/enqueue->receive-q q + {:op :error + :status 400 + :client-event-id client-event-id + :original-event (merge original-event + debug-info) + :type (keyword (name type)) + :message message + :hint hint + :session-id sess-id}) + + (::ex/session-missing + ::ex/socket-missing + ::ex/socket-error) (tracer/record-exception-span! instant-ex {:name "receive-worker/socket-unreachable"}) (do (tracer/add-exception! instant-ex {:escaping? false}) - (rs/try-send-event! store-conn sess-id - {:op :error - :status 500 - :client-event-id client-event-id - :original-event original-event - :type (keyword (name type)) - :message message - :hint hint}))))) - -(defn- handle-uncaught-err [store-conn session original-event root-err] + (receive-queue/enqueue->receive-q q + {:op :error + :status 500 + :client-event-id client-event-id + :original-event (merge original-event + debug-info) + :type (keyword (name type)) + :message message + :hint hint + :session-id sess-id}))))) + +(defn- handle-uncaught-err [session original-event root-err debug-info] (let [sess-id (:session/id session) + q (:receive-q (:session/socket session)) {:keys [client-event-id]} original-event] (tracer/add-exception! root-err {:escaping? false}) - (rs/try-send-event! store-conn sess-id - {:op :error - :client-event-id client-event-id - :status 500 - :original-event original-event - :message (str "Yikes, something broke on our end! Sorry about that." - " Please ping us (Joe and Stopa) on Discord and let us know!")}))) + + (receive-queue/enqueue->receive-q q + {:op :error + :client-event-id client-event-id + :status 500 + :original-event (merge original-event + debug-info) + :message (str "Yikes, something broke on our end! Sorry about that." + " Please ping us (Joe and Stopa) on Discord and let us know!") + :session-id sess-id}))) (defn handle-receive-attrs [store-conn session event metadata] (let [{:keys [session/socket]} session @@ -430,8 +461,13 @@ :attributes (handle-receive-attrs store-conn session event metadata)} (let [pending-handlers (:pending-handlers (:session/socket session)) in-progress-stmts (atom #{}) + debug-info (atom nil) event-fut (binding [sql/*in-progress-stmts* in-progress-stmts] - (ua/vfuture (handle-event store-conn eph-store-atom session event))) + (ua/vfuture (handle-event store-conn + eph-store-atom + session + event + debug-info))) pending-handler {:future event-fut :op (:op event) :in-progress-stmts in-progress-stmts @@ -462,10 +498,14 @@ instant-ex (ex/find-instant-exception e) root-err (root-cause e)] (cond - instant-ex (handle-instant-exception - store-conn session original-event instant-ex) - :else (handle-uncaught-err - store-conn session original-event root-err)))) + instant-ex (handle-instant-exception session + original-event + instant-ex + @debug-info) + :else (handle-uncaught-err session + original-event + root-err + @debug-info)))) (finally (swap! pending-handlers disj pending-handler))))))) @@ -563,24 +603,27 @@ (grouped-queue/inflight-queue-reserve 1 inflight-q))) +(defn start-receive-worker [store-conn eph-store-atom receive-q stop-signal id] + (ua/vfut-bg + (loop [] + (if @stop-signal + (tracer/record-info! {:name "receive-worker/shutdown-complete" + :attributes {:worker-n id}}) + (do (grouped-queue/process-polling! + receive-q + {:reserve-fn receive-worker-reserve-fn + :process-fn (fn [group-key batch] + (straight-jacket-process-receive-q-batch store-conn + eph-store-atom + batch + {:worker-n id + :batch-size (count batch) + :group-key group-key}))}) + (recur)))))) + (defn start-receive-workers [store-conn eph-store-atom receive-q stop-signal] (doseq [n (range num-receive-workers)] - (ua/vfut-bg - (loop [] - (if @stop-signal - (tracer/record-info! {:name "receive-worker/shutdown-complete" - :attributes {:worker-n n}}) - (do (grouped-queue/process-polling! - receive-q - {:reserve-fn receive-worker-reserve-fn - :process-fn (fn [group-key batch] - (straight-jacket-process-receive-q-batch store-conn - eph-store-atom - batch - {:worker-n n - :batch-size (count batch) - :group-key group-key}))}) - (recur))))))) + (start-receive-worker store-conn eph-store-atom receive-q stop-signal n))) ;; ----------------- ;; Websocket Interop @@ -675,6 +718,9 @@ :refresh-presence [:refresh-presence session-id room-id] + :error + [:error session-id] + nil))) (comment diff --git a/server/src/instant/reactive/store.clj b/server/src/instant/reactive/store.clj index d9189ffb3..dc2c766ea 100644 --- a/server/src/instant/reactive/store.clj +++ b/server/src/instant/reactive/store.clj @@ -442,9 +442,19 @@ (or (symbol? dq-part) (symbol? iv-part)) true (set? dq-part) (intersects? iv-part dq-part) - (and (map? dq-part) (contains? dq-part :not)) - (let [not-val (:not dq-part)] - (some (partial not= not-val) iv-part)))) + (map? dq-part) + (if-let [{:keys [op value]} (:$comparator dq-part)] + (let [f (case op + :$gt > + :$gte >= + :$lt < + :$lte <=)] + (some (fn [v] + (f v value)) + iv-part)) + (when (contains? dq-part :$not) + (let [not-val (:$not dq-part)] + (some (partial not= not-val) iv-part)))))) (defn match-topic? [iv-topic dq-topic] diff --git a/server/src/instant/util/json.clj b/server/src/instant/util/json.clj index 3b91ff70d..576f1498f 100644 --- a/server/src/instant/util/json.clj +++ b/server/src/instant/util/json.clj @@ -25,3 +25,17 @@ (def <-json "Converts a JSON string to a Clojure data structure." cheshire/parse-string) + +(defn json-type-of-clj [v] + (cond (string? v) + "string" + (number? v) + "number" + (boolean? v) + "boolean" + (nil? v) + "null" + (or (vector? v) + (list? v)) + "array" + :else "object")) diff --git a/server/src/instant/util/string.clj b/server/src/instant/util/string.clj index ce5835522..7891283e9 100644 --- a/server/src/instant/util/string.clj +++ b/server/src/instant/util/string.clj @@ -45,3 +45,19 @@ (recur (inc found-idx) (conj idxes found-idx)) idxes))) + +(defn join-in-sentence + "Joins items in the list in a sentence + ['a'] => 'a' + ['a', 'b'] => 'a and b' + ['a', 'b', 'c'] => 'a, b, and c'" + [ls] + (case (count ls) + 0 "" + 1 (format "%s" (first ls)) + 2 (format "%s and %s" + (first ls) + (second ls)) + (format "%s, and %s" + (string/join ", " (butlast ls)) + (last ls)))) diff --git a/server/test/instant/admin/routes_test.clj b/server/test/instant/admin/routes_test.clj index 9a09d94d7..6e2cdc78d 100644 --- a/server/test/instant/admin/routes_test.clj +++ b/server/test/instant/admin/routes_test.clj @@ -177,6 +177,79 @@ (attr-model/get-by-app-id app-id)))))))))) +(deftest strong-init-and-inference + (with-empty-app + (fn [{app-id :id admin-token :admin-token :as _app}] + (let [goal-id (str (UUID/randomUUID)) + user-id (str (UUID/randomUUID)) + goal-owner-attr-id (str (UUID/randomUUID)) + + add-links [["add-attr" + {:id goal-owner-attr-id + :forward-identity [(UUID/randomUUID) "goals" "owner"] + :reverse-identity [(UUID/randomUUID) "users" "ownedGoals"] + :value-type "ref" + :cardinality "one" + :unique? false + :index? false}]] + add-objects [["update" "goals" + goal-id + {"title" "get fit"}] + ["update" "users" + user-id + {"name" "stopa"}] + ["link" "goals" + goal-id + {"owner" user-id}]] + _add-links-ret (transact-post + {:body {:steps add-links} + :headers {"app-id" (str app-id) + "authorization" (str "Bearer " admin-token)}}) + _add-objects-ret (transact-post + {:body {:steps add-objects} + :headers {"app-id" (str app-id) + "authorization" (str "Bearer " admin-token)}})] + (let [q (query-post + {:body {:query {:goals {:owner {}}}} + :headers {"app-id" (str app-id) + "authorization" (str "Bearer " admin-token)}}) + goal (-> q :body (get "goals") first) + owner-part (get goal "owner")] + + (is (= "get fit" (get goal "title"))) + (is (= 1 (count owner-part))) + (is (= "stopa" (get (first owner-part) "name")))) + + (testing "cardinality inference works" + (let [q (query-post + {:body {:query {:goals {:owner {}}} + :inference? true} + :headers {"app-id" (str app-id) + "authorization" (str "Bearer " admin-token)}}) + goal (-> q :body (get "goals") first) + owner (get goal "owner")] + (is (= "get fit" (get goal "title"))) + (is (= "stopa" (get owner "name"))))) + + (testing "throw-missing-attrs works" + (let [{:keys [status body]} (transact-post + {:body {:steps [["update" "goals" + goal-id + {"myFavoriteColor" "purple"}]] + :throw-on-missing-attrs? true} + :headers {"app-id" (str app-id) + "authorization" (str "Bearer " admin-token)}})] + + (is (= 400 status)) + (is (= #{"goals.myFavoriteColor"} + (-> body + :hint + :errors + first + :hint + :attributes + set))))))))) + (deftest refresh-tokens-test (with-empty-app (let [email "stopa@instantdb.com"] @@ -289,8 +362,7 @@ (is (= 200 (:status refresh-ret))) (is (some? token)) - - ;; retrieve user by refresh token +;; retrieve user by refresh token (let [get-user-ret (app-users-get {:params {:refresh_token token} :headers {"app-id" app-id @@ -370,8 +442,7 @@ (is (= 200 (:status refresh-ret))) (is (some? token)) - - ;; delete user by refresh token +;; delete user by refresh token (let [delete-user-ret (app-users-delete {:params {:refresh_token token} :headers {"app-id" app-id @@ -633,8 +704,8 @@ "authorization" (str "Bearer " admin-token)}}) :body) user (-> query-result - (get "users") - first) + (get "users") + first) tasks (-> user (get "tasks"))] (is (= "Stepan" (get user "name"))) @@ -673,7 +744,7 @@ (defn tx-validation-err [attrs steps] (try - (admin-model/->tx-steps! attrs steps) + (admin-model/->tx-steps! {:attrs attrs} steps) (catch clojure.lang.ExceptionInfo e (-> e ex-data ::ex/hint :errors first)))) diff --git a/server/test/instant/db/datalog_test.clj b/server/test/instant/db/datalog_test.clj index 7136a4e12..ac9ff7491 100644 --- a/server/test/instant/db/datalog_test.clj +++ b/server/test/instant/db/datalog_test.clj @@ -30,14 +30,14 @@ (deftest patterns (testing "named patterns are verbose raw patterns" - (is (= '([:pattern {:idx :eav, :e [:variable ?a], :a [:variable ?b], :v [:variable ?c] :created-at [:any _]}] - [:pattern {:idx :ea, :e [:variable ?c], :a [:variable ?d], :v [:variable ?e] :created-at [:any _]}]) + (is (= '([:pattern {:idx [:keyword :eav], :e [:variable ?a], :a [:variable ?b], :v [:variable ?c] :created-at [:any _]}] + [:pattern {:idx [:keyword :ea], :e [:variable ?c], :a [:variable ?d], :v [:variable ?e] :created-at [:any _]}]) (d/->named-patterns '[[:eav ?a ?b ?c] [:ea ?c ?d ?e]])))) (testing "named patterns coerce values into sets" - (is (= '([:pattern {:idx :av, :e [:any _], :a [:variable ?a], :v [:constant #{5}] :created-at [:any _]}]) + (is (= '([:pattern {:idx [:keyword :av], :e [:any _], :a [:variable ?a], :v [:constant #{5}] :created-at [:any _]}]) (d/->named-patterns '[[:av _ ?a 5]])))) (testing "named patterns add wildcards for missing params" - (is (= '([:pattern {:idx :vae, :e [:any _], :a [:any _], :v [:any _] :created-at [:any _]}]) + (is (= '([:pattern {:idx [:keyword :vae], :e [:any _], :a [:any _], :v [:any _] :created-at [:any _]}]) (d/->named-patterns '[[:vae]]))))) (deftest pats->coarse-topics diff --git a/server/test/instant/db/instaql_test.clj b/server/test/instant/db/instaql_test.clj index 213e60cf7..68a8ca811 100644 --- a/server/test/instant/db/instaql_test.clj +++ b/server/test/instant/db/instaql_test.clj @@ -9,8 +9,10 @@ [instant.db.model.attr :as attr-model] [instant.db.model.triple :as triple-model] [instant.db.transaction :as tx] - [instant.fixtures - :refer [with-empty-app with-zeneca-app with-zeneca-byop]] + [instant.fixtures :refer [with-empty-app + with-zeneca-app + with-zeneca-checked-data-app + with-zeneca-byop]] [instant.jdbc.aurora :as aurora] [instant.jdbc.sql :as sql] [instant.model.app :as app-model] @@ -21,6 +23,7 @@ [instant.util.instaql :refer [instaql-nodes->object-tree]] [instant.util.test :refer [instant-ex-data pretty-perm-q]]) (:import + (java.time Instant) (java.util UUID))) (def ^:private r (delay (resolvers/make-zeneca-resolver))) @@ -57,7 +60,7 @@ (when (seq aggregates) {:aggregate aggregates})))) -(defn- is-pretty-eq? +(defmacro is-pretty-eq? "InstaQL will execute in parallel. This means that it _is_ possible for nodes @@ -71,15 +74,16 @@ This checks equality strictly based on the set of topics and triples in the result" [pretty-a pretty-b] - (testing "(topics is-pretty-eq?)" - (is (= (set (mapcat :topics pretty-a)) - (set (mapcat :topics pretty-b))))) - (testing "(triples is-pretty-eq?)" - (is (= (set (mapcat :triples pretty-a)) - (set (mapcat :triples pretty-b))))) - (testing "(aggregate is-pretty-eq?)" - (is (= (set (remove nil? (mapcat :aggregate pretty-a))) - (set (remove nil? (mapcat :aggregate pretty-b))))))) + `(do + (testing "(topics is-pretty-eq?)" + (is (= (set (mapcat :topics ~pretty-a)) + (set (mapcat :topics ~pretty-b))))) + (testing "(triples is-pretty-eq?)" + (is (= (set (mapcat :triples ~pretty-a)) + (set (mapcat :triples ~pretty-b))))) + (testing "(aggregate is-pretty-eq?)" + (is (= (set (remove nil? (mapcat :aggregate ~pretty-a))) + (set (remove nil? (mapcat :aggregate ~pretty-b)))))))) (defn- query-pretty ([q] @@ -123,12 +127,12 @@ :in [0 :option-map :where-conds 0 1]} (validation-err {:users {:$ {:where {:handle {:is "stopa"}}}}}))) (is (= '{:expected uuid? - :in ["users" [:$ :where "bookshelves"]] + :in ["users" :$ :where "bookshelves"] :message "Expected bookshelves to be a uuid, got \"hello\""} (validation-err {:users {:$ {:where {:bookshelves "hello"}}}}))) (is (= '{:expected uuid? - :in ["users" [:$ :where "bookshelves"]] + :in ["users" :$ :where "bookshelves"] :message "Expected bookshelves to match on a uuid, found \"hello\" in [\"hello\",\"00000000-0000-0000-0000-000000000000\"]"} (validation-err {:users {:$ {:where {:bookshelves {:in ["00000000-0000-0000-0000-000000000000" @@ -196,6 +200,63 @@ {:$ {:aggregate :count} :bookshelves {}}}))))) +(deftest validations-on-checked-data + (with-empty-app + (fn [app] + (testing "checked-data-types" + (tx/transact! aurora/conn-pool + (attr-model/get-by-app-id (:id app)) + (:id app) + (for [t [:string :number :boolean :date]] + [:add-attr {:id (random-uuid) + :forward-identity [(random-uuid) "etype" (name t)] + :unique? false + :index? true + :value-type :blob + :checked-data-type t + :cardinality :one}])) + (let [ctx (let [attrs (attr-model/get-by-app-id (:id app))] + {:db {:conn-pool aurora/conn-pool} + :app-id (:id app) + :attrs attrs})] + (is (= '{:expected? string?, + :in ["etype" :$ :where "string"], + :message + "The data type of `etype.string` is `string`, but the query got the value `1` of type `number`."} + (validation-err ctx {:etype {:$ {:where {:string 1}}}}))) + (is (= '{:expected? number?, + :in ["etype" :$ :where "number"], + :message + "The data type of `etype.number` is `number`, but the query got the value `\"hello\"` of type `string`."} + (validation-err ctx {:etype {:$ {:where {:number "hello"}}}}))) + (is (= '{:expected? boolean?, + :in ["etype" :$ :where "boolean"], + :message + "The data type of `etype.boolean` is `boolean`, but the query got the value `0` of type `number`."} + (validation-err ctx {:etype {:$ {:where {:boolean 0}}}}))) + (is (= '{:expected? timestamp?, + :in ["etype" :$ :where "date"], + :message + "The data type of `etype.date` is `date`, but the query got value `9999999999999999999999` of type `number`."} + (validation-err ctx {:etype {:$ {:where {:date 9999999999999999999999}}}}))) + (is (= '{:expected? date-string?, + :in ["etype" :$ :where "date"], + :message + "The data type of `etype.date` is `date`, but the query got value `\"tomorrow\"` of type `string`."} + (validation-err ctx {:etype {:$ {:where {:date "tomorrow"}}}}))) + + (is (= '{:expected? string? + :in ["etype" :$ :where "string" :$gt], + :message + "The data type of `etype.string` is `string`, but the query got the value `10` of type `number`."} + (validation-err ctx {:etype {:$ {:where {:string {:$gt 10}}}}}))) + + (is (= '{:expected? boolean? + :in ["etype" :$ :where "boolean" :$gt], + :message + "The data type of `etype.boolean` is `boolean`, but the query got the value `1` of type `number`."} + (validation-err ctx {:etype {:$ {:where {:boolean {:$gt 1}}}}})))))))) + (deftest pagination (testing "limit" (is-pretty-eq? (query-pretty {:users {:$ {:limit 2 @@ -1955,6 +2016,116 @@ ("eid-joe-averbukh" :users/email "joe@instantdb.com") ("eid-joe-averbukh" :users/createdAt "2021-01-07 18:51:23.742637"))})))) +(deftest comparators + (with-empty-app + (fn [app] + (let [attr-ids {:string (random-uuid) + :number (random-uuid) + :boolean (random-uuid) + :date (random-uuid)} + label-attr-id (random-uuid) + labels ["a" "b" "c" "d" "e"] + make-ctx (fn [] + (let [attrs (attr-model/get-by-app-id (:id app))] + {:db {:conn-pool aurora/conn-pool} + :app-id (:id app) + :attrs attrs})) + run-query (fn [return-field q] + (let [ctx (make-ctx) + r (resolvers/make-resolver {:conn-pool aurora/conn-pool} + (:id app) + [["books" "field"] + ["authors" "field"]])] + (->> (iq/permissioned-query ctx q) + (instaql-nodes->object-tree ctx) + (#(get % "etype")) + (map #(get % (name return-field))) + set))) + run-explain (fn [data-type value] + (-> (d/explain (make-ctx) + {:children + {:pattern-groups + [{:patterns + [[{:idx-key :ave, :data-type data-type} + '?etype-0 + (get attr-ids data-type) + {:$comparator {:op :$gt, :value value, :data-type data-type}}]]}]}}) + (get "QUERY PLAN") + first + (get-in ["Plan" "Plans" 0 "Index Name"])))] + (tx/transact! aurora/conn-pool + (attr-model/get-by-app-id (:id app)) + (:id app) + (concat + [[:add-attr {:id (random-uuid) + :forward-identity [(random-uuid) "etype" "id"] + :unique? true + :index? true + :value-type :blob + :checked-data-type :string + :cardinality :one}] + [:add-attr {:id label-attr-id + :forward-identity [(random-uuid) "etype" "label"] + :unique? true + :index? true + :value-type :blob + :checked-data-type :string + :cardinality :one}]] + (for [[t attr-id] attr-ids] + [:add-attr {:id attr-id + :forward-identity [(random-uuid) "etype" (name t)] + :unique? false + :index? true + :value-type :blob + :checked-data-type t + :cardinality :one}]) + (mapcat + (fn [i] + (let [id (random-uuid)] + [[:add-triple id label-attr-id (nth labels i)] + [:add-triple id (:string attr-ids) (str i)] + [:add-triple id (:number attr-ids) i] + [:add-triple id (:date attr-ids) i] + [:add-triple id (:boolean attr-ids) (zero? (mod i 2))]])) + (range (count labels))))) + (testing "string" + (is (= #{"3" "4"} (run-query :string {:etype {:$ {:where {:string {:$gt "2"}}}}}))) + (is (= #{"2" "3" "4"} (run-query :string {:etype {:$ {:where {:string {:$gte "2"}}}}}))) + (is (= #{"0" "1"} (run-query :string {:etype {:$ {:where {:string {:$lt "2"}}}}}))) + (is (= #{"0" "1" "2"} (run-query :string {:etype {:$ {:where {:string {:$lte "2"}}}}}))) + + (testing "uses index" + (is (= "triples_string_trgm_gist_idx" (run-explain :string "2"))))) + + (testing "number" + (is (= #{3 4} (run-query :number {:etype {:$ {:where {:number {:$gt 2}}}}}))) + (is (= #{2 3 4} (run-query :number {:etype {:$ {:where {:number {:$gte 2}}}}}))) + (is (= #{0 1} (run-query :number {:etype {:$ {:where {:number {:$lt 2}}}}}))) + (is (= #{0 1 2} (run-query :number {:etype {:$ {:where {:number {:$lte 2}}}}}))) + + (testing "uses index" + (is (= "triples_number_type_idx" (run-explain :number 2))))) + + (testing "date" + (is (= #{3 4} (run-query :date {:etype {:$ {:where {:date {:$gt 2}}}}}))) + (is (= #{2 3 4} (run-query :date {:etype {:$ {:where {:date {:$gte 2}}}}}))) + (is (= #{0 1} (run-query :date {:etype {:$ {:where {:date {:$lt 2}}}}}))) + (is (= #{0 1 2} (run-query :date {:etype {:$ {:where {:date {:$lte 2}}}}}))) + + (testing "uses index" + (is (= "triples_date_type_idx" (run-explain :date (Instant/ofEpochMilli 2)))))) + + (testing "boolean" + (is (= #{} (run-query :boolean {:etype {:$ {:where {:boolean {:$gt true}}}}}))) + (is (= #{true} (run-query :boolean {:etype {:$ {:where {:boolean {:$gt false}}}}}))) + (is (= #{true} (run-query :boolean {:etype {:$ {:where {:boolean {:$gte true}}}}}))) + (is (= #{} (run-query :boolean {:etype {:$ {:where {:boolean {:$lt false}}}}}))) + (is (= #{false} (run-query :boolean {:etype {:$ {:where {:boolean {:$lt true}}}}}))) + (is (= #{false true} (run-query :boolean {:etype {:$ {:where {:boolean {:$lte true}}}}}))) + + (testing "uses index" + (is (= "triples_boolean_type_idx" (run-explain :boolean true))))))))) + (deftest child-forms (testing "no child where" (is-pretty-eq? @@ -2534,145 +2705,148 @@ (app-model/delete-by-id! {:id app-id})) (deftest read-perms - (with-zeneca-app - (fn [{app-id :id :as _app} _r] - (testing "no perms returns full" - (rule-model/put! - aurora/conn-pool - {:app-id app-id :code {}}) - (is - (= #{"alex" "joe" "stopa" "nicolegf"} - (->> (pretty-perm-q - {:app-id app-id :current-user nil} - {:users {}}) - :users - (map :handle) - set)))) - (testing "false returns nothing" - (rule-model/put! - aurora/conn-pool - {:app-id app-id :code {:users {:allow {:view "false"}}}}) - (is - (empty? - (->> (pretty-perm-q - {:app-id app-id :current-user nil} - {:users {}}) - :users - (map :handle) - set)))) - (testing "property equality" - (rule-model/put! - aurora/conn-pool - {:app-id app-id :code {:users {:allow {:view "data.handle == 'stopa'"}}}}) - (is - (= - #{"stopa"} - (->> (pretty-perm-q - {:app-id app-id :current-user nil} - {:users {}}) - :users - (map :handle) - set)))) - (testing "bind" - (rule-model/put! - aurora/conn-pool - {:app-id app-id :code {:users {:allow {:view "data.handle != handle"} - :bind ["handle" "'stopa'"]}}}) - (is - (= - #{"alex" "joe" "nicolegf"} - (->> (pretty-perm-q - {:app-id app-id :current-user nil} - {:users {}}) - :users - (map :handle) - set)))) - (testing "ref" - (rule-model/put! - aurora/conn-pool - {:app-id app-id :code {:bookshelves {:allow {:view "handle in data.ref('users.handle')"} - :bind ["handle" "'alex'"]}}}) - (is - (= - #{"Short Stories" "Nonfiction"} - (->> (pretty-perm-q - {:app-id app-id :current-user nil} - {:bookshelves {}}) - :bookshelves - (map :name) - set)))) - (testing "auth required" - (rule-model/put! - aurora/conn-pool - {:app-id app-id :code {:users {:allow {:view "auth.id != null"}}}}) - (is - (empty? - (->> (pretty-perm-q - {:app-id app-id :current-user nil} - {:users {}}) - :users - (map :handle) - set)))) - - (testing "null shouldn't evaluate to true" - (rule-model/put! - aurora/conn-pool - {:app-id app-id :code {:users {:allow {:view "auth.isAdmin"}}}}) - (is - (empty? - (->> (pretty-perm-q - {:app-id app-id :current-user nil} - {:users {}}) - :users - (map :handle) - set)))) - (testing "can only view authed user data" - (rule-model/put! - aurora/conn-pool - {:app-id app-id :code {:users {:allow {:view "auth.handle == data.handle"}}}}) - (is - (= #{"stopa"} - (->> (pretty-perm-q - {:app-id app-id :current-user {:handle "stopa"}} - {:users {}}) - :users - (map :handle) - set)))) - - (testing "page-info is filtered" - (is - (= {:start-cursor ["eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili"], - :end-cursor ["eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili"] - :has-next-page? false, - :has-previous-page? false} - (let [r (resolvers/make-zeneca-resolver app-id)] - (->> (iq/permissioned-query - {:db {:conn-pool aurora/conn-pool} - :app-id app-id - :attrs (attr-model/get-by-app-id app-id) - :datalog-query-fn d/query - :current-user {:handle "stopa"}} - {:users {:$ {:limit 10}}}) - first - :data - :datalog-result - :page-info - (resolvers/walk-friendly r) - ;; remove timestamps - (#(update % :start-cursor drop-last)) - (#(update % :end-cursor drop-last))))))) - - (testing "bad rules produce a permission evaluation exception" - (rule-model/put! - aurora/conn-pool - {:app-id app-id :code {:users {:allow {:view "auth.handle in data.nonexistent"}}}}) - - (is - (= ::ex/permission-evaluation-failed - (::ex/type (instant-ex-data - (pretty-perm-q - {:app-id app-id :current-user {:handle "stopa"}} - {:users {}}))))))))) + (doseq [[app-fn description] [[with-zeneca-app "without checked attrs"] + [with-zeneca-checked-data-app "with checked attrs"]]] + (testing description + (app-fn + (fn [{app-id :id :as _app} _r] + (testing "no perms returns full" + (rule-model/put! + aurora/conn-pool + {:app-id app-id :code {}}) + (is + (= #{"alex" "joe" "stopa" "nicolegf"} + (->> (pretty-perm-q + {:app-id app-id :current-user nil} + {:users {}}) + :users + (map :handle) + set)))) + (testing "false returns nothing" + (rule-model/put! + aurora/conn-pool + {:app-id app-id :code {:users {:allow {:view "false"}}}}) + (is + (empty? + (->> (pretty-perm-q + {:app-id app-id :current-user nil} + {:users {}}) + :users + (map :handle) + set)))) + (testing "property equality" + (rule-model/put! + aurora/conn-pool + {:app-id app-id :code {:users {:allow {:view "data.handle == 'stopa'"}}}}) + (is + (= + #{"stopa"} + (->> (pretty-perm-q + {:app-id app-id :current-user nil} + {:users {}}) + :users + (map :handle) + set)))) + (testing "bind" + (rule-model/put! + aurora/conn-pool + {:app-id app-id :code {:users {:allow {:view "data.handle != handle"} + :bind ["handle" "'stopa'"]}}}) + (is + (= + #{"alex" "joe" "nicolegf"} + (->> (pretty-perm-q + {:app-id app-id :current-user nil} + {:users {}}) + :users + (map :handle) + set)))) + (testing "ref" + (rule-model/put! + aurora/conn-pool + {:app-id app-id :code {:bookshelves {:allow {:view "handle in data.ref('users.handle')"} + :bind ["handle" "'alex'"]}}}) + (is + (= + #{"Short Stories" "Nonfiction"} + (->> (pretty-perm-q + {:app-id app-id :current-user nil} + {:bookshelves {}}) + :bookshelves + (map :name) + set)))) + (testing "auth required" + (rule-model/put! + aurora/conn-pool + {:app-id app-id :code {:users {:allow {:view "auth.id != null"}}}}) + (is + (empty? + (->> (pretty-perm-q + {:app-id app-id :current-user nil} + {:users {}}) + :users + (map :handle) + set)))) + + (testing "null shouldn't evaluate to true" + (rule-model/put! + aurora/conn-pool + {:app-id app-id :code {:users {:allow {:view "auth.isAdmin"}}}}) + (is + (empty? + (->> (pretty-perm-q + {:app-id app-id :current-user nil} + {:users {}}) + :users + (map :handle) + set)))) + (testing "can only view authed user data" + (rule-model/put! + aurora/conn-pool + {:app-id app-id :code {:users {:allow {:view "auth.handle == data.handle"}}}}) + (is + (= #{"stopa"} + (->> (pretty-perm-q + {:app-id app-id :current-user {:handle "stopa"}} + {:users {}}) + :users + (map :handle) + set)))) + + (testing "page-info is filtered" + (is + (= {:start-cursor ["eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili"], + :end-cursor ["eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili"] + :has-next-page? false, + :has-previous-page? false} + (let [r (resolvers/make-zeneca-resolver app-id)] + (->> (iq/permissioned-query + {:db {:conn-pool aurora/conn-pool} + :app-id app-id + :attrs (attr-model/get-by-app-id app-id) + :datalog-query-fn d/query + :current-user {:handle "stopa"}} + {:users {:$ {:limit 10}}}) + first + :data + :datalog-result + :page-info + (resolvers/walk-friendly r) + ;; remove timestamps + (#(update % :start-cursor drop-last)) + (#(update % :end-cursor drop-last))))))) + + (testing "bad rules produce a permission evaluation exception" + (rule-model/put! + aurora/conn-pool + {:app-id app-id :code {:users {:allow {:view "auth.handle in data.nonexistent"}}}}) + + (is + (= ::ex/permission-evaluation-failed + (::ex/type (instant-ex-data + (pretty-perm-q + {:app-id app-id :current-user {:handle "stopa"}} + {:users {}}))))))))))) (deftest coarse-topics [] (let [{:keys [patterns]} @@ -2724,6 +2898,68 @@ :aggregate [{:count 4} {:count 392}]})))) +(deftest namespaces-that-share-eids [] + (with-empty-app + (fn [app] + (let [book-id-aid (random-uuid) + book-field-aid (random-uuid) + author-id-aid (random-uuid) + author-field-aid (random-uuid) + shared-eid (random-uuid) + run-query (fn [{:keys [admin?]} q] + (let [ctx (let [attrs (attr-model/get-by-app-id (:id app))] + {:db {:conn-pool aurora/conn-pool} + :app-id (:id app) + :attrs attrs + :admin? admin?}) + r (resolvers/make-resolver {:conn-pool aurora/conn-pool} + (:id app) + [["books" "field"] + ["authors" "field"]])] + (->> (iq/permissioned-query ctx q) + (instaql-nodes->object-tree ctx))))] + (tx/transact! aurora/conn-pool + (attr-model/get-by-app-id (:id app)) + (:id app) + [[:add-attr {:id book-id-aid + :forward-identity [(random-uuid) "books" "id"] + :unique? true + :index? true + :value-type :blob + :cardinality :one}] + [:add-attr {:id book-field-aid + :forward-identity [(random-uuid) "books" "field"] + :unique? false + :index? false + :value-type :blob + :cardinality :one}] + [:add-attr {:id author-id-aid + :forward-identity [(random-uuid) "authors" "id"] + :unique? true + :index? true + :value-type :blob + :cardinality :one}] + [:add-attr {:id author-field-aid + :forward-identity [(random-uuid) "authors" "field"] + :unique? false + :index? false + :value-type :blob + :cardinality :one}] + [:add-triple shared-eid book-id-aid (str shared-eid)] + [:add-triple shared-eid book-field-aid "book"] + [:add-triple shared-eid author-id-aid (str shared-eid)] + [:add-triple shared-eid author-field-aid "author"]]) + (rule-model/put! aurora/conn-pool + {:app-id (:id app) :code {:books {:allow {:view "false"}} + :authors {:allow {:view "true"}}}}) + (is (= {"books" [{"field" "book", "id" (str shared-eid)}] + "authors" [{"field" "author", "id" (str shared-eid)}]} + (run-query {:admin? true} {:books {} :authors {}}))) + + (is (= {"books" [] + "authors" [{"field" "author", "id" (str shared-eid)}]} + (run-query {:admin? false} {:books {} :authors {}}))))))) + ;; ----------- ;; Users table @@ -2770,7 +3006,7 @@ (is-pretty-eq? (query-pretty' {:$users {:$ {:where {:email "first@example.com"}}}}) [{:topics - [[:av '_ #{:$users/email} #{"first@example.com"}] + [[:ave '_ #{:$users/email} #{"first@example.com"}] '-- [:ea #{first-id} #{:$users/email :$users/id} '_]], :triples @@ -2842,7 +3078,7 @@ r1 {:books {:$ {:where {"$user-creator.email" "alex@instantdb.com"}}}}) [{:topics - [[:av '_ #{:$users/email} #{"alex@instantdb.com"}] + [[:ave '_ #{:$users/email} #{"alex@instantdb.com"}] [:vae '_ #{:books/$user-creator} #{"eid-alex"}] '-- [:ea diff --git a/server/test/instant/reactive/session_test.clj b/server/test/instant/reactive/session_test.clj index 253cda843..317b958b4 100644 --- a/server/test/instant/reactive/session_test.clj +++ b/server/test/instant/reactive/session_test.clj @@ -15,6 +15,7 @@ [instant.db.transaction :as tx] [instant.db.instaql :as iq] [instant.lib.ring.websocket :as ws] + [instant.grouped-queue :as grouped-queue] [instant.reactive.ephemeral :as eph] [instant.reactive.query :as rq] [instant.reactive.receive-queue :as receive-queue]) @@ -34,7 +35,7 @@ (defn- with-session [f] (let [sess-id (UUID/randomUUID) fake-ws-conn (a/chan 1) - receive-q (LinkedBlockingQueue.) + receive-q (grouped-queue/create {:group-fn session/group-fn}) room-refresh-ch (a/chan (a/sliding-buffer 1)) store-conn (rs/init-store) eph-store-atom (atom {}) @@ -48,7 +49,8 @@ :receive-q receive-q :ping-job (future) :pending-handlers (atom #{})} - query-reactive rq/instaql-query-reactive!] + query-reactive rq/instaql-query-reactive! + stop-signal (atom false)] (session/on-open store-conn socket) (session/on-open store-conn second-socket) @@ -65,10 +67,13 @@ (let [res (query-reactive store-conn base-ctx instaql-query return-type)] (swap! *instaql-query-results* assoc-in [session-id instaql-query] res) res))] + (session/start-receive-worker store-conn eph-store-atom receive-q stop-signal 0) (f store-conn eph-store-atom {:socket socket :second-socket second-socket}) + (session/on-close store-conn eph-store-atom socket) - (session/on-close store-conn eph-store-atom second-socket))))) + (session/on-close store-conn eph-store-atom second-socket) + (reset! stop-signal true))))) (defn- blocking-send-msg [{:keys [ws-conn id]} msg] (session/handle-receive *store-conn* *eph-store-atom* (rs/get-session @*store-conn* id) msg {})