-
Notifications
You must be signed in to change notification settings - Fork 1.2k
chore: db migrations and types for functions configs #6960
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
33d54f4
1c676b3
06a73d2
d2539b0
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| /** | ||
| * @param {import('knex').Knex} knex | ||
| */ | ||
| exports.up = async function (knex) { | ||
| await knex.raw(` | ||
| ALTER TABLE _nango_configs | ||
| ADD CONSTRAINT _nango_configs_id_environment_id_unique UNIQUE (id, environment_id); | ||
|
|
||
| CREATE TABLE IF NOT EXISTS function_configs ( | ||
| id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, | ||
| nango_config_id INTEGER NOT NULL, | ||
| environment_id INTEGER NOT NULL, | ||
| name TEXT NOT NULL, | ||
| current_version_id INTEGER, | ||
| enabled BOOLEAN NOT NULL DEFAULT TRUE, | ||
| created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| deleted_at TIMESTAMPTZ, | ||
| CONSTRAINT function_configs_nango_config_environment_fkey | ||
| FOREIGN KEY (nango_config_id, environment_id) | ||
| REFERENCES _nango_configs(id, environment_id) ON DELETE CASCADE | ||
| ); | ||
|
|
||
| CREATE INDEX IF NOT EXISTS function_configs_nango_config_id_idx | ||
| ON function_configs (nango_config_id); | ||
|
|
||
| CREATE INDEX IF NOT EXISTS function_configs_environment_id_idx | ||
| ON function_configs (environment_id); | ||
|
|
||
| CREATE INDEX IF NOT EXISTS function_configs_current_version_id_idx | ||
| ON function_configs (current_version_id); | ||
|
|
||
| CREATE UNIQUE INDEX IF NOT EXISTS function_configs_unique_idx | ||
| ON function_configs (nango_config_id, name) | ||
| WHERE deleted_at IS NULL; | ||
|
|
||
| CREATE TABLE IF NOT EXISTS function_config_versions ( | ||
| id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, | ||
| function_config_id INTEGER NOT NULL REFERENCES function_configs(id) ON DELETE CASCADE, | ||
| description TEXT NOT NULL, | ||
| file_location TEXT NOT NULL, | ||
| version TEXT NOT NULL, | ||
| source TEXT NOT NULL CONSTRAINT function_config_versions_source_check CHECK (source IN ('catalog', 'standalone', 'repo')), | ||
| trigger JSONB NOT NULL, | ||
| requires JSONB NOT NULL, | ||
| capabilities JSONB NOT NULL, | ||
| limits JSONB NOT NULL, | ||
| input_schema_ref TEXT, | ||
| output_schema_ref TEXT, | ||
| model_schema_refs TEXT[] NOT NULL, | ||
| metadata_schema_ref TEXT, | ||
| checkpoint_schema_ref TEXT, | ||
| json_schema JSONB NOT NULL, | ||
| created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| deleted_at TIMESTAMPTZ | ||
| ); | ||
|
|
||
| CREATE INDEX IF NOT EXISTS function_config_versions_function_config_id_idx | ||
| ON function_config_versions (function_config_id); | ||
|
|
||
| CREATE UNIQUE INDEX IF NOT EXISTS function_config_versions_config_version_unique_idx | ||
| ON function_config_versions (function_config_id, version) | ||
| WHERE deleted_at IS NULL; | ||
|
|
||
| CREATE UNIQUE INDEX IF NOT EXISTS function_config_versions_id_config_unique_idx | ||
| ON function_config_versions (id, function_config_id); | ||
|
|
||
| ALTER TABLE function_configs | ||
| ADD CONSTRAINT function_configs_current_version_id_fkey | ||
| FOREIGN KEY (current_version_id, id) REFERENCES function_config_versions(id, function_config_id) | ||
| ON DELETE SET NULL (current_version_id) DEFERRABLE INITIALLY DEFERRED; | ||
|
|
||
| CREATE TABLE IF NOT EXISTS function_instances ( | ||
| id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, | ||
| nango_connection_id INTEGER NOT NULL REFERENCES _nango_connections(id) ON DELETE CASCADE, | ||
| function_config_id INTEGER NOT NULL REFERENCES function_configs(id) ON DELETE CASCADE, | ||
| name TEXT NOT NULL, | ||
| variant VARCHAR(255) NOT NULL DEFAULT 'base', | ||
| last_run_at TIMESTAMPTZ, | ||
| frequency TEXT, | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What's the frequency in this new model?
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. the same as it is right now, with one caveat: it can be null now. There is no plan to change how frequency works (other than support for cron syntax but from a data pov it is still some sort of string)
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yeah but it's in the trigger now, right? I'm guessing this is here in function_instances to adapt to the scheduler. |
||
| created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, | ||
| deleted_at TIMESTAMPTZ | ||
| ); | ||
|
|
||
| CREATE INDEX IF NOT EXISTS function_instances_connection_id_idx | ||
| ON function_instances (nango_connection_id); | ||
|
|
||
| CREATE INDEX IF NOT EXISTS function_instances_function_config_id_idx | ||
| ON function_instances (function_config_id); | ||
|
|
||
| CREATE UNIQUE INDEX IF NOT EXISTS function_instances_config_connection_variant_unique_idx | ||
| ON function_instances (function_config_id, nango_connection_id, variant) | ||
| WHERE deleted_at IS NULL; | ||
|
|
||
| CREATE UNIQUE INDEX IF NOT EXISTS function_instances_id_config_unique_idx | ||
| ON function_instances (id, function_config_id); | ||
| `); | ||
| }; | ||
|
|
||
| /** | ||
| * @param {import('knex').Knex} knex | ||
| */ | ||
| exports.down = async function () {}; | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -11,8 +11,12 @@ export interface DeletedCorrect { | |
| deleted_at: Date | null; | ||
| deleted: boolean; | ||
| } | ||
| export interface DeletedAt { | ||
| deleted_at: Date | null; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No redundant boolean??????
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 😜 |
||
| } | ||
|
|
||
| export interface TimestampsAndDeleted extends Timestamps, Deleted {} | ||
| export interface TimestampsAndDeletedCorrect extends Timestamps, DeletedCorrect {} | ||
| export interface TimestampsAndDeletedAt extends Timestamps, DeletedAt {} | ||
|
|
||
| export type Tags = Record<string, string>; | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| import type { TimestampsAndDeletedAt } from '../db.js'; | ||
| import type { FunctionCapabilities } from '../functions/capabilities.js'; | ||
| import type { FunctionLimits, FunctionTriggerDefinition, Requires } from '../functions/config.js'; | ||
| import type { FunctionSource } from '../syncConfigs/db.js'; | ||
| import type { JSONSchema7 } from 'json-schema'; | ||
|
|
||
| export interface DBFunctionConfig extends TimestampsAndDeletedAt { | ||
| id: number; | ||
| nango_config_id: number; | ||
| environment_id: number; | ||
| name: string; | ||
| current_version_id: number | null; | ||
| enabled: boolean; | ||
| } | ||
|
|
||
| export interface DBFunctionConfigVersion extends TimestampsAndDeletedAt { | ||
| id: number; | ||
| function_config_id: number; | ||
| description: string; | ||
| file_location: string; | ||
| version: string; | ||
| source: FunctionSource; | ||
| trigger: FunctionTriggerDefinition; | ||
| requires: Requires; | ||
| capabilities: FunctionCapabilities; | ||
| limits: FunctionLimits; | ||
| input_schema_ref: string | null; | ||
| output_schema_ref: string | null; | ||
| model_schema_refs: string[]; | ||
| metadata_schema_ref: string | null; | ||
| checkpoint_schema_ref: string | null; | ||
| json_schema: JSONSchema7; | ||
| } | ||
|
|
||
| export interface DBFunctionInstance extends TimestampsAndDeletedAt { | ||
| id: number; | ||
| nango_connection_id: number; | ||
| function_config_id: number; | ||
| name: string; | ||
| variant: string; | ||
| last_run_at: Date | null; | ||
| frequency: string | null; | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| import type { OnEventType } from '../scripts/on-events/api.js'; | ||
|
|
||
| /** | ||
| * Declares where the debounce/coalescing key is extracted from: | ||
| * - `{ body: '$.portalId' }`: dot notation path into the body | ||
| * - `{ header: 'x-goog-resource-id' }`: flat, case-insensitive header lookup | ||
| */ | ||
| export type DebounceKeySource = { body: string } | { header: string }; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Function config types now have two sources of truth, and their semantics already diverge. Move the shared declarations to Prompt for AI agents
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. as I said next PR will align types across runner-sdk and db. I want to keep the PR small |
||
|
|
||
| /** Coalesces a burst of inbound HTTP requests into a single function run within a sliding window. */ | ||
| export interface DebounceOptions { | ||
| /** Events sharing the same resolved key coalesce together. */ | ||
| keyBy?: DebounceKeySource | DebounceKeySource[]; | ||
| /** Sliding window in milliseconds. */ | ||
| windowMs: number; | ||
| /** When exceeded, the window stops sliding. */ | ||
| maxEntities?: number; | ||
| /** Which payloads from the coalesced window the handler receives. */ | ||
| take?: 'latest' | 'first' | 'all'; | ||
| } | ||
|
|
||
| /** | ||
| * Declares how a function is initiated from outside; `kind` discriminates the source. | ||
| * - `none`: no external trigger; the function is invoke-only | ||
| * - `schedule`: a periodic schedule | ||
| * - `http`: an incoming HTTP call or webhook request | ||
| * - `event`: an internal Nango lifecycle event | ||
| */ | ||
| export type FunctionTriggerDefinition = | ||
| | { kind: 'none' } | ||
| | { kind: 'schedule'; frequency: string; autoStart?: boolean } | ||
| | { kind: 'http'; subscriptions?: string[]; debounce?: DebounceOptions } | ||
| | { kind: 'event'; events: OnEventType[] }; | ||
|
|
||
| export type FunctionConcurrencyLimit = 1 | 'max'; | ||
|
|
||
| export interface FunctionLimits { | ||
| concurrency?: { | ||
| perConnection: FunctionConcurrencyLimit; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Scheduled function configs can persist Prompt for AI agents
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. will be enforced at deploy time |
||
| }; | ||
| } | ||
|
|
||
| export type Requires = { connection?: true; outbound?: boolean; invoke?: boolean } | { connection: false; outbound?: false; invoke?: boolean }; | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P1: Function instances can pair a connection with a function config from another environment because these independent foreign keys do not enforce a shared environment. This permits cross-environment function execution if a caller supplies mismatched IDs; store
environment_idand enforce composite foreign keys (or otherwise constrain both references to the same environment).Prompt for AI agents
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
on purpose, to be enforced app-side by the data access layer. Adding composite keys over several tables including existing connection table isn't worth it imho. let me know if you disagree
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks for the feedback! I've saved this as a new learning.