feat(ui): introduce kv() and secret() pebble autocompletions

closes kestra-io/kestra-ee#77

ui/src/components/inputs/MonacoEditor.vue

@@ -2,7 +2,7 @@
     <div class="ks-monaco-editor" />
 </template>
 
-<script>
+<script lang="ts">
     import {defineComponent} from "vue";
     import {mapActions, mapMutations, mapState} from "vuex";
 
@@ -20,8 +20,10 @@
     import {editorViewTypes} from "../../utils/constants";
     import Utils from "../../utils/utils";
     import YamlUtils from "../../utils/yamlUtils";
-    import {FlowAutoCompletion, YamlNoAutoCompletion} from "override/services/autoCompletionProvider";
+    import {QUOTE, YamlNoAutoCompletion} from "../../services/autoCompletionProvider.js"
+    import {FlowAutoCompletion} from "override/services/flowAutoCompletionProvider.js";
     import RegexProvider from "../../utils/regex";
+    import type {Position} from "monaco-editor"
 
     window.MonacoEnvironment = {
         getWorker(moduleId, label) {
@@ -218,15 +220,15 @@
 
             const NO_SUGGESTIONS = {suggestions: []};
 
-            let yamlAutoCompletionProvider;
+            let yamlAutoCompletionProvider: YamlNoAutoCompletion;
             if (this.schemaType === "flow") {
                 yamlAutoCompletionProvider = new FlowAutoCompletion(this.$store);
             } else {
                 yamlAutoCompletionProvider = new YamlNoAutoCompletion();
             }
 
             const endOfWordColumn = (position, model) => {
-                return position.column + (model.findNextMatch(RegexProvider.beforeSeparator, position, true, false, null, true)?.matches[0].length ?? 0);
+                return position.column + (model.findNextMatch(RegexProvider.beforeSeparator(), position, true, false, null, true)?.matches[0].length ?? 0);
             }
 
             this.autoCompletionProviders.push(monaco.languages.registerCompletionItemProvider("yaml", {
@@ -236,7 +238,7 @@
                     const cursorPosition = model.getOffsetAt(position);
                     const parsed = YamlUtils.parse(source, false);
 
-                    const currentWord = model.findPreviousMatch(RegexProvider.beforeSeparator, position, true, false, null, true);
+                    const currentWord = model.findPreviousMatch(RegexProvider.beforeSeparator(), position, true, false, null, true);
                     const elementUnderCursor = YamlUtils.localizeElementAtIndex(source, cursorPosition);
                     if (elementUnderCursor?.key === undefined) {
                         return NO_SUGGESTIONS;
@@ -246,7 +248,7 @@
                     const autoCompletions = await yamlAutoCompletionProvider.valueAutoCompletion(source, parsed, elementUnderCursor);
                     return {
                         suggestions: autoCompletions.map(autoCompletion => {
-                            const [label, isKey] = autoCompletion.split(":");
+                            const [label, isKey] = autoCompletion.split(":") as [string, string | undefined];
                             let insertText = label;
                             const endColumn = endOfWordColumn(position, model);
                             if (isKey === undefined) {
@@ -276,19 +278,27 @@
                 }
             }));
 
-            const propertySuggestion = (label, position) => ({
-                kind: monaco.languages.CompletionItemKind.Property,
-                label,
-                insertText: label,
-                range: {
-                    startLineNumber: position.lineNumber,
-                    endLineNumber: position.lineNumber,
-                    startColumn: position.startColumn,
-                    endColumn: position.endColumn
+            const propertySuggestion = (value: string, position: Position, kind: monaco.languages.CompletionItemKind | undefined) => {
+                let label = value.split("(")[0];
+                if (label.startsWith(QUOTE) && label.endsWith(QUOTE)) {
+                    label = label.substring(1, label.length - 1);
                 }
-            });
 
-            this.autoCompletionProviders.push(monaco.languages.registerCompletionItemProvider("yaml", {
+                return ({
+                    kind: kind ?? (value.includes("(") ? monaco.languages.CompletionItemKind.Function : monaco.languages.CompletionItemKind.Property),
+                    label: label,
+                    insertText: value,
+                    insertTextRules: value.includes("${1:") ? monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet : undefined,
+                    range: {
+                        startLineNumber: position.lineNumber,
+                        endLineNumber: position.lineNumber,
+                        startColumn: position.startColumn,
+                        endColumn: position.endColumn
+                    }
+                });
+            };
+
+            this.autoCompletionProviders.push(monaco.languages.registerCompletionItemProvider(["yaml", "plaintext"], {
                 triggerCharacters: ["{"],
                 async provideCompletionItems(model, position) {
                     // Not a subfield access
@@ -309,6 +319,48 @@
                 }
             }));
 
+            this.autoCompletionProviders.push(monaco.languages.registerCompletionItemProvider(["yaml", "plaintext"], {
+                triggerCharacters: ["("],
+                async provideCompletionItems(model, position) {
+                    const source = model.getValue();
+                    const parsed = YamlUtils.parse(source, false);
+
+                    const functionMatcher = model.findPreviousMatch(RegexProvider.capturePebbleFunction + "$", position, true, false, null, true);
+                    if (functionMatcher === null) {
+                        return NO_SUGGESTIONS;
+                    }
+
+                    const QUOTES = ["\"", "'"];
+                    const wordStartOffset = functionMatcher.matches?.[3]?.length
+                        ?? model.findPreviousMatch(RegexProvider.beforeSeparator(QUOTES) + "$", position, true, false, null, true).matches[0].length;
+                    const startOfWordColumn = position.column - wordStartOffset;
+                    return {
+                        suggestions: (await yamlAutoCompletionProvider.functionAutoCompletion(
+                            parsed,
+                            functionMatcher.matches[1],
+                            Object.fromEntries(functionMatcher.matches?.[2]?.split(/ *, */)?.map(arg => arg.split(/ *= */)) ?? []))
+                        ).map(s => {
+                            const suggestion = propertySuggestion(s, {
+                                lineNumber: position.lineNumber,
+                                startColumn: startOfWordColumn,
+                                endColumn: endOfWordColumn(position, model)
+                            }, monaco.languages.CompletionItemKind.Value);
+
+                            // If the inserted value is a string (surrounded by quotes), we remove them if there is already one
+                            if (suggestion.insertText.startsWith(QUOTE) && suggestion.insertText.endsWith(QUOTE)) {
+                                const lineContent = model.getLineContent(position.lineNumber);
+                                suggestion.insertText = suggestion.insertText.substring(
+                                    QUOTES.includes(lineContent.charAt(startOfWordColumn - 2)) ? 1 : 0,
+                                    suggestion.insertText.length - (QUOTES.includes(lineContent.charAt(endOfWordColumn)) ? 1 : 0)
+                                );
+                            }
+
+                            return suggestion;
+                        })
+                    };
+                }
+            }))
+
             this.autoCompletionProviders.push(monaco.languages.registerCompletionItemProvider(["yaml", "plaintext"], {
                 triggerCharacters: ["."],
                 async provideCompletionItems(model, position) {
@@ -330,7 +382,7 @@
                             }))
                     };
                 }
-            }))
+            }));
 
             // Exposing functions globally for testing purposes
             window.pasteToEditor = (textToPaste) => {

ui/src/override/services/autoCompletionProvider.ts → ui/src/override/services/flowAutoCompletionProvider.ts

@@ -1,26 +1,14 @@
 import type {Store} from "vuex";
 import type {JSONSchema} from "@kestra-io/ui-libs";
 import YamlUtils, {YamlElement} from "../../utils/yamlUtils";
-
-export class YamlNoAutoCompletion {
-    rootFieldAutoCompletion(): Promise<string[]> {
-        return Promise.resolve([]);
-    }
-
-    nestedFieldAutoCompletion(_source: string, _parsed?: any, _parentField?: string): Promise<string[]> {
-        return Promise.resolve([])
-    }
-
-    valueAutoCompletion(_source: string, _parsed?: any, _yamlElement: YamlElement | undefined): Promise<string[]> {
-        return Promise.resolve([]);
-    }
-}
+import {QUOTE, YamlNoAutoCompletion} from "../../services/autoCompletionProvider";
+import RegexProvider from "../../utils/regex";
 
 function distinct<T>(val: T[] | undefined): T[] {
     return Array.from(new Set(val ?? []));
 }
 
-export class FlowAutoCompletion extends YamlNoAutoCompletion{
+export class FlowAutoCompletion extends YamlNoAutoCompletion {
     store: Store<Record<string, any>>;
     flowsInputsCache: Record<string, string[]> = {};
 
@@ -43,7 +31,9 @@ export class FlowAutoCompletion extends YamlNoAutoCompletion{
             "envs",
             "globals",
             "parents",
-            "error"
+            "error",
+            "secret(namespace=${1:flow.namespace}, key=" + QUOTE + "${2:MY_SECRET}" + QUOTE + ")",
+            "kv(namespace=${1:flow.namespace}, key=" + QUOTE + "${2:my_key}" + QUOTE + ")"
         ]);
     }
 
@@ -183,4 +173,44 @@ export class FlowAutoCompletion extends YamlNoAutoCompletion{
 
         return Promise.resolve([]);
     }
+
+    private extractArgValue(arg) {
+        if (arg === undefined) {
+            return undefined;
+        }
+
+        const captureValue = new RegExp("^" + RegexProvider.captureStringValue + "$").exec(arg);
+        if (!captureValue) {
+            return undefined;
+        }
+
+        return captureValue?.[1];
+    }
+
+    async functionAutoCompletion(parsed: any | undefined, functionName: string, args: Record<string, string>): Promise<string[]> {
+        let namespaceArg = args.namespace;
+        if (namespaceArg === undefined || namespaceArg === "flow.namespace") {
+           namespaceArg = parsed?.namespace === undefined ? "" : QUOTE + parsed.namespace + QUOTE;
+        }
+        switch (functionName) {
+            case "secret": {
+                const namespace = this.extractArgValue(namespaceArg);
+                if (namespace === undefined) {
+                    return Promise.resolve([]);
+                }
+                return Array.from(Object.entries(await this.store.dispatch("namespace/inheritedSecrets", {id: namespace})).reduce((acc, [_, nsSecrets]: [string, string[]]) => {
+                    nsSecrets.forEach(secret => acc.add(QUOTE + secret + QUOTE));
+                    return acc;
+                }, new Set()));
+            }
+            case "kv": {
+                const namespace = this.extractArgValue(namespaceArg);
+                if (namespace === undefined) {
+                    return Promise.resolve([]);
+                }
+                return (await this.store.dispatch("namespace/kvsList", {id: namespace})).map(kv => QUOTE + kv.key + QUOTE);
+            }
+        }
+        return Promise.resolve([]);
+    }
 }

ui/src/services/autoCompletionProvider.ts

@@ -0,0 +1,21 @@
+import {YamlElement} from "../utils/yamlUtils";
+
+export const QUOTE = "'";
+
+export class YamlNoAutoCompletion {
+    rootFieldAutoCompletion(): Promise<string[]> {
+        return Promise.resolve([]);
+    }
+
+    nestedFieldAutoCompletion(_source: string, _parsed: any | undefined, _parentField: string): Promise<string[]> {
+        return Promise.resolve([])
+    }
+
+    valueAutoCompletion(_source: string, _parsed: any | undefined, _yamlElement: YamlElement | undefined): Promise<string[]> {
+        return Promise.resolve([]);
+    }
+
+    functionAutoCompletion(_parsed: any | undefined, _functionName: string, _args: Record<string, string>): Promise<string[]> {
+        return Promise.resolve([]);
+    }
+}

ui/src/stores/namespaces.js

@@ -15,8 +15,9 @@ export default {
     namespaced: true,
     state: {
         datatypeNamespaces: undefined,
-        namespaces: undefined,       
+        namespaces: undefined,
         namespace: undefined,
+        inheritedSecrets: undefined,
         kvs: undefined,
     },
     actions: {
@@ -85,6 +86,15 @@ export default {
                 });
         },
 
+        inheritedSecrets({commit}, item) {
+            return this.$http.get(`${apiUrl(this)}/namespaces/${item.id}/inherited-secrets`, {validateStatus: (status) => status === 200 || status === 404})
+                .then(response => {
+                    commit("setInheritedSecrets", response.data)
+
+                    return response.data;
+                });
+        },
+
         // Create a directory
         async createDirectory(_, payload) {
             const URL = `${base.call(this, payload.namespace)}/files/directory?path=${slashPrefix(payload.path)}`;
@@ -176,7 +186,7 @@ export default {
     mutations: {
         setDatatypeNamespaces(state, datatypeNamespaces) {
             state.datatypeNamespaces = datatypeNamespaces;
-        },    
+        },
         setNamespaces(state, namespaces) {
             state.namespaces = namespaces
         },
@@ -186,5 +196,8 @@ export default {
         setKvs(state, kvs) {
             state.kvs = kvs
         },
+        setInheritedSecrets(state, secrets) {
+            state.inheritedSecrets = secrets
+        },
     },
 };

ui/src/utils/regex.ts

@@ -1,10 +1,17 @@
 const pebbleStart = "\\{\\{ *";
-const fieldWithoutDotCapture = "([^}:~. ]*)(?![^}\\s])";
-const dotAccessedFieldWithParentCapture = "([^}:~ ]+)\\." + fieldWithoutDotCapture;
-const maybeTextFollowedBySeparator = "(?:[^~}: ]*[~ ]+)*";
+const fieldWithoutDotCapture = "([^\\(\\)}:~. ]*)(?![^\\(\\)}\\s])";
+const dotAccessedFieldWithParentCapture = "([^\\(\\)},:~ ]+)\\." + fieldWithoutDotCapture;
+const maybeTextFollowedBySeparator = "(?:[^~},: ]*[~ ]+)*";
+const maybeParams = "((?:[^\\n\\(\\)~},:= ]+=[^\\n~},:= ]+(?: *, *)?)+)?['\"]?([^\\n\\(\\)~},:= ]*)?";
+const functionWithMaybeParams = "([^\\n\\(\\)},:~ ]+)\\(" + maybeParams
 
 export default {
-    beforeSeparator: "([^}:\\s]*)",
+    beforeSeparator: (additionalSeparators = []) => `([^}:\\s${additionalSeparators.join("")}]*)`,
+    /** [fullMatch, dotForbiddenField] */
     capturePebbleVarRoot: `${pebbleStart}${maybeTextFollowedBySeparator}${fieldWithoutDotCapture}`,
+    /** [fullMatch, parentFieldMaybeIncludingDots, childField] */
     capturePebbleVarParent: `${pebbleStart}${maybeTextFollowedBySeparator}${dotAccessedFieldWithParentCapture}`,
+    /** [fullMatch, functionName, textBetweenParenthesis, maybeTypedWordStart] */
+    capturePebbleFunction: `${pebbleStart}${maybeTextFollowedBySeparator}${functionWithMaybeParams}`,
+    captureStringValue: "^[\"']([^\"']+)[\"']$"
 }

ui/tests/unit/services/flowAutoCompletionProvider.spec.ts

@@ -1,6 +1,6 @@
 import type {Store} from "vuex";
 import {describe, expect, it, Mock, vi} from "vitest"
-import {FlowAutoCompletion} from "override/services/autoCompletionProvider.ts";
+import {FlowAutoCompletion} from "override/services/flowAutoCompletionProvider";
 import YamlUtils from "../../../src/utils/yamlUtils";
 
 const defaultFlow = `inputs:
@@ -98,6 +98,18 @@ const mockedStore: MockStore<Record<string, any>> = {
             } else {
                 return Promise.reject("404")
             }
+        } else if (type === "namespace/inheritedSecrets") {
+            if (payload.id === "my.namespace") {
+                return Promise.resolve({"my.namespace": ["myFirstSecret", "mySecondSecret"], "my": ["myInheritedSecret"]});
+            } else if (payload.id === "another.namespace") {
+                return Promise.resolve({"another.namespace": ["anotherNsFirstSecret", "anotherNsSecondSecret"]});
+            }
+        } else if (type === "namespace/kvsList") {
+            if (payload.id === "my.namespace") {
+                return Promise.resolve([{key: "myFirstKv"}, {key: "mySecondKv"}]);
+            } else if (payload.id === "another.namespace") {
+                return Promise.resolve([{key: "anotherNsFirstKv"}, {key: "anotherNsSecondKv"}]);
+            }
         }
         return Promise.reject("404")
     })
@@ -121,7 +133,9 @@ describe("FlowAutoCompletionProvider", () => {
             "envs",
             "globals",
             "parents",
-            "error"
+            "error",
+            "secret(namespace=${1:flow.namespace}, key='${2:MY_SECRET}')",
+            "kv(namespace=${1:flow.namespace}, key='${2:my_key}')",
         ]);
     })
 
@@ -159,4 +173,11 @@ describe("FlowAutoCompletionProvider", () => {
         // With newline already inserted
         expect(await provider.valueAutoCompletion(defaultFlow.substring(0, firstInputIndex) + "\n        " + defaultFlow.substring(firstInputIndex, defaultFlow.length), parsed, YamlUtils.localizeElementAtIndex(defaultFlow, firstInputIndex))).toEqual(["second-input:"]);
     })
+
+    it("function autocompletions", async () => {
+        expect(await provider.functionAutoCompletion(parsed, "secret", {})).toEqual(["'myFirstSecret'", "'mySecondSecret'", "'myInheritedSecret'"]);
+        expect(await provider.functionAutoCompletion(parsed, "secret", {namespace: "'another.namespace'"})).toEqual(["'anotherNsFirstSecret'", "'anotherNsSecondSecret'"]);
+        expect(await provider.functionAutoCompletion(parsed, "kv", {})).toEqual(["'myFirstKv'", "'mySecondKv'"]);
+        expect(await provider.functionAutoCompletion(parsed, "kv", {namespace: "'another.namespace'"})).toEqual(["'anotherNsFirstKv'", "'anotherNsSecondKv'"]);
+    })
 })