fix+docs: Fix cli docs autogen and update autogenerated docs (#319)

cli/tracecat_cli/dev.py

@@ -5,6 +5,7 @@
 import shutil
 import subprocess
 import tempfile
+from collections import defaultdict
 from itertools import chain
 from pathlib import Path
 from typing import Any
@@ -418,6 +419,10 @@ def generate_udf_docs():
 ## Integrations
 Note that the fully qualified namespace for each Integration UDF is prefixed with `integrations.`.
 {integrations_secrets_table}
+
+## ETL Actions
+Note that the fully qualified namespace for each ETL UDF is prefixed with `etl.`.
+{etl_secrets_table}
 """
 
 
@@ -434,31 +439,22 @@ def generate_secret_tables(
     registry.init()
 
     # Table of core UDFs required secrets
-    core_udfs_secrets = []
-    integrations_secrets = []
+    secrets = defaultdict(list)
     # Get UDF -> Secrets mapping
-    for key, udf in registry.filter():
-        match key.split("."):
-            case ["integrations", *stem, func]:
-                integrations_secrets.append(
-                    (
-                        ".".join(stem) if stem else "-",
-                        func,
-                        ", ".join(wrap(s.name, "`") for s in udf.secrets)
-                        if udf.secrets
-                        else "-",
-                    )
-                )
-            case ["core", *stem, func]:
-                core_udfs_secrets.append(
-                    (
-                        ".".join(stem) if stem else "-",
-                        func,
-                        ", ".join(wrap(s.name, "`") for s in udf.secrets)
-                        if udf.secrets
-                        else "-",
-                    )
-                )
+    blacklist = {"example"}
+    for key, udf in registry:
+        top_level_ns, *stem, func = key.split(".")
+        if top_level_ns in blacklist:
+            continue
+        secrets[top_level_ns].append(
+            (
+                ".".join(stem) if stem else "-",
+                func,
+                ", ".join(wrap(s.name, "`") for s in udf.secrets)
+                if udf.secrets
+                else "-",
+            )
+        )
 
     # Get all secrets -> secret keys
     api_credentials = set()
@@ -475,10 +471,14 @@ def generate_secret_tables(
             header=("Secret Name", "Secret Keys"), rows=list(api_credentials)
         ),
         core_udfs_secrets_table=create_markdown_table(
-            header=("Sub-namespace", "Function", "Secrets"), rows=core_udfs_secrets
+            header=("Sub-namespace", "Function", "Secrets"), rows=secrets["core"]
         ),
         integrations_secrets_table=create_markdown_table(
-            header=("Sub-namespace", "Function", "Secrets"), rows=integrations_secrets
+            header=("Sub-namespace", "Function", "Secrets"),
+            rows=secrets["integrations"],
+        ),
+        etl_secrets_table=create_markdown_table(
+            header=("Sub-namespace", "Function", "Secrets"), rows=secrets["etl"]
         ),
     )
 

docs/api-reference/reference/auth/auth:databaselogin.mdx

@@ -0,0 +1,3 @@
+---
+openapi: post /auth/login
+---

docs/api-reference/reference/auth/auth:databaselogout.mdx

@@ -0,0 +1,3 @@
+---
+openapi: post /auth/logout
+---

docs/api-reference/reference/auth/oauth:googledatabaseauthorize.mdx

@@ -0,0 +1,3 @@
+---
+openapi: get /auth/oauth/authorize
+---

docs/api-reference/reference/auth/oauth:googledatabasecallback.mdx

@@ -0,0 +1,3 @@
+---
+openapi: get /auth/oauth/callback
+---

docs/api-reference/reference/auth/register:register.mdx

@@ -0,0 +1,3 @@
+---
+openapi: post /auth/register
+---

docs/api-reference/reference/auth/reset:forgot-password.mdx

@@ -0,0 +1,3 @@
+---
+openapi: post /auth/forgot-password
+---

docs/api-reference/reference/auth/reset:reset-password.mdx

@@ -0,0 +1,3 @@
+---
+openapi: post /auth/reset-password
+---

docs/api-reference/reference/auth/verify:request-token.mdx

@@ -0,0 +1,3 @@
+---
+openapi: post /auth/request-verify-token
+---

docs/api-reference/reference/auth/verify:verify.mdx

@@ -0,0 +1,3 @@
+---
+openapi: post /auth/verify
+---

docs/api-reference/reference/public/check-health.mdx

@@ -0,0 +1,3 @@
+---
+openapi: get /health
+---

docs/api-reference/reference/public/incoming-webhook-wait.mdx

@@ -0,0 +1,3 @@
+---
+openapi: post /webhooks/{path}/{secret}/wait
+---

docs/api-reference/reference/secrets/delete-secret-by-id.mdx

@@ -0,0 +1,3 @@
+---
+openapi: delete /secrets/{secret_id}
+---

docs/api-reference/reference/secrets/update-secret-by-id.mdx

@@ -0,0 +1,3 @@
+---
+openapi: post /secrets/{secret_id}
+---

docs/api-reference/reference/users/search-user.mdx

@@ -0,0 +1,3 @@
+---
+openapi: get /users/search
+---

docs/api-reference/reference/users/users:current-user.mdx

@@ -0,0 +1,3 @@
+---
+openapi: get /users/me
+---

docs/api-reference/reference/users/users:delete-user.mdx

@@ -0,0 +1,3 @@
+---
+openapi: delete /users/{id}
+---

docs/api-reference/reference/users/users:patch-current-user.mdx

@@ -0,0 +1,3 @@
+---
+openapi: patch /users/me
+---

docs/api-reference/reference/users/users:patch-user.mdx

@@ -0,0 +1,3 @@
+---
+openapi: patch /users/{id}
+---

docs/api-reference/reference/users/users:user.mdx

@@ -0,0 +1,3 @@
+---
+openapi: get /users/{id}
+---

docs/api-reference/reference/validation/validate-workflow.mdx

@@ -0,0 +1,3 @@
+---
+openapi: post /validate-workflow
+---

docs/api-reference/reference/workflow-executions/cancel-workflow-execution.mdx

@@ -0,0 +1,3 @@
+---
+openapi: post /workflow-executions/{execution_id}/cancel
+---

docs/api-reference/reference/workflow-executions/create-workflow-execution.mdx

@@ -0,0 +1,3 @@
+---
+openapi: post /workflow-executions
+---

docs/api-reference/reference/workflow-executions/list-workflow-execution-event-history.mdx

@@ -0,0 +1,3 @@
+---
+openapi: get /workflow-executions/{execution_id}/history
+---

docs/api-reference/reference/workflow-executions/terminate-workflow-execution.mdx

@@ -0,0 +1,3 @@
+---
+openapi: post /workflow-executions/{execution_id}/terminate
+---

docs/api-reference/reference/workflows/create-workflow-definition.mdx

@@ -0,0 +1,3 @@
+---
+openapi: post /workflows/{workflow_id}/definition
+---

docs/api-reference/reference/workspaces/create-workspace-membership.mdx

@@ -0,0 +1,3 @@
+---
+openapi: post /workspaces/{workspace_id}/memberships
+---

docs/api-reference/reference/workspaces/create-workspace.mdx

@@ -0,0 +1,3 @@
+---
+openapi: post /workspaces
+---

docs/api-reference/reference/workspaces/delete-workspace-membership.mdx

@@ -0,0 +1,3 @@
+---
+openapi: delete /workspaces/{workspace_id}/memberships/{user_id}
+---

docs/api-reference/reference/workspaces/delete-workspace.mdx

@@ -0,0 +1,3 @@
+---
+openapi: delete /workspaces/{workspace_id}
+---

docs/api-reference/reference/workspaces/get-workspace-membership.mdx

@@ -0,0 +1,3 @@
+---
+openapi: get /workspaces/{workspace_id}/memberships/{user_id}
+---

docs/api-reference/reference/workspaces/get-workspace.mdx

@@ -0,0 +1,3 @@
+---
+openapi: get /workspaces/{workspace_id}
+---

docs/api-reference/reference/workspaces/list-workspace-memberships.mdx

@@ -0,0 +1,3 @@
+---
+openapi: get /workspaces/{workspace_id}/memberships
+---

docs/api-reference/reference/workspaces/list-workspaces.mdx

@@ -0,0 +1,3 @@
+---
+openapi: get /workspaces
+---

docs/api-reference/reference/workspaces/search-workspaces.mdx

@@ -0,0 +1,3 @@
+---
+openapi: get /workspaces/search
+---

docs/api-reference/reference/workspaces/update-workspace.mdx

@@ -0,0 +1,3 @@
+---
+openapi: patch /workspaces/{workspace_id}
+---

docs/integrations/secrets_cheatsheet.mdx

@@ -6,33 +6,36 @@ description: A cheatsheet of all the secrets required by the UDFs and integratio
 The secret keys required by each secret are listed below.
 | Secret Name | Secret Keys |
 | --- | --- |
-| `wiz` | `WIZ_API_URL`, `WIZ_AUTH_URL`, `WIZ_CLIENT_ID`, `WIZ_CLIENT_SECRET` |
-| `elastic` | `ELASTIC_API_KEY`, `ELASTIC_API_URL` |
-| `microsoft_defender_cloud` | `MICROSOFT_GRAPH_CLIENT_ID`, `MICROSOFT_GRAPH_CLIENT_SECRET`, `MICROSOFT_GRAPH_TENANT_ID` |
-| `datadog` | `DD_API_KEY`, `DD_APP_KEY`, `DD_REGION` |
 | `aws_guardduty` | `AWS_ACCESS_KEY_ID`, `AWS_REGION`, `AWS_SECRET_ACCESS_KEY` |
-| `sentinel_one` | `SENTINEL_ONE_API_TOKEN`, `SENTINEL_ONE_BASE_URL` |
 | `microsoft_defender_endpoint` | `MICROSOFT_GRAPH_CLIENT_ID`, `MICROSOFT_GRAPH_CLIENT_SECRET`, `MICROSOFT_GRAPH_TENANT_ID` |
+| `okta` | `OKTA_API_TOKEN`, `OKTA_BASE_URL` |
 | `crowdstrike` | `CROWDSTRIKE_CLIENT_ID`, `CROWDSTRIKE_CLIENT_SECRET` |
-| `virustotal` | `VIRUSTOTAL_API_KEY` |
+| `slack` | `SLACK_BOT_TOKEN` |
+| `microsoft_defender_cloud` | `MICROSOFT_GRAPH_CLIENT_ID`, `MICROSOFT_GRAPH_CLIENT_SECRET`, `MICROSOFT_GRAPH_TENANT_ID` |
+| `datadog` | `DD_API_KEY`, `DD_APP_KEY`, `DD_REGION` |
 | `resend_api_key` | `RESEND_API_KEY` |
 | `openai` | `OPENAI_API_KEY` |
-| `slack_chatops` | `SLACK_BOT_TOKEN` |
+| `virustotal` | `VIRUSTOTAL_API_KEY` |
+| `sentinel_one` | `SENTINEL_ONE_API_TOKEN`, `SENTINEL_ONE_BASE_URL` |
+| `wiz` | `WIZ_API_URL`, `WIZ_AUTH_URL`, `WIZ_CLIENT_ID`, `WIZ_CLIENT_SECRET` |
+| `elastic` | `ELASTIC_API_KEY`, `ELASTIC_API_URL` |
+| `ldap` | `LDAP_BIND_DN`, `LDAP_BIND_PASS` |
 
 ## Core Actions
 Note that the fully qualified namespace for each Core Action UDF is prefixed with `core.`.
 | Sub-namespace | Function | Secrets |
 | --- | --- | --- |
-| extraction | extract_emails | - |
-| extraction | extract_ipv4_addresses | - |
+| - | send_email_smtp | - |
 | - | open_case | - |
 | condition | regex | - |
 | condition | compare | - |
 | condition | membership | - |
-| - | send_email | `resend_api_key` |
 | - | http_request | - |
 | - | ai_action | `openai` |
 | transform | reshape | - |
+| transform | filter | - |
+| transform | build_reference_table | - |
+| workflow | execute | - |
 
 ## Integrations
 Note that the fully qualified namespace for each Integration UDF is prefixed with `integrations.`.
@@ -41,18 +44,43 @@ Note that the fully qualified namespace for each Integration UDF is prefixed wit
 | aws.guardduty | list_guardduty_alerts | `aws_guardduty` |
 | microsoft_defender | list_defender_cloud_alerts | `microsoft_defender_cloud` |
 | wiz | list_wiz_alerts | `wiz` |
-| chat.slack | post_slack_message | `slack_chatops` |
-| chat.slack | list_slack_users | `slack_chatops` |
+| chat.slack | post_slack_message | `slack` |
+| chat.slack | list_slack_conversations | `slack` |
+| chat.slack | list_slack_users | `slack` |
+| chat.slack | tag_slack_users | `slack` |
 | crowdstrike | list_crowdstrike_alerts | `crowdstrike` |
 | crowdstrike | list_crowdstrike_detects | `crowdstrike` |
 | crowdstrike | update_crowdstrike_alert_status | `crowdstrike` |
 | crowdstrike | update_crowdstrike_detect_status | `crowdstrike` |
 | microsoft_defender | list_defender_endpoint_alerts | `microsoft_defender_endpoint` |
 | sentinel_one | list_sentinelone_alerts | `sentinel_one` |
 | sentinel_one | update_sentinelone_alert_status | `sentinel_one` |
+| sentinel_one | get_sentinelone_agents_by_username | `sentinel_one` |
+| sentinel_one | get_sentinelone_agents_by_hostname | `sentinel_one` |
+| sentinel_one | isolate_sentinelone_agent | `sentinel_one` |
+| sentinel_one | unisolate_sentinelone_agent | `sentinel_one` |
+| sentinel_one | get_sentinel_one_firewall_rule | `sentinel_one` |
+| sentinel_one | update_sentinel_one_firewall_rule | `sentinel_one` |
+| email.resend | send_email_resend | `resend_api_key` |
 | virustotal | analyze_url | `virustotal` |
 | virustotal | analyze_ip_address | `virustotal` |
 | virustotal | analyze_malware_sample | `virustotal` |
+| ldap | find_ldap_users | `ldap` |
+| ldap | disable_ad_user | `ldap` |
+| ldap | enable_ad_user | `ldap` |
+| okta | find_okta_users | `okta` |
+| okta | suspend_okta_user | `okta` |
+| okta | unsuspend_okta_user | `okta` |
+| okta | expire_okta_sessions | `okta` |
+| okta | list_okta_user_events | `okta` |
 | datadog | list_datadog_alerts | `datadog` |
 | elastic | list_elastic_alerts | `elastic` |
 | sinks | write_to_database | - |
+
+## ETL Actions
+Note that the fully qualified namespace for each ETL UDF is prefixed with `etl.`.
+| Sub-namespace | Function | Secrets |
+| --- | --- | --- |
+| extraction | extract_emails | - |
+| extraction | extract_ipv4_addresses | - |
+| extraction | extract_urls | - |

docs/integrations/udfs/core_open_case.mdx

@@ -37,11 +37,6 @@ _No secrets required._
         },
         "Tag": {
             "properties": {
-                "is_ai_generated": {
-                    "default": false,
-                    "title": "Is Ai Generated",
-                    "type": "boolean"
-                },
                 "tag": {
                     "title": "Tag",
                     "type": "string"

docs/integrations/udfs/core_send_email_smtp.mdx

@@ -0,0 +1,63 @@
+---
+title: Send Email (SMTP)
+description: core.send_email_smtp
+---
+
+Perform a send email action using SMTP.
+
+This is the [JSONSchema7](https://json-schema.org/draft-07/json-schema-release-notes) definition for the `core.send_email_smtp` integration.
+
+
+## Secrets
+_No secrets required._
+
+## Inputs
+
+<CodeGroup>
+```json JSONSchema7 Definition
+{
+    "additionalProperties": false,
+    "properties": {
+        "body": {
+            "title": "Body",
+            "type": "string"
+        },
+        "recipients": {
+            "items": {
+                "type": "string"
+            },
+            "title": "Recipients",
+            "type": "array"
+        },
+        "sender": {
+            "default": "[email protected]",
+            "title": "Sender",
+            "type": "string"
+        },
+        "subject": {
+            "title": "Subject",
+            "type": "string"
+        }
+    },
+    "required": [
+        "recipients",
+        "subject",
+        "body"
+    ],
+    "title": "CoreSendEmailSmtp",
+    "type": "object"
+}
+```
+
+</CodeGroup>
+
+## Response
+
+<CodeGroup>
+```json JSONSchema7 Definition
+{
+    "type": "object"
+}
+```
+
+</CodeGroup>