diff --git a/skills/agentfeeds/LICENSE b/skills/agentfeeds/LICENSE new file mode 100644 index 00000000..c9dad29d --- /dev/null +++ b/skills/agentfeeds/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Agent Feeds contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/skills/agentfeeds/SKILL.md b/skills/agentfeeds/SKILL.md new file mode 100644 index 00000000..5d9ce1bb --- /dev/null +++ b/skills/agentfeeds/SKILL.md @@ -0,0 +1,136 @@ +--- +name: agentfeeds +description: Use Agent Feeds for ambient awareness from continuously refreshed local streams under ~/.agentfeeds. Use at session start to install/check background refresh and insert a compact stream brief, and before web search or expensive source-specific queries when a prompt may be covered by changing local context such as RSS/news, GitHub, calendars, weather, local files, personal sources, templates, subscriptions, or subscribed stream state. +version: 0.1.1 +author: verkyyi +license: MIT +metadata: + hermes: + tags: [Productivity, AI Agents, Personal Context, Local First] +--- + +# Agent Feeds + +Agent Feeds is a local-first ambient context layer for agents. A background fetcher keeps changing stream state warm on disk so agents can answer from local, inspectable context before re-searching, querying, processing, or asking the user to repeat information. + +Use this skill at session start, when managing feeds/subscriptions/templates, and before web search or expensive source-specific work if subscribed local state may already cover the prompt. + +Requires shell access, Python 3.11+, and either `pip` or `uv` for setup. Background polling is supported on macOS, Linux, FreeBSD, and WSL-style POSIX environments. The bundle includes a frozen template catalog for first use; network access is needed for setup, remote catalog updates, and public feed refreshes. + +## Command Map + +Use the bundled scripts from the skill root: + +```bash +python3 scripts/setup.py +python3 scripts/agentfeeds.py brief +python3 scripts/agentfeeds.py search --json +python3 scripts/agentfeeds.py streams health --json +python3 scripts/agentfeeds.py streams read --limit 20 --json +python3 scripts/agentfeeds.py streams find --json +python3 scripts/agentfeeds.py templates find +python3 scripts/agentfeeds.py subscribe [key=value ...] --dry-run --json +python3 scripts/agentfeeds.py subscribe [key=value ...] +python3 scripts/agentfeeds.py refresh --stream +``` + +`python3 scripts/agentfeeds.py` is the agent-facing CLI. `python3 scripts/agentfeeds_fetch.py` is an internal refresh worker used by polling and wrappers; prefer `agentfeeds.py refresh` in agent instructions. Runtime state defaults to `~/.agentfeeds/`; treat the file layout as an implementation detail except when debugging or editing a scaffolded local template. + +Vocabulary: + +- Template: reusable feed definition. Some templates are ready to subscribe with no parameters; others require parameters. +- Subscription: configured active instance of a template. +- Stream: refreshed readable data for an active subscription. + +References to load only when needed: + +- Runtime setup details: `references/runtime-setup.md` +- Template authoring details: `references/template-authoring.md` +- Background refresh details: `references/background-refresh.md` +- macOS personal source setup: `references/macos-personal-sources.md` + +Built-in templates come from the standalone catalog repo `https://github.com/verkyyi/agentfeeds-catalog` and are cached locally; user-local templates live under `~/.agentfeeds/templates/`. +The release bundle includes a frozen built-in catalog fallback so first-run discovery does not depend on GitHub being reachable. + +## Session Start + +At the start of each session: + +1. If the bundled CLI fails because dependencies are missing, run `python3 scripts/setup.py`. +2. Check background refresh with `python3 scripts/agentfeeds.py admin polling status --json`; if missing, run `python3 scripts/agentfeeds.py admin polling install`. +3. Check stream health with `python3 scripts/agentfeeds.py streams health --json`. +4. Generate stable compact context with `python3 scripts/agentfeeds.py brief`. +5. If the host supports prompt slots, place the exact brief output in a system-level or persistent context slot so stable stream metadata can benefit from model-side prompt caching. + +The default brief avoids volatile timestamps. Use `python3 scripts/agentfeeds.py brief --include-freshness` only for freshness/debugging questions. + +If health reports errors, missing state, or stale state, continue with available local context but tell the user ambient awareness is degraded when it affects the answer. + +## Answering Flow + +When a user prompt may be covered by subscribed changing context: + +1. Search local state first: `python3 scripts/agentfeeds.py search --json`. +2. If matches are non-stale and answer the prompt, read the matching stream with `python3 scripts/agentfeeds.py streams read --limit 20 --json` and answer from local state. +3. Refresh on demand only when the user asks about current/time-bounded data, the stream is older than 2x its poll interval, or the user explicitly asks to refresh. Use `python3 scripts/agentfeeds.py refresh --stream `, then search/read again. +4. If health shows a fetch error or missing state, explain the degraded source and ask for reconfiguration only when needed. +5. Use web search or source-specific external tools only when local streams do not cover the prompt, are stale and cannot refresh, or the user explicitly asks for outside/current web information beyond subscribed data. + +Use `streams find` only for stream metadata discovery. Use top-level `search` for content snippets. + +## Subscribe And Manage + +When the user asks to subscribe to a source: + +1. Search built-ins first with `python3 scripts/agentfeeds.py templates find `. +2. Inspect likely matches with `python3 scripts/agentfeeds.py templates show --json`. +3. Prefer a built-in template when it fits the source shape and auth model. +4. Collect only required parameters, preview with `python3 scripts/agentfeeds.py subscribe [key=value ...] --dry-run --json`, then subscribe with `python3 scripts/agentfeeds.py subscribe [key=value ...]`. +5. Confirm with `python3 scripts/agentfeeds.py streams health --json` and, if useful, one stream read. + +Default to answering first. Subscribe only when the user explicitly asks to subscribe, asks about a recurring topic, or a follow-up would clearly benefit from warm local state. If the user names a category rather than a source, list candidate templates and ask which source they mean. + +For unsubscribe: + +```bash +python3 scripts/agentfeeds.py streams list +python3 scripts/agentfeeds.py unsubscribe +``` + +If the user names a template instead of a concrete subscription, list matching active streams and ask which one to remove. + +## Template Strategy + +Balance built-in templates and local authoring this way: + +- Use built-in templates for common source classes, stable public APIs, shared schemas, and anything many operators would reuse. +- Use parameterized built-ins for source families: RSS/Atom URLs, GitHub repos, iCalendar URLs, weather coordinates, public JSON APIs. +- Use local templates for private files, private dashboards, one-off APIs, local tools, experimental sources, and operator-specific commands. +- Do not author a local template until built-ins have been checked and no suitable template fits. +- If a local template proves broadly useful, suggest upstreaming it to the catalog rather than keeping many near-duplicate local templates. +- Prefer local/private read-only sources before suggesting public feeds when the user wants personal context. + +When no built-in template fits: + +```bash +python3 scripts/agentfeeds.py admin templates adapters +python3 scripts/agentfeeds.py admin templates scaffold +python3 scripts/agentfeeds.py admin templates validate +python3 scripts/agentfeeds.py admin templates test key=value +``` + +Read `references/template-authoring.md` before editing scaffolded template YAML. + +For `local_command` templates, use argv arrays only. Only create command templates for explicitly requested or approved read-only commands. Avoid commands that mutate files, cloud resources, accounts, or external services. New `local_command` templates are pending and cannot run until approved by the operator. + +After scaffolding or installing a `local_command` template, tell the user to run `python3 scripts/agentfeeds.py admin templates approve-command [key=value ...]` themselves in an interactive terminal. Do not approve on the user's behalf, even if asked. Explain that approval is tied to the exact template and command digest, and edits revoke it. + +For macOS personal context, prefer the built-in `mac/*` templates from the bundled catalog. They do not require local-command approval, but macOS may ask the user to grant Calendar, Reminders, Automation, or Full Disk Access permissions on first refresh. + +## Safety Rules + +- Use `subscribe` and `unsubscribe` for subscription changes. +- Use `agentfeeds.py refresh` for refreshes. +- Do not hand-write state or status files. +- Do not include secret values in template YAML. Use `{{secret:name}}` references and tell the user to set values with `python3 scripts/agentfeeds.py admin secrets set `. +- Treat Agent Feeds as warm changing context, not durable memory, semantic search, or a data warehouse. diff --git a/skills/agentfeeds/agents/openai.yaml b/skills/agentfeeds/agents/openai.yaml new file mode 100644 index 00000000..a7e60ef0 --- /dev/null +++ b/skills/agentfeeds/agents/openai.yaml @@ -0,0 +1,7 @@ +interface: + display_name: "Agent Feeds" + short_description: "Use warm local feeds for ambient personal context" + default_prompt: "Use $agentfeeds to check background refresh, insert my stream brief, and help subscribe local or Mac personal sources." + +policy: + allow_implicit_invocation: true diff --git a/skills/agentfeeds/assets/agentfeeds-demo.gif b/skills/agentfeeds/assets/agentfeeds-demo.gif new file mode 100644 index 00000000..47eb7b51 Binary files /dev/null and b/skills/agentfeeds/assets/agentfeeds-demo.gif differ diff --git a/skills/agentfeeds/catalog/INDEX.json b/skills/agentfeeds/catalog/INDEX.json new file mode 100644 index 00000000..f2151469 --- /dev/null +++ b/skills/agentfeeds/catalog/INDEX.json @@ -0,0 +1,609 @@ +{ + "generated_at": "2026-05-03T22:31:45Z", + "spec_version": "0.3", + "stream_count": 25, + "streams": [ + { + "auth": "none", + "catalog_order": 10, + "catalog_tier": 1, + "description": "Today's agenda from every calendar configured in Calendar.app.", + "id": "mac/calendar-today", + "mode": "event", + "parameters": [], + "path": "catalog/streams/mac/calendar-today.yaml", + "platforms": [ + "macos" + ], + "quality_tier": "experimental", + "requires": [ + "calendar_tcc" + ], + "tags": [ + "mac", + "calendar", + "agenda", + "eventkit", + "tcc", + "no-auth" + ], + "title": "Today's Calendar.app agenda", + "type": "ical.event" + }, + { + "auth": "none", + "catalog_order": 20, + "catalog_tier": 1, + "description": "Next 7 days from every calendar configured in Calendar.app.", + "id": "mac/calendar-upcoming", + "mode": "event", + "parameters": [], + "path": "catalog/streams/mac/calendar-upcoming.yaml", + "platforms": [ + "macos" + ], + "quality_tier": "experimental", + "requires": [ + "calendar_tcc" + ], + "tags": [ + "mac", + "calendar", + "upcoming", + "eventkit", + "tcc", + "no-auth" + ], + "title": "Upcoming Calendar.app events", + "type": "ical.event" + }, + { + "auth": "none", + "catalog_order": 30, + "catalog_tier": 1, + "description": "Open reminders with due dates and list grouping from Reminders.app.", + "id": "mac/reminders-pending", + "mode": "event", + "parameters": [], + "path": "catalog/streams/mac/reminders-pending.yaml", + "platforms": [ + "macos" + ], + "quality_tier": "experimental", + "requires": [ + "reminders_tcc" + ], + "tags": [ + "mac", + "reminders", + "tasks", + "tcc", + "no-auth" + ], + "title": "Pending Reminders.app items", + "type": "mac.reminder" + }, + { + "auth": "none", + "catalog_order": 40, + "catalog_tier": 1, + "description": "Recently modified Notes.app notes with titles and snippets.", + "id": "mac/notes-recent", + "mode": "event", + "parameters": [], + "path": "catalog/streams/mac/notes-recent.yaml", + "platforms": [ + "macos" + ], + "quality_tier": "experimental", + "requires": [ + "notes_automation_tcc" + ], + "tags": [ + "mac", + "notes", + "recent", + "automation", + "tcc", + "no-auth" + ], + "title": "Recent Notes.app notes", + "type": "mac.note" + }, + { + "auth": "none", + "catalog_order": 50, + "catalog_tier": 1, + "description": "Unread Mail.app subjects, senders, and snippets without message bodies.", + "id": "mac/mail-unread", + "mode": "event", + "parameters": [], + "path": "catalog/streams/mac/mail-unread.yaml", + "platforms": [ + "macos" + ], + "quality_tier": "experimental", + "requires": [ + "mail_automation_tcc" + ], + "tags": [ + "mac", + "mail", + "inbox", + "unread", + "automation", + "tcc", + "no-auth" + ], + "title": "Unread Mail.app messages", + "type": "mac.mail-message" + }, + { + "auth": "none", + "catalog_order": 60, + "catalog_tier": 1, + "description": "Recent iMessage conversations with unread counts and last-message snippets.", + "id": "mac/imessage-unread", + "mode": "event", + "parameters": [], + "path": "catalog/streams/mac/imessage-unread.yaml", + "platforms": [ + "macos" + ], + "quality_tier": "experimental", + "requires": [ + "full_disk_access" + ], + "tags": [ + "mac", + "imessage", + "messages", + "unread", + "sqlite", + "tcc", + "no-auth" + ], + "title": "Unread iMessage conversations", + "type": "mac.imessage-thread" + }, + { + "auth": "none", + "catalog_order": 10, + "catalog_tier": 2, + "description": "Read-only snapshot of one local text, Markdown, or JSON file.", + "id": "local/file", + "mode": "snapshot", + "parameters": [ + "path" + ], + "path": "catalog/streams/local/file.yaml", + "quality_tier": "verified", + "tags": [ + "local", + "file", + "markdown", + "text", + "json", + "no-auth" + ], + "title": "Local file", + "type": "local.file" + }, + { + "auth": "none", + "catalog_order": 20, + "catalog_tier": 2, + "description": "Last modified files in a watched directory.", + "id": "local/directory-recent", + "mode": "event", + "parameters": [ + "path" + ], + "path": "catalog/streams/local/directory-recent.yaml", + "quality_tier": "experimental", + "tags": [ + "local", + "files", + "directory", + "recent", + "no-auth" + ], + "title": "Recent files in a directory", + "type": "local.directory-entry" + }, + { + "auth": "none", + "catalog_order": 30, + "catalog_tier": 2, + "description": "Recently modified Markdown notes from a local vault directory.", + "id": "local/markdown-vault", + "mode": "event", + "parameters": [ + "path" + ], + "path": "catalog/streams/local/markdown-vault.yaml", + "quality_tier": "experimental", + "tags": [ + "local", + "markdown", + "obsidian", + "notes", + "vault", + "no-auth" + ], + "title": "Markdown vault", + "type": "local.markdown-document" + }, + { + "auth": "none", + "catalog_order": 40, + "catalog_tier": 2, + "description": "Current branch, dirty files, and ahead or behind counts for a watched repo.", + "id": "local/git-status", + "mode": "snapshot", + "parameters": [ + "path" + ], + "path": "catalog/streams/local/git-status.yaml", + "quality_tier": "experimental", + "tags": [ + "local", + "git", + "dev", + "repo", + "status", + "no-auth" + ], + "title": "Local Git status", + "type": "local.git-status" + }, + { + "auth": "none", + "catalog_order": 50, + "catalog_tier": 2, + "description": "Items saved to Safari Reading List from the local bookmarks file.", + "id": "mac/safari-reading-list", + "mode": "event", + "parameters": [], + "path": "catalog/streams/mac/safari-reading-list.yaml", + "platforms": [ + "macos" + ], + "quality_tier": "experimental", + "tags": [ + "mac", + "safari", + "reading-list", + "bookmarks", + "no-auth" + ], + "title": "Safari Reading List", + "type": "mac.reading-list-item" + }, + { + "auth": "none", + "catalog_order": 60, + "catalog_tier": 2, + "description": "Last modified items in the local Downloads folder.", + "id": "mac/finder-recent-downloads", + "mode": "event", + "parameters": [], + "path": "catalog/streams/mac/finder-recent-downloads.yaml", + "platforms": [ + "macos" + ], + "quality_tier": "experimental", + "tags": [ + "mac", + "finder", + "downloads", + "files", + "recent", + "no-auth" + ], + "title": "Recent Downloads", + "type": "local.directory-entry" + }, + { + "auth": "bearer_token", + "catalog_order": 10, + "catalog_tier": 3, + "description": "Authenticated GitHub notifications for the current user.", + "id": "dev/github-notifications", + "mode": "event", + "parameters": [], + "path": "catalog/streams/dev/github-notifications.yaml", + "quality_tier": "experimental", + "requires": [ + "keychain_bearer_token" + ], + "tags": [ + "dev", + "github", + "notifications", + "keychain", + "auth" + ], + "title": "My GitHub notifications", + "type": "github.notification" + }, + { + "auth": "bearer_token", + "catalog_order": 20, + "catalog_tier": 3, + "description": "Pull requests authored by or assigned for review to the current GitHub user.", + "id": "dev/github-prs-mine", + "mode": "event", + "parameters": [], + "path": "catalog/streams/dev/github-prs-mine.yaml", + "quality_tier": "experimental", + "requires": [ + "keychain_bearer_token" + ], + "tags": [ + "dev", + "github", + "pull-request", + "review", + "keychain", + "auth" + ], + "title": "My GitHub pull requests", + "type": "github.pull-request" + }, + { + "auth": "bearer_token", + "catalog_order": 30, + "catalog_tier": 3, + "description": "Linear issues assigned to the authenticated user.", + "id": "tasks/linear-mine", + "mode": "event", + "parameters": [], + "path": "catalog/streams/tasks/linear-mine.yaml", + "quality_tier": "experimental", + "requires": [ + "keychain_bearer_token" + ], + "tags": [ + "tasks", + "linear", + "issues", + "assigned", + "keychain", + "auth" + ], + "title": "My Linear issues", + "type": "linear.issue" + }, + { + "auth": "bearer_token", + "catalog_order": 40, + "catalog_tier": 3, + "description": "Todoist tasks due today for the authenticated user.", + "id": "tasks/todoist-today", + "mode": "event", + "parameters": [], + "path": "catalog/streams/tasks/todoist-today.yaml", + "quality_tier": "experimental", + "requires": [ + "keychain_bearer_token" + ], + "tags": [ + "tasks", + "todoist", + "today", + "keychain", + "auth" + ], + "title": "Today's Todoist tasks", + "type": "todoist.task" + }, + { + "auth": "bearer_token", + "catalog_order": 50, + "catalog_tier": 3, + "description": "Recently edited Notion pages visible to the authenticated integration.", + "id": "notes/notion-recent", + "mode": "event", + "parameters": [], + "path": "catalog/streams/notes/notion-recent.yaml", + "quality_tier": "experimental", + "requires": [ + "keychain_bearer_token" + ], + "tags": [ + "notes", + "notion", + "recent", + "keychain", + "auth" + ], + "title": "Recent Notion pages", + "type": "notion.page" + }, + { + "auth": "none", + "catalog_order": 10, + "catalog_tier": 4, + "description": "Subscribe to any public RSS or Atom URL.", + "id": "news/rss-generic", + "mode": "event", + "parameters": [ + "url" + ], + "path": "catalog/streams/news/rss-generic.yaml", + "quality_tier": "verified", + "tags": [ + "news", + "rss", + "atom", + "free", + "no-auth" + ], + "title": "Generic RSS feed", + "type": "rss.item" + }, + { + "auth": "none", + "catalog_order": 20, + "catalog_tier": 4, + "description": "Events from a public iCalendar URL.", + "id": "calendar/ics", + "mode": "event", + "parameters": [ + "url" + ], + "path": "catalog/streams/calendar/ics.yaml", + "quality_tier": "verified", + "tags": [ + "calendar", + "ics", + "ical", + "events", + "free", + "no-auth" + ], + "title": "iCalendar feed", + "type": "ical.event" + }, + { + "auth": "none", + "catalog_order": 30, + "catalog_tier": 4, + "description": "Free, no-API-key weather observations for any latitude and longitude.", + "id": "weather/openmeteo-current", + "mode": "snapshot", + "parameters": [ + "lat", + "lon" + ], + "path": "catalog/streams/weather/openmeteo-current.yaml", + "quality_tier": "verified", + "tags": [ + "weather", + "free", + "no-auth", + "global" + ], + "title": "Current weather conditions (Open-Meteo)", + "type": "weather.observation" + }, + { + "auth": "none", + "catalog_order": 40, + "catalog_tier": 4, + "description": "Free, no-API-key 7-day forecast for any latitude and longitude.", + "id": "weather/openmeteo-forecast", + "mode": "snapshot", + "parameters": [ + "lat", + "lon" + ], + "path": "catalog/streams/weather/openmeteo-forecast.yaml", + "quality_tier": "verified", + "tags": [ + "weather", + "forecast", + "free", + "no-auth", + "global" + ], + "title": "7-day weather forecast (Open-Meteo)", + "type": "weather.forecast" + }, + { + "auth": "none", + "catalog_order": 50, + "catalog_tier": 4, + "description": "Current currency exchange rates for a base currency via a no-auth public API.", + "id": "finance/exchangerate", + "mode": "snapshot", + "parameters": [ + "base" + ], + "path": "catalog/streams/finance/exchangerate.yaml", + "quality_tier": "verified", + "tags": [ + "finance", + "currency", + "exchange-rate", + "free", + "no-auth" + ], + "title": "Current exchange rates", + "type": "finance.exchange-rate" + }, + { + "auth": "none", + "catalog_order": 60, + "catalog_tier": 4, + "description": "Recent pull requests for a public GitHub repository.", + "id": "dev/github-prs", + "mode": "event", + "parameters": [ + "owner", + "repo", + "state" + ], + "path": "catalog/streams/dev/github-prs.yaml", + "quality_tier": "verified", + "tags": [ + "dev", + "github", + "pull-request", + "prs", + "free", + "no-auth" + ], + "title": "GitHub repository pull requests", + "type": "github.pull-request" + }, + { + "auth": "none", + "catalog_order": 70, + "catalog_tier": 4, + "description": "Recent issues for a public GitHub repository.", + "id": "dev/github-issues", + "mode": "event", + "parameters": [ + "owner", + "repo", + "state" + ], + "path": "catalog/streams/dev/github-issues.yaml", + "quality_tier": "verified", + "tags": [ + "dev", + "github", + "issues", + "free", + "no-auth" + ], + "title": "GitHub repository issues", + "type": "github.issue" + }, + { + "auth": "none", + "catalog_order": 80, + "catalog_tier": 4, + "description": "Latest release events for a public GitHub repository.", + "id": "dev/github-releases", + "mode": "event", + "parameters": [ + "owner", + "repo" + ], + "path": "catalog/streams/dev/github-releases.yaml", + "quality_tier": "verified", + "tags": [ + "dev", + "github", + "releases", + "free", + "no-auth" + ], + "title": "GitHub repository releases", + "type": "github.release" + } + ] +} diff --git a/skills/agentfeeds/catalog/schemas/envelope.v0.3.json b/skills/agentfeeds/catalog/schemas/envelope.v0.3.json new file mode 100644 index 00000000..7503976b --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/envelope.v0.3.json @@ -0,0 +1,53 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/envelope.v0.3.json", + "title": "Agent Feeds Event Envelope", + "type": "object", + "additionalProperties": false, + "required": [ + "specversion", + "id", + "source", + "type", + "time", + "schema_url", + "schema_version", + "mode", + "data" + ], + "properties": { + "specversion": { + "const": "agentfeeds/0.3" + }, + "id": { + "type": "string", + "minLength": 1 + }, + "source": { + "type": "string", + "pattern": "^feed://" + }, + "type": { + "type": "string", + "minLength": 1 + }, + "time": { + "type": "string", + "format": "date-time" + }, + "schema_url": { + "type": "string", + "format": "uri" + }, + "schema_version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "mode": { + "enum": ["snapshot", "event", "delta"] + }, + "data": { + "type": "object" + } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/finance.exchange-rate.v1.json b/skills/agentfeeds/catalog/schemas/event-types/finance.exchange-rate.v1.json new file mode 100644 index 00000000..1beca07f --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/finance.exchange-rate.v1.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/finance.exchange-rate.v1.json", + "title": "Exchange Rates", + "type": "object", + "required": ["base", "rates"], + "properties": { + "base": { "type": "string" }, + "date": { "type": "string" }, + "rates": { + "type": "object", + "additionalProperties": { "type": "number" } + } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/github.issue.v1.json b/skills/agentfeeds/catalog/schemas/event-types/github.issue.v1.json new file mode 100644 index 00000000..02a2cf26 --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/github.issue.v1.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/github.issue.v1.json", + "title": "GitHub Issue", + "type": "object", + "required": ["number", "title", "state", "html_url", "created_at", "updated_at"], + "properties": { + "number": { "type": "integer" }, + "title": { "type": "string" }, + "state": { "type": "string" }, + "html_url": { "type": "string" }, + "user": { "type": ["string", "null"] }, + "labels": { + "type": "array", + "items": { "type": "string" } + }, + "created_at": { "type": "string" }, + "updated_at": { "type": "string" }, + "closed_at": { "type": ["string", "null"] }, + "body": { "type": ["string", "null"] } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/github.notification.v1.json b/skills/agentfeeds/catalog/schemas/event-types/github.notification.v1.json new file mode 100644 index 00000000..1ac32883 --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/github.notification.v1.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/github.notification.v1.json", + "title": "GitHub Notification", + "type": "object", + "required": ["id", "reason", "unread", "updated_at", "repository", "subject_title"], + "properties": { + "id": { "type": "string" }, + "reason": { "type": "string" }, + "unread": { "type": "boolean" }, + "updated_at": { "type": "string" }, + "repository": { "type": "string" }, + "subject_title": { "type": "string" }, + "subject_type": { "type": ["string", "null"] }, + "url": { "type": ["string", "null"] } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/github.pull-request.v1.json b/skills/agentfeeds/catalog/schemas/event-types/github.pull-request.v1.json new file mode 100644 index 00000000..2bc38ba9 --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/github.pull-request.v1.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/github.pull-request.v1.json", + "title": "GitHub Pull Request", + "type": "object", + "required": ["number", "title", "state", "html_url", "created_at", "updated_at"], + "properties": { + "number": { "type": "integer" }, + "title": { "type": "string" }, + "state": { "type": "string" }, + "html_url": { "type": "string" }, + "user": { "type": ["string", "null"] }, + "draft": { "type": "boolean" }, + "head_ref": { "type": ["string", "null"] }, + "base_ref": { "type": ["string", "null"] }, + "created_at": { "type": "string" }, + "updated_at": { "type": "string" }, + "closed_at": { "type": ["string", "null"] }, + "merged_at": { "type": ["string", "null"] }, + "body": { "type": ["string", "null"] } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/github.release.v1.json b/skills/agentfeeds/catalog/schemas/event-types/github.release.v1.json new file mode 100644 index 00000000..a85bd582 --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/github.release.v1.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/github.release.v1.json", + "title": "GitHub Release", + "type": "object", + "required": ["tag_name", "name", "published_at", "html_url"], + "properties": { + "tag_name": { "type": "string" }, + "name": { "type": ["string", "null"] }, + "published_at": { "type": "string" }, + "html_url": { "type": "string" }, + "body": { "type": ["string", "null"] } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/ical-event.v1.json b/skills/agentfeeds/catalog/schemas/event-types/ical-event.v1.json new file mode 100644 index 00000000..acbf9c69 --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/ical-event.v1.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/ical-event.v1.json", + "title": "iCalendar Event", + "type": "object", + "required": ["uid", "summary"], + "properties": { + "uid": { "type": "string" }, + "summary": { "type": "string" }, + "starts_at": { "type": ["string", "null"] }, + "ends_at": { "type": ["string", "null"] }, + "location": { "type": ["string", "null"] } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/linear.issue.v1.json b/skills/agentfeeds/catalog/schemas/event-types/linear.issue.v1.json new file mode 100644 index 00000000..1283579d --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/linear.issue.v1.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/linear.issue.v1.json", + "title": "Linear Issue", + "type": "object", + "required": ["id", "identifier", "title", "state", "url", "updated_at"], + "properties": { + "id": { "type": "string" }, + "identifier": { "type": "string" }, + "title": { "type": "string" }, + "state": { "type": "string" }, + "priority": { "type": ["integer", "null"] }, + "team": { "type": ["string", "null"] }, + "url": { "type": "string" }, + "updated_at": { "type": "string" }, + "due_at": { "type": ["string", "null"] } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/local.command.v1.json b/skills/agentfeeds/catalog/schemas/event-types/local.command.v1.json new file mode 100644 index 00000000..0d2e1b6f --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/local.command.v1.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/local.command.v1.json", + "title": "Local Command Snapshot", + "type": "object", + "required": ["command", "exit_code", "stdout", "ran_at"], + "properties": { + "command": { + "type": "array", + "items": { "type": "string" } + }, + "cwd": { "type": ["string", "null"] }, + "exit_code": { "type": "integer" }, + "stdout": { "type": "string" }, + "stderr": { "type": "string" }, + "stdout_truncated": { "type": "boolean" }, + "stderr_truncated": { "type": "boolean" }, + "parsed_json": {}, + "transformed": {}, + "started_at": { "type": "string" }, + "ran_at": { "type": "string" } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/local.directory-entry.v1.json b/skills/agentfeeds/catalog/schemas/event-types/local.directory-entry.v1.json new file mode 100644 index 00000000..4951b74b --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/local.directory-entry.v1.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/local.directory-entry.v1.json", + "title": "Local Directory Entry", + "type": "object", + "required": ["path", "name", "kind", "modified_at"], + "properties": { + "path": { "type": "string" }, + "name": { "type": "string" }, + "kind": { "enum": ["file", "directory", "symlink", "other"] }, + "extension": { "type": ["string", "null"] }, + "size_bytes": { "type": ["integer", "null"] }, + "modified_at": { "type": "string" } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/local.file.v1.json b/skills/agentfeeds/catalog/schemas/event-types/local.file.v1.json new file mode 100644 index 00000000..76266e42 --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/local.file.v1.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/local.file.v1.json", + "title": "Local File Snapshot", + "type": "object", + "required": ["path", "name", "content", "size_bytes", "sha256", "modified_at"], + "properties": { + "path": { "type": "string" }, + "name": { "type": "string" }, + "extension": { "type": "string" }, + "content": { "type": "string" }, + "size_bytes": { "type": "integer" }, + "sha256": { "type": "string" }, + "modified_at": { "type": "string" } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/local.git-status.v1.json b/skills/agentfeeds/catalog/schemas/event-types/local.git-status.v1.json new file mode 100644 index 00000000..8c0ef190 --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/local.git-status.v1.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/local.git-status.v1.json", + "title": "Local Git Status", + "type": "object", + "required": ["path", "branch", "clean", "dirty_files", "ahead", "behind"], + "properties": { + "path": { "type": "string" }, + "branch": { "type": "string" }, + "clean": { "type": "boolean" }, + "dirty_files": { + "type": "array", + "items": { "type": "string" } + }, + "ahead": { "type": "integer" }, + "behind": { "type": "integer" } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/local.markdown-document.v1.json b/skills/agentfeeds/catalog/schemas/event-types/local.markdown-document.v1.json new file mode 100644 index 00000000..e7e4ed80 --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/local.markdown-document.v1.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/local.markdown-document.v1.json", + "title": "Local Markdown Document", + "type": "object", + "required": ["path", "title", "snippet", "modified_at", "frontmatter"], + "properties": { + "path": { "type": "string" }, + "title": { "type": "string" }, + "snippet": { "type": "string" }, + "modified_at": { "type": "string" }, + "frontmatter": { + "type": "object", + "additionalProperties": true + }, + "tags": { + "type": "array", + "items": { "type": "string" } + } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/mac.imessage-thread.v1.json b/skills/agentfeeds/catalog/schemas/event-types/mac.imessage-thread.v1.json new file mode 100644 index 00000000..243ba76c --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/mac.imessage-thread.v1.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/mac.imessage-thread.v1.json", + "title": "iMessage Conversation", + "type": "object", + "required": ["thread_id", "display_name", "unread_count", "last_message_at", "snippet"], + "properties": { + "thread_id": { "type": "string" }, + "display_name": { "type": "string" }, + "participants": { + "type": "array", + "items": { "type": "string" } + }, + "unread_count": { "type": "integer" }, + "last_message_at": { "type": "string" }, + "snippet": { "type": "string" } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/mac.mail-message.v1.json b/skills/agentfeeds/catalog/schemas/event-types/mac.mail-message.v1.json new file mode 100644 index 00000000..bbf350e2 --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/mac.mail-message.v1.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/mac.mail-message.v1.json", + "title": "macOS Mail Message", + "type": "object", + "required": ["id", "subject", "sender", "received_at", "unread"], + "properties": { + "id": { "type": "string" }, + "subject": { "type": "string" }, + "sender": { "type": "string" }, + "from_email": { "type": ["string", "null"] }, + "received_at": { "type": "string" }, + "unread": { "type": "boolean" }, + "mailbox": { "type": ["string", "null"] }, + "account": { "type": ["string", "null"] }, + "snippet": { "type": ["string", "null"] } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/mac.note.v1.json b/skills/agentfeeds/catalog/schemas/event-types/mac.note.v1.json new file mode 100644 index 00000000..65d6cdd2 --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/mac.note.v1.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/mac.note.v1.json", + "title": "macOS Note", + "type": "object", + "required": ["id", "title", "snippet", "modified_at"], + "properties": { + "id": { "type": "string" }, + "title": { "type": "string" }, + "snippet": { "type": "string" }, + "modified_at": { "type": "string" }, + "folder": { "type": ["string", "null"] }, + "account": { "type": ["string", "null"] } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/mac.reading-list-item.v1.json b/skills/agentfeeds/catalog/schemas/event-types/mac.reading-list-item.v1.json new file mode 100644 index 00000000..632ab35e --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/mac.reading-list-item.v1.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/mac.reading-list-item.v1.json", + "title": "Safari Reading List Item", + "type": "object", + "required": ["title", "url"], + "properties": { + "title": { "type": "string" }, + "url": { "type": "string" }, + "added_at": { "type": ["string", "null"] }, + "preview_text": { "type": ["string", "null"] } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/mac.reminder.v1.json b/skills/agentfeeds/catalog/schemas/event-types/mac.reminder.v1.json new file mode 100644 index 00000000..b257dc9c --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/mac.reminder.v1.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/mac.reminder.v1.json", + "title": "macOS Reminder", + "type": "object", + "required": ["id", "title", "completed", "list_name"], + "properties": { + "id": { "type": "string" }, + "title": { "type": "string" }, + "completed": { "type": "boolean" }, + "list_name": { "type": "string" }, + "due_at": { "type": ["string", "null"] }, + "priority": { "type": ["integer", "null"] }, + "notes_snippet": { "type": ["string", "null"] }, + "updated_at": { "type": ["string", "null"] }, + "url": { "type": ["string", "null"] } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/notion.page.v1.json b/skills/agentfeeds/catalog/schemas/event-types/notion.page.v1.json new file mode 100644 index 00000000..619d996b --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/notion.page.v1.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/notion.page.v1.json", + "title": "Notion Page", + "type": "object", + "required": ["id", "title", "url", "last_edited_at"], + "properties": { + "id": { "type": "string" }, + "title": { "type": "string" }, + "url": { "type": "string" }, + "last_edited_at": { "type": "string" }, + "created_at": { "type": ["string", "null"] }, + "parent_type": { "type": ["string", "null"] } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/rss-item.v1.json b/skills/agentfeeds/catalog/schemas/event-types/rss-item.v1.json new file mode 100644 index 00000000..9bd22039 --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/rss-item.v1.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/rss-item.v1.json", + "title": "RSS Item", + "type": "object", + "required": ["title"], + "properties": { + "title": { "type": "string" }, + "link": { "type": ["string", "null"] }, + "summary": { "type": ["string", "null"] }, + "published": { "type": ["string", "null"] }, + "id": { "type": ["string", "null"] } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/todoist.task.v1.json b/skills/agentfeeds/catalog/schemas/event-types/todoist.task.v1.json new file mode 100644 index 00000000..0148195f --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/todoist.task.v1.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/todoist.task.v1.json", + "title": "Todoist Task", + "type": "object", + "required": ["id", "content", "url", "priority", "due"], + "properties": { + "id": { "type": "string" }, + "content": { "type": "string" }, + "description": { "type": ["string", "null"] }, + "url": { "type": "string" }, + "priority": { "type": "integer" }, + "project_id": { "type": ["string", "null"] }, + "due": { + "type": "object", + "additionalProperties": true + } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/weather.forecast.v1.json b/skills/agentfeeds/catalog/schemas/event-types/weather.forecast.v1.json new file mode 100644 index 00000000..523eeec0 --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/weather.forecast.v1.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/weather.forecast.v1.json", + "title": "Weather Forecast", + "type": "object", + "required": ["generated_at", "daily"], + "properties": { + "generated_at": { "type": "string" }, + "daily": { + "type": "array", + "items": { "type": "object" } + } + } +} diff --git a/skills/agentfeeds/catalog/schemas/event-types/weather.observation.v1.json b/skills/agentfeeds/catalog/schemas/event-types/weather.observation.v1.json new file mode 100644 index 00000000..28fe3ed5 --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/event-types/weather.observation.v1.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/weather.observation.v1.json", + "title": "Weather Observation", + "type": "object", + "required": ["observed_at"], + "properties": { + "temperature_c": { "type": "number" }, + "humidity_pct": { "type": "number" }, + "wind_kph": { "type": "number" }, + "conditions_code": { "type": ["integer", "string"] }, + "observed_at": { "type": "string" } + } +} diff --git a/skills/agentfeeds/catalog/schemas/stream-definition.v0.3.json b/skills/agentfeeds/catalog/schemas/stream-definition.v0.3.json new file mode 100644 index 00000000..9badfe63 --- /dev/null +++ b/skills/agentfeeds/catalog/schemas/stream-definition.v0.3.json @@ -0,0 +1,121 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://agentfeeds.dev/schemas/stream-definition.v0.3.json", + "title": "Agent Feeds Stream Definition", + "type": "object", + "additionalProperties": true, + "required": [ + "id", + "title", + "description", + "type", + "mode", + "schema_url", + "schema_version", + "source_uri_template", + "adapter", + "recommended_poll_interval_seconds", + "auth", + "tags", + "quality_tier" + ], + "properties": { + "id": { + "type": "string", + "pattern": "^[a-z0-9-]+/[a-z0-9-]+$" + }, + "title": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string", + "minLength": 1 + }, + "type": { + "type": "string", + "minLength": 1 + }, + "mode": { + "enum": ["snapshot", "event", "delta"] + }, + "schema_url": { + "type": "string", + "format": "uri" + }, + "schema_version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "parameters": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "type", "description", "required"], + "properties": { + "name": { "type": "string" }, + "type": { "enum": ["string", "number", "integer", "boolean"] }, + "description": { "type": "string" }, + "required": { "type": "boolean" } + } + }, + "default": [] + }, + "source_uri_template": { + "type": "string", + "pattern": "^feed://" + }, + "adapter": { + "type": "object", + "required": ["kind"], + "properties": { + "kind": { + "enum": [ + "json_http", + "paginated_json_http", + "rss", + "ical", + "local_file", + "filesystem_scan", + "markdown_scan", + "git_status", + "local_command", + "apple_automation", + "sqlite_query", + "plist_reading_list" + ] + } + } + }, + "recommended_poll_interval_seconds": { + "type": "integer", + "minimum": 60 + }, + "auth": { + "enum": ["none", "bearer_token"] + }, + "tags": { + "type": "array", + "items": { "type": "string" } + }, + "catalog_tier": { + "type": "integer", + "minimum": 1 + }, + "catalog_order": { + "type": "integer", + "minimum": 1 + }, + "platforms": { + "type": "array", + "items": { "type": "string" } + }, + "requires": { + "type": "array", + "items": { "type": "string" } + }, + "quality_tier": { + "enum": ["verified", "community", "experimental"] + } + } +} diff --git a/skills/agentfeeds/catalog/streams/calendar/ics.yaml b/skills/agentfeeds/catalog/streams/calendar/ics.yaml new file mode 100644 index 00000000..7c6bd909 --- /dev/null +++ b/skills/agentfeeds/catalog/streams/calendar/ics.yaml @@ -0,0 +1,23 @@ +id: calendar/ics +title: iCalendar feed +description: Events from a public iCalendar URL. +type: ical.event +mode: event +schema_url: https://agentfeeds.dev/schemas/ical-event.v1.json +schema_version: 1.0.0 +parameters: + - name: url + type: string + description: Public iCalendar URL + required: true +source_uri_template: "feed://calendar.local/ics?url={url}" +adapter: + kind: ical + url: "{url}" +recommended_poll_interval_seconds: 1800 +auth: none +tags: [calendar, ics, ical, events, free, no-auth] +quality_tier: verified +catalog_tier: 4 +catalog_order: 20 +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/dev/github-issues.yaml b/skills/agentfeeds/catalog/streams/dev/github-issues.yaml new file mode 100644 index 00000000..240249a4 --- /dev/null +++ b/skills/agentfeeds/catalog/streams/dev/github-issues.yaml @@ -0,0 +1,50 @@ +id: dev/github-issues +title: GitHub repository issues +description: Recent issues for a public GitHub repository. +type: github.issue +mode: event +schema_url: https://agentfeeds.dev/schemas/github.issue.v1.json +schema_version: 1.0.0 +parameters: + - name: owner + type: string + description: GitHub repository owner + required: true + - name: repo + type: string + description: GitHub repository name + required: true + - name: state + type: string + description: Issue state, such as open, closed, or all + required: true +source_uri_template: "feed://api.github.com/repos/{owner}/{repo}/issues?state={state}" +adapter: + kind: paginated_json_http + url: "https://api.github.com/repos/{owner}/{repo}/issues?state={state}&per_page=30" + method: GET + headers: + Accept: application/vnd.github+json + transform: + language: jmespath + expression: | + [?pull_request==`null`].{ + number: number, + title: title, + state: state, + html_url: html_url, + user: user.login, + labels: labels[].name, + created_at: created_at, + updated_at: updated_at, + closed_at: closed_at, + body: body + } + id_from: number +recommended_poll_interval_seconds: 900 +auth: none +tags: [dev, github, issues, free, no-auth] +quality_tier: verified +catalog_tier: 4 +catalog_order: 70 +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/dev/github-notifications.yaml b/skills/agentfeeds/catalog/streams/dev/github-notifications.yaml new file mode 100644 index 00000000..8a317d28 --- /dev/null +++ b/skills/agentfeeds/catalog/streams/dev/github-notifications.yaml @@ -0,0 +1,38 @@ +id: dev/github-notifications +title: My GitHub notifications +description: Authenticated GitHub notifications for the current user. +type: github.notification +mode: event +schema_url: https://agentfeeds.dev/schemas/github.notification.v1.json +schema_version: 1.0.0 +parameters: [] +source_uri_template: "feed://api.github.com/notifications" +adapter: + kind: paginated_json_http + url: "https://api.github.com/notifications?per_page=50" + method: GET + auth_service: github + headers: + Accept: application/vnd.github+json + transform: + language: jmespath + expression: | + [].{ + id: id, + reason: reason, + unread: unread, + updated_at: updated_at, + repository: repository.full_name, + subject_title: subject.title, + subject_type: subject.type, + url: subject.url + } + id_from: id +recommended_poll_interval_seconds: 300 +auth: bearer_token +tags: [dev, github, notifications, keychain, auth] +quality_tier: experimental +catalog_tier: 3 +catalog_order: 10 +requires: [keychain_bearer_token] +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/dev/github-prs-mine.yaml b/skills/agentfeeds/catalog/streams/dev/github-prs-mine.yaml new file mode 100644 index 00000000..da875b05 --- /dev/null +++ b/skills/agentfeeds/catalog/streams/dev/github-prs-mine.yaml @@ -0,0 +1,43 @@ +id: dev/github-prs-mine +title: My GitHub pull requests +description: Pull requests authored by or assigned for review to the current GitHub user. +type: github.pull-request +mode: event +schema_url: https://agentfeeds.dev/schemas/github.pull-request.v1.json +schema_version: 1.0.0 +parameters: [] +source_uri_template: "feed://api.github.com/search/issues?q=type:pr+involves:@me" +adapter: + kind: paginated_json_http + url: "https://api.github.com/search/issues?q=type:pr+involves:@me+state:open&per_page=50" + method: GET + auth_service: github + headers: + Accept: application/vnd.github+json + transform: + language: jmespath + expression: | + items[].{ + number: number, + title: title, + state: state, + html_url: html_url, + user: user.login, + draft: false, + head_ref: null, + base_ref: null, + created_at: created_at, + updated_at: updated_at, + closed_at: closed_at, + merged_at: null, + body: body + } + id_from: html_url +recommended_poll_interval_seconds: 600 +auth: bearer_token +tags: [dev, github, pull-request, review, keychain, auth] +quality_tier: experimental +catalog_tier: 3 +catalog_order: 20 +requires: [keychain_bearer_token] +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/dev/github-prs.yaml b/skills/agentfeeds/catalog/streams/dev/github-prs.yaml new file mode 100644 index 00000000..7b92957a --- /dev/null +++ b/skills/agentfeeds/catalog/streams/dev/github-prs.yaml @@ -0,0 +1,53 @@ +id: dev/github-prs +title: GitHub repository pull requests +description: Recent pull requests for a public GitHub repository. +type: github.pull-request +mode: event +schema_url: https://agentfeeds.dev/schemas/github.pull-request.v1.json +schema_version: 1.0.0 +parameters: + - name: owner + type: string + description: GitHub repository owner + required: true + - name: repo + type: string + description: GitHub repository name + required: true + - name: state + type: string + description: Pull request state, such as open, closed, or all + required: true +source_uri_template: "feed://api.github.com/repos/{owner}/{repo}/pulls?state={state}" +adapter: + kind: paginated_json_http + url: "https://api.github.com/repos/{owner}/{repo}/pulls?state={state}&per_page=30" + method: GET + headers: + Accept: application/vnd.github+json + transform: + language: jmespath + expression: | + [].{ + number: number, + title: title, + state: state, + html_url: html_url, + user: user.login, + draft: draft, + head_ref: head.ref, + base_ref: base.ref, + created_at: created_at, + updated_at: updated_at, + closed_at: closed_at, + merged_at: merged_at, + body: body + } + id_from: number +recommended_poll_interval_seconds: 900 +auth: none +tags: [dev, github, pull-request, prs, free, no-auth] +quality_tier: verified +catalog_tier: 4 +catalog_order: 60 +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/dev/github-releases.yaml b/skills/agentfeeds/catalog/streams/dev/github-releases.yaml new file mode 100644 index 00000000..4a091579 --- /dev/null +++ b/skills/agentfeeds/catalog/streams/dev/github-releases.yaml @@ -0,0 +1,41 @@ +id: dev/github-releases +title: GitHub repository releases +description: Latest release events for a public GitHub repository. +type: github.release +mode: event +schema_url: https://agentfeeds.dev/schemas/github.release.v1.json +schema_version: 1.0.0 +parameters: + - name: owner + type: string + description: GitHub repository owner + required: true + - name: repo + type: string + description: GitHub repository name + required: true +source_uri_template: "feed://api.github.com/repos/{owner}/{repo}/releases" +adapter: + kind: paginated_json_http + url: "https://api.github.com/repos/{owner}/{repo}/releases" + method: GET + headers: + Accept: application/vnd.github+json + transform: + language: jmespath + expression: | + [].{ + tag_name: tag_name, + name: name, + published_at: published_at, + html_url: html_url, + body: body + } + id_from: tag_name +recommended_poll_interval_seconds: 3600 +auth: none +tags: [dev, github, releases, free, no-auth] +quality_tier: verified +catalog_tier: 4 +catalog_order: 80 +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/finance/exchangerate.yaml b/skills/agentfeeds/catalog/streams/finance/exchangerate.yaml new file mode 100644 index 00000000..36ca6e9c --- /dev/null +++ b/skills/agentfeeds/catalog/streams/finance/exchangerate.yaml @@ -0,0 +1,34 @@ +id: finance/exchangerate +title: Current exchange rates +description: Current currency exchange rates for a base currency via a no-auth public API. +type: finance.exchange-rate +mode: snapshot +schema_url: https://agentfeeds.dev/schemas/finance.exchange-rate.v1.json +schema_version: 1.0.0 +parameters: + - name: base + type: string + description: Base currency code such as USD or EUR + required: true +source_uri_template: "feed://open.er-api.com/v6/latest/{base}" +adapter: + kind: json_http + url: "https://open.er-api.com/v6/latest/{base}" + method: GET + headers: {} + transform: + language: jmespath + expression: | + { + base: base_code, + date: time_last_update_utc, + rates: rates + } + id_from: "join('-', [base_code, to_string(time_last_update_unix)])" +recommended_poll_interval_seconds: 3600 +auth: none +tags: [finance, currency, exchange-rate, free, no-auth] +quality_tier: verified +catalog_tier: 4 +catalog_order: 50 +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/local/directory-recent.yaml b/skills/agentfeeds/catalog/streams/local/directory-recent.yaml new file mode 100644 index 00000000..33ad8e02 --- /dev/null +++ b/skills/agentfeeds/catalog/streams/local/directory-recent.yaml @@ -0,0 +1,25 @@ +id: local/directory-recent +title: Recent files in a directory +description: Last modified files in a watched directory. +type: local.directory-entry +mode: event +schema_url: https://agentfeeds.dev/schemas/local.directory-entry.v1.json +schema_version: 1.0.0 +parameters: + - name: path + type: string + description: Absolute or home-relative directory path + required: true +source_uri_template: "feed://local.directory/recent?path={path}" +adapter: + kind: filesystem_scan + path: "{path}" + order_by: modified_at + limit: 25 +recommended_poll_interval_seconds: 300 +auth: none +tags: [local, files, directory, recent, no-auth] +quality_tier: experimental +catalog_tier: 2 +catalog_order: 20 +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/local/file.yaml b/skills/agentfeeds/catalog/streams/local/file.yaml new file mode 100644 index 00000000..e7714679 --- /dev/null +++ b/skills/agentfeeds/catalog/streams/local/file.yaml @@ -0,0 +1,23 @@ +id: local/file +title: Local file +description: Read-only snapshot of one local text, Markdown, or JSON file. +type: local.file +mode: snapshot +schema_url: https://agentfeeds.dev/schemas/local.file.v1.json +schema_version: 1.0.0 +parameters: + - name: path + type: string + description: Absolute or home-relative file path + required: true +source_uri_template: "feed://local.file/file?path={path}" +adapter: + kind: local_file + path: "{path}" +recommended_poll_interval_seconds: 300 +auth: none +tags: [local, file, markdown, text, json, no-auth] +quality_tier: verified +catalog_tier: 2 +catalog_order: 10 +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/local/git-status.yaml b/skills/agentfeeds/catalog/streams/local/git-status.yaml new file mode 100644 index 00000000..ce2b209f --- /dev/null +++ b/skills/agentfeeds/catalog/streams/local/git-status.yaml @@ -0,0 +1,23 @@ +id: local/git-status +title: Local Git status +description: Current branch, dirty files, and ahead or behind counts for a watched repo. +type: local.git-status +mode: snapshot +schema_url: https://agentfeeds.dev/schemas/local.git-status.v1.json +schema_version: 1.0.0 +parameters: + - name: path + type: string + description: Absolute or home-relative Git repository path + required: true +source_uri_template: "feed://local.git/status?path={path}" +adapter: + kind: git_status + path: "{path}" +recommended_poll_interval_seconds: 300 +auth: none +tags: [local, git, dev, repo, status, no-auth] +quality_tier: experimental +catalog_tier: 2 +catalog_order: 40 +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/local/markdown-vault.yaml b/skills/agentfeeds/catalog/streams/local/markdown-vault.yaml new file mode 100644 index 00000000..3836537b --- /dev/null +++ b/skills/agentfeeds/catalog/streams/local/markdown-vault.yaml @@ -0,0 +1,26 @@ +id: local/markdown-vault +title: Markdown vault +description: Recently modified Markdown notes from a local vault directory. +type: local.markdown-document +mode: event +schema_url: https://agentfeeds.dev/schemas/local.markdown-document.v1.json +schema_version: 1.0.0 +parameters: + - name: path + type: string + description: Absolute or home-relative vault directory path + required: true +source_uri_template: "feed://local.markdown/vault?path={path}" +adapter: + kind: markdown_scan + path: "{path}" + parse_frontmatter: true + order_by: modified_at + limit: 25 +recommended_poll_interval_seconds: 300 +auth: none +tags: [local, markdown, obsidian, notes, vault, no-auth] +quality_tier: experimental +catalog_tier: 2 +catalog_order: 30 +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/mac/calendar-today.yaml b/skills/agentfeeds/catalog/streams/mac/calendar-today.yaml new file mode 100644 index 00000000..14df5b23 --- /dev/null +++ b/skills/agentfeeds/catalog/streams/mac/calendar-today.yaml @@ -0,0 +1,46 @@ +id: mac/calendar-today +title: Today's Calendar.app agenda +description: Today's agenda from every calendar configured in Calendar.app. +type: ical.event +mode: event +schema_url: https://agentfeeds.dev/schemas/ical-event.v1.json +schema_version: 1.0.0 +parameters: [] +source_uri_template: "feed://mac.calendar/today" +adapter: + kind: apple_automation + tcc_permission: Calendar + script: | + set startDate to current date + set hours of startDate to 0 + set minutes of startDate to 0 + set seconds of startDate to 0 + set endDate to startDate + (1 * days) + set rows to {} + tell application "Calendar" + repeat with cal in calendars + set calName to name of cal + set matches to every event of cal whose start date is greater than or equal to startDate and start date is less than endDate + repeat with ev in matches + set evLocation to "" + try + set evLocation to location of ev + end try + set end of rows to (uid of ev) & tab & (summary of ev) & tab & ((start date of ev) as string) & tab & ((end date of ev) as string) & tab & evLocation & tab & calName + end repeat + end repeat + end tell + set AppleScript's text item delimiters to linefeed + return rows as text + columns: [uid, summary, starts_at, ends_at, location, calendar_name] + id_column: uid + time_column: starts_at +recommended_poll_interval_seconds: 900 +auth: none +tags: [mac, calendar, agenda, eventkit, tcc, no-auth] +quality_tier: experimental +catalog_tier: 1 +catalog_order: 10 +platforms: [macos] +requires: [calendar_tcc] +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/mac/calendar-upcoming.yaml b/skills/agentfeeds/catalog/streams/mac/calendar-upcoming.yaml new file mode 100644 index 00000000..18977b5f --- /dev/null +++ b/skills/agentfeeds/catalog/streams/mac/calendar-upcoming.yaml @@ -0,0 +1,46 @@ +id: mac/calendar-upcoming +title: Upcoming Calendar.app events +description: Next 7 days from every calendar configured in Calendar.app. +type: ical.event +mode: event +schema_url: https://agentfeeds.dev/schemas/ical-event.v1.json +schema_version: 1.0.0 +parameters: [] +source_uri_template: "feed://mac.calendar/upcoming?days=7" +adapter: + kind: apple_automation + tcc_permission: Calendar + script: | + set startDate to current date + set hours of startDate to 0 + set minutes of startDate to 0 + set seconds of startDate to 0 + set endDate to startDate + (7 * days) + set rows to {} + tell application "Calendar" + repeat with cal in calendars + set calName to name of cal + set matches to every event of cal whose start date is greater than or equal to startDate and start date is less than endDate + repeat with ev in matches + set evLocation to "" + try + set evLocation to location of ev + end try + set end of rows to (uid of ev) & tab & (summary of ev) & tab & ((start date of ev) as string) & tab & ((end date of ev) as string) & tab & evLocation & tab & calName + end repeat + end repeat + end tell + set AppleScript's text item delimiters to linefeed + return rows as text + columns: [uid, summary, starts_at, ends_at, location, calendar_name] + id_column: uid + time_column: starts_at +recommended_poll_interval_seconds: 3600 +auth: none +tags: [mac, calendar, upcoming, eventkit, tcc, no-auth] +quality_tier: experimental +catalog_tier: 1 +catalog_order: 20 +platforms: [macos] +requires: [calendar_tcc] +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/mac/finder-recent-downloads.yaml b/skills/agentfeeds/catalog/streams/mac/finder-recent-downloads.yaml new file mode 100644 index 00000000..61ff02e1 --- /dev/null +++ b/skills/agentfeeds/catalog/streams/mac/finder-recent-downloads.yaml @@ -0,0 +1,22 @@ +id: mac/finder-recent-downloads +title: Recent Downloads +description: Last modified items in the local Downloads folder. +type: local.directory-entry +mode: event +schema_url: https://agentfeeds.dev/schemas/local.directory-entry.v1.json +schema_version: 1.0.0 +parameters: [] +source_uri_template: "feed://mac.finder/recent-downloads" +adapter: + kind: filesystem_scan + path: "~/Downloads" + order_by: modified_at + limit: 25 +recommended_poll_interval_seconds: 300 +auth: none +tags: [mac, finder, downloads, files, recent, no-auth] +quality_tier: experimental +catalog_tier: 2 +catalog_order: 60 +platforms: [macos] +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/mac/imessage-unread.yaml b/skills/agentfeeds/catalog/streams/mac/imessage-unread.yaml new file mode 100644 index 00000000..da85c749 --- /dev/null +++ b/skills/agentfeeds/catalog/streams/mac/imessage-unread.yaml @@ -0,0 +1,39 @@ +id: mac/imessage-unread +title: Unread iMessage conversations +description: Recent iMessage conversations with unread counts and last-message snippets. +type: mac.imessage-thread +mode: event +schema_url: https://agentfeeds.dev/schemas/mac.imessage-thread.v1.json +schema_version: 1.0.0 +parameters: [] +source_uri_template: "feed://mac.imessage/unread?limit=25" +adapter: + kind: sqlite_query + database: "~/Library/Messages/chat.db" + read_only: true + tcc_permission: Full Disk Access + query: | + SELECT chat.ROWID, chat.display_name, message.text, message.date + FROM chat + JOIN chat_message_join ON chat.ROWID = chat_message_join.chat_id + JOIN message ON message.ROWID = chat_message_join.message_id + WHERE message.is_read = 0 + ORDER BY message.date DESC + LIMIT 25 + columns: [thread_id, display_name, snippet, last_message_at] + timestamp_columns: + last_message_at: mac_absolute_ns + static: + participants: [] + unread_count: 1 + id_column: thread_id + time_column: last_message_at +recommended_poll_interval_seconds: 900 +auth: none +tags: [mac, imessage, messages, unread, sqlite, tcc, no-auth] +quality_tier: experimental +catalog_tier: 1 +catalog_order: 60 +platforms: [macos] +requires: [full_disk_access] +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/mac/mail-unread.yaml b/skills/agentfeeds/catalog/streams/mac/mail-unread.yaml new file mode 100644 index 00000000..c69241e5 --- /dev/null +++ b/skills/agentfeeds/catalog/streams/mac/mail-unread.yaml @@ -0,0 +1,47 @@ +id: mac/mail-unread +title: Unread Mail.app messages +description: Unread Mail.app subjects, senders, and snippets without message bodies. +type: mac.mail-message +mode: event +schema_url: https://agentfeeds.dev/schemas/mac.mail-message.v1.json +schema_version: 1.0.0 +parameters: [] +source_uri_template: "feed://mac.mail/unread?limit=50" +adapter: + kind: apple_automation + tcc_permission: Automation + script: | + set rows to {} + tell application "Mail" + set matches to messages of inbox whose read status is false + set maxItems to count of matches + if maxItems > 50 then set maxItems to 50 + repeat with i from 1 to maxItems + set msg to item i of matches + set previewText to "" + try + set previewText to content of msg + if length of previewText > 300 then set previewText to text 1 thru 300 of previewText + end try + set end of rows to (message id of msg) & tab & (subject of msg) & tab & (sender of msg) & tab & ((date received of msg) as string) & tab & previewText + end repeat + end tell + set AppleScript's text item delimiters to linefeed + return rows as text + columns: [id, subject, sender, received_at, snippet] + static: + unread: true + mailbox: Inbox + from_email: null + account: null + id_column: id + time_column: received_at +recommended_poll_interval_seconds: 900 +auth: none +tags: [mac, mail, inbox, unread, automation, tcc, no-auth] +quality_tier: experimental +catalog_tier: 1 +catalog_order: 50 +platforms: [macos] +requires: [mail_automation_tcc] +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/mac/notes-recent.yaml b/skills/agentfeeds/catalog/streams/mac/notes-recent.yaml new file mode 100644 index 00000000..3d1d1869 --- /dev/null +++ b/skills/agentfeeds/catalog/streams/mac/notes-recent.yaml @@ -0,0 +1,45 @@ +id: mac/notes-recent +title: Recent Notes.app notes +description: Recently modified Notes.app notes with titles and snippets. +type: mac.note +mode: event +schema_url: https://agentfeeds.dev/schemas/mac.note.v1.json +schema_version: 1.0.0 +parameters: [] +source_uri_template: "feed://mac.notes/recent?limit=20" +adapter: + kind: apple_automation + tcc_permission: Automation + script: | + set rows to {} + tell application "Notes" + set allNotes to notes + set maxItems to count of allNotes + if maxItems > 20 then set maxItems to 20 + repeat with i from 1 to maxItems + set n to item i of allNotes + set bodyText to plaintext of n + if length of bodyText > 300 then set bodyText to text 1 thru 300 of bodyText + set folderName to "" + try + set folderName to name of container of n + end try + set end of rows to (id of n) & tab & (name of n) & tab & bodyText & tab & ((modification date of n) as string) & tab & folderName + end repeat + end tell + set AppleScript's text item delimiters to linefeed + return rows as text + columns: [id, title, snippet, modified_at, folder] + static: + account: null + id_column: id + time_column: modified_at +recommended_poll_interval_seconds: 900 +auth: none +tags: [mac, notes, recent, automation, tcc, no-auth] +quality_tier: experimental +catalog_tier: 1 +catalog_order: 40 +platforms: [macos] +requires: [notes_automation_tcc] +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/mac/reminders-pending.yaml b/skills/agentfeeds/catalog/streams/mac/reminders-pending.yaml new file mode 100644 index 00000000..8b60f908 --- /dev/null +++ b/skills/agentfeeds/catalog/streams/mac/reminders-pending.yaml @@ -0,0 +1,60 @@ +id: mac/reminders-pending +title: Pending Reminders.app items +description: Open reminders with due dates and list grouping from Reminders.app. +type: mac.reminder +mode: event +schema_url: https://agentfeeds.dev/schemas/mac.reminder.v1.json +schema_version: 1.0.0 +parameters: [] +source_uri_template: "feed://mac.reminders/pending" +adapter: + kind: apple_automation + tcc_permission: Reminders + script: | + set rows to {} + tell application "Reminders" + repeat with listRef in lists + set listName to name of listRef + set matches to every reminder of listRef whose completed is false + repeat with remRef in matches + set bodyText to "" + set dueText to "" + set priorityValue to 0 + try + set bodyText to body of remRef + end try + try + set dueText to (due date of remRef) as string + end try + try + set priorityValue to priority of remRef + end try + set end of rows to (id of remRef) & tab & (name of remRef) & tab & ((completed of remRef) as string) & tab & listName & tab & dueText & tab & (priorityValue as string) & tab & bodyText + end repeat + end repeat + end tell + set AppleScript's text item delimiters to linefeed + return rows as text + columns: + - id + - title + - name: completed + type: boolean + - list_name + - due_at + - name: priority + type: integer + - notes_snippet + static: + url: null + id_column: id + time_column: due_at +recommended_poll_interval_seconds: 900 +auth: none +tags: [mac, reminders, tasks, tcc, no-auth] +quality_tier: experimental +catalog_tier: 1 +catalog_order: 30 +platforms: [macos] +requires: [reminders_tcc] +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/mac/safari-reading-list.yaml b/skills/agentfeeds/catalog/streams/mac/safari-reading-list.yaml new file mode 100644 index 00000000..6f75e037 --- /dev/null +++ b/skills/agentfeeds/catalog/streams/mac/safari-reading-list.yaml @@ -0,0 +1,21 @@ +id: mac/safari-reading-list +title: Safari Reading List +description: Items saved to Safari Reading List from the local bookmarks file. +type: mac.reading-list-item +mode: event +schema_url: https://agentfeeds.dev/schemas/mac.reading-list-item.v1.json +schema_version: 1.0.0 +parameters: [] +source_uri_template: "feed://mac.safari/reading-list" +adapter: + kind: plist_reading_list + path: "~/Library/Safari/Bookmarks.plist" + limit: 50 +recommended_poll_interval_seconds: 3600 +auth: none +tags: [mac, safari, reading-list, bookmarks, no-auth] +quality_tier: experimental +catalog_tier: 2 +catalog_order: 50 +platforms: [macos] +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/news/rss-generic.yaml b/skills/agentfeeds/catalog/streams/news/rss-generic.yaml new file mode 100644 index 00000000..10892f6d --- /dev/null +++ b/skills/agentfeeds/catalog/streams/news/rss-generic.yaml @@ -0,0 +1,23 @@ +id: news/rss-generic +title: Generic RSS feed +description: Subscribe to any public RSS or Atom URL. +type: rss.item +mode: event +schema_url: https://agentfeeds.dev/schemas/rss-item.v1.json +schema_version: 1.0.0 +parameters: + - name: url + type: string + description: Public RSS or Atom feed URL + required: true +source_uri_template: "feed://rss.local/generic?url={url}" +adapter: + kind: rss + url: "{url}" +recommended_poll_interval_seconds: 900 +auth: none +tags: [news, rss, atom, free, no-auth] +quality_tier: verified +catalog_tier: 4 +catalog_order: 10 +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/notes/notion-recent.yaml b/skills/agentfeeds/catalog/streams/notes/notion-recent.yaml new file mode 100644 index 00000000..f96f1018 --- /dev/null +++ b/skills/agentfeeds/catalog/streams/notes/notion-recent.yaml @@ -0,0 +1,45 @@ +id: notes/notion-recent +title: Recent Notion pages +description: Recently edited Notion pages visible to the authenticated integration. +type: notion.page +mode: event +schema_url: https://agentfeeds.dev/schemas/notion.page.v1.json +schema_version: 1.0.0 +parameters: [] +source_uri_template: "feed://api.notion.com/pages/recent" +adapter: + kind: json_http + url: "https://api.notion.com/v1/search" + method: POST + auth_service: notion + headers: + Content-Type: application/json + Notion-Version: "2022-06-28" + body: + page_size: 50 + filter: + property: object + value: page + sort: + direction: descending + timestamp: last_edited_time + transform: + language: jmespath + expression: | + results[].{ + id: id, + title: properties.*.title[0].plain_text | [0], + url: url, + last_edited_at: last_edited_time, + created_at: created_time, + parent_type: parent.type + } + id_from: id +recommended_poll_interval_seconds: 900 +auth: bearer_token +tags: [notes, notion, recent, keychain, auth] +quality_tier: experimental +catalog_tier: 3 +catalog_order: 50 +requires: [keychain_bearer_token] +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/tasks/linear-mine.yaml b/skills/agentfeeds/catalog/streams/tasks/linear-mine.yaml new file mode 100644 index 00000000..5841aa64 --- /dev/null +++ b/skills/agentfeeds/catalog/streams/tasks/linear-mine.yaml @@ -0,0 +1,58 @@ +id: tasks/linear-mine +title: My Linear issues +description: Linear issues assigned to the authenticated user. +type: linear.issue +mode: event +schema_url: https://agentfeeds.dev/schemas/linear.issue.v1.json +schema_version: 1.0.0 +parameters: [] +source_uri_template: "feed://api.linear.app/graphql/issues/mine" +adapter: + kind: json_http + url: "https://api.linear.app/graphql" + method: POST + auth_service: linear + headers: + Content-Type: application/json + body: + query: | + query AgentFeedsMyIssues { + viewer { + assignedIssues(first: 50) { + nodes { + id + identifier + title + priority + url + updatedAt + dueDate + state { name } + team { name } + } + } + } + } + transform: + language: jmespath + expression: | + data.viewer.assignedIssues.nodes[].{ + id: id, + identifier: identifier, + title: title, + state: state.name, + priority: priority, + team: team.name, + url: url, + updated_at: updatedAt, + due_at: dueDate + } + id_from: id +recommended_poll_interval_seconds: 600 +auth: bearer_token +tags: [tasks, linear, issues, assigned, keychain, auth] +quality_tier: experimental +catalog_tier: 3 +catalog_order: 30 +requires: [keychain_bearer_token] +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/tasks/todoist-today.yaml b/skills/agentfeeds/catalog/streams/tasks/todoist-today.yaml new file mode 100644 index 00000000..a262a57a --- /dev/null +++ b/skills/agentfeeds/catalog/streams/tasks/todoist-today.yaml @@ -0,0 +1,35 @@ +id: tasks/todoist-today +title: Today's Todoist tasks +description: Todoist tasks due today for the authenticated user. +type: todoist.task +mode: event +schema_url: https://agentfeeds.dev/schemas/todoist.task.v1.json +schema_version: 1.0.0 +parameters: [] +source_uri_template: "feed://api.todoist.com/tasks/today" +adapter: + kind: json_http + url: "https://api.todoist.com/rest/v2/tasks?filter=today" + method: GET + auth_service: todoist + transform: + language: jmespath + expression: | + [].{ + id: id, + content: content, + description: description, + url: url, + priority: priority, + project_id: project_id, + due: due + } + id_from: id +recommended_poll_interval_seconds: 600 +auth: bearer_token +tags: [tasks, todoist, today, keychain, auth] +quality_tier: experimental +catalog_tier: 3 +catalog_order: 40 +requires: [keychain_bearer_token] +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/weather/openmeteo-current.yaml b/skills/agentfeeds/catalog/streams/weather/openmeteo-current.yaml new file mode 100644 index 00000000..9e95c12b --- /dev/null +++ b/skills/agentfeeds/catalog/streams/weather/openmeteo-current.yaml @@ -0,0 +1,40 @@ +id: weather/openmeteo-current +title: Current weather conditions (Open-Meteo) +description: Free, no-API-key weather observations for any latitude and longitude. +type: weather.observation +mode: snapshot +schema_url: https://agentfeeds.dev/schemas/weather.observation.v1.json +schema_version: 1.0.0 +parameters: + - name: lat + type: number + description: Latitude (-90 to 90) + required: true + - name: lon + type: number + description: Longitude (-180 to 180) + required: true +source_uri_template: "feed://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lon}" +adapter: + kind: json_http + url: "https://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lon}¤t=temperature_2m,relative_humidity_2m,wind_speed_10m,weather_code" + method: GET + headers: {} + transform: + language: jmespath + expression: | + { + temperature_c: current.temperature_2m, + humidity_pct: current.relative_humidity_2m, + wind_kph: current.wind_speed_10m, + conditions_code: current.weather_code, + observed_at: current.time + } + id_from: "join('-', [current.time, to_string(latitude), to_string(longitude)])" +recommended_poll_interval_seconds: 600 +auth: none +tags: [weather, free, no-auth, global] +quality_tier: verified +catalog_tier: 4 +catalog_order: 30 +contributed_by: agentfeeds diff --git a/skills/agentfeeds/catalog/streams/weather/openmeteo-forecast.yaml b/skills/agentfeeds/catalog/streams/weather/openmeteo-forecast.yaml new file mode 100644 index 00000000..7983b1df --- /dev/null +++ b/skills/agentfeeds/catalog/streams/weather/openmeteo-forecast.yaml @@ -0,0 +1,37 @@ +id: weather/openmeteo-forecast +title: 7-day weather forecast (Open-Meteo) +description: Free, no-API-key 7-day forecast for any latitude and longitude. +type: weather.forecast +mode: snapshot +schema_url: https://agentfeeds.dev/schemas/weather.forecast.v1.json +schema_version: 1.0.0 +parameters: + - name: lat + type: number + description: Latitude (-90 to 90) + required: true + - name: lon + type: number + description: Longitude (-180 to 180) + required: true +source_uri_template: "feed://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lon}&daily=temperature_2m_max,temperature_2m_min,weather_code" +adapter: + kind: json_http + url: "https://api.open-meteo.com/v1/forecast?latitude={lat}&longitude={lon}&daily=temperature_2m_max,temperature_2m_min,weather_code&forecast_days=7" + method: GET + headers: {} + transform: + language: jmespath + expression: | + { + generated_at: timezone, + daily: daily + } + id_from: "join('-', ['forecast', to_string(latitude), to_string(longitude)])" +recommended_poll_interval_seconds: 3600 +auth: none +tags: [weather, forecast, free, no-auth, global] +quality_tier: verified +catalog_tier: 4 +catalog_order: 40 +contributed_by: agentfeeds diff --git a/skills/agentfeeds/pyproject.toml b/skills/agentfeeds/pyproject.toml new file mode 100644 index 00000000..81feef3e --- /dev/null +++ b/skills/agentfeeds/pyproject.toml @@ -0,0 +1,38 @@ +[project] +name = "agentfeeds-skill" +version = "0.1.1" +description = "Local-first ambient context streams for compatible personal agents" +readme = "SKILL.md" +requires-python = ">=3.11" +license = "MIT" +dependencies = [ + "feedparser>=6.0.11", + "icalendar>=5.0.13", + "jmespath>=1.0.1", + "jsonschema>=4.22.0", + "PyYAML>=6.0.1", + "requests>=2.32.3", +] + +[project.scripts] +agentfeeds = "agentfeeds_runtime.commands:main" +agentfeeds-fetch = "agentfeeds_runtime.fetcher:main" +agentfeeds-install-poll = "agentfeeds_runtime.polling.install:main" +agentfeeds-uninstall-poll = "agentfeeds_runtime.polling.uninstall:main" + +[build-system] +requires = ["setuptools>=69"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["scripts/lib"] +include = ["agentfeeds_runtime*"] + +[dependency-groups] +dev = [ + "pillow>=10.0.0", + "pytest>=8.2.0", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/skills/agentfeeds/references/background-refresh.md b/skills/agentfeeds/references/background-refresh.md new file mode 100644 index 00000000..b6f157e9 --- /dev/null +++ b/skills/agentfeeds/references/background-refresh.md @@ -0,0 +1,35 @@ +# Background Refresh + +Background refresh is required for normal Agent Feeds use. It keeps subscriptions warm between conversations so the agent can answer from local state instead of rerunning source-specific fetching, searching, querying, or processing on demand. + +Check scheduler status: + +```bash +python3 scripts/agentfeeds.py admin polling status --json +python3 scripts/agentfeeds.py streams health --json +``` + +Install or update background polling: + +```bash +python3 scripts/agentfeeds.py admin polling install +``` + +Uninstall polling only when the user no longer wants ambient refresh: + +```bash +python3 scripts/agentfeeds.py admin polling uninstall +``` + +On macOS, polling uses launchd. On Linux, FreeBSD, and WSL-style POSIX environments, polling uses a tagged crontab block. Native Windows polling is not currently supported. The runtime computes the shortest configured subscription interval and floors it at 5 minutes. + +Agents should try to verify or install polling at session start. If the scheduler is unsupported or unavailable, report that Agent Feeds can still refresh explicitly but ambient refresh is degraded. + +Use stream health to distinguish scheduler setup from per-stream fetch problems. `streams health --json` reports missing state, stale state, last success, last error, and consecutive failure count for each active subscription. + +Explicit refresh remains useful for immediate freshness: + +```bash +python3 scripts/agentfeeds.py refresh --stream +python3 scripts/agentfeeds.py refresh --all +``` diff --git a/skills/agentfeeds/references/macos-personal-sources.md b/skills/agentfeeds/references/macos-personal-sources.md new file mode 100644 index 00000000..26bf71ed --- /dev/null +++ b/skills/agentfeeds/references/macos-personal-sources.md @@ -0,0 +1,71 @@ +# macOS Personal Sources + +Use this reference when the user wants local personal context from macOS apps such as Calendar, Reminders, Notes, Mail, Messages, Safari, Finder, or local folders. + +The public catalog includes built-in `mac/*` templates. Prefer these before creating operator-local templates. + +## Discover + +Find available Mac templates: + +```bash +python3 scripts/agentfeeds.py templates find mac +``` + +Useful built-ins include: + +- `mac/calendar-today`: today's Calendar.app agenda +- `mac/calendar-upcoming`: next 7 days of Calendar.app events +- `mac/reminders-pending`: pending Reminders.app items +- `mac/notes-recent`: recently modified Notes.app notes +- `mac/mail-unread`: unread Mail.app messages +- `mac/imessage-unread`: unread iMessage conversations +- `mac/safari-reading-list`: Safari Reading List items +- `mac/finder-recent-downloads`: recent items in Downloads + +## Subscribe + +Subscribe only the sources the user asks for: + +```bash +python3 scripts/agentfeeds.py subscribe mac/calendar-today +python3 scripts/agentfeeds.py subscribe mac/reminders-pending +python3 scripts/agentfeeds.py subscribe mac/mail-unread +``` + +Then check health: + +```bash +python3 scripts/agentfeeds.py streams health --json +``` + +## Permissions + +Mac templates are read-only, but the host process may need user-granted macOS permissions: + +- Calendar templates may require Calendar permission. +- Reminders templates may require Reminders permission. +- Notes and Mail templates may require Automation permission for the relevant app. +- iMessage reads `~/Library/Messages/chat.db` and may require Full Disk Access. +- Safari Reading List reads `~/Library/Safari/Bookmarks.plist`. +- Finder Downloads reads `~/Downloads`. + +If a source fails with a macOS permission error, tell the user which permission is needed, then refresh that one stream: + +```bash +python3 scripts/agentfeeds.py refresh --stream +``` + +## Answering + +For questions like "what is on my calendar today?", "what reminders are still open?", or "what recent mail needs attention?", search local state first: + +```bash +python3 scripts/agentfeeds.py search --json +``` + +Read the matching stream before answering: + +```bash +python3 scripts/agentfeeds.py streams read --limit 20 --json +``` diff --git a/skills/agentfeeds/references/runtime-setup.md b/skills/agentfeeds/references/runtime-setup.md new file mode 100644 index 00000000..923cdeed --- /dev/null +++ b/skills/agentfeeds/references/runtime-setup.md @@ -0,0 +1,53 @@ +# Runtime Setup + +Run setup from the skill root before doing real work from a fresh checkout: + +```bash +python3 scripts/setup.py +``` + +The setup script installs an editable Python runtime into: + +```text +~/.agentfeeds/runtime-venv/ +``` + +The script wrappers re-exec through that virtual environment when it exists: + +```bash +python3 scripts/agentfeeds.py --help +python3 scripts/agentfeeds_fetch.py --help +``` + +If Python cannot create a venv with `pip`, `scripts/setup.py` falls back to `uv pip install` when `uv` is available. If both `pip` and `uv` are missing, install one of them and rerun setup. + +Console entry points installed by the package remain acceptable for local development: + +```bash +agentfeeds --help +agentfeeds-fetch --help +``` + +For portable skill usage, prefer the bundled scripts. + +After setup, verify or install background refresh: + +```bash +python3 scripts/agentfeeds.py admin polling status --json +python3 scripts/agentfeeds.py admin polling install +python3 scripts/agentfeeds.py streams health --json +``` + +At session start, generate the compact prompt brief: + +```bash +python3 scripts/agentfeeds.py brief +``` + +Use the default brief for stable prompt/context slots. It intentionally avoids volatile timestamps; use `--include-freshness` only for freshness debugging. + +When a user prompt may be covered by existing ambient context, search local state before rerunning source-specific work: + +```bash +python3 scripts/agentfeeds.py search --json +``` diff --git a/skills/agentfeeds/references/template-authoring.md b/skills/agentfeeds/references/template-authoring.md new file mode 100644 index 00000000..727f4b2f --- /dev/null +++ b/skills/agentfeeds/references/template-authoring.md @@ -0,0 +1,71 @@ +# Template Authoring + +Use local templates when no built-in template fits a private file, dashboard, API, or approved read-only command. + +Built-in templates are sourced from the standalone catalog repository: + +```text +https://github.com/verkyyi/agentfeeds-catalog +``` + +The skill bundle includes a frozen catalog snapshot for offline first-run discovery. Updated catalog files are cached under `~/.agentfeeds/catalog-cache/` by the refresh worker. User-local templates live under `~/.agentfeeds/templates/` and are merged into discovery at runtime. + +Start by checking built-ins: + +```bash +python3 scripts/agentfeeds.py templates find +python3 scripts/agentfeeds.py templates show +``` + +List supported adapter kinds: + +```bash +python3 scripts/agentfeeds.py admin templates adapters +``` + +Scaffold a draft: + +```bash +python3 scripts/agentfeeds.py admin templates scaffold +``` + +The scaffold command prints the generated file paths. Edit the template YAML, then validate and dry-run it: + +```bash +python3 scripts/agentfeeds.py admin templates validate +python3 scripts/agentfeeds.py admin templates test key=value +``` + +Local templates live under the Agent Feeds runtime root: + +```text +~/.agentfeeds/templates/streams/ +~/.agentfeeds/templates/schemas/event-types/ +``` + +Use `local_command` only for explicitly approved read-only commands. Commands must be argv arrays, not shell strings. Avoid commands that mutate files, cloud resources, accounts, or external services. New command templates are written with `pending: true` and cannot run until the operator approves them. + +Local command templates require an interactive command digest approval before they can run: + +```bash +python3 scripts/agentfeeds.py admin templates approve-command [key=value ...] +``` + +Tell the user to run approval themselves in a terminal. Do not approve on the user's behalf. If the command, parameters, or template YAML change, approve the new digest before testing, subscribing, or refreshing. + +For templates that need secrets, store only references in YAML: + +```yaml +headers: + Authorization: "Bearer {{secret:github_token}}" +``` + +Tell the user to set the value with: + +```bash +python3 scripts/agentfeeds.py admin secrets set github_token +``` + +On macOS this uses Keychain when available; other platforms fall back to a local 0600 secret file under the Agent Feeds root. + +macOS personal sources are built-in catalog templates under `mac/*`. Prefer those before creating operator-local templates. diff --git a/skills/agentfeeds/scripts/agentfeeds.py b/skills/agentfeeds/scripts/agentfeeds.py new file mode 100755 index 00000000..bcc9a000 --- /dev/null +++ b/skills/agentfeeds/scripts/agentfeeds.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Run the Agent Feeds management CLI from the skill checkout.""" + +from __future__ import annotations + +import sys +import os +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +VENV_PYTHON = Path.home() / ".agentfeeds" / "runtime-venv" / "bin" / "python" +if VENV_PYTHON.exists() and Path(sys.executable).resolve() != VENV_PYTHON.resolve(): + os.execv(str(VENV_PYTHON), [str(VENV_PYTHON), __file__, *sys.argv[1:]]) +LIB = ROOT / "scripts" / "lib" +if str(LIB) not in sys.path: + sys.path.insert(0, str(LIB)) + +from agentfeeds_runtime.commands import main # noqa: E402 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/agentfeeds/scripts/agentfeeds_fetch.py b/skills/agentfeeds/scripts/agentfeeds_fetch.py new file mode 100755 index 00000000..725d897b --- /dev/null +++ b/skills/agentfeeds/scripts/agentfeeds_fetch.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Run the Agent Feeds refresh worker from the skill checkout.""" + +from __future__ import annotations + +import sys +import os +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +VENV_PYTHON = Path.home() / ".agentfeeds" / "runtime-venv" / "bin" / "python" +if VENV_PYTHON.exists() and Path(sys.executable).resolve() != VENV_PYTHON.resolve(): + os.execv(str(VENV_PYTHON), [str(VENV_PYTHON), __file__, *sys.argv[1:]]) +LIB = ROOT / "scripts" / "lib" +if str(LIB) not in sys.path: + sys.path.insert(0, str(LIB)) + +from agentfeeds_runtime.fetcher import main # noqa: E402 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/agentfeeds/scripts/lib/agentfeeds_runtime/__init__.py b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/__init__.py new file mode 100644 index 00000000..33e82b34 --- /dev/null +++ b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/__init__.py @@ -0,0 +1,3 @@ +"""Agent Feeds reference implementation.""" + +__version__ = "0.0.0" diff --git a/skills/agentfeeds/scripts/lib/agentfeeds_runtime/__main__.py b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/__main__.py new file mode 100644 index 00000000..e1fae542 --- /dev/null +++ b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/__main__.py @@ -0,0 +1,9 @@ +"""Run the Agent Feeds management CLI with `python -m agentfeeds_runtime`.""" + +from __future__ import annotations + +from agentfeeds_runtime.commands import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/__init__.py b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/__init__.py new file mode 100644 index 00000000..01220b8b --- /dev/null +++ b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/__init__.py @@ -0,0 +1,41 @@ +"""Adapter registry for Agent Feeds streams.""" + +from __future__ import annotations + +from agentfeeds_runtime.adapters.http import fetch_json +from agentfeeds_runtime.adapters.ical import fetch_ical +from agentfeeds_runtime.adapters.local_sources import fetch_filesystem_scan, fetch_git_status, fetch_markdown_scan +from agentfeeds_runtime.adapters.local_command import fetch_local_command +from agentfeeds_runtime.adapters.local_file import fetch_local_file +from agentfeeds_runtime.adapters.mac_native import fetch_apple_automation, fetch_plist_reading_list, fetch_sqlite_query +from agentfeeds_runtime.adapters.rss import fetch_rss + + +def run_adapter(stream: dict, parameters: dict, *, validate_parameters, source_uri_for, substitute) -> tuple[str, list[dict]]: + validate_parameters(stream, parameters) + stream_uri = source_uri_for(stream, parameters) + adapter = substitute(stream["adapter"], parameters) + kind = adapter["kind"] + if kind in {"json_http", "paginated_json_http"}: + return stream_uri, fetch_json(stream, adapter, stream_uri) + if kind == "rss": + return stream_uri, fetch_rss(stream, adapter, stream_uri) + if kind == "ical": + return stream_uri, fetch_ical(stream, adapter, stream_uri) + if kind == "local_file": + return stream_uri, fetch_local_file(stream, adapter, stream_uri) + if kind == "filesystem_scan": + return stream_uri, fetch_filesystem_scan(stream, adapter, stream_uri) + if kind == "markdown_scan": + return stream_uri, fetch_markdown_scan(stream, adapter, stream_uri) + if kind == "git_status": + return stream_uri, fetch_git_status(stream, adapter, stream_uri) + if kind == "local_command": + return stream_uri, fetch_local_command(stream, adapter, stream_uri) + if kind == "apple_automation": + return stream_uri, fetch_apple_automation(stream, adapter, stream_uri) + if kind == "sqlite_query": + return stream_uri, fetch_sqlite_query(stream, adapter, stream_uri) + if kind == "plist_reading_list": + return stream_uri, fetch_plist_reading_list(stream, adapter, stream_uri) + raise ValueError(f"unsupported adapter kind: {kind}") diff --git a/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/common.py b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/common.py new file mode 100644 index 00000000..723aa01f --- /dev/null +++ b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/common.py @@ -0,0 +1,40 @@ +"""Shared adapter helpers.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import UTC, datetime + +import jmespath + +from agentfeeds_runtime.constants import AGENTFEEDS_VERSION + + +def now_utc() -> str: + return datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def stable_hash(value: object) -> str: + encoded = json.dumps(value, sort_keys=True, default=str).encode("utf-8") + return hashlib.sha256(encoded).hexdigest()[:16] + + +def jmespath_search(expression: str | None, document: object) -> object: + if not expression: + return None + return jmespath.search(expression, document) + + +def envelope(stream: dict, stream_uri: str, event_id: object, data: dict, event_time: str | None = None) -> dict: + return { + "specversion": AGENTFEEDS_VERSION, + "id": str(event_id or stable_hash(data)), + "source": stream_uri, + "type": stream["type"], + "time": event_time or now_utc(), + "schema_url": stream["schema_url"], + "schema_version": stream["schema_version"], + "mode": stream["mode"], + "data": data, + } diff --git a/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/http.py b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/http.py new file mode 100644 index 00000000..8239b50c --- /dev/null +++ b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/http.py @@ -0,0 +1,38 @@ +"""HTTP JSON adapters.""" + +from __future__ import annotations + +import requests + +from agentfeeds_runtime.adapters.common import envelope, jmespath_search, stable_hash +from agentfeeds_runtime.constants import REQUEST_TIMEOUT_SECONDS + + +def fetch_json(stream: dict, adapter: dict, stream_uri: str) -> list[dict]: + response = requests.request( + adapter.get("method", "GET"), + adapter["url"], + headers=adapter.get("headers") or {}, + json=adapter.get("body"), + timeout=REQUEST_TIMEOUT_SECONDS, + ) + response.raise_for_status() + raw = response.json() + expression = adapter.get("transform", {}).get("expression") + transformed = jmespath_search(expression, raw) if expression else raw + + if adapter["kind"] == "json_http" and stream["mode"] == "snapshot": + if not isinstance(transformed, dict): + raise ValueError(f"{stream['id']}: json_http transform must produce an object") + event_id = jmespath_search(adapter.get("id_from"), raw) or stable_hash(transformed) + return [envelope(stream, stream_uri, event_id, transformed)] + + if not isinstance(transformed, list): + raise ValueError(f"{stream['id']}: {adapter['kind']} event transform must produce an array") + events = [] + for item in transformed: + if not isinstance(item, dict): + raise ValueError(f"{stream['id']}: {adapter['kind']} event items must be objects") + event_id = jmespath_search(adapter.get("id_from"), item) or stable_hash(item) + events.append(envelope(stream, stream_uri, event_id, item)) + return events diff --git a/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/ical.py b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/ical.py new file mode 100644 index 00000000..c6055591 --- /dev/null +++ b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/ical.py @@ -0,0 +1,35 @@ +"""iCalendar adapter.""" + +from __future__ import annotations + +import icalendar +import requests + +from agentfeeds_runtime.adapters.common import envelope, stable_hash +from agentfeeds_runtime.constants import REQUEST_TIMEOUT_SECONDS + + +def serialize_ical_value(value: object) -> str | None: + if value is None: + return None + decoded = getattr(value, "dt", value) + if hasattr(decoded, "isoformat"): + return decoded.isoformat() + return str(decoded) + + +def fetch_ical(stream: dict, adapter: dict, stream_uri: str) -> list[dict]: + response = requests.get(adapter["url"], timeout=REQUEST_TIMEOUT_SECONDS) + response.raise_for_status() + calendar = icalendar.Calendar.from_ical(response.content) + events = [] + for component in calendar.walk("VEVENT"): + data = { + "uid": str(component.get("uid", "")), + "summary": str(component.get("summary", "")), + "starts_at": serialize_ical_value(component.get("dtstart")), + "ends_at": serialize_ical_value(component.get("dtend")), + "location": str(component.get("location")) if component.get("location") else None, + } + events.append(envelope(stream, stream_uri, data["uid"] or stable_hash(data), data)) + return events diff --git a/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/local_command.py b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/local_command.py new file mode 100644 index 00000000..79bd820b --- /dev/null +++ b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/local_command.py @@ -0,0 +1,113 @@ +"""Local command adapter.""" + +from __future__ import annotations + +import json +import os +import signal +import subprocess +from pathlib import Path + +from agentfeeds_runtime.adapters.common import envelope, jmespath_search, now_utc, stable_hash +from agentfeeds_runtime.constants import COMMAND_MAX_OUTPUT_BYTES, COMMAND_TIMEOUT_SECONDS + + +def _decode_limited(raw: bytes, limit: int) -> tuple[str, bool]: + truncated = len(raw) > limit + return raw[:limit].decode("utf-8", errors="replace"), truncated + + +def run_local_command(stream: dict, adapter: dict) -> dict: + command = adapter.get("command") + if not isinstance(command, list) or not command or not all(isinstance(item, str) for item in command): + raise ValueError(f"{stream['id']}: local_command adapter.command must be a non-empty string array") + + timeout_seconds = int(adapter.get("timeout_seconds") or COMMAND_TIMEOUT_SECONDS) + max_output_bytes = int(adapter.get("max_output_bytes") or COMMAND_MAX_OUTPUT_BYTES) + cwd = adapter.get("cwd") + if cwd is not None: + cwd = str(Path(str(cwd)).expanduser()) + + started_at = now_utc() + process = subprocess.Popen( + command, + cwd=cwd, + env={key: os.environ[key] for key in ("HOME", "PATH", "USER", "SHELL") if key in os.environ}, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=os.name == "posix", + ) + timed_out = False + try: + stdout_raw, stderr_raw = process.communicate(timeout=timeout_seconds) + except subprocess.TimeoutExpired: + timed_out = True + if os.name == "posix": + os.killpg(process.pid, signal.SIGKILL) + else: # pragma: no cover - native Windows is not a supported polling target yet. + process.kill() + stdout_raw, stderr_raw = process.communicate() + ran_at = now_utc() + stdout, stdout_truncated = _decode_limited(stdout_raw, max_output_bytes) + stderr, stderr_truncated = _decode_limited(stderr_raw, max_output_bytes) + + parsed_json = None + transformed = None + if adapter.get("parse") == "json" and not timed_out: + parsed_json = json.loads(stdout or "null") + expression = adapter.get("transform", {}).get("expression") + transformed = jmespath_search(expression, parsed_json) if expression else parsed_json + + return { + "command": command, + "cwd": cwd, + "exit_code": process.returncode, + "timed_out": timed_out, + "stdout": stdout, + "stderr": stderr, + "stdout_truncated": stdout_truncated, + "stderr_truncated": stderr_truncated, + "parsed_json": parsed_json, + "transformed": transformed, + "started_at": started_at, + "ran_at": ran_at, + } + + +def fetch_local_command_events(stream: dict, adapter: dict, stream_uri: str, result: dict) -> list[dict]: + if adapter.get("parse") != "json": + raise ValueError(f"{stream['id']}: local_command event streams require parse: json") + + items_expression = adapter.get("items_from") or "@" + items = jmespath_search(items_expression, result["parsed_json"]) + if not isinstance(items, list): + raise ValueError(f"{stream['id']}: local_command items_from must produce an array") + + transform_expression = adapter.get("transform", {}).get("expression") + id_expression = adapter.get("id_from") + time_expression = adapter.get("time_from") + events = [] + for item in items: + transformed = jmespath_search(transform_expression, item) if transform_expression else item + if not isinstance(transformed, dict): + raise ValueError(f"{stream['id']}: local_command event transform must produce an object") + event_id = jmespath_search(id_expression, item) if id_expression else None + if event_id is None: + event_id = stable_hash(item) + event_time = jmespath_search(time_expression, item) if time_expression else None + if event_time is None: + event_time = result["ran_at"] + events.append(envelope(stream, stream_uri, event_id, transformed, str(event_time) if event_time else None)) + return events + + +def fetch_local_command(stream: dict, adapter: dict, stream_uri: str) -> list[dict]: + result = run_local_command(stream, adapter) + + if stream["mode"] == "event": + return fetch_local_command_events(stream, adapter, stream_uri, result) + if stream["mode"] != "snapshot": + raise ValueError(f"{stream['id']}: local_command supports snapshot and event modes") + + data = result + return [envelope(stream, stream_uri, stable_hash(data), data, result["ran_at"])] diff --git a/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/local_file.py b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/local_file.py new file mode 100644 index 00000000..fe8e8b21 --- /dev/null +++ b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/local_file.py @@ -0,0 +1,33 @@ +"""Local file adapter.""" + +from __future__ import annotations + +import hashlib +from datetime import UTC, datetime +from pathlib import Path + +from agentfeeds_runtime.adapters.common import envelope + + +def fetch_local_file(stream: dict, adapter: dict, stream_uri: str) -> list[dict]: + path = Path(adapter["path"]).expanduser() + if not path.is_absolute(): + path = path.resolve() + if not path.exists(): + raise FileNotFoundError(f"{stream['id']}: local file not found: {path}") + if not path.is_file(): + raise ValueError(f"{stream['id']}: local path is not a file: {path}") + + raw = path.read_bytes() + stat = path.stat() + modified_at = datetime.fromtimestamp(stat.st_mtime, UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") + data = { + "path": str(path), + "name": path.name, + "extension": path.suffix.lstrip("."), + "content": raw.decode("utf-8", errors="replace"), + "size_bytes": stat.st_size, + "sha256": hashlib.sha256(raw).hexdigest(), + "modified_at": modified_at, + } + return [envelope(stream, stream_uri, data["sha256"], data, modified_at)] diff --git a/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/local_sources.py b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/local_sources.py new file mode 100644 index 00000000..f1a7b195 --- /dev/null +++ b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/local_sources.py @@ -0,0 +1,125 @@ +"""Local filesystem and repository adapters.""" + +from __future__ import annotations + +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +from agentfeeds_runtime.adapters.common import envelope, stable_hash + + +def _iso_mtime(path: Path) -> str: + return datetime.fromtimestamp(path.stat().st_mtime, UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _resolve_path(value: str) -> Path: + path = Path(value).expanduser() + return path if path.is_absolute() else path.resolve() + + +def _entry_kind(path: Path) -> str: + if path.is_symlink(): + return "symlink" + if path.is_file(): + return "file" + if path.is_dir(): + return "directory" + return "other" + + +def _directory_entry(path: Path) -> dict: + stat = path.stat() + return { + "path": str(path), + "name": path.name, + "kind": _entry_kind(path), + "extension": path.suffix.lstrip(".") or None, + "size_bytes": stat.st_size if path.is_file() else None, + "modified_at": _iso_mtime(path), + } + + +def fetch_filesystem_scan(stream: dict, adapter: dict, stream_uri: str) -> list[dict]: + root = _resolve_path(adapter["path"]) + if not root.exists(): + raise FileNotFoundError(f"{stream['id']}: local directory not found: {root}") + if not root.is_dir(): + raise ValueError(f"{stream['id']}: local path is not a directory: {root}") + limit = int(adapter.get("limit") or 25) + entries = [_directory_entry(path) for path in root.iterdir()] + entries.sort(key=lambda item: item["modified_at"], reverse=True) + return [envelope(stream, stream_uri, stable_hash(item), item, item["modified_at"]) for item in entries[:limit]] + + +def _parse_frontmatter(text: str) -> tuple[dict, str]: + if not text.startswith("---\n"): + return {}, text + end = text.find("\n---\n", 4) + if end == -1: + return {}, text + raw = text[4:end] + body = text[end + 5 :] + data = {} + for line in raw.splitlines(): + if ":" not in line: + continue + key, value = line.split(":", 1) + data[key.strip()] = value.strip().strip("\"'") + return data, body + + +def fetch_markdown_scan(stream: dict, adapter: dict, stream_uri: str) -> list[dict]: + root = _resolve_path(adapter["path"]) + if not root.exists(): + raise FileNotFoundError(f"{stream['id']}: markdown vault not found: {root}") + if not root.is_dir(): + raise ValueError(f"{stream['id']}: markdown vault path is not a directory: {root}") + limit = int(adapter.get("limit") or 25) + docs = [] + for path in root.rglob("*.md"): + text = path.read_text(encoding="utf-8", errors="replace") + frontmatter, body = _parse_frontmatter(text) if adapter.get("parse_frontmatter") else ({}, text) + title = frontmatter.get("title") or next((line.lstrip("# ").strip() for line in body.splitlines() if line.strip()), path.stem) + snippet = " ".join(line.strip() for line in body.splitlines() if line.strip())[:500] + tags = frontmatter.get("tags") or "" + if isinstance(tags, str): + tags = [item.strip() for item in tags.strip("[]").split(",") if item.strip()] + item = { + "path": str(path), + "title": title, + "snippet": snippet, + "modified_at": _iso_mtime(path), + "frontmatter": frontmatter, + "tags": tags, + } + docs.append(item) + docs.sort(key=lambda item: item["modified_at"], reverse=True) + return [envelope(stream, stream_uri, stable_hash(item), item, item["modified_at"]) for item in docs[:limit]] + + +def _git(repo: Path, *args: str) -> str: + return subprocess.run(["git", "-C", str(repo), *args], check=False, text=True, capture_output=True).stdout.strip() + + +def fetch_git_status(stream: dict, adapter: dict, stream_uri: str) -> list[dict]: + repo = _resolve_path(adapter["path"]) + if not (repo / ".git").exists(): + raise ValueError(f"{stream['id']}: path is not a git repository: {repo}") + branch = _git(repo, "branch", "--show-current") or "HEAD" + dirty_files = [line[3:] for line in _git(repo, "status", "--porcelain").splitlines() if line] + ahead = behind = 0 + upstream = _git(repo, "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{upstream}") + if upstream: + counts = _git(repo, "rev-list", "--left-right", "--count", f"{upstream}...HEAD").split() + if len(counts) == 2: + behind, ahead = int(counts[0]), int(counts[1]) + data = { + "path": str(repo), + "branch": branch, + "clean": not dirty_files, + "dirty_files": dirty_files, + "ahead": ahead, + "behind": behind, + } + return [envelope(stream, stream_uri, stable_hash(data), data)] diff --git a/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/mac_native.py b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/mac_native.py new file mode 100644 index 00000000..7a9d7263 --- /dev/null +++ b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/mac_native.py @@ -0,0 +1,158 @@ +"""macOS native read-only adapters.""" + +from __future__ import annotations + +import platform +import plistlib +import sqlite3 +import subprocess +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from agentfeeds_runtime.adapters.common import envelope, stable_hash + + +def _require_macos(stream: dict) -> None: + if platform.system() != "Darwin": + raise RuntimeError(f"{stream['id']}: this template requires macOS") + + +def _osascript(stream: dict, script: str) -> str: + _require_macos(stream) + result = subprocess.run(["osascript", "-e", script], check=False, text=True, capture_output=True, timeout=30) + if result.returncode: + raise RuntimeError(f"{stream['id']}: osascript failed: {result.stderr.strip()}") + return result.stdout + + +def _rows(output: str, min_parts: int) -> list[list[str]]: + rows = [] + for line in output.splitlines(): + parts = line.split("\t") + if len(parts) >= min_parts: + rows.append(parts) + return rows + + +def _convert(value: object, value_type: str | None) -> object: + if value == "": + return None + if value_type == "boolean": + return str(value).lower() == "true" + if value_type == "integer": + try: + return int(value) + except (TypeError, ValueError): + return None + if value_type == "number": + try: + return float(value) + except (TypeError, ValueError): + return None + return value + + +def _column_name(column: object) -> str: + return column["name"] if isinstance(column, dict) else str(column) + + +def _column_type(column: object) -> str | None: + return column.get("type") if isinstance(column, dict) else None + + +def _row_data(columns: list[object], values: list[object], static: dict | None = None) -> dict: + data = dict(static or {}) + for column, value in zip(columns, values, strict=False): + data[_column_name(column)] = _convert(value, _column_type(column)) + return data + + +def fetch_apple_automation(stream: dict, adapter: dict, stream_uri: str) -> list[dict]: + columns = adapter.get("columns") or [] + if not columns: + raise ValueError(f"{stream['id']}: adapter.columns is required for apple_automation") + script = adapter["script"] + rows = _rows(_osascript(stream, script), len(columns)) + id_column = adapter.get("id_column") + time_column = adapter.get("time_column") + static = adapter.get("static") or {} + events = [] + for row in rows: + data = _row_data(columns, row, static) + event_id = data.get(id_column) if id_column else None + event_time = data.get(time_column) if time_column else None + events.append(envelope(stream, stream_uri, event_id or stable_hash(data), data, event_time)) + return events + + +def _mac_absolute_epoch(value: float) -> str: + dt = datetime(2001, 1, 1, tzinfo=UTC) + timedelta(seconds=float(value)) + return dt.replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _walk_reading_list(node: object) -> list[dict]: + items = [] + if isinstance(node, dict): + uri = node.get("URLString") + extra = node.get("ReadingList") or {} + if uri and extra: + items.append( + { + "title": node.get("URIDictionary", {}).get("title") or uri, + "url": uri, + "added_at": _mac_absolute_epoch(extra["DateAdded"]) if extra.get("DateAdded") else None, + "preview_text": extra.get("PreviewText"), + } + ) + for value in node.values(): + items.extend(_walk_reading_list(value)) + elif isinstance(node, list): + for value in node: + items.extend(_walk_reading_list(value)) + return items + + +def fetch_plist_reading_list(stream: dict, adapter: dict, stream_uri: str) -> list[dict]: + path = Path(adapter["path"]).expanduser() + if not path.exists(): + raise FileNotFoundError(f"{stream['id']}: Safari bookmarks file not found: {path}") + payload = plistlib.loads(path.read_bytes()) + items = _walk_reading_list(payload) + items.sort(key=lambda item: item.get("added_at") or "", reverse=True) + limit = int(adapter.get("limit") or 50) + return [envelope(stream, stream_uri, item["url"], item, item.get("added_at")) for item in items[:limit]] + + +def _convert_sqlite_row(adapter: dict, columns: list[object], row: tuple) -> dict: + data = _row_data(columns, list(row), adapter.get("static") or {}) + for column, encoding in (adapter.get("timestamp_columns") or {}).items(): + if encoding == "mac_absolute_ns" and data.get(column) is not None: + data[column] = _mac_absolute_epoch(float(data[column]) / 1_000_000_000) + elif encoding == "mac_absolute_seconds" and data.get(column) is not None: + data[column] = _mac_absolute_epoch(float(data[column])) + return data + + +def fetch_sqlite_query(stream: dict, adapter: dict, stream_uri: str) -> list[dict]: + if adapter.get("tcc_permission"): + _require_macos(stream) + path = Path(adapter["database"]).expanduser() + if not path.exists(): + raise FileNotFoundError(f"{stream['id']}: SQLite database not found: {path}") + columns = adapter.get("columns") or [] + if not columns: + raise ValueError(f"{stream['id']}: adapter.columns is required for sqlite_query") + connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + try: + rows = connection.execute(adapter["query"], adapter.get("params") or []).fetchall() + finally: + connection.close() + id_column = adapter.get("id_column") + time_column = adapter.get("time_column") + events = [] + for row in rows: + data = _convert_sqlite_row(adapter, columns, row) + event_id = data.get(id_column) if id_column else None + event_time = data.get(time_column) if time_column else None + events.append(envelope(stream, stream_uri, event_id or stable_hash(data), data, event_time)) + return events diff --git a/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/rss.py b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/rss.py new file mode 100644 index 00000000..2da2ccfc --- /dev/null +++ b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/adapters/rss.py @@ -0,0 +1,25 @@ +"""RSS and Atom adapter.""" + +from __future__ import annotations + +import feedparser + +from agentfeeds_runtime.adapters.common import envelope, stable_hash + + +def fetch_rss(stream: dict, adapter: dict, stream_uri: str) -> list[dict]: + parsed = feedparser.parse(adapter["url"]) + if parsed.bozo: + raise ValueError(f"{stream['id']}: failed to parse RSS feed: {parsed.bozo_exception}") + events = [] + for entry in parsed.entries: + data = { + "title": entry.get("title", ""), + "link": entry.get("link"), + "summary": entry.get("summary"), + "published": entry.get("published"), + "id": entry.get("id") or entry.get("guid") or entry.get("link"), + } + event_id = data["id"] or stable_hash(data) + events.append(envelope(stream, stream_uri, event_id, data)) + return events diff --git a/skills/agentfeeds/scripts/lib/agentfeeds_runtime/commands.py b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/commands.py new file mode 100644 index 00000000..8b066983 --- /dev/null +++ b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/commands.py @@ -0,0 +1,1635 @@ +"""User-facing Agent Feeds management CLI.""" + +from __future__ import annotations + +import argparse +import getpass +import hashlib +import json +import os +import plistlib +import platform +import re +import subprocess +import sys +from collections import Counter +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +import yaml + +from agentfeeds_runtime import fetcher as fetch +from agentfeeds_runtime.polling import install as polling_install +from agentfeeds_runtime.polling import uninstall as polling_uninstall + + +INSTANCE_ID_PATTERN = re.compile(r"^[a-z0-9-]+/[a-z0-9][a-z0-9-]*$") +ADAPTER_KINDS = { + "local_file": "Read one local text, Markdown, or JSON file as a snapshot.", + "filesystem_scan": "Scan a local directory and emit recent file entries.", + "markdown_scan": "Scan a local Markdown directory and emit recent documents.", + "git_status": "Read local Git branch, dirty files, and ahead/behind status.", + "local_command": "Run an argv-only local command for a snapshot or JSON-derived events.", + "json_http": "Fetch one HTTP JSON document and transform it into a snapshot.", + "paginated_json_http": "Fetch an HTTP JSON array and transform it into event items.", + "rss": "Fetch an RSS or Atom feed as event items.", + "ical": "Fetch an iCalendar URL as event items.", + "apple_automation": "Run read-only AppleScript automation and map tab-delimited rows to events.", + "sqlite_query": "Run a read-only SQLite query and map rows to events.", + "plist_reading_list": "Read Safari-style Reading List entries from a property-list file.", +} +POLLING_LABEL = "dev.agentfeeds.fetch" +POLLING_BEGIN_MARKER = "# BEGIN Agent Feeds polling" +POLLING_END_MARKER = "# END Agent Feeds polling" +SECRET_REF_PATTERN = re.compile(r"\{\{secret:([A-Za-z_][A-Za-z0-9_.-]*)\}\}") +QUERY_TOKEN_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") +QUERY_STOPWORDS = { + "about", + "after", + "also", + "and", + "are", + "can", + "could", + "did", + "does", + "for", + "from", + "has", + "have", + "how", + "into", + "latest", + "me", + "my", + "now", + "of", + "on", + "please", + "say", + "show", + "tell", + "that", + "the", + "this", + "to", + "was", + "what", + "when", + "where", + "who", + "why", + "with", +} + + +def parse_value(value: str) -> Any: + lowered = value.lower() + if lowered == "true": + return True + if lowered == "false": + return False + try: + if "." in value: + return float(value) + return int(value) + except ValueError: + return value + + +def parse_params(items: list[str]) -> dict[str, Any]: + params = {} + for item in items: + if "=" not in item: + raise SystemExit(f"parameters must be key=value, got: {item}") + key, value = item.split("=", 1) + if not key: + raise SystemExit(f"parameter key cannot be empty: {item}") + params[key] = parse_value(value) + return params + + +def save_subscriptions(root: Path, config: dict) -> None: + root.mkdir(parents=True, exist_ok=True) + path = root / "subscriptions.yaml" + tmp_path = path.with_suffix(".yaml.tmp") + tmp_path.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") + tmp_path.replace(path) + + +def secret_refs_in_template(value: object) -> set[str]: + refs: set[str] = set() + if isinstance(value, str): + refs.update(SECRET_REF_PATTERN.findall(value)) + elif isinstance(value, list): + for item in value: + refs.update(secret_refs_in_template(item)) + elif isinstance(value, dict): + for item in value.values(): + refs.update(secret_refs_in_template(item)) + return refs + + +def active_subscriptions(root: Path) -> dict: + fetch.ensure_root(root) + return fetch.load_subscriptions(root) + + +def stream_summary(root: Path, stream_id: str) -> dict: + return fetch.load_stream_definition(root, stream_id) + + +def template_id_for(subscription: dict) -> str: + return str(subscription["template"]) + + +def state_path_for_subscription(root: Path, subscription: dict) -> Path | None: + try: + stream = fetch.load_stream_definition(root, template_id_for(subscription)) + stream_uri = fetch.source_uri_for(stream, subscription.get("parameters") or {}) + return fetch.state_path_for_stream(stream_uri, root) + except Exception: + return None + + +def _slugify(value: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") + return slug or "feed" + + +def _hash_suffix(value: object) -> str: + encoded = json.dumps(value, sort_keys=True, default=str).encode("utf-8") + return hashlib.sha256(encoded).hexdigest()[:6] + + +def _template_id_parts(template_id: str) -> tuple[str, str]: + if not INSTANCE_ID_PATTERN.match(template_id): + raise ValueError("template id must look like category/name using lowercase letters, numbers, and hyphens") + return tuple(template_id.split("/", 1)) # type: ignore[return-value] + + +def _title_from_slug(slug: str) -> str: + return slug.replace("-", " ").title() + + +def _schema_payload(template_id: str, mode: str) -> dict: + category, name = _template_id_parts(template_id) + type_name = f"{category}.{name.replace('-', '.')}" + data = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": f"https://agentfeeds.dev/schemas/{type_name}.v1.json", + "title": _title_from_slug(name), + "type": "object", + "required": ["title"], + "properties": { + "title": {"type": "string"}, + "url": {"type": ["string", "null"]}, + "content": {"type": ["string", "null"]}, + "updated_at": {"type": ["string", "null"]}, + "stdout": {"type": ["string", "null"]}, + "transformed": {}, + }, + } + if mode == "event": + data["required"] = ["title"] + return data + + +def scaffold_stream(template_id: str, adapter_kind: str) -> tuple[dict, dict, Path, Path]: + if adapter_kind not in ADAPTER_KINDS: + raise ValueError(f"unsupported adapter kind: {adapter_kind}") + category, name = _template_id_parts(template_id) + title = _title_from_slug(name) + type_name = f"{category}.{name.replace('-', '.')}" + mode = ( + "event" + if adapter_kind in { + "paginated_json_http", + "rss", + "ical", + "filesystem_scan", + "markdown_scan", + "apple_automation", + "sqlite_query", + "plist_reading_list", + } + else "snapshot" + ) + schema_name = f"{type_name}.v1.json" + stream_path = Path(category) / f"{name}.yaml" + schema_path = Path(schema_name) + + parameters = [] + source_uri_template = f"feed://{type_name}/source" + adapter: dict[str, object] = {"kind": adapter_kind} + transform = { + "language": "jmespath", + "expression": "{title: title, url: url, content: body, updated_at: updated_at}", + } + + if adapter_kind == "local_file": + parameters = [{"name": "path", "type": "string", "description": "Local file path", "required": True}] + source_uri_template = f"feed://{type_name}/file?path={{path}}" + adapter = {"kind": "local_file", "path": "{path}"} + elif adapter_kind == "filesystem_scan": + parameters = [{"name": "path", "type": "string", "description": "Local directory path", "required": True}] + source_uri_template = f"feed://{type_name}/directory?path={{path}}" + adapter = {"kind": "filesystem_scan", "path": "{path}", "order_by": "modified_at", "limit": 25} + elif adapter_kind == "markdown_scan": + parameters = [{"name": "path", "type": "string", "description": "Markdown directory path", "required": True}] + source_uri_template = f"feed://{type_name}/markdown?path={{path}}" + adapter = {"kind": "markdown_scan", "path": "{path}", "parse_frontmatter": True, "order_by": "modified_at", "limit": 25} + elif adapter_kind == "git_status": + parameters = [{"name": "path", "type": "string", "description": "Git repository path", "required": True}] + source_uri_template = f"feed://{type_name}/git?path={{path}}" + adapter = {"kind": "git_status", "path": "{path}"} + elif adapter_kind == "local_command": + parameters = [] + source_uri_template = f"feed://{type_name}/command" + adapter = { + "kind": "local_command", + "command": ["echo", "{\"title\":\"example\",\"status\":\"ok\"}"], + "timeout_seconds": 20, + "max_output_bytes": 1048576, + "parse": "json", + "transform": { + "language": "jmespath", + "expression": "{title: title, content: status}", + }, + } + type_name = "local.command" + schema_name = "local.command.v1.json" + schema_path = Path(schema_name) + elif adapter_kind == "json_http": + parameters = [{"name": "url", "type": "string", "description": "JSON API URL", "required": True}] + source_uri_template = f"feed://{type_name}/json?url={{url}}" + adapter = {"kind": "json_http", "url": "{url}", "method": "GET", "headers": {}, "transform": transform} + elif adapter_kind == "paginated_json_http": + parameters = [{"name": "url", "type": "string", "description": "JSON API URL", "required": True}] + source_uri_template = f"feed://{type_name}/items?url={{url}}" + adapter = {"kind": "paginated_json_http", "url": "{url}", "method": "GET", "headers": {}, "transform": transform, "id_from": "url"} + elif adapter_kind == "rss": + parameters = [{"name": "url", "type": "string", "description": "RSS or Atom URL", "required": True}] + source_uri_template = f"feed://{type_name}/rss?url={{url}}" + adapter = {"kind": "rss", "url": "{url}"} + type_name = "rss.item" + schema_name = "rss-item.v1.json" + schema_path = Path(schema_name) + elif adapter_kind == "ical": + parameters = [{"name": "url", "type": "string", "description": "iCalendar URL", "required": True}] + source_uri_template = f"feed://{type_name}/ics?url={{url}}" + adapter = {"kind": "ical", "url": "{url}"} + type_name = "ical.event" + schema_name = "ical-event.v1.json" + schema_path = Path(schema_name) + elif adapter_kind == "apple_automation": + source_uri_template = f"feed://{type_name}/apple-automation" + adapter = { + "kind": "apple_automation", + "tcc_permission": "Automation", + "script": "set rows to {\"example-id\" & tab & \"Example title\"}\nset AppleScript's text item delimiters to linefeed\nreturn rows as text", + "columns": ["id", "title"], + "id_column": "id", + } + elif adapter_kind == "sqlite_query": + parameters = [{"name": "database", "type": "string", "description": "SQLite database path", "required": True}] + source_uri_template = f"feed://{type_name}/sqlite?database={{database}}" + adapter = { + "kind": "sqlite_query", + "database": "{database}", + "tcc_permission": "Full Disk Access", + "query": "SELECT 1 AS id, 'Example title' AS title", + "columns": ["id", "title"], + "id_column": "id", + } + elif adapter_kind == "plist_reading_list": + parameters = [{"name": "path", "type": "string", "description": "Property-list file path", "required": True}] + source_uri_template = f"feed://{type_name}/plist-reading-list?path={{path}}" + adapter = {"kind": "plist_reading_list", "path": "{path}", "limit": 50} + + stream = { + "id": template_id, + "title": title, + "description": f"Draft template for {title}.", + "type": type_name, + "mode": mode, + "schema_url": f"https://agentfeeds.dev/schemas/{schema_name}", + "schema_version": "1.0.0", + "parameters": parameters, + "source_uri_template": source_uri_template, + "adapter": adapter, + "recommended_poll_interval_seconds": 300, + "auth": "none", + "tags": [category, adapter_kind.replace("_", "-")], + "quality_tier": "experimental", + "contributed_by": "local", + } + if adapter_kind == "local_command": + stream["pending"] = True + return stream, _schema_payload(template_id, mode), stream_path, schema_path + + +def _domain_slug(domain: str) -> str: + return _slugify(domain.removeprefix("www.").rstrip(".").replace(".", "-")) + + +def _domain_title(domain: str, suffix: str = "RSS feed") -> str: + base = domain.removeprefix("www.").split(".")[0] + return f"{base.replace('-', ' ').title()} {suffix}".strip() + + +def _default_identity(template: dict, params: dict[str, Any]) -> tuple[str, str]: + template_id = template["id"] + parameters = template.get("parameters") or [] + if not parameters: + return template_id, template["title"] + + category = template_id.split("/", 1)[0] + title = template["title"] + + if template_id == "local/file" and params.get("path"): + path = Path(str(params["path"])).expanduser() + name = path.name or "file" + return f"{category}/{_slugify(name)}", name + + if params.get("owner") and params.get("repo"): + owner = _slugify(str(params["owner"])) + repo = _slugify(str(params["repo"])) + suffixes = { + "dev/github-issues": "issues", + "dev/github-prs": "prs", + "dev/github-releases": "releases", + } + suffix = suffixes.get(template_id, template_id.split("/", 1)[1]) + return f"{category}/{owner}-{repo}-{suffix}", f"{params['owner']}/{params['repo']} {suffix}" + + if template_id == "calendar/ics" and params.get("url"): + parsed = urlparse(str(params["url"])) + if parsed.netloc: + return f"{category}/{_domain_slug(parsed.netloc)}", f"{_domain_title(parsed.netloc, 'calendar')}" + + if template_id == "news/rss-generic" and params.get("url"): + domain = urlparse(str(params["url"])).netloc.lower() or None + if domain: + return f"{category}/{_domain_slug(domain)}", _domain_title(domain) + + if params.get("url"): + parsed = urlparse(str(params["url"])) + if parsed.netloc: + return f"{category}/{_domain_slug(parsed.netloc)}", _domain_title(parsed.netloc) + + if params.get("base"): + base = str(params["base"]).upper() + return f"{category}/{_slugify(base)}-exchange-rates", f"{base} exchange rates" + + if params.get("lat") is not None and params.get("lon") is not None: + lat = _slugify(str(params["lat"])) + lon = _slugify(str(params["lon"])) + tail = _slugify(template_id.split("/", 1)[1]) + return f"{category}/{tail}-{lat}-{lon}", title + + return f"{category}/{_slugify(template_id)}-{_hash_suffix(params)}", title + + +def _append_collision_suffix(instance_id: str, template: dict, params: dict[str, Any], existing_ids: set[str]) -> str: + if instance_id not in existing_ids: + return instance_id + + path = "" + if params.get("url"): + parsed_path = urlparse(str(params["url"])).path.strip("/") + if parsed_path: + path = _slugify(parsed_path.split("/")[-2] if parsed_path.endswith("/") else parsed_path.rsplit("/", 1)[-1]) + if path: + candidate = f"{instance_id}-{path}" + if candidate not in existing_ids: + return candidate + + return f"{instance_id}-{_hash_suffix({'template': template['id'], 'parameters': params})}" + + +def materialize_subscription( + template: dict, + params: dict[str, Any], + existing_ids: set[str], + instance_id: str | None = None, + title: str | None = None, +) -> dict: + default_id, default_title = _default_identity(template, params) + if instance_id: + resolved_id = instance_id + elif not template.get("parameters"): + resolved_id = default_id + else: + resolved_id = _append_collision_suffix(default_id, template, params, existing_ids) + resolved_title = title or default_title + if not INSTANCE_ID_PATTERN.match(resolved_id): + raise ValueError( + "subscription id must look like category/name using lowercase letters, numbers, and hyphens" + ) + if resolved_id in existing_ids: + raise ValueError(f"subscription id already exists: {resolved_id}") + + subscription = { + "id": resolved_id, + "title": resolved_title, + "template": template["id"], + } + if params: + subscription["parameters"] = params + return subscription + + +def subscription_preview(root: Path, stream: dict, subscription: dict) -> dict: + parameters = subscription.get("parameters") or {} + stream_uri = fetch.source_uri_for(stream, parameters) + state_path = fetch.state_path_for_stream(stream_uri, root) + preview = { + "subscription": subscription, + "stream": stream_uri, + "state_path": str(state_path.relative_to(root)), + "requires_secrets": sorted(secret_refs_in_template(stream)), + "next_actions": [], + } + if stream.get("pending") and stream.get("adapter", {}).get("kind") == "local_command": + preview["next_actions"].append( + { + "action": "approve_local_command", + "template_id": stream["id"], + "command": f"python3 scripts/agentfeeds.py admin templates approve-command {stream['id']}", + "reason": "local_command template is pending operator approval", + } + ) + return preview + + +def local_template_file(root: Path, template_id: str) -> Path | None: + for path in sorted(fetch.template_streams_root(root).glob("**/*.yaml")): + try: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except yaml.YAMLError: + continue + if payload.get("id") == template_id: + return path + return None + + +def cmd_templates_search(args: argparse.Namespace) -> int: + fetch.ensure_root(args.root) + index = fetch.load_catalog_index(args.root) + query = " ".join(args.query).lower().strip() + streams = index.get("streams") or [] + if query: + streams = [ + stream + for stream in streams + if query + in " ".join( + [ + stream.get("id", ""), + stream.get("title", ""), + stream.get("description", ""), + " ".join(stream.get("tags") or []), + stream.get("type", ""), + ] + ).lower() + ] + for stream in streams: + params = ", ".join(stream.get("parameters") or []) or "none" + source = f", source: {stream.get('source')}" if args.verbose and stream.get("source") else "" + print(f"{stream['id']}: {stream['title']} [params: {params}, mode: {stream['mode']}{source}]") + if args.verbose: + print(f" {stream.get('description', '')}") + return 0 + + +def cmd_templates_show(args: argparse.Namespace) -> int: + fetch.ensure_root(args.root) + try: + stream = fetch.load_stream_definition(args.root, args.template_id) + except Exception as exc: + print(str(exc), file=sys.stderr) + return 1 + + result = { + "id": stream["id"], + "title": stream["title"], + "description": stream["description"], + "type": stream["type"], + "mode": stream["mode"], + "parameters": stream.get("parameters") or [], + "auth": stream.get("auth"), + "quality_tier": stream.get("quality_tier"), + "tags": stream.get("tags") or [], + "recommended_poll_interval_seconds": stream.get("recommended_poll_interval_seconds"), + } + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + required = [ + parameter["name"] + for parameter in result["parameters"] + if parameter.get("required") + ] + optional = [ + parameter["name"] + for parameter in result["parameters"] + if not parameter.get("required") + ] + print(f"{result['id']}: {result['title']}") + print(result["description"]) + print(f"Type: {result['type']}") + print(f"Mode: {result['mode']}") + print(f"Required parameters: {', '.join(required) or 'none'}") + print(f"Optional parameters: {', '.join(optional) or 'none'}") + print(f"Auth: {result['auth']}") + print(f"Quality: {result['quality_tier']}") + return 0 + + +def cmd_subscribe(args: argparse.Namespace) -> int: + config = active_subscriptions(args.root) + config.setdefault("version", fetch.SPEC_VERSION) + config.setdefault("defaults", {"poll_interval_seconds": 600, "history_limit": 50}) + subscriptions = config.setdefault("subscriptions", []) + + try: + stream = fetch.load_stream_definition(args.root, args.template_id) + params = parse_params(args.parameters) + fetch.validate_parameters(stream, params) + except Exception as exc: + print(str(exc), file=sys.stderr) + return 2 + + existing_ids = {str(item.get("id")) for item in subscriptions} + try: + if args.update: + match = next((item for item in subscriptions if item.get("id") == args.update), None) + if not match: + raise ValueError(f"subscription id not found for update: {args.update}") + subscription = dict(match) + subscription["template"] = stream["id"] + if args.title: + subscription["title"] = args.title + elif "title" not in subscription: + subscription["title"] = stream.get("title") or args.update + if params: + subscription["parameters"] = params + elif "parameters" in subscription: + subscription.pop("parameters") + else: + subscription = materialize_subscription(stream, params, existing_ids, args.instance_id, args.title) + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 2 + if args.poll_interval_seconds: + subscription["poll_interval_seconds"] = args.poll_interval_seconds + if args.history_limit: + subscription["history_limit"] = args.history_limit + + preview = subscription_preview(args.root, stream, subscription) + if preview["requires_secrets"]: + preview["next_actions"].append( + { + "action": "set_secret", + "names": preview["requires_secrets"], + "command": "python3 scripts/agentfeeds.py admin secrets set ", + } + ) + if args.dry_run: + if args.json: + print(json.dumps(preview, indent=2, sort_keys=True)) + else: + print(f"Subscription: {subscription['id']} ({subscription['title']})") + print(f"Template: {subscription['template']}") + print(f"State path: {preview['state_path']}") + if preview["requires_secrets"]: + print(f"Secrets: {', '.join(preview['requires_secrets'])}") + return 0 + + pending_approval = next((action for action in preview["next_actions"] if action["action"] == "approve_local_command"), None) + if pending_approval: + if args.json: + print(json.dumps({**preview, "error": "local_command template is pending operator approval"}, indent=2, sort_keys=True)) + else: + print(f"{stream['id']}: local_command template is pending operator approval", file=sys.stderr) + print(f"Next: {pending_approval['command']}", file=sys.stderr) + return 2 + + if args.update: + for index, item in enumerate(subscriptions): + if item.get("id") == args.update: + subscriptions[index] = subscription + break + else: + subscriptions.append(subscription) + save_subscriptions(args.root, config) + if args.json: + result = {**preview, "created": not bool(args.update), "updated": bool(args.update)} + print(json.dumps(result, indent=2, sort_keys=True)) + else: + print(f"{'Updated' if args.update else 'Subscribed'}: {subscription['id']} ({subscription['title']})") + + if args.no_fetch: + fetch.regenerate_catalog(args.root) + return 0 + return fetch.main(["--root", str(args.root), "--once", subscription["id"]]) + + +def cmd_unsubscribe(args: argparse.Namespace) -> int: + config = active_subscriptions(args.root) + subscriptions = config.get("subscriptions") or [] + matches = [ + subscription + for subscription in subscriptions + if subscription.get("id") == args.subscription_id + ] + if not matches: + print(f"No matching subscription: {args.subscription_id}", file=sys.stderr) + return 1 + + remove_keys = {id(match) for match in matches} + state_paths = [state_path_for_subscription(args.root, match) for match in matches] + config["subscriptions"] = [ + subscription for subscription in subscriptions if id(subscription) not in remove_keys + ] + save_subscriptions(args.root, config) + + if not args.keep_state: + for path in state_paths: + if path and path.exists() and args.root in path.parents: + path.unlink() + + fetch.regenerate_catalog(args.root) + print(f"Unsubscribed: {args.subscription_id}") + return 0 + + +def state_status(root: Path, subscription: dict, defaults: dict) -> dict: + stream = stream_summary(root, template_id_for(subscription)) + path = state_path_for_subscription(root, subscription) + interval = fetch.poll_interval(subscription, stream, defaults) + payload = fetch.load_existing_state(path) if path else None + meta = (payload or {}).get("_meta", {}) + updated = fetch.parse_utc(meta.get("last_updated")) + stale = True + due = True + if updated: + age = datetime.now(UTC) - updated + due = age >= timedelta(seconds=interval) + stale = age > timedelta(seconds=interval * 2) + return { + "id": subscription["id"], + "title": subscription.get("title") or stream.get("title"), + "template": subscription.get("template"), + "parameters": subscription.get("parameters") or {}, + "path": str(path.relative_to(root)) if path else "", + "exists": bool(path and path.exists()), + "last_updated": meta.get("last_updated"), + "next_poll_due": meta.get("next_poll_due"), + "due": due, + "stale": stale, + "mode": stream.get("mode"), + } + + +def _stream_rows(root: Path) -> list[dict]: + config = active_subscriptions(root) + defaults = config.get("defaults") or {} + return [state_status(root, subscription, defaults) for subscription in config.get("subscriptions") or []] + + +def _stream_matches(row: dict, query: str) -> bool: + haystack = " ".join( + [ + row.get("id", ""), + row.get("title", ""), + row.get("template", ""), + " ".join(f"{key}={value}" for key, value in sorted((row.get("parameters") or {}).items())), + row.get("mode", ""), + ] + ).lower() + return query.lower() in haystack + + +def _print_stream_rows(rows: list[dict]) -> None: + if not rows: + print("No active streams.") + return + for row in rows: + freshness = "stale" if row["stale"] else "due" if row["due"] else "fresh" + exists = "ok" if row["exists"] else "missing" + print(f"{row['id']}: {row['title']} [{freshness}, {exists}, updated={row['last_updated'] or 'never'}]") + + +def _active_fetch_error(status: dict) -> bool: + if int(status.get("consecutive_failures") or 0) <= 0: + return False + error_at = fetch.parse_utc(status.get("last_error_at")) + success_at = fetch.parse_utc(status.get("last_success_at")) + return bool(error_at and (not success_at or error_at >= success_at)) + + +def _health_state(row: dict, status: dict) -> str: + if _active_fetch_error(status): + return "error" + if not row["exists"]: + return "missing" + if row["stale"]: + return "stale" + if row["due"]: + return "due" + return "ok" + + +def _stream_health(root: Path) -> dict: + rows = _stream_rows(root) + streams = [] + counts = Counter() + for row in rows: + status = fetch.load_fetch_status(root, row["id"]) + state = _health_state(row, status) + counts[state] += 1 + streams.append( + { + **row, + "health": state, + "last_attempt_at": status.get("last_attempt_at"), + "last_success_at": status.get("last_success_at"), + "last_error_at": status.get("last_error_at"), + "last_error": status.get("last_error"), + "consecutive_failures": int(status.get("consecutive_failures") or 0), + } + ) + summary = { + "total": len(streams), + "ok": counts.get("ok", 0), + "due": counts.get("due", 0), + "stale": counts.get("stale", 0), + "missing": counts.get("missing", 0), + "error": counts.get("error", 0), + } + summary["healthy"] = summary["total"] > 0 and summary["missing"] == 0 and summary["error"] == 0 and summary["stale"] == 0 + next_actions = [] + if summary["error"]: + next_actions.append( + { + "action": "inspect_errors", + "command": "python3 scripts/agentfeeds.py streams health --json", + "reason": "one or more streams have active fetch errors", + } + ) + stale_stream = next((row for row in streams if row["health"] == "stale"), None) + if stale_stream: + next_actions.append( + { + "action": "refresh_stream", + "subscription_id": stale_stream["id"], + "command": f"python3 scripts/agentfeeds.py refresh --stream {stale_stream['id']}", + "reason": "stream state is stale", + } + ) + missing_stream = next((row for row in streams if row["health"] == "missing"), None) + if missing_stream: + next_actions.append( + { + "action": "refresh_stream", + "subscription_id": missing_stream["id"], + "command": f"python3 scripts/agentfeeds.py refresh --stream {missing_stream['id']}", + "reason": "stream has no local state file yet", + } + ) + return {"summary": summary, "streams": streams, "next_actions": next_actions} + + +def _print_health(result: dict) -> None: + summary = result["summary"] + print( + "Streams: " + f"{summary['total']} total, {summary['ok']} ok, {summary['due']} due, " + f"{summary['stale']} stale, {summary['missing']} missing, {summary['error']} error" + ) + for row in result["streams"]: + print(f"{row['id']}: {row['health']} [updated={row['last_updated'] or 'never'}]") + if row.get("last_error"): + print(f" error: {row['last_error']}") + + +def cmd_streams_health(args: argparse.Namespace) -> int: + result = _stream_health(args.root) + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + _print_health(result) + return 0 + + +def _brief_entries(root: Path, max_streams: int, include_freshness: bool) -> tuple[list[dict], bool]: + rows = _stream_rows(root) + entries = [] + for row in rows[:max_streams]: + entry = {"id": row["id"], "title": row["title"]} + if include_freshness: + entry.update( + { + "freshness": "stale" if row["stale"] else "due" if row["due"] else "fresh", + "exists": row["exists"], + "last_updated": row["last_updated"], + } + ) + entries.append(entry) + return entries, len(rows) > max_streams + + +def _brief_health_summary(health: dict) -> str | None: + summary = health["summary"] + degraded = summary["stale"] or summary["missing"] or summary["error"] + if not degraded: + return None + parts = [] + for key in ("stale", "missing", "error"): + if summary[key]: + parts.append(f"{summary[key]} {key}") + return "Ambient health: degraded (" + ", ".join(parts) + ")" + + +def render_brief(entries: list[dict], truncated: bool, include_freshness: bool, health: dict | None = None) -> str: + lines = [""] + if health: + health_line = _brief_health_summary(health) + if health_line: + lines.append(health_line) + if entries: + lines.append("Available local streams:") + for entry in entries: + line = f"- {entry['id']}: {entry['title']}" + if include_freshness: + line += f" [{entry['freshness']}, updated={entry['last_updated'] or 'never'}]" + lines.append(line) + if truncated: + lines.append("- ...") + else: + lines.append("No active local streams.") + lines.append("") + return "\n".join(lines) + + +def cmd_brief(args: argparse.Namespace) -> int: + fetch.ensure_root(args.root) + entries, truncated = _brief_entries(args.root, args.max_streams, args.include_freshness) + health = _stream_health(args.root) + brief = render_brief(entries, truncated, args.include_freshness, health) + if args.json: + print( + json.dumps( + { + "brief": brief, + "health": health["summary"], + "streams": entries, + "truncated": truncated, + "stable": not args.include_freshness, + "recommended_prompt_slot": "system", + }, + indent=2, + sort_keys=True, + ) + ) + return 0 + print(brief) + return 0 + + +def _query_terms(query: str) -> list[str]: + terms = [] + seen = set() + for raw in QUERY_TOKEN_PATTERN.findall(query.lower()): + term = raw.strip("._-") + if len(term) < 2 or term in QUERY_STOPWORDS or term in seen: + continue + seen.add(term) + terms.append(term) + return terms + + +def _iter_text_fields(value: object, path: str = "data", depth: int = 0, max_depth: int = 20): + if depth > max_depth: + return + if value is None: + return + if isinstance(value, dict): + for key, item in value.items(): + yield from _iter_text_fields(item, f"{path}.{key}", depth + 1, max_depth) + return + if isinstance(value, list): + for index, item in enumerate(value): + yield from _iter_text_fields(item, f"{path}[{index}]", depth + 1, max_depth) + return + if isinstance(value, (str, int, float, bool)): + text = str(value) + if text: + yield path, text + + +def _best_field_match(value: object, terms: list[str], match_mode: str, max_field_chars: int) -> dict | None: + best = None + fields = [(path, text) for path, text in _iter_text_fields(value)] + for path, text in fields: + searchable = text[:max_field_chars] + lowered = searchable.lower() + matched_terms = [term for term in terms if term in lowered] + if match_mode == "all" and len(matched_terms) != len(terms): + continue + if match_mode == "any" and not matched_terms: + continue + occurrences = sum(lowered.count(term) for term in matched_terms) + score = len(matched_terms) * 100 + occurrences + candidate = { + "path": path, + "text": searchable, + "score": score, + "matched_terms": matched_terms, + } + if best is None or candidate["score"] > best["score"]: + best = candidate + if best is None and fields: + combined = " ".join(text for _path, text in fields)[:max_field_chars] + lowered = combined.lower() + matched_terms = [term for term in terms if term in lowered] + if (match_mode == "all" and len(matched_terms) == len(terms)) or (match_mode == "any" and matched_terms): + occurrences = sum(lowered.count(term) for term in matched_terms) + best = { + "path": "data", + "text": combined, + "score": len(matched_terms) * 100 + occurrences, + "matched_terms": matched_terms, + } + return best + + +def _snippet(text: str, terms: list[str], max_chars: int) -> str: + compact = re.sub(r"\s+", " ", text).strip() + if len(compact) <= max_chars: + return compact + lowered = compact.lower() + positions = [lowered.find(term) for term in terms if lowered.find(term) >= 0] + center = min(positions) if positions else 0 + start = max(0, center - max_chars // 3) + end = min(len(compact), start + max_chars) + start = max(0, end - max_chars) + prefix = "..." if start else "" + suffix = "..." if end < len(compact) else "" + return f"{prefix}{compact[start:end].strip()}{suffix}" + + +def _search_items_for_payload(payload: dict) -> list[dict]: + data = payload.get("data") + meta = payload.get("_meta") or {} + if isinstance(data, list): + items = [] + for index, event in enumerate(data): + if isinstance(event, dict): + items.append( + { + "kind": "event", + "index": index, + "id": event.get("id"), + "time": event.get("time"), + "data": event.get("data", event), + } + ) + return items + return [ + { + "kind": "snapshot", + "index": None, + "id": None, + "time": meta.get("last_updated"), + "data": data, + } + ] + + +def _search_state(root: Path, query: str, *, match_mode: str, limit: int, max_field_chars: int, snippet_chars: int) -> dict: + terms = _query_terms(query) + if not terms: + raise ValueError("search query has no usable terms") + + matches = [] + missing_streams = 0 + for row in _stream_rows(root): + if not row["exists"] or not row.get("path"): + missing_streams += 1 + continue + payload = fetch.load_existing_state(root / row["path"]) + if not payload: + continue + for item in _search_items_for_payload(payload): + best = _best_field_match(item["data"], terms, match_mode, max_field_chars) + if not best: + continue + matches.append( + { + "subscription_id": row["id"], + "title": row["title"], + "template": row["template"], + "mode": row["mode"], + "stale": row["stale"], + "last_updated": row["last_updated"], + "item_kind": item["kind"], + "item_index": item["index"], + "item_id": item["id"], + "item_time": item["time"], + "path": best["path"], + "matched_terms": best["matched_terms"], + "score": best["score"], + "snippet": _snippet(best["text"], terms, snippet_chars), + } + ) + matches.sort( + key=lambda match: ( + match["score"], + match["item_time"] or match["last_updated"] or "", + match["subscription_id"], + ), + reverse=True, + ) + return { + "query": query, + "terms": terms, + "match": match_mode, + "matches": matches[:limit], + "total_matches": len(matches), + "missing_streams": missing_streams, + } + + +def cmd_search(args: argparse.Namespace) -> int: + fetch.ensure_root(args.root) + query = " ".join(args.query).strip() + if not query: + print("search requires a query", file=sys.stderr) + return 2 + try: + result = _search_state( + args.root, + query, + match_mode=args.match, + limit=args.limit, + max_field_chars=args.max_field_chars, + snippet_chars=args.snippet_chars, + ) + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 2 + + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + if not result["matches"]: + print("No matching local stream state.") + return 0 + for match in result["matches"]: + stale = "stale" if match["stale"] else "fresh" + item = f" item={match['item_id']}" if match["item_id"] else "" + print(f"{match['subscription_id']}: {match['title']} [{stale}{item}]") + print(f" {match['path']}: {match['snippet']}") + return 0 + + +def cmd_streams_list(args: argparse.Namespace) -> int: + rows = _stream_rows(args.root) + if args.json: + print(json.dumps({"streams": rows}, indent=2, sort_keys=True)) + return 0 + _print_stream_rows(rows) + return 0 + + +def cmd_streams_search(args: argparse.Namespace) -> int: + query = " ".join(args.query).strip() + rows = _stream_rows(args.root) + if query: + rows = [row for row in rows if _stream_matches(row, query)] + if args.json: + print(json.dumps({"streams": rows}, indent=2, sort_keys=True)) + return 0 + _print_stream_rows(rows) + return 0 + + +def _subscription_by_id(root: Path, subscription_id: str) -> tuple[dict, dict]: + config = active_subscriptions(root) + for subscription in config.get("subscriptions") or []: + if subscription.get("id") == subscription_id: + return config, subscription + raise KeyError(f"No matching subscription: {subscription_id}") + + +def _stream_detail(root: Path, subscription_id: str) -> dict: + config, subscription = _subscription_by_id(root, subscription_id) + defaults = config.get("defaults") or {} + row = state_status(root, subscription, defaults) + stream = fetch.load_stream_definition(root, template_id_for(subscription)) + parameters = subscription.get("parameters") or {} + stream_uri = fetch.source_uri_for(stream, parameters) + path = fetch.state_path_for_stream(stream_uri, root) + payload = fetch.load_existing_state(path) if path.exists() else None + meta = (payload or {}).get("_meta", {}) + data = (payload or {}).get("data") + data_summary = {"kind": type(data).__name__} + if isinstance(data, list): + data_summary["count"] = len(data) + elif isinstance(data, dict): + data_summary["keys"] = sorted(data.keys()) + return { + **row, + "template": row.get("template"), + "stream": stream_uri, + "state_path": str(path.relative_to(root)), + "meta": meta, + "data_summary": data_summary, + } + + +def cmd_streams_show(args: argparse.Namespace) -> int: + try: + result = _stream_detail(args.root, args.subscription_id) + except Exception as exc: + print(str(exc), file=sys.stderr) + return 1 + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + print(f"{result['id']}: {result['title']}") + print(f"Template: {result['template']}") + print(f"Mode: {result['mode']}") + print(f"Freshness: {'stale' if result['stale'] else 'due' if result['due'] else 'fresh'}") + print(f"Updated: {result['last_updated'] or 'never'}") + print(f"State path: {result['state_path']}") + print(f"Stream: {result['stream']}") + print(f"Data: {json.dumps(result['data_summary'], sort_keys=True)}") + return 0 + + +def _limited_data(data: object, limit: int | None) -> object: + if limit is None: + return data + if isinstance(data, list): + return data[:limit] + return data + + +def cmd_streams_read(args: argparse.Namespace) -> int: + try: + detail = _stream_detail(args.root, args.subscription_id) + path = args.root / detail["state_path"] + payload = fetch.load_existing_state(path) + if payload is None: + raise FileNotFoundError(f"state file not found or invalid: {detail['state_path']}") + except Exception as exc: + print(str(exc), file=sys.stderr) + return 1 + + data = _limited_data(payload.get("data"), args.limit) + result = { + "id": detail["id"], + "title": detail["title"], + "template": detail["template"], + "state_path": detail["state_path"], + "stale": detail["stale"], + "last_updated": detail["last_updated"], + "data": data, + } + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + print(f"{result['id']}: {result['title']}") + print(f"Template: {result['template']}") + print(f"State path: {result['state_path']}") + print(f"Updated: {result['last_updated'] or 'never'}") + print(f"Stale: {'yes' if result['stale'] else 'no'}") + print("Data:") + print(json.dumps(data, indent=2, sort_keys=True)) + return 0 + + +def cmd_refresh(args: argparse.Namespace) -> int: + if args.all: + return fetch.main(["--root", str(args.root), "--all"]) + if not args.subscription_id: + print("refresh requires a subscription id or --all", file=sys.stderr) + return 2 + return fetch.main(["--root", str(args.root), "--stream", args.subscription_id]) + + +def _current_crontab() -> str: + try: + result = subprocess.run(["crontab", "-l"], check=False, text=True, capture_output=True) + except FileNotFoundError: + return "" + return "" if result.returncode else result.stdout + + +def _cron_block_present(text: str) -> bool: + return POLLING_BEGIN_MARKER in text and POLLING_END_MARKER in text + + +def _argv_uses_root(argv: object, root: Path) -> bool: + if not isinstance(argv, list): + return False + items = [str(item) for item in argv] + if "--root" in items: + index = items.index("--root") + return index + 1 < len(items) and Path(items[index + 1]).expanduser() == root + return root == fetch.DEFAULT_ROOT + + +def _cron_block_uses_root(text: str, root: Path) -> bool: + if not _cron_block_present(text): + return False + if "--root" in text: + return str(root) in text + return root == fetch.DEFAULT_ROOT + + +def _polling_status(root: Path) -> dict: + system = platform.system() + status: dict[str, object] = { + "root": str(root), + "platform": system, + "installed": False, + "method": "unsupported", + "fetcher": None, + "logs": str(root / "logs"), + } + try: + status["fetcher"] = polling_install.fetcher_path() + status["fetcher_available"] = True + except Exception as exc: # noqa: BLE001 - status should be diagnostic, not fatal. + status["fetcher_available"] = False + status["fetcher_error"] = str(exc) + + if system == "Darwin": + plist_path = Path.home() / "Library" / "LaunchAgents" / f"{POLLING_LABEL}.plist" + installed = False + if plist_path.exists(): + try: + payload = plistlib.loads(plist_path.read_bytes()) + installed = _argv_uses_root(payload.get("ProgramArguments"), root) + except Exception: + installed = root == fetch.DEFAULT_ROOT + status.update({"method": "launchd", "installed": installed, "path": str(plist_path)}) + return status + if system in {"Linux", "FreeBSD"}: + text = _current_crontab() + status.update({"method": "cron", "installed": _cron_block_uses_root(text, root)}) + return status + return status + + +def _print_polling_status(status: dict) -> None: + print(f"Polling: {'installed' if status['installed'] else 'not installed'}") + print(f"Method: {status['method']}") + print(f"Root: {status['root']}") + print(f"Fetcher: {status.get('fetcher') or status.get('fetcher_error') or 'unknown'}") + print(f"Logs: {status['logs']}") + + +def cmd_polling_status(args: argparse.Namespace) -> int: + fetch.ensure_root(args.root) + status = _polling_status(args.root) + if args.json: + print(json.dumps(status, indent=2, sort_keys=True)) + return 0 + _print_polling_status(status) + return 0 + + +def cmd_polling_install(args: argparse.Namespace) -> int: + fetch.ensure_root(args.root) + return polling_install.main(["--root", str(args.root)]) + + +def cmd_polling_uninstall(_args: argparse.Namespace) -> int: + return polling_uninstall.main([]) + + +def cmd_templates_list(args: argparse.Namespace) -> int: + fetch.ensure_root(args.root) + index = fetch.load_catalog_index(args.root) + for stream in index.get("streams") or []: + params = ", ".join(stream.get("parameters") or []) or "none" + print(f"{stream['id']}: {stream['title']} [params: {params}, source: {stream.get('source', 'builtin')}]") + return 0 + + +def cmd_templates_path(args: argparse.Namespace) -> int: + fetch.ensure_root(args.root) + print(fetch.templates_root(args.root)) + return 0 + + +def cmd_templates_validate(args: argparse.Namespace) -> int: + fetch.ensure_root(args.root) + try: + paths = fetch.validate_template_tree(args.root) + except Exception as exc: + print(str(exc), file=sys.stderr) + return 1 + if not paths: + print(f"No local templates found in {fetch.template_streams_root(args.root)}") + return 0 + for path in paths: + print(f"valid: {path}") + return 0 + + +def cmd_templates_adapters(_args: argparse.Namespace) -> int: + for kind, description in ADAPTER_KINDS.items(): + print(f"{kind}: {description}") + return 0 + + +def cmd_templates_scaffold(args: argparse.Namespace) -> int: + fetch.ensure_root(args.root) + try: + stream, schema, stream_rel_path, schema_rel_path = scaffold_stream(args.template_id, args.adapter_kind) + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 2 + + stream_path = fetch.template_streams_root(args.root) / stream_rel_path + schema_path = fetch.template_schemas_root(args.root) / schema_rel_path + if stream_path.exists() and not args.force: + print(f"template already exists: {stream_path}", file=sys.stderr) + return 1 + + stream_path.parent.mkdir(parents=True, exist_ok=True) + stream_path.write_text(yaml.safe_dump(stream, sort_keys=False), encoding="utf-8") + + if args.adapter_kind not in {"rss", "ical", "local_command"} and (args.force or not schema_path.exists()): + schema_path.parent.mkdir(parents=True, exist_ok=True) + schema_path.write_text(json.dumps(schema, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + print(f"wrote: {stream_path}") + if args.adapter_kind in {"rss", "ical", "local_command"}: + print(f"schema: built-in {stream['schema_url']}") + else: + print(f"wrote: {schema_path}") + print("Next: edit the draft, then run `python3 scripts/agentfeeds.py admin templates validate`.") + return 0 + + +def _event_sample(events: list[dict]) -> object: + if not events: + return None + return events[0].get("data") + + +def cmd_templates_test(args: argparse.Namespace) -> int: + fetch.ensure_root(args.root) + try: + stream = fetch.load_stream_definition(args.root, args.template_id) + params = parse_params(args.parameters) + fetch.validate_parameters(stream, params) + stream_uri, events = fetch.run_adapter(stream, params, args.root) + state_path = fetch.state_path_for_stream(stream_uri, args.root) + except Exception as exc: + print(str(exc), file=sys.stderr) + return 1 + + result = { + "template": stream["id"], + "title": stream["title"], + "mode": stream["mode"], + "stream": stream_uri, + "state_path": str(state_path.relative_to(args.root)), + "event_count": len(events), + "sample": _event_sample(events), + } + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + print(f"Template: {result['template']}") + print(f"Title: {result['title']}") + print(f"Mode: {result['mode']}") + print(f"Stream: {result['stream']}") + print(f"State path: {result['state_path']}") + print(f"Events: {result['event_count']}") + print("Sample:") + print(json.dumps(result["sample"], indent=2, sort_keys=True)) + return 0 + + +def cmd_templates_approve_command(args: argparse.Namespace) -> int: + fetch.ensure_root(args.root) + try: + stream = fetch.load_stream_definition(args.root, args.template_id) + params = parse_params(args.parameters) + fetch.validate_parameters(stream, params) + adapter = fetch.substitute(stream["adapter"], params) + adapter = fetch.resolve_secret_refs(args.root, adapter) + if adapter.get("kind") != "local_command": + raise ValueError(f"{stream['id']}: template is not a local_command template") + if not sys.stdin.isatty(): + raise PermissionError("local_command approval must be run by the operator in an interactive terminal") + output = sys.stderr if args.json else sys.stdout + print("Local command approval required.", file=output) + print(f"Template: {stream['id']}", file=output) + print(f"Command: {json.dumps(adapter.get('command'))}", file=output) + print(f"CWD: {adapter.get('cwd') or os.getcwd()}", file=output) + if args.json: + print("Type APPROVE to approve this exact command: ", end="", file=sys.stderr) + typed = input() + else: + typed = input("Type APPROVE to approve this exact command: ") + if typed != "APPROVE": + raise PermissionError("approval cancelled") + stream_path = local_template_file(args.root, stream["id"]) + if stream_path: + payload = yaml.safe_load(stream_path.read_text(encoding="utf-8")) or {} + payload["pending"] = False + stream_path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + stream = fetch.load_stream_definition(args.root, args.template_id) + approval = fetch.write_local_command_approval(args.root, stream, adapter) + path = fetch.local_command_approval_path(args.root, stream["id"]) + except Exception as exc: + print(str(exc), file=sys.stderr) + return 1 + + result = { + "template": stream["id"], + "approval_path": str(path.relative_to(args.root)), + "digest": approval["digest"], + "command": approval["command"], + "cwd": approval.get("cwd"), + } + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + print(f"Approved: {result['template']}") + print(f"Path: {result['approval_path']}") + print(f"Digest: {result['digest']}") + print(f"Command: {json.dumps(result['command'])}") + return 0 + + +def cmd_secrets_set(args: argparse.Namespace) -> int: + fetch.ensure_root(args.root) + if not sys.stdin.isatty(): + print("secret input must be run by the user in an interactive terminal", file=sys.stderr) + return 1 + value = getpass.getpass(f"Secret value for {args.name}: ") + fetch.write_secret(args.root, args.name, value) + print(f"Secret set: {args.name}") + return 0 + + +def cmd_secrets_list(args: argparse.Namespace) -> int: + fetch.ensure_root(args.root) + names = sorted(path.stem for path in (args.root / "secrets").glob("*.txt")) + if args.json: + print(json.dumps({"secrets": names}, indent=2, sort_keys=True)) + return 0 + for name in names: + print(name) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Manage Agent Feeds subscriptions") + parser.add_argument("--root", type=Path, default=fetch.DEFAULT_ROOT, help="agentfeeds root directory") + subparsers = parser.add_subparsers(dest="command", required=True) + + brief = subparsers.add_parser("brief", help="print compact stable stream context for prompt injection") + brief.add_argument("--max-streams", type=int, default=20) + brief.add_argument("--include-freshness", action="store_true", help="include volatile freshness metadata") + brief.add_argument("--json", action="store_true") + brief.set_defaults(func=cmd_brief) + + search = subparsers.add_parser("search", help="search existing local stream state") + search.add_argument("query", nargs="+") + search.add_argument("--match", choices=["all", "any"], default="all", help="require all query terms or any term") + search.add_argument("--limit", type=int, default=10, help="maximum matching items to return") + search.add_argument("--max-field-chars", type=int, default=20000, help="maximum characters searched per field") + search.add_argument("--snippet-chars", type=int, default=240, help="maximum snippet characters") + search.add_argument("--json", action="store_true") + search.set_defaults(func=cmd_search) + + subscribe = subparsers.add_parser("subscribe", help="add a subscription") + subscribe.add_argument("template_id") + subscribe.add_argument("parameters", nargs="*", help="template parameters as key=value") + subscribe.add_argument("--id", dest="instance_id", help="concrete subscription id to create") + subscribe.add_argument("--title", help="concrete subscription title") + subscribe.add_argument("--poll-interval-seconds", type=int) + subscribe.add_argument("--history-limit", type=int) + subscribe.add_argument("--no-fetch", action="store_true") + subscribe.add_argument("--dry-run", action="store_true", help="preview subscription id, state path, and requirements without writing") + subscribe.add_argument("--update", metavar="SUBSCRIPTION_ID", help="update an existing subscription in place") + subscribe.add_argument("--json", action="store_true", help="print machine-readable output") + subscribe.set_defaults(func=cmd_subscribe) + + unsubscribe = subparsers.add_parser("unsubscribe", help="remove a subscription") + unsubscribe.add_argument("subscription_id") + unsubscribe.add_argument("--keep-state", action="store_true") + unsubscribe.set_defaults(func=cmd_unsubscribe) + + refresh = subparsers.add_parser("refresh", help="refresh subscriptions") + refresh.add_argument("--stream", dest="subscription_id", help="refresh one subscription id") + refresh.add_argument("--all", action="store_true") + refresh.set_defaults(func=cmd_refresh) + + streams = subparsers.add_parser("streams", help="inspect and read active streams") + stream_subparsers = streams.add_subparsers(dest="stream_command", required=True) + streams_list = stream_subparsers.add_parser("list", help="list active streams") + streams_list.add_argument("--json", action="store_true") + streams_list.set_defaults(func=cmd_streams_list) + streams_find = stream_subparsers.add_parser("find", help="find active streams by metadata") + streams_find.add_argument("query", nargs="*") + streams_find.add_argument("--json", action="store_true") + streams_find.set_defaults(func=cmd_streams_search) + streams_health = stream_subparsers.add_parser("health", help="show stream freshness and fetch errors") + streams_health.add_argument("--json", action="store_true") + streams_health.set_defaults(func=cmd_streams_health) + streams_read = stream_subparsers.add_parser("read", help="read active stream data") + streams_read.add_argument("subscription_id") + streams_read.add_argument("--limit", type=int, default=20, help="limit event-list data rows") + streams_read.add_argument("--json", action="store_true", help="print machine-readable output") + streams_read.set_defaults(func=cmd_streams_read) + + templates = subparsers.add_parser("templates", help="browse and test feed templates") + template_subparsers = templates.add_subparsers(dest="template_command", required=True) + template_search = template_subparsers.add_parser("find", help="find built-in and local templates") + template_search.add_argument("query", nargs="*") + template_search.add_argument("-v", "--verbose", action="store_true") + template_search.set_defaults(func=cmd_templates_search) + template_show = template_subparsers.add_parser("show", help="show one template") + template_show.add_argument("template_id") + template_show.add_argument("--json", action="store_true") + template_show.set_defaults(func=cmd_templates_show) + + admin = subparsers.add_parser("admin", help="advanced setup, diagnostics, and template authoring") + admin_subparsers = admin.add_subparsers(dest="admin_command", required=True) + admin_polling = admin_subparsers.add_parser("polling", help="manage background refresh") + admin_polling_subparsers = admin_polling.add_subparsers(dest="polling_command", required=True) + admin_polling_status = admin_polling_subparsers.add_parser("status", help="show background refresh status") + admin_polling_status.add_argument("--json", action="store_true") + admin_polling_status.set_defaults(func=cmd_polling_status) + admin_polling_subparsers.add_parser("install", help="install or update background refresh").set_defaults(func=cmd_polling_install) + admin_polling_subparsers.add_parser("uninstall", help="remove background refresh").set_defaults(func=cmd_polling_uninstall) + + admin_templates = admin_subparsers.add_parser("templates", help="template authoring tools") + admin_template_subparsers = admin_templates.add_subparsers(dest="template_command", required=True) + admin_template_subparsers.add_parser("adapters", help="list scaffoldable adapter kinds").set_defaults(func=cmd_templates_adapters) + admin_template_subparsers.add_parser("path", help="print the local template directory").set_defaults(func=cmd_templates_path) + admin_template_subparsers.add_parser("list", help="list built-in and local templates").set_defaults(func=cmd_templates_list) + admin_scaffold = admin_template_subparsers.add_parser("scaffold", help="create a draft local template") + admin_scaffold.add_argument("adapter_kind", choices=sorted(ADAPTER_KINDS)) + admin_scaffold.add_argument("template_id", metavar="template_id") + admin_scaffold.add_argument("--force", action="store_true", help="overwrite an existing draft") + admin_scaffold.set_defaults(func=cmd_templates_scaffold) + admin_test = admin_template_subparsers.add_parser("test", help="run a template once without writing state") + admin_test.add_argument("template_id", metavar="template_id") + admin_test.add_argument("parameters", nargs="*", help="template parameters as key=value") + admin_test.add_argument("--json", action="store_true", help="print machine-readable output") + admin_test.set_defaults(func=cmd_templates_test) + admin_approve = admin_template_subparsers.add_parser("approve-command", help="operator-only approval for local_command templates") + admin_approve.add_argument("template_id", metavar="template_id") + admin_approve.add_argument("parameters", nargs="*", help="template parameters as key=value") + admin_approve.add_argument("--json", action="store_true", help="print machine-readable output") + admin_approve.set_defaults(func=cmd_templates_approve_command) + admin_template_subparsers.add_parser("validate", help="validate local templates").set_defaults(func=cmd_templates_validate) + + admin_streams = admin_subparsers.add_parser("streams", help="advanced stream diagnostics") + admin_stream_subparsers = admin_streams.add_subparsers(dest="stream_command", required=True) + admin_stream_show = admin_stream_subparsers.add_parser("show", help="show active stream metadata") + admin_stream_show.add_argument("subscription_id") + admin_stream_show.add_argument("--json", action="store_true") + admin_stream_show.set_defaults(func=cmd_streams_show) + + admin_secrets = admin_subparsers.add_parser("secrets", help="manage local secret values") + admin_secret_subparsers = admin_secrets.add_subparsers(dest="secret_command", required=True) + secret_set = admin_secret_subparsers.add_parser("set", help="set a local secret value") + secret_set.add_argument("name") + secret_set.set_defaults(func=cmd_secrets_set) + secret_list = admin_secret_subparsers.add_parser("list", help="list local secret names") + secret_list.add_argument("--json", action="store_true") + secret_list.set_defaults(func=cmd_secrets_list) + + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + args.root = args.root.expanduser() + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/agentfeeds/scripts/lib/agentfeeds_runtime/constants.py b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/constants.py new file mode 100644 index 00000000..23f433ef --- /dev/null +++ b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/constants.py @@ -0,0 +1,6 @@ +"""Shared runtime constants for Agent Feeds.""" + +AGENTFEEDS_VERSION = "agentfeeds/0.3" +REQUEST_TIMEOUT_SECONDS = 20 +COMMAND_TIMEOUT_SECONDS = 20 +COMMAND_MAX_OUTPUT_BYTES = 1024 * 1024 diff --git a/skills/agentfeeds/scripts/lib/agentfeeds_runtime/fetcher.py b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/fetcher.py new file mode 100755 index 00000000..776b842d --- /dev/null +++ b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/fetcher.py @@ -0,0 +1,915 @@ +#!/usr/bin/env python3 +"""Reference fetcher for Agent Feeds v0.3.""" + +from __future__ import annotations + +import argparse +import contextlib +import hashlib +import json +import os +import re +import subprocess +import sys +from datetime import UTC, datetime, timedelta +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +import jsonschema +import requests + +from agentfeeds_runtime.adapters import run_adapter as run_adapter_impl +from agentfeeds_runtime.constants import REQUEST_TIMEOUT_SECONDS + +try: + import fcntl +except ImportError: # pragma: no cover - Windows import compatibility. + fcntl = None + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + + +SPEC_VERSION = "0.3" +DEFAULT_ROOT = Path.home() / ".agentfeeds" +DEFAULT_CATALOG_BASE_URL = "https://raw.githubusercontent.com/verkyyi/agentfeeds-catalog/main" +PARAMETER_PATTERN = re.compile(r"{([A-Za-z_][A-Za-z0-9_]*)}") +SECRET_REF_PATTERN = re.compile(r"\{\{secret:([A-Za-z_][A-Za-z0-9_.-]*)\}\}") +SENSITIVE_OUTPUT_PATTERN = re.compile( + r"(?i)(authorization|token|api[-_ ]?key|secret|cookie)([\"'\s:=]+)([^\"'\s,}]+)" +) +UNSAFE_NAME_PATTERN = re.compile(r"[^A-Za-z0-9._-]+") +LOCK_FILE_NAME = "agentfeeds-fetch.lock" + + +def now_utc() -> str: + return datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def parse_utc(value: str | None) -> datetime | None: + if not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(UTC) + except ValueError: + return None + + +def safe_name_part(value: str, fallback: str = "part", max_chars: int = 96) -> str: + compact = UNSAFE_NAME_PATTERN.sub("-", value).strip("-._") + if not compact: + compact = fallback + if len(compact) > max_chars: + compact = compact[:max_chars].rstrip("-._") or fallback + return compact + + +def query_name_part(query: str) -> str: + compact = safe_name_part(query.replace("&", "-"), "query", 96) + digest = hashlib.sha256(query.encode("utf-8")).hexdigest()[:12] + return f"{compact}.{digest}" + + +def state_path_for_stream(stream_uri: str, root: Path = DEFAULT_ROOT) -> Path: + parsed = urlparse(stream_uri) + if parsed.scheme != "feed" or not parsed.netloc: + raise ValueError(f"invalid feed URI: {stream_uri}") + + path = ".".join(safe_name_part(part, "path") for part in parsed.path.split("/") if part) + name = path or "index" + if parsed.netloc == "local.file": + params = parse_qs(parsed.query, keep_blank_values=True) + local_path = (params.get("path") or [""])[0] + stem = safe_name_part(Path(local_path).name, "file") + digest = hashlib.sha256(parsed.query.encode("utf-8")).hexdigest()[:12] + return root / "state" / parsed.netloc / f"{name}.{stem}.{digest}.json" + if parsed.query: + name = f"{name}.{query_name_part(parsed.query)}" + return root / "state" / parsed.netloc / f"{name}.json" + + +def atomic_write_json(path: Path, payload: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = path.with_suffix(path.suffix + ".tmp") + with tmp_path.open("w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2, sort_keys=True) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + tmp_path.replace(path) + + +def status_path_for_subscription(root: Path, subscription_id: str) -> Path: + parts = [part for part in subscription_id.split("/") if part] + if not parts: + parts = ["unknown"] + safe_parts = [re.sub(r"[^A-Za-z0-9._-]+", "-", part).strip("-") or "unknown" for part in parts] + safe_parts[-1] = f"{safe_parts[-1]}.json" + return root / "status" / "subscriptions" / Path(*safe_parts) + + +def load_fetch_status(root: Path, subscription_id: str) -> dict: + return load_existing_state(status_path_for_subscription(root, subscription_id)) or {} + + +def redact_sensitive_text(value: str | None) -> str | None: + if value is None: + return None + value = re.sub(r"(?i)(authorization[\"'\s:=]+)[^,\n}]+", r"\1[redacted]", value) + return SENSITIVE_OUTPUT_PATTERN.sub(lambda match: f"{match.group(1)}{match.group(2)}[redacted]", value) + + +def write_fetch_status( + root: Path, + subscription: dict, + stream: dict | None, + *, + attempted_at: str, + succeeded: bool, + error: str | None = None, + state_path: Path | None = None, +) -> None: + subscription_id = str(subscription.get("id", "")) + previous = load_fetch_status(root, subscription_id) + payload = { + "subscription_id": subscription_id, + "template_id": template_id_for(subscription) if subscription.get("template") else None, + "title": subscription_title(subscription, stream or {}) if stream else str(subscription.get("title") or subscription_id), + "last_attempt_at": attempted_at, + "last_success_at": attempted_at if succeeded else previous.get("last_success_at"), + "last_error_at": None if succeeded else attempted_at, + "last_error": None if succeeded else redact_sensitive_text(error), + "consecutive_failures": 0 if succeeded else int(previous.get("consecutive_failures") or 0) + 1, + "state_path": str(state_path.relative_to(root)) if state_path and root in state_path.parents else previous.get("state_path"), + } + atomic_write_json(status_path_for_subscription(root, subscription_id), payload) + + +@contextlib.contextmanager +def fetch_lock(root: Path): + if fcntl is None: + raise RuntimeError("Agent Feeds background locking requires a POSIX platform such as macOS, Linux, or WSL") + path = root / LOCK_FILE_NAME + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a+", encoding="utf-8") as handle: + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + yield False + return + try: + handle.seek(0) + handle.truncate() + handle.write(f"pid={os.getpid()} acquired_at={now_utc()}\n") + handle.flush() + os.fsync(handle.fileno()) + yield True + finally: + handle.seek(0) + handle.truncate() + handle.flush() + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + + +def ensure_root(root: Path) -> None: + (root / "approvals" / "local-command").mkdir(parents=True, exist_ok=True) + (root / "catalog-cache").mkdir(parents=True, exist_ok=True) + (root / "secrets").mkdir(parents=True, exist_ok=True) + (root / "state").mkdir(parents=True, exist_ok=True) + (root / "templates" / "streams").mkdir(parents=True, exist_ok=True) + (root / "templates" / "schemas" / "event-types").mkdir(parents=True, exist_ok=True) + subscriptions = root / "subscriptions.yaml" + if not subscriptions.exists(): + subscriptions.write_text( + 'version: "0.3"\n' + "defaults:\n" + " poll_interval_seconds: 600\n" + " history_limit: 50\n" + "subscriptions: []\n", + encoding="utf-8", + ) + + +def load_subscriptions(root: Path) -> dict: + if yaml is None: + raise RuntimeError("PyYAML is required to read subscriptions.yaml") + path = root / "subscriptions.yaml" + if not path.exists(): + return {"version": SPEC_VERSION, "subscriptions": []} + return yaml.safe_load(path.read_text(encoding="utf-8")) or {} + + +def repo_root() -> Path: + return Path(__file__).resolve().parents[3] + + +def templates_root(root: Path) -> Path: + return root / "templates" + + +def template_streams_root(root: Path) -> Path: + return templates_root(root) / "streams" + + +def template_schemas_root(root: Path) -> Path: + return templates_root(root) / "schemas" / "event-types" + + +def catalog_cache_root(root: Path) -> Path: + return root / "catalog-cache" + + +def cached_catalog_file(root: Path, relative_path: str) -> Path: + return catalog_cache_root(root) / relative_path + + +def bundled_catalog_root() -> Path: + return repo_root() / "catalog" + + +def local_catalog_root() -> Path | None: + configured = os.environ.get("AGENTFEEDS_CATALOG_DIR") + candidates = [] + if configured: + candidates.append(Path(configured).expanduser()) + candidates.append(repo_root()) + + for candidate in candidates: + if (candidate / "catalog" / "INDEX.json").exists(): + return candidate + if candidate.name == "catalog" and (candidate / "INDEX.json").exists(): + return candidate.parent + return None + + +def catalog_base_url() -> str: + return os.environ.get("AGENTFEEDS_CATALOG_BASE_URL", DEFAULT_CATALOG_BASE_URL).rstrip("/") + + +def write_text_atomic(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = path.with_suffix(path.suffix + ".tmp") + tmp_path.write_text(content, encoding="utf-8") + tmp_path.replace(path) + + +def copy_catalog_cache_from_dir(root: Path, source_root: Path) -> None: + cache_root = catalog_cache_root(root) + catalog_root = source_root / "catalog" + index_text = (catalog_root / "INDEX.json").read_text(encoding="utf-8") + write_text_atomic(cache_root / "INDEX.json", index_text) + write_text_atomic(cache_root / "catalog" / "INDEX.json", index_text) + + for path in sorted(catalog_root.glob("**/*")): + if path.is_file(): + relative_path = path.relative_to(source_root) + write_text_atomic(cache_root / relative_path, path.read_text(encoding="utf-8")) + + +def download_catalog_cache(root: Path) -> None: + cache_root = catalog_cache_root(root) + base_url = catalog_base_url() + index_response = requests.get(f"{base_url}/catalog/INDEX.json", timeout=REQUEST_TIMEOUT_SECONDS) + index_response.raise_for_status() + index_text = index_response.text + index = json.loads(index_text) + + write_text_atomic(cache_root / "INDEX.json", index_text) + write_text_atomic(cache_root / "catalog" / "INDEX.json", index_text) + + schema_names = {"stream-definition.v0.3.json", "envelope.v0.3.json"} + for stream in index.get("streams", []): + relative_path = stream["path"] + response = requests.get(f"{base_url}/{relative_path}", timeout=REQUEST_TIMEOUT_SECONDS) + response.raise_for_status() + stream_text = response.text + write_text_atomic(cache_root / relative_path, stream_text) + stream_payload = yaml.safe_load(stream_text) + schema_names.add(stream_payload["schema_url"].rstrip("/").split("/")[-1]) + + for name in sorted(schema_names): + if name in {"stream-definition.v0.3.json", "envelope.v0.3.json"}: + relative_path = f"catalog/schemas/{name}" + else: + relative_path = f"catalog/schemas/event-types/{name}" + response = requests.get(f"{base_url}/{relative_path}", timeout=REQUEST_TIMEOUT_SECONDS) + response.raise_for_status() + write_text_atomic(cache_root / relative_path, response.text) + + +def update_catalog_cache(root: Path) -> None: + source_root = local_catalog_root() + if source_root is not None: + copy_catalog_cache_from_dir(root, source_root) + return + download_catalog_cache(root) + + +def load_catalog_index(root: Path) -> dict: + cache = catalog_cache_root(root) / "INDEX.json" + if not cache.exists(): + update_catalog_cache(root) + index = json.loads(cache.read_text(encoding="utf-8")) + streams = {stream["id"]: {**stream, "source": stream.get("source") or "builtin"} for stream in index.get("streams", [])} + for path in sorted(template_streams_root(root).glob("**/*.yaml")): + stream = stream_summary(path, root) + if stream["id"] in streams: + continue + streams[stream["id"]] = stream + merged = {**index, "streams": sorted(streams.values(), key=lambda item: item["id"])} + merged["stream_count"] = len(merged["streams"]) + return merged + + +def stream_summary(path: Path, root: Path) -> dict: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + try: + rel_path = str(path.relative_to(catalog_cache_root(root))) + source = "builtin" + except ValueError: + try: + rel_path = str(path.relative_to(repo_root())) + source = "builtin" + except ValueError: + rel_path = str(path) + source = "local" + try: + path.relative_to(template_streams_root(root)) + rel_path = str(path) + source = "local" + except ValueError: + pass + return { + "id": data["id"], + "title": data["title"], + "description": data["description"], + "type": data["type"], + "mode": data["mode"], + "tags": data.get("tags", []), + "parameters": [param["name"] for param in data.get("parameters", [])], + "auth": data["auth"], + "quality_tier": data["quality_tier"], + "path": rel_path, + "source": source, + } + + +def stream_definition_schema_path(root: Path) -> Path: + candidates = [ + cached_catalog_file(root, "catalog/schemas/stream-definition.v0.3.json"), + bundled_catalog_root() / "schemas" / "stream-definition.v0.3.json", + ] + for path in candidates: + if path.exists(): + return path + update_catalog_cache(root) + cached = cached_catalog_file(root, "catalog/schemas/stream-definition.v0.3.json") + if cached.exists(): + return cached + raise FileNotFoundError("stream definition schema not found in catalog cache") + + +def builtin_stream_paths(root: Path): + yield from (catalog_cache_root(root) / "catalog" / "streams").glob("**/*.yaml") + yield from (bundled_catalog_root() / "streams").glob("**/*.yaml") + + +def cached_event_schemas_root(root: Path) -> Path: + return cached_catalog_file(root, "catalog/schemas/event-types") + + +def load_stream_definition(root: Path, stream_id: str, _refreshed: bool = False) -> dict: + index = load_catalog_index(root) + match = next((item for item in index.get("streams", []) if item.get("id") == stream_id), None) + if not match: + raise KeyError(f"stream id not found in catalog: {stream_id}") + + candidate_paths = [] + if match.get("path"): + match_path = Path(match["path"]) + if match_path.is_absolute(): + candidate_paths.append(match_path) + else: + candidate_paths.append(repo_root() / match["path"]) + candidate_paths.append(root / "catalog-cache" / match["path"]) + candidate_paths.append(templates_root(root) / match["path"]) + candidate_paths.extend(template_streams_root(root).glob("**/*.yaml")) + candidate_paths.extend(builtin_stream_paths(root)) + + for path in candidate_paths: + if not path.exists(): + continue + data = yaml.safe_load(path.read_text(encoding="utf-8")) + if data.get("id") == stream_id: + return data + if match.get("source") != "local" and not _refreshed: + update_catalog_cache(root) + return load_stream_definition(root, stream_id, _refreshed=True) + raise FileNotFoundError(f"stream definition not found for {stream_id}") + + +def schema_path_for_url(root: Path, schema_url: str) -> Path: + name = schema_url.rstrip("/").split("/")[-1] + for path in [ + template_schemas_root(root) / name, + cached_event_schemas_root(root) / name, + bundled_catalog_root() / "schemas" / "event-types" / name, + ]: + if path.exists(): + return path + update_catalog_cache(root) + cached = cached_event_schemas_root(root) / name + if cached.exists(): + return cached + raise FileNotFoundError(f"referenced schema not found locally: {schema_url}") + + +def validate_stream_file(path: Path, root: Path = DEFAULT_ROOT) -> None: + stream = yaml.safe_load(path.read_text(encoding="utf-8")) + schema = json.loads(stream_definition_schema_path(root).read_text(encoding="utf-8")) + jsonschema.validate(stream, schema) + + schema_path = schema_path_for_url(root, stream["schema_url"]) + json.loads(schema_path.read_text(encoding="utf-8")) + + adapter_kind = stream["adapter"]["kind"] + if adapter_kind in {"json_http", "paginated_json_http"}: + for required in ("url", "method", "transform"): + if required not in stream["adapter"]: + raise ValueError(f"{path}: adapter.{required} is required for {adapter_kind}") + if adapter_kind in {"rss", "ical"} and "url" not in stream["adapter"]: + raise ValueError(f"{path}: adapter.url is required for {adapter_kind}") + path_based_adapters = {"local_file", "filesystem_scan", "markdown_scan", "git_status", "plist_reading_list"} + if adapter_kind in path_based_adapters and "path" not in stream["adapter"]: + raise ValueError(f"{path}: adapter.path is required for {adapter_kind}") + if adapter_kind == "local_command": + command = stream["adapter"].get("command") + if not isinstance(command, list) or not command or not all(isinstance(item, str) for item in command): + raise ValueError(f"{path}: adapter.command must be a non-empty string array for local_command") + if stream["mode"] == "event" and stream["adapter"].get("parse") != "json": + raise ValueError(f"{path}: local_command event streams require adapter.parse: json") + if adapter_kind == "apple_automation": + for required in ("script", "columns"): + if required not in stream["adapter"]: + raise ValueError(f"{path}: adapter.{required} is required for apple_automation") + if adapter_kind == "sqlite_query": + for required in ("database", "query", "columns"): + if required not in stream["adapter"]: + raise ValueError(f"{path}: adapter.{required} is required for sqlite_query") + tcc_adapters = {"apple_automation", "sqlite_query"} + if adapter_kind in tcc_adapters and "tcc_permission" not in stream["adapter"]: + raise ValueError(f"{path}: adapter.tcc_permission is required for {adapter_kind}") + + +def validate_template_tree(root: Path) -> list[Path]: + stream_paths = sorted(template_streams_root(root).glob("**/*.yaml")) + seen: dict[str, Path] = {} + index = load_catalog_index(root) + builtin_ids = {stream["id"] for stream in index.get("streams", []) if stream.get("source") != "local"} + for path in stream_paths: + validate_stream_file(path, root) + stream_id = yaml.safe_load(path.read_text(encoding="utf-8"))["id"] + if stream_id in builtin_ids: + raise ValueError(f"{path}: local template id conflicts with built-in template: {stream_id}") + if stream_id in seen: + raise ValueError(f"{path}: duplicate local template id also defined in {seen[stream_id]}: {stream_id}") + seen[stream_id] = path + return stream_paths + + +def substitute(value: object, parameters: dict) -> object: + if isinstance(value, str): + return PARAMETER_PATTERN.sub( + lambda match: str(parameters[match.group(1)]) + if match.group(1) in parameters + else match.group(0), + value, + ) + if isinstance(value, list): + return [substitute(item, parameters) for item in value] + if isinstance(value, dict): + return {key: substitute(item, parameters) for key, item in value.items()} + return value + + +def source_uri_for(stream: dict, parameters: dict) -> str: + return substitute(stream["source_uri_template"], parameters) + + +def secret_path(root: Path, name: str) -> Path: + safe = safe_name_part(name, "secret") + return root / "secrets" / f"{safe}.txt" + + +def read_secret(root: Path, name: str) -> str: + if sys.platform == "darwin": + result = subprocess.run( + ["security", "find-generic-password", "-s", "agentfeeds", "-a", name, "-w"], + check=False, + text=True, + capture_output=True, + ) + if result.returncode == 0: + return result.stdout.rstrip("\n") + path = secret_path(root, name) + if not path.exists(): + raise KeyError(f"secret {name} not set; user can run `python3 scripts/agentfeeds.py admin secrets set {name}`") + return path.read_text(encoding="utf-8").rstrip("\n") + + +def write_secret(root: Path, name: str, value: str) -> None: + if sys.platform == "darwin": + result = subprocess.run( + ["security", "add-generic-password", "-U", "-s", "agentfeeds", "-a", name, "-w", value], + check=False, + text=True, + capture_output=True, + ) + if result.returncode == 0: + return + path = secret_path(root, name) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(value.rstrip("\n") + "\n", encoding="utf-8") + os.chmod(path, 0o600) + + +def resolve_secret_refs(root: Path, value: object) -> object: + if isinstance(value, str): + return SECRET_REF_PATTERN.sub(lambda match: read_secret(root, match.group(1)), value) + if isinstance(value, list): + return [resolve_secret_refs(root, item) for item in value] + if isinstance(value, dict): + return {key: resolve_secret_refs(root, item) for key, item in value.items()} + return value + + +def add_auth_service_secret(adapter: dict) -> dict: + service = adapter.get("auth_service") + if not service: + return adapter + resolved = {**adapter} + headers = dict(resolved.get("headers") or {}) + headers.setdefault("Authorization", f"Bearer {{{{secret:{service}_token}}}}") + resolved["headers"] = headers + return resolved + + +def validate_parameters(stream: dict, parameters: dict) -> None: + missing = [ + parameter["name"] + for parameter in stream.get("parameters", []) + if parameter.get("required") and parameter["name"] not in parameters + ] + if missing: + raise ValueError(f"{stream['id']}: missing required parameters: {', '.join(missing)}") + + +def local_command_approval_digest(stream: dict, adapter: dict) -> str: + clean_stream = {key: value for key, value in stream.items() if not key.startswith("__") and key != "pending"} + payload = { + "stream": clean_stream, + "command": adapter.get("command"), + "cwd": adapter.get("cwd"), + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def local_command_approval_path(root: Path, stream_id: str) -> Path: + parts = [part for part in stream_id.split("/") if part] + if not parts: + parts = ["unknown"] + safe_parts = [safe_name_part(part, "unknown") for part in parts] + safe_parts[-1] = f"{safe_parts[-1]}.json" + return root / "approvals" / "local-command" / Path(*safe_parts) + + +def write_local_command_approval(root: Path, stream: dict, adapter: dict) -> dict: + digest = local_command_approval_digest(stream, adapter) + payload = { + "version": SPEC_VERSION, + "stream_id": stream["id"], + "digest": digest, + "command": adapter.get("command"), + "cwd": adapter.get("cwd"), + "approved_at": now_utc(), + } + atomic_write_json(local_command_approval_path(root, stream["id"]), payload) + return payload + + +def require_local_command_approval(root: Path, stream: dict, adapter: dict) -> None: + approval_path = local_command_approval_path(root, stream["id"]) + approval = load_existing_state(approval_path) + digest = local_command_approval_digest(stream, adapter) + if not approval or approval.get("digest") != digest: + rel_path = approval_path.relative_to(root) + raise PermissionError( + f"{stream['id']}: local_command is not approved for this command digest; " + f"review the command and run `python3 scripts/agentfeeds.py admin templates approve-command {stream['id']}` " + f"to write {rel_path}" + ) + + +def publisher_for(stream_uri: str) -> str: + return urlparse(stream_uri).netloc + + +def run_adapter(stream: dict, parameters: dict, root: Path | None = None) -> tuple[str, list[dict]]: + validate_parameters(stream, parameters) + adapter = substitute(stream["adapter"], parameters) + adapter = add_auth_service_secret(adapter) + adapter = resolve_secret_refs(root, adapter) if root is not None else adapter + if adapter.get("kind") == "local_command": + if root is None: + raise PermissionError(f"{stream['id']}: local_command execution requires an Agent Feeds root for approval lookup") + if stream.get("pending"): + raise PermissionError(f"{stream['id']}: local_command template is pending operator approval") + require_local_command_approval(root, stream, adapter) + resolved_stream = {**stream, "adapter": adapter} + return run_adapter_impl( + resolved_stream, + parameters, + validate_parameters=validate_parameters, + source_uri_for=source_uri_for, + substitute=substitute, + ) + + +def value_at_path(payload: dict, dotted_path: str) -> object: + current: object = payload + for part in dotted_path.split("."): + if not isinstance(current, dict) or part not in current: + return None + current = current[part] + return current + + +def comparison_matches(actual: object, expected: object) -> bool: + if not isinstance(expected, dict): + return actual == expected + for operator, value in expected.items(): + if operator == "gte" and not (actual is not None and actual >= value): + return False + if operator == "gt" and not (actual is not None and actual > value): + return False + if operator == "lte" and not (actual is not None and actual <= value): + return False + if operator == "lt" and not (actual is not None and actual < value): + return False + if operator == "eq" and actual != value: + return False + return True + + +def event_matches_filter(event: dict, filters: dict | None) -> bool: + if not filters: + return True + for path, expected in filters.items(): + payload = event["data"] if path.startswith("data.") else event + lookup = path.removeprefix("data.") + if not comparison_matches(value_at_path(payload, lookup), expected): + return False + return True + + +def load_existing_state(path: Path) -> dict | None: + if not path.exists(): + return None + try: + return json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return None + + +def poll_interval(subscription: dict, stream: dict, defaults: dict) -> int: + return int( + subscription.get("poll_interval_seconds") + or defaults.get("poll_interval_seconds") + or stream.get("recommended_poll_interval_seconds") + or 600 + ) + + +def template_id_for(subscription: dict) -> str: + return str(subscription["template"]) + + +def subscription_title(subscription: dict, stream: dict) -> str: + return str(subscription.get("title") or stream.get("title") or subscription["id"]) + + +def state_path_for_subscription(root: Path, subscription: dict) -> Path: + stream = load_stream_definition(root, template_id_for(subscription)) + stream_uri = source_uri_for(stream, subscription.get("parameters") or {}) + return state_path_for_stream(stream_uri, root) + + +def is_due(path: Path, interval_seconds: int, force: bool) -> bool: + if force or not path.exists(): + return True + existing = load_existing_state(path) + if not existing: + return True + updated = parse_utc(existing.get("_meta", {}).get("last_updated")) + if not updated: + return True + return datetime.now(UTC) - updated >= timedelta(seconds=interval_seconds) + + +def schema_validation_enabled() -> bool: + value = os.environ.get("AGENTFEEDS_VALIDATE") + if value is None: + return True + return value.lower() not in {"0", "false", "no", "off"} + + +def validate_events_for_stream(root: Path, stream: dict, events: list[dict]) -> None: + if not schema_validation_enabled(): + return + schema = json.loads(schema_path_for_url(root, stream["schema_url"]).read_text(encoding="utf-8")) + for index, event in enumerate(events): + try: + jsonschema.validate(event.get("data"), schema) + except jsonschema.ValidationError as exc: + raise ValueError(f"{stream['id']}: event {index} does not match {stream['schema_url']}: {exc.message}") from exc + + +def state_payload( + subscription: dict, + stream: dict, + stream_uri: str, + events: list[dict], + existing: dict | None, + interval_seconds: int, + history_limit: int, +) -> dict: + updated_at = now_utc() + next_due = ( + datetime.now(UTC).replace(microsecond=0) + timedelta(seconds=interval_seconds) + ).isoformat().replace("+00:00", "Z") + meta = { + "stream": stream_uri, + "type": stream["type"], + "mode": stream["mode"], + "last_updated": updated_at, + "next_poll_due": next_due, + "schema_url": stream["schema_url"], + "schema_version": stream["schema_version"], + "publisher": publisher_for(stream_uri), + "stale": False, + "subscription_id": subscription["id"], + "template_id": stream["id"], + "title": subscription_title(subscription, stream), + } + + if stream["mode"] == "snapshot": + if not events: + raise ValueError(f"{stream['id']}: snapshot adapter produced no events") + return {"_meta": meta, "data": events[0]["data"]} + + if stream["mode"] == "event": + old_events = existing.get("data", []) if existing else [] + merged = [] + seen = set() + for event in [*events, *old_events]: + event_id = event.get("id") + if event_id in seen: + continue + seen.add(event_id) + merged.append(event) + return {"_meta": meta, "data": merged[:history_limit]} + + if stream["mode"] == "delta": + current = existing.get("data", {}) if existing else {} + for event in events: + current.update(event["data"]) + return {"_meta": meta, "data": current} + + raise ValueError(f"unsupported mode: {stream['mode']}") + + +def regenerate_catalog(root: Path) -> None: + lines = [ + "# Agent Feeds - Active Subscriptions", + "", + "This file lists data streams currently subscribed. Prefer `python3 scripts/agentfeeds.py streams read --json` for normal agent access.", + "", + ] + state_entries = [] + subscriptions = load_subscriptions(root).get("subscriptions") or [] + for subscription in subscriptions: + stream = load_stream_definition(root, template_id_for(subscription)) + parameters = subscription.get("parameters") or {} + stream_uri = source_uri_for(stream, parameters) + path = state_path_for_stream(stream_uri, root) + payload = load_existing_state(path) if path.exists() else {} + meta = (payload or {}).get("_meta", {}) + state_entries.append((subscription, stream, stream_uri, path, meta)) + if not state_entries: + lines.extend(["No active state files found.", ""]) + + for subscription, stream, stream_uri, path, meta in state_entries: + title = subscription_title(subscription, stream) + rel_path = path.relative_to(root) + lines.extend( + [ + f"## {title}", + f"- **ID:** `{subscription.get('id', '')}`", + f"- **Template:** `{template_id_for(subscription)}`", + f"- **Stream:** `{meta.get('stream') or stream_uri}`", + f"- **Path:** `{rel_path}`", + f"- **Updated:** {meta.get('last_updated', 'never')}", + f"- **Stale:** {'yes' if meta.get('stale') else 'no'}", + f"- **Mode:** {meta.get('mode') or stream.get('mode', 'unknown')}", + "", + ] + ) + + lines.extend( + [ + "---", + f"*Last regenerated: {now_utc()}.*", + "", + ] + ) + (root / "catalog.md").write_text("\n".join(lines), encoding="utf-8") + + +def run_fetch(args: argparse.Namespace, root: Path) -> int: + subscriptions = load_subscriptions(root) + defaults = subscriptions.get("defaults") or {} + active = subscriptions.get("subscriptions") or [] + if args.once: + active = [sub for sub in active if sub.get("id") == args.once] + if args.stream: + active = [sub for sub in active if sub.get("id") == args.stream] + + if not active: + regenerate_catalog(root) + return 0 + + force = args.all or bool(args.stream) or bool(args.once) + failures = 0 + for subscription in active: + attempted_at = now_utc() + stream = None + path = None + try: + stream = load_stream_definition(root, template_id_for(subscription)) + parameters = subscription.get("parameters") or {} + stream_uri = source_uri_for(stream, parameters) + path = state_path_for_stream(stream_uri, root) + interval_seconds = poll_interval(subscription, stream, defaults) + if not is_due(path, interval_seconds, force): + continue + + stream_uri, events = run_adapter(stream, parameters, root) + events = [event for event in events if event_matches_filter(event, subscription.get("filter"))] + validate_events_for_stream(root, stream, events) + history_limit = int(subscription.get("history_limit") or defaults.get("history_limit") or 50) + existing = load_existing_state(path) + payload = state_payload(subscription, stream, stream_uri, events, existing, interval_seconds, history_limit) + atomic_write_json(path, payload) + write_fetch_status(root, subscription, stream, attempted_at=attempted_at, succeeded=True, state_path=path) + except Exception as exc: # noqa: BLE001 - keep cron-friendly failure reporting. + failures += 1 + write_fetch_status(root, subscription, stream, attempted_at=attempted_at, succeeded=False, error=str(exc), state_path=path) + print(f"{subscription.get('id', '')}: {exc}", file=sys.stderr) + + regenerate_catalog(root) + return 1 if failures else 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Fetch Agent Feeds subscriptions") + parser.add_argument("--all", action="store_true", help="refresh every subscription") + parser.add_argument("--stream", help="refresh one subscription id") + parser.add_argument("--once", help="one-shot fetch for a subscription id") + parser.add_argument("--regenerate-catalog", action="store_true", help="regenerate catalog.md without polling") + parser.add_argument("--update-catalog", action="store_true", help="refresh the local catalog cache") + parser.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="agentfeeds root directory") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + root = args.root.expanduser() + ensure_root(root) + + if args.update_catalog: + update_catalog_cache(root) + if args.regenerate_catalog: + regenerate_catalog(root) + return 0 + with fetch_lock(root) as acquired: + if not acquired: + print(f"agentfeeds-fetch already running for {root}; skipping", file=sys.stderr) + return 1 + return run_fetch(args, root) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/agentfeeds/scripts/lib/agentfeeds_runtime/polling/__init__.py b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/polling/__init__.py new file mode 100644 index 00000000..d126ee63 --- /dev/null +++ b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/polling/__init__.py @@ -0,0 +1 @@ +"""Polling scheduler helpers for Agent Feeds.""" diff --git a/skills/agentfeeds/scripts/lib/agentfeeds_runtime/polling/install.py b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/polling/install.py new file mode 100755 index 00000000..0b261149 --- /dev/null +++ b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/polling/install.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Install background polling for Agent Feeds v0.3.""" + +from __future__ import annotations + +import os +import argparse +import platform +import plistlib +import shlex +import shutil +import subprocess +from pathlib import Path + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + + +LABEL = "dev.agentfeeds.fetch" +BEGIN_MARKER = "# BEGIN Agent Feeds polling" +END_MARKER = "# END Agent Feeds polling" +DEFAULT_ROOT = Path.home() / ".agentfeeds" +MIN_INTERVAL_SECONDS = 300 +STABLE_PATH = f"{Path.home()}/.local/bin:/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" + + +def load_subscriptions(root: Path) -> dict: + path = root / "subscriptions.yaml" + if not path.exists() or yaml is None: + return {} + return yaml.safe_load(path.read_text(encoding="utf-8")) or {} + + +def poll_interval_seconds(root: Path) -> int: + config = load_subscriptions(root) + defaults = config.get("defaults") or {} + intervals = [] + if defaults.get("poll_interval_seconds"): + intervals.append(int(defaults["poll_interval_seconds"])) + for subscription in config.get("subscriptions") or []: + if subscription.get("poll_interval_seconds"): + intervals.append(int(subscription["poll_interval_seconds"])) + return max(min(intervals or [600]), MIN_INTERVAL_SECONDS) + + +def fetcher_path() -> str: + venv_candidate = Path.home() / ".agentfeeds" / "runtime-venv" / "bin" / "agentfeeds-fetch" + if venv_candidate.exists(): + return str(venv_candidate) + candidate = Path.home() / ".local" / "bin" / "agentfeeds-fetch" + if candidate.exists(): + return str(candidate) + command = shutil.which("agentfeeds-fetch") + if command: + return command + raise FileNotFoundError("agentfeeds-fetch not found; run `python3 scripts/setup.py` first") + + +def install_launchd(root: Path, interval: int, fetcher: str) -> None: + launch_agents = Path.home() / "Library" / "LaunchAgents" + launch_agents.mkdir(parents=True, exist_ok=True) + logs = root / "logs" + logs.mkdir(parents=True, exist_ok=True) + plist_path = launch_agents / f"{LABEL}.plist" + payload = { + "Label": LABEL, + "ProgramArguments": [fetcher, "--root", str(root), "--all"], + "StartInterval": interval, + "RunAtLoad": True, + "StandardOutPath": str(logs / "poll.out.log"), + "StandardErrorPath": str(logs / "poll.err.log"), + "EnvironmentVariables": { + "PATH": STABLE_PATH, + }, + } + plist_path.write_bytes(plistlib.dumps(payload)) + + domain = f"gui/{os.getuid()}" + subprocess.run(["launchctl", "bootout", domain, str(plist_path)], check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + subprocess.run(["launchctl", "bootstrap", domain, str(plist_path)], check=True) + subprocess.run(["launchctl", "enable", f"{domain}/{LABEL}"], check=False) + subprocess.run(["launchctl", "kickstart", "-k", f"{domain}/{LABEL}"], check=False) + print(f"installed launchd polling: {plist_path}") + print(f"interval: {interval} seconds") + print(f"logs: {logs}") + + +def current_crontab() -> str: + result = subprocess.run(["crontab", "-l"], check=False, text=True, capture_output=True) + return "" if result.returncode else result.stdout + + +def without_existing_block(text: str) -> str: + lines = text.splitlines() + output = [] + skipping = False + for line in lines: + if line == BEGIN_MARKER: + skipping = True + continue + if line == END_MARKER: + skipping = False + continue + if not skipping: + output.append(line) + return "\n".join(output).strip() + + +def install_cron(root: Path, interval: int, fetcher: str) -> None: + minutes = max(interval // 60, 5) + logs = root / "logs" + logs.mkdir(parents=True, exist_ok=True) + command = ( + f"*/{minutes} * * * * " + f"{shlex.quote(fetcher)} --root {shlex.quote(str(root))} --all " + f">> {shlex.quote(str(logs / 'poll.out.log'))} " + f"2>> {shlex.quote(str(logs / 'poll.err.log'))}" + ) + existing = without_existing_block(current_crontab()) + new_crontab = "\n".join(line for line in [existing, BEGIN_MARKER, command, END_MARKER, ""] if line) + subprocess.run(["crontab", "-"], input=new_crontab + "\n", text=True, check=True) + print(f"installed cron polling every {minutes} minutes") + print(f"logs: {logs}") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Install Agent Feeds background polling") + parser.add_argument("--root", type=Path, default=DEFAULT_ROOT, help="agentfeeds root directory") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + root = args.root.expanduser() + root.mkdir(parents=True, exist_ok=True) + interval = poll_interval_seconds(root) + fetcher = fetcher_path() + system = platform.system() + if system == "Darwin": + install_launchd(root, interval, fetcher) + return 0 + if system in {"Linux", "FreeBSD"}: + install_cron(root, interval, fetcher) + return 0 + print(f"unsupported platform for polling installer: {system}") + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/agentfeeds/scripts/lib/agentfeeds_runtime/polling/uninstall.py b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/polling/uninstall.py new file mode 100755 index 00000000..1b3e514d --- /dev/null +++ b/skills/agentfeeds/scripts/lib/agentfeeds_runtime/polling/uninstall.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Uninstall background polling for Agent Feeds v0.3.""" + +from __future__ import annotations + +import os +import argparse +import platform +import subprocess +from pathlib import Path + + +LABEL = "dev.agentfeeds.fetch" +BEGIN_MARKER = "# BEGIN Agent Feeds polling" +END_MARKER = "# END Agent Feeds polling" + + +def uninstall_launchd() -> None: + plist_path = Path.home() / "Library" / "LaunchAgents" / f"{LABEL}.plist" + domain = f"gui/{os.getuid()}" + subprocess.run(["launchctl", "bootout", domain, str(plist_path)], check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if plist_path.exists(): + plist_path.unlink() + print(f"removed launchd polling: {plist_path}") + + +def current_crontab() -> str: + result = subprocess.run(["crontab", "-l"], check=False, text=True, capture_output=True) + return "" if result.returncode else result.stdout + + +def without_existing_block(text: str) -> str: + lines = text.splitlines() + output = [] + skipping = False + for line in lines: + if line == BEGIN_MARKER: + skipping = True + continue + if line == END_MARKER: + skipping = False + continue + if not skipping: + output.append(line) + return "\n".join(output).strip() + + +def uninstall_cron() -> None: + cleaned = without_existing_block(current_crontab()) + subprocess.run(["crontab", "-"], input=(cleaned + "\n") if cleaned else "", text=True, check=True) + print("removed cron polling") + + +def build_parser() -> argparse.ArgumentParser: + return argparse.ArgumentParser(description="Uninstall Agent Feeds background polling") + + +def main(argv: list[str] | None = None) -> int: + build_parser().parse_args(argv) + system = platform.system() + if system == "Darwin": + uninstall_launchd() + return 0 + if system in {"Linux", "FreeBSD"}: + uninstall_cron() + return 0 + print(f"unsupported platform for polling uninstaller: {system}") + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/agentfeeds/scripts/setup.py b/skills/agentfeeds/scripts/setup.py new file mode 100755 index 00000000..4db118be --- /dev/null +++ b/skills/agentfeeds/scripts/setup.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Install the Agent Feeds runtime into a local virtual environment.""" + +from __future__ import annotations + +import argparse +import shutil +import subprocess +import sys +import venv +from pathlib import Path + + +DEFAULT_VENV = Path.home() / ".agentfeeds" / "runtime-venv" + + +def venv_python(path: Path) -> Path: + if sys.platform == "win32": + return path / "Scripts" / "python.exe" + return path / "bin" / "python" + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Install Agent Feeds skill runtime") + parser.add_argument("--venv", type=Path, default=DEFAULT_VENV, help="runtime virtualenv path") + return parser + + +def create_venv(path: Path) -> None: + if venv_python(path).exists(): + return + try: + venv.EnvBuilder(with_pip=True).create(path) + return + except Exception: + if path.exists(): + import shutil as _shutil + + _shutil.rmtree(path) + uv = shutil.which("uv") + if not uv: + raise + subprocess.run([uv, "venv", "--python", sys.executable, str(path)], check=True) + + +def install_runtime(python: Path, root: Path) -> None: + has_pip = subprocess.run( + [str(python), "-m", "pip", "--version"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ).returncode == 0 + if has_pip: + subprocess.run([str(python), "-m", "pip", "install", "--upgrade", "pip"], check=True) + subprocess.run([str(python), "-m", "pip", "install", "-e", str(root)], check=True) + return + uv = shutil.which("uv") + if not uv: + raise RuntimeError(f"pip is unavailable in {python}; install pip or install uv") + subprocess.run([uv, "pip", "install", "--python", str(python), "-e", str(root)], check=True) + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + root = Path(__file__).resolve().parents[1] + venv_path = args.venv.expanduser() + venv_path.parent.mkdir(parents=True, exist_ok=True) + create_venv(venv_path) + python = venv_python(venv_path) + install_runtime(python, root) + print(f"installed Agent Feeds runtime: {venv_path}") + print(f"management CLI: {python} {root / 'scripts' / 'agentfeeds.py'}") + print(f"refresh worker: {python} {root / 'scripts' / 'agentfeeds_fetch.py'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())