Merge pull request #173 from boettiger-lab/refactor/drop-mcp-preload

refactor: drop MCP get_collection preload; move schema to call-time (#171)

Author: cboettig

app/dataset-catalog.js

@@ -16,8 +16,6 @@ export class DatasetCatalog {
     constructor() {
         /** @type {Map<string, DatasetEntry>} keyed by collection ID */
         this.datasets = new Map();
-        /** @type {Map<string, Object>} MCP get_collection results, keyed by collection ID */
-        this.mcpCollections = new Map();
         this.catalogUrl = null;
         this.titilerUrl = null;
     }
@@ -507,20 +505,6 @@ export class DatasetCatalog {
         return doc?.href || null;
     }
 
-    // ---- MCP preload ----
-
-    /**
-     * Store a structured collection dict from MCP get_collection.
-     * Called at startup to cache per-asset schemas and parquet paths
-     * for system prompt generation.
-     *
-     * @param {string} id - Collection ID
-     * @param {Object} data - Structured STAC collection from MCP
-     */
-    setMcpCollection(id, data) {
-        this.mcpCollections.set(id, data);
-    }
-
     // ---- Public API ----
 
     /**
@@ -610,53 +594,12 @@ export class DatasetCatalog {
         return sections.join('\n---\n\n');
     }
 
-    /**
-     * Extract SQL-relevant parquet assets from MCP data.
-     * Prefers H3 hex assets over full GeoParquet — when both exist,
-     * the hex is what DuckDB queries should use (partitioned, H3-indexed).
-     * @private
-     */
-    _getSqlAssets(mcpData) {
-        const assets = mcpData.assets || {};
-        const all = [];
-        let hasHex = false;
-
-        for (const [assetId, asset] of Object.entries(assets)) {
-            const type = asset.type || '';
-            const href = asset.href || '';
-            if (!type.includes('parquet') && !href.endsWith('.parquet') &&
-                !href.includes('/hex/') && !href.endsWith('/')) continue;
-            if (type.includes('pmtiles') || type.includes('tiff')) continue;
-
-            let s3Path = href;
-            if (s3Path.endsWith('/')) s3Path = s3Path.replace(/\/+$/, '') + '/**';
-
-            const isHex = href.includes('/hex/') || href.includes('/hex') ||
-                          (asset.title || '').toLowerCase().includes('hex') ||
-                          (asset.title || '').toLowerCase().includes('h3');
-            if (isHex) hasHex = true;
-
-            all.push({ assetId, title: asset.title || assetId, s3Path, isHex, asset });
-        }
-
-        // If hex assets exist, omit full GeoParquet (model doesn't need it for SQL)
-        return hasHex ? all.filter(a => a.isHex) : all;
-    }
-
     /**
      * Render SQL asset paths only (no columns) for the system prompt.
-     * Uses MCP data when available, falls back to local extraction.
+     * Uses the client-direct parquetAssets extracted during load().
      * @private
      */
     _renderSqlPaths(ds) {
-        const mcpData = this.mcpCollections.get(ds.id);
-        if (mcpData) {
-            const sqlAssets = this._getSqlAssets(mcpData);
-            if (sqlAssets.length === 0) return '';
-            const lines = sqlAssets.map(a => `- ${a.title}: \`read_parquet('${a.s3Path}')\``);
-            return '\n**SQL assets:**\n' + lines.join('\n') + '\n';
-        }
-        // Fallback: local parquet assets
         if (ds.parquetAssets.length === 0) return '';
         let out = '\n**SQL assets:**\n';
         for (const pa of ds.parquetAssets) {
@@ -665,123 +608,6 @@ export class DatasetCatalog {
         return out;
     }
 
-    /**
-     * Format schema info for the get_schema tool.
-     * Returns a compact, tabular representation optimized for LLM consumption:
-     * paths, column headers with representative values, and coded value lists.
-     *
-     * Uses MCP get_collection data (correct per-asset schemas).
-     * Falls back to local catalog data when MCP data is unavailable.
-     *
-     * @param {string} id - Collection ID
-     * @returns {string|null} Formatted schema text, or null if dataset not found
-     */
-    formatSchema(id) {
-        const ds = this.datasets.get(id);
-        if (!ds) return null;
-
-        const mcpData = this.mcpCollections.get(id);
-        if (mcpData) {
-            return this._formatSchemaFromMcp(ds, mcpData);
-        }
-        return this._formatSchemaFallback(ds);
-    }
-
-    /** @private */
-    _formatSchemaFromMcp(ds, mcpData) {
-        const sqlAssets = this._getSqlAssets(mcpData);
-        const collCols = (mcpData['table:columns'] || []).filter(c =>
-            !['geometry', 'geom', 'bbox'].includes(c.name?.toLowerCase())
-        );
-
-        // Determine which columns to use: per-asset if available, else collection-level
-        const sections = [];
-        for (const { title, s3Path, asset } of sqlAssets) {
-            const assetCols = (asset['table:columns'] || []).filter(c =>
-                !['geometry', 'geom', 'bbox'].includes(c.name?.toLowerCase())
-            );
-            // Use per-asset columns if available, fall back to collection-level
-            const cols = assetCols.length > 0 ? assetCols : collCols;
-            sections.push(this._renderOneAssetSchema(title, s3Path, cols));
-        }
-
-        // If no SQL assets found at all, render collection-level columns standalone
-        if (sections.length === 0 && collCols.length > 0) {
-            sections.push(this._renderOneAssetSchema(ds.title, null, collCols));
-        }
-
-        if (sections.length === 0) return `No schema available for ${ds.id}. Try get_stac_details("${ds.id}").`;
-        return sections.join('\n');
-    }
-
-    /**
-     * Render one asset's schema in tabular format.
-     * @private
-     */
-    _renderOneAssetSchema(title, s3Path, cols) {
-        let out = `${title}:\n`;
-        if (s3Path) out += `  read_parquet('${s3Path}')\n\n`;
-
-        if (cols.length === 0) return out;
-
-        // Column headers + sample values row (like SELECT * LIMIT 1 output)
-        const names = cols.map(c => c.name);
-        const samples = cols.map(c => {
-            if (c.values?.length > 0) return String(c.values[0]);
-            return `(${c.type || '?'})`;
-        });
-        out += '  ' + names.join(' | ') + '\n';
-        out += '  ' + samples.join(' | ') + '\n';
-
-        // Coded values
-        const coded = cols.filter(c => c.values?.length > 0);
-        if (coded.length > 0) {
-            out += '\n';
-            for (const c of coded) {
-                out += `  ${c.name}: ${c.values.join(', ')}\n`;
-            }
-        }
-
-        // Columns with descriptions but no coded values — show a compact hint
-        const described = cols.filter(c => !c.values?.length && c.description);
-        if (described.length > 0) {
-            out += '\n';
-            for (const c of described) {
-                out += `  ${c.name}: ${c.description}\n`;
-            }
-        }
-
-        // If no coded values at all, suggest SELECT * LIMIT 1 to see real data
-        if (coded.length === 0 && s3Path) {
-            out += `\n  Tip: run SELECT * FROM read_parquet('${s3Path}') LIMIT 1 to see sample values.\n`;
-        }
-
-        return out;
-    }
-
-    /** @private */
-    _formatSchemaFallback(ds) {
-        if (ds.parquetAssets.length === 0 && ds.columns.length === 0) {
-            return `No schema available for ${ds.id}. Try get_stac_details("${ds.id}").`;
-        }
-        let out = '';
-        for (const pa of ds.parquetAssets) {
-            out += `${pa.title}:\n  read_parquet('${pa.s3Path}')\n\n`;
-        }
-        if (ds.columns.length > 0) {
-            const names = ds.columns.map(c => c.name);
-            out += '  ' + names.join(' | ') + '\n';
-            const coded = ds.columns.filter(c => c.values?.length > 0);
-            if (coded.length > 0) {
-                out += '\n';
-                for (const c of coded) {
-                    out += `  ${c.name}: ${c.values.join(', ')}\n`;
-                }
-            }
-        }
-        return out;
-    }
-
     /**
      * Generate a flat list of all map layer IDs and their configs.
      * Used by MapManager to create layers.

app/main.js

@@ -216,37 +216,6 @@ async function main() {
         }], mcp);
     }
 
-    /* ── 5b. Preload MCP collection data for system prompt ──────────── */
-    // Call get_collection for each configured dataset to get correct per-asset
-    // schemas and parquet paths. This data enriches the system prompt.
-    // Falls back gracefully if MCP is unavailable.
-    try {
-        const collectionIds = appConfig.collections.map(c =>
-            typeof c === 'string' ? c : c.collection_id
-        );
-        const preloadResults = await Promise.allSettled(
-            collectionIds.map(id =>
-                mcp.callTool('get_collection', { collection_id: id })
-            )
-        );
-        let preloaded = 0;
-        for (let i = 0; i < collectionIds.length; i++) {
-            const r = preloadResults[i];
-            if (r.status === 'fulfilled' && r.value) {
-                try {
-                    const data = JSON.parse(r.value);
-                    if (!data.error) {
-                        catalog.setMcpCollection(collectionIds[i], data);
-                        preloaded++;
-                    }
-                } catch { /* not JSON — skip */ }
-            }
-        }
-        console.log(`[main] Preloaded ${preloaded}/${collectionIds.length} collections from MCP`);
-    } catch (err) {
-        console.warn('[main] MCP preload failed, using local catalog data:', err.message);
-    }
-
     /* ── 6. Build system prompt ────────────────────────────────────────── */
     const basePrompt = await fetchText('system-prompt.md');
     const catalogText = catalog.generatePromptCatalog();

app/map-tools.js

@@ -282,23 +282,36 @@ Vector layers: ${vectorLayerIds().join(', ')}`,
         // ---- Dataset Knowledge Tools ----
         {
             name: 'get_schema',
-            description: 'Get column names, types, sample values, and coded value lists for a dataset — formatted like SELECT * LIMIT 1 output. Also includes the read_parquet() path. **Call this before your first SQL query against a dataset.** Instant, no approval needed. For datasets outside your app, use `get_stac_details` instead.',
+            description: 'Get column names, types, sample values, and coded value lists for a dataset — formatted like SELECT * LIMIT 1 output. Also includes the read_parquet() path. **Call this before your first SQL query against a dataset.** For datasets outside your app, use `get_stac_details` instead.',
             inputSchema: {
                 type: 'object',
                 properties: {
                     dataset_id: { type: 'string', description: 'Collection ID of the dataset' }
                 },
                 required: ['dataset_id']
             },
-            execute: (args) => {
-                const result = catalog.formatSchema(args.dataset_id);
-                if (result === null) {
+            execute: async (args) => {
+                if (!catalog.get(args.dataset_id)) {
                     return JSON.stringify({
                         success: false,
                         error: `Dataset not found: ${args.dataset_id}. Available: ${catalog.getIds().join(', ')}. For datasets outside this app, use get_stac_details.`
                     });
                 }
-                return result;
+                if (!mcpClient) {
+                    return JSON.stringify({
+                        success: false,
+                        error: 'Schema service unavailable: MCP client not configured.'
+                    });
+                }
+                try {
+                    const raw = await mcpClient.callTool('get_stac_details', { dataset_id: args.dataset_id });
+                    return typeof raw === 'string' ? raw : JSON.stringify(raw);
+                } catch (err) {
+                    return JSON.stringify({
+                        success: false,
+                        error: `Schema service unavailable: ${err.message || err}. Try again, or call get_stac_details directly.`
+                    });
+                }
             },
         },
 

app/system-prompt.md

@@ -50,6 +50,8 @@ This applies equally when styling (e.g., building a `match` expression to color
 
 The dataset catalog below lists `read_parquet()` paths for every pre-loaded dataset. **These paths are authoritative — never guess, construct, or modify S3 paths.** Use them directly in SQL.
 
+**When a dataset has both a hex-indexed parquet path and a full GeoParquet path, prefer the hex path for SQL queries.** The hex path is partitioned by H3 cell and dramatically faster for spatial aggregations and joins. Asset titles make the distinction clear (e.g. `"SVI 2022 hex"` vs `"SVI 2022"`).
+
 **Before your first SQL query against a dataset, call `get_schema(dataset_id)`.** It returns column names, types, representative values, and coded value lists — instant, no approval needed. You don't need to call it again for follow-up queries on the same dataset unless you're unsure about column names.
 
 For datasets outside your app config, use `get_stac_details(collection_id)` instead.