scraper.js

const { execSync } = require('child_process');
const fs = require('fs');
const path = require('path');

const TOKEN = fs.readFileSync(path.join(__dirname, '.token'), 'utf8').trim();
const COOKIES = JSON.parse(fs.readFileSync(path.join(__dirname, 'cookies.txt'), 'utf8'))
  .map(c => c.name + '=' + c.value).join('; ');
const PROJECTS_DIR = path.join(__dirname, 'projects');
const DELAY_MS = 1200;
const NO_PROJECT = '[No Project]';

function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }

function curlGet(urlPath) {
  const url = 'https://chatgpt.com/backend-api' + urlPath;
  const result = execSync(
    `curl -s "${url}" `
    + `-H "Authorization: Bearer ${TOKEN}" `
    + `-H "Cookie: ${COOKIES}" `
    + `-H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36" `
    + `-H "Accept: application/json" `
    + `-H "Referer: https://chatgpt.com/" `
    + `-H "Sec-Fetch-Dest: empty" `
    + `-H "Sec-Fetch-Mode: cors" `
    + `-H "Sec-Fetch-Site: same-origin"`,
    { maxBuffer: 50 * 1024 * 1024 }
  );
  return JSON.parse(result.toString());
}

function curlDownload(url, outputPath) {
  execSync(
    `curl -s -o "${outputPath}" "${url}" `
    + `-H "Authorization: Bearer ${TOKEN}" `
    + `-H "Cookie: ${COOKIES}" `
    + `-H "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36" `
    + `-H "Accept: */*" `
    + `-H "Referer: https://chatgpt.com/"`,
    { maxBuffer: 50 * 1024 * 1024 }
  );
}

function safeName(name) {
  return (name || 'Untitled').replace(/[<>:"/\\|?*]/g, '_').replace(/\s+/g, ' ').trim();
}

function downloadImage(fileId, assetsDir, originalName) {
  try {
    // Step 1: get signed download URL
    const dlInfo = curlGet('/files/download/' + fileId + '?inline=false');
    if (!dlInfo.download_url) return null;

    // Step 2: download the actual file
    if (!fs.existsSync(assetsDir)) fs.mkdirSync(assetsDir, { recursive: true });
    const ext = path.extname(originalName) || '.jpg';
    const filename = safeName(path.basename(originalName, ext)) + ext;
    const outPath = path.join(assetsDir, filename);
    curlDownload(dlInfo.download_url, outPath);

    const stats = fs.statSync(outPath);
    if (stats.size < 100) {
      fs.unlinkSync(outPath);
      return null;
    }
    return filename;
  } catch (e) {
    console.log('    Image download failed:', e.message.substring(0, 100));
    return null;
  }
}

function extractMessages(conv) {
  const messages = [];
  let nodeId = conv.current_node;
  while (nodeId) {
    const node = conv.mapping[nodeId];
    if (node.message && node.message.content && node.message.author) {
      const msg = node.message;
      const role = msg.author.role;
      if (role === 'user' || role === 'assistant') {
        const contentType = msg.content.content_type;
        const parts = msg.content.parts || [];

        // Extract text parts
        const textParts = parts.filter(p => typeof p === 'string');
        const text = textParts.join('\n');

        // Extract image references
        const images = parts
          .filter(p => typeof p === 'object' && p && p.content_type === 'image_asset_pointer')
          .map(p => ({
            fileId: (p.asset_pointer || '').replace('sediment://', ''),
            width: p.width,
            height: p.height,
            sizeBytes: p.size_bytes
          }));

        // Get attachment metadata for filenames
        const attachments = (msg.metadata && msg.metadata.attachments) || [];

        // Get citation data
        const contentRefs = (msg.metadata && msg.metadata.content_references) || [];
        const searchResultGroups = (msg.metadata && msg.metadata.search_result_groups) || [];

        if (text.trim() || images.length > 0) {
          messages.unshift({ role, text, images, attachments, contentRefs, searchResultGroups, contentType });
        }
      }
    }
    nodeId = node.parent;
  }
  return messages;
}

function processCitations(text, contentRefs, searchResultGroups) {
  // Build a lookup of ref_index -> URL from search_result_groups
  const refUrlMap = {};
  for (const grp of (searchResultGroups || [])) {
    for (const entry of (grp.entries || [])) {
      if (entry.ref_id && entry.url) {
        const key = entry.ref_id.ref_index;
        const cleanUrl = entry.url.replace(/[?&]utm_source=chatgpt\.com$/, '');
        if (!refUrlMap[key]) refUrlMap[key] = cleanUrl;
      }
    }
  }

  // Strip invisible Unicode characters (PUA/surrogate chars ChatGPT injects around citations)
  let processedText = text.replace(/[\uE000-\uF8FF\uD800-\uDFFF]/g, '');

  const hasRefs = Object.keys(refUrlMap).length > 0 || (contentRefs && contentRefs.length > 0);
  if (!hasRefs && !processedText.match(/citeturn/)) return { text: processedText, footnotes: [] };
  const footnoteList = [];
  let footnoteNum = 1;
  const usedUrls = new Set();

  // Parse citation blocks: "citeturnXXXtypeN(turnXXXtypeN)*"
  processedText = processedText.replace(/cite((?:turn\d+(?:search|view)\d+)+)/g, (fullMatch, inner) => {
    const refPattern = /turn\d+(search|view)(\d+)/g;
    let refMatch;
    const fnNums = [];
    while ((refMatch = refPattern.exec(inner)) !== null) {
      const refIndex = parseInt(refMatch[2]);
      const url = refUrlMap[refIndex];
      if (url && !usedUrls.has(url)) {
        usedUrls.add(url);
        footnoteList.push({ num: footnoteNum, urls: [url] });
        fnNums.push(`[^${footnoteNum}]`);
        footnoteNum++;
      } else if (url) {
        const existing = footnoteList.find(f => f.urls[0] === url);
        if (existing) fnNums.push(`[^${existing.num}]`);
      }
    }
    return fnNums.length > 0 ? fnNums.join('') : '';
  });

  // Clean up any remaining cite patterns
  processedText = processedText.replace(/cite(?:turn\d+\w+)+/g, '');

  // Clean up sources_footnote markers from content_references
  for (const ref of (contentRefs || [])) {
    if (ref.type === 'sources_footnote' && ref.matched_text) {
      processedText = processedText.replace(ref.matched_text, '');
    }
  }

  return { text: processedText.trimEnd(), footnotes: footnoteList };
}

function buildMarkdown(conv, messages, assetsFolder) {
  let ct = conv.create_time, ut = conv.update_time;
  try { ct = new Date(ct * 1000).toISOString().replace('T', ' ').replace(/\.\d+Z/, ' UTC'); } catch(e) {}
  try { ut = new Date(ut * 1000).toISOString().replace('T', ' ').replace(/\.\d+Z/, ' UTC'); } catch(e) {}

  let md = `# ${conv.title || 'Untitled'}\n\n`;
  md += `**Created:** ${ct}\n`;
  md += `**Updated:** ${ut}\n`;
  md += `**Model:** ${conv.default_model_slug || 'unknown'}\n`;
  md += `**Conversation ID:** ${conv.conversation_id}\n\n---\n\n`;

  const allFootnotes = [];

  for (const msg of messages) {
    md += msg.role === 'user' ? '## You\n\n' : '## ChatGPT\n\n';

    // Add image references
    for (const img of msg.images) {
      const att = msg.attachments.find(a => a.id === img.fileId);
      const filename = att ? att.name : img.fileId + '.jpg';
      if (assetsFolder) {
        md += `![${filename}](${assetsFolder}/${safeName(path.basename(filename, path.extname(filename))) + path.extname(filename)})\n\n`;
      } else {
        md += `[Image: ${filename} (${img.width}x${img.height})]\n\n`;
      }
    }

    // Process citations and add text
    const { text, footnotes } = processCitations(msg.text, msg.contentRefs, msg.searchResultGroups);
    if (text.trim()) md += text + '\n\n';

    for (const fn of footnotes) {
      allFootnotes.push(fn);
    }
  }

  // Add footnotes section
  if (allFootnotes.length > 0) {
    md += '---\n\n### References\n\n';
    for (const fn of allFootnotes) {
      md += `[^${fn.num}]: ${fn.urls.join(' , ')}\n`;
    }
    md += '\n';
  }

  return md;
}

async function saveConversation(convId, folder) {
  const conv = curlGet('/conversation/' + convId);
  const msgs = extractMessages(conv);
  const safeTitle = safeName(conv.title);
  const dir = path.join(PROJECTS_DIR, safeName(folder));
  if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });

  // Check if any messages have images
  const hasImages = msgs.some(m => m.images.length > 0);
  let assetsFolder = null;
  let assetsDirPath = null;

  if (hasImages) {
    assetsFolder = '_assets - ' + safeTitle;
    assetsDirPath = path.join(dir, assetsFolder);
  }

  // Download images
  let imageCount = 0;
  if (hasImages) {
    for (const msg of msgs) {
      for (const img of msg.images) {
        const att = msg.attachments.find(a => a.id === img.fileId);
        const originalName = att ? att.name : img.fileId + '.jpg';
        const result = downloadImage(img.fileId, assetsDirPath, originalName);
        if (result) imageCount++;
        await sleep(500);
      }
    }
  }

  const md = buildMarkdown(conv, msgs, assetsFolder);
  const filepath = path.join(dir, safeTitle + '.md');
  fs.writeFileSync(filepath, md, 'utf8');

  return { title: conv.title, messages: msgs.length, size: md.length, images: imageCount };
}

function downloadProjectFile(fileId, gizmoId, outDir, fileName) {
  try {
    const dlInfo = curlGet('/files/' + fileId + '/download?gizmo_id=' + gizmoId);
    if (!dlInfo.download_url) return false;
    if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true });
    const outPath = path.join(outDir, safeName(path.basename(fileName, path.extname(fileName))) + path.extname(fileName));
    curlDownload(dlInfo.download_url, outPath);
    const stats = fs.statSync(outPath);
    if (stats.size < 50) { fs.unlinkSync(outPath); return false; }
    return true;
  } catch (e) {
    console.log('    File download failed:', e.message.substring(0, 100));
    return false;
  }
}

function saveMemories() {
  console.log('\nSaving memories...');
  try {
    let allMemories = [];
    const data = curlGet('/memories?limit=500');
    allMemories = data.memories || [];
    console.log(`  ${allMemories.length} memories found`);

    let md = '# ChatGPT Memories\n\n';
    md += `**Exported:** ${new Date().toISOString().replace('T', ' ').replace(/\.\d+Z/, ' UTC')}\n`;
    md += `**Count:** ${allMemories.length}\n\n---\n\n`;
    for (const m of allMemories) {
      md += `- **[${m.updated_at}]** ${m.content}`;
      if (m.gizmo_id) md += ` _(project: ${m.gizmo_id})_`;
      md += '\n';
    }

    const outDir = path.join(PROJECTS_DIR, '_account');
    if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true });
    fs.writeFileSync(path.join(outDir, 'memories.md'), md, 'utf8');
    // Also save raw JSON for machine use
    fs.writeFileSync(path.join(outDir, 'memories.json'), JSON.stringify(allMemories, null, 2), 'utf8');
    console.log('  Saved to _account/memories.md + memories.json');
  } catch (e) {
    console.log('  Failed:', e.message.substring(0, 100));
  }
}

function saveCustomInstructions() {
  console.log('\nSaving custom instructions...');
  try {
    const ci = curlGet('/user_system_messages');
    let md = '# ChatGPT Custom Instructions\n\n';
    md += `**Exported:** ${new Date().toISOString().replace('T', ' ').replace(/\.\d+Z/, ' UTC')}\n`;
    md += `**Enabled:** ${ci.enabled}\n\n---\n\n`;
    md += '## About You (what ChatGPT knows about you)\n\n';
    md += (ci.about_user_message || '_(empty)_') + '\n\n';
    md += '## Response Preferences (how ChatGPT should respond)\n\n';
    md += (ci.about_model_message || '_(empty)_') + '\n\n';

    const outDir = path.join(PROJECTS_DIR, '_account');
    if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true });
    fs.writeFileSync(path.join(outDir, 'custom_instructions.md'), md, 'utf8');
    fs.writeFileSync(path.join(outDir, 'custom_instructions.json'), JSON.stringify(ci, null, 2), 'utf8');
    console.log('  Saved to _account/custom_instructions.md + .json');
  } catch (e) {
    console.log('  Failed:', e.message.substring(0, 100));
  }
}

async function main() {
  const testMode = process.argv.includes('--test');
  const testConvId = process.argv.find((a, i) => process.argv[i-1] === '--conv');

  if (testConvId) {
    console.log('Testing single conversation:', testConvId);
    const result = await saveConversation(testConvId, NO_PROJECT);
    console.log('Result:', result);
    return;
  }

  console.log('Starting ChatGPT scraper...\n');

  // 1. Get all projects (and their files metadata)
  console.log('Fetching projects...');
  let allProjects = [];
  let allSidebarItems = [];
  let cursor = null;
  do {
    let url = '/gizmos/snorlax/sidebar?owned_only=true&conversations_per_gizmo=0';
    if (cursor) url += '&cursor=' + encodeURIComponent(cursor);
    const data = curlGet(url);
    for (const item of data.items) {
      allProjects.push({ id: item.gizmo.gizmo.id, name: item.gizmo.gizmo.display.name });
      allSidebarItems.push(item);
    }
    cursor = data.cursor;
  } while (cursor);
  console.log(`Found ${allProjects.length} projects\n`);

  // 2. Download project files
  console.log('Downloading project files...');
  let totalProjectFiles = 0;
  for (const item of allSidebarItems) {
    const g = item.gizmo;
    const files = g.files || [];
    const projName = g.gizmo.display.name.trim();
    if (files.length === 0) continue;

    console.log(`\n  ${projName}: ${files.length} files`);
    const filesDir = path.join(PROJECTS_DIR, safeName(projName), '_project_files');
    for (const f of files) {
      await sleep(DELAY_MS);
      const ok = downloadProjectFile(f.file_id, g.gizmo.id, filesDir, f.name);
      if (ok) {
        totalProjectFiles++;
        console.log(`    ✓ ${f.name} (${f.size} bytes)`);
      } else {
        console.log(`    ✗ ${f.name}`);
      }
    }
  }
  console.log(`\n  Total project files downloaded: ${totalProjectFiles}`);

  // 3. Scrape each project's conversations
  for (const proj of allProjects) {
    console.log(`\nProject: ${proj.name}`);
    let convIds = [];
    let pcursor = '0';
    do {
      await sleep(DELAY_MS);
      const data = curlGet('/gizmos/' + proj.id + '/conversations?cursor=' + pcursor);
      for (const item of data.items) {
        convIds.push({ id: item.id, title: item.title });
      }
      pcursor = data.cursor || null;
    } while (pcursor);
    console.log(`  ${convIds.length} conversations`);

    for (const c of convIds) {
      await sleep(DELAY_MS);
      try {
        const result = await saveConversation(c.id, proj.name.trim());
        console.log(`  ✓ ${result.title} (${result.messages} msgs, ${result.images} imgs, ${result.size} bytes)`);
      } catch(e) {
        if (e.message.includes('RATE_LIMITED') || e.message.includes('429')) {
          console.log('  ⚠ Rate limited, waiting 30s...');
          await sleep(30000);
          try {
            const result = await saveConversation(c.id, proj.name.trim());
            console.log(`  ✓ ${result.title} (${result.messages} msgs, ${result.images} imgs, ${result.size} bytes)`);
          } catch(e2) {
            console.log(`  ✗ ${c.title}: ${e2.message.substring(0, 100)}`);
          }
        } else {
          console.log(`  ✗ ${c.title}: ${e.message.substring(0, 100)}`);
        }
      }
    }
  }

  // 4. Scrape non-project conversations
  console.log('\n\nNon-project conversations:');
  let offset = 0;
  let total = 0;
  do {
    await sleep(DELAY_MS);
    const data = curlGet('/conversations?offset=' + offset + '&limit=28&order=updated');
    total = data.total;
    for (const item of data.items) {
      await sleep(DELAY_MS);
      try {
        const result = await saveConversation(item.id, NO_PROJECT);
        console.log(`  ✓ ${result.title} (${result.messages} msgs, ${result.images} imgs, ${result.size} bytes)`);
      } catch(e) {
        if (e.message.includes('RATE_LIMITED') || e.message.includes('429')) {
          console.log('  ⚠ Rate limited, waiting 30s...');
          await sleep(30000);
          try {
            const result = await saveConversation(item.id, NO_PROJECT);
            console.log(`  ✓ ${result.title} (${result.messages} msgs, ${result.images} imgs, ${result.size} bytes)`);
          } catch(e2) {
            console.log(`  ✗ ${item.title}: ${e2.message.substring(0, 100)}`);
          }
        } else {
          console.log(`  ✗ ${item.title}: ${e.message.substring(0, 100)}`);
        }
      }
    }
    offset += data.items.length;
  } while (offset < total);

  // 5. Scrape archived conversations
  console.log('\n\nArchived conversations:');
  try {
    const arch = curlGet('/conversations?is_archived=true&offset=0&limit=100');
    console.log(`  ${arch.total} archived conversations`);
    for (const item of arch.items) {
      await sleep(DELAY_MS);
      try {
        const result = await saveConversation(item.id, '[Archived]');
        console.log(`  ✓ ${result.title} (${result.messages} msgs, ${result.images} imgs, ${result.size} bytes)`);
      } catch(e) {
        console.log(`  ✗ ${item.title}: ${e.message.substring(0, 100)}`);
      }
    }
  } catch(e) {
    console.log('  Failed:', e.message.substring(0, 100));
  }

  // 6. Save account-level data
  saveMemories();
  saveCustomInstructions();

  console.log('\n\nDone!');
}

main().catch(e => console.error('Fatal:', e));