From a751a0c8c4cc74e3a889f41f70515ae4db428470 Mon Sep 17 00:00:00 2001 From: darthjaja Date: Tue, 12 May 2026 20:12:40 +0000 Subject: [PATCH 1/2] fix(twitter): repair search and tweets readback --- cli-manifest.json | 8 +- clis/twitter/post.js | 27 ++- clis/twitter/post.test.js | 30 +++ clis/twitter/search.js | 356 +++++++++++++++++++----------------- clis/twitter/search.test.js | 355 +++++++---------------------------- clis/twitter/shared.js | 125 +++++++++++-- clis/twitter/shared.test.js | 35 +++- clis/twitter/tweets.js | 160 +++++++++++----- clis/twitter/tweets.test.js | 102 +++++++++++ 9 files changed, 672 insertions(+), 526 deletions(-) diff --git a/cli-manifest.json b/cli-manifest.json index c3caa8d85..6805f80df 100644 --- a/cli-manifest.json +++ b/cli-manifest.json @@ -23244,7 +23244,9 @@ "columns": [ "status", "message", - "text" + "text", + "id", + "url" ], "type": "js", "modulePath": "twitter/post.js", @@ -23460,7 +23462,7 @@ "description": "Search Twitter/X for tweets, with optional --from / --has / --exclude / --product filters mapped to X's search operators", "access": "read", "domain": "x.com", - "strategy": "intercept", + "strategy": "cookie", "browser": true, "args": [ { @@ -23553,7 +23555,7 @@ "type": "js", "modulePath": "twitter/search.js", "sourceFile": "twitter/search.js", - "navigateBefore": true, + "navigateBefore": "https://x.com", "siteSession": "persistent" }, { diff --git a/clis/twitter/post.js b/clis/twitter/post.js index 134e02e9f..2c5473849 100644 --- a/clis/twitter/post.js +++ b/clis/twitter/post.js @@ -161,12 +161,25 @@ async function submitTweet(page, text) { const normalize = s => String(s || '').replace(/\u00a0/g, ' ').replace(/\s+/g, ' ').trim(); const expectedText = normalize(expected); const visible = (el) => !!el && (el.offsetParent !== null || el.getClientRects().length > 0); + const statusUrl = (root = document) => { + const links = Array.from(root.querySelectorAll('a[href*="/status/"]')); + for (const link of links) { + const href = link.href || link.getAttribute('href') || ''; + if (!href) continue; + try { + const url = new URL(href, window.location.origin); + const match = url.pathname.match(/^\\/(?:[^/]+|i)\\/status\\/(\\d+)/); + if (match) return { url: url.href, id: match[1] }; + } catch {} + } + return {}; + }; for (let i = 0; i < ${JSON.stringify(iterations)}; i++) { await new Promise(r => setTimeout(r, ${JSON.stringify(SUBMIT_POLL_MS)})); const toasts = Array.from(document.querySelectorAll('[role="alert"], [data-testid="toast"]')) .filter((el) => visible(el)); const successToast = toasts.find((el) => /sent|posted|your post was sent|your tweet was sent/i.test(el.textContent || '')); - if (successToast) return { ok: true, message: 'Tweet posted successfully.' }; + if (successToast) return { ok: true, message: 'Tweet posted successfully.', ...statusUrl(successToast) }; const alert = toasts.find((el) => /failed|error|try again|not sent|could not/i.test(el.textContent || '')); if (alert) return { ok: false, message: (alert.textContent || 'Tweet failed to post.').trim() }; @@ -175,7 +188,7 @@ async function submitTweet(page, text) { const hasMedia = !!document.querySelector('[data-testid="attachments"], [data-testid="tweetPhoto"]') || document.querySelectorAll('img[src^="blob:"], video[src^="blob:"]').length > 0; if (!composerStillHasText && !hasMedia) { - return { ok: true, message: 'Tweet posted successfully.' }; + return { ok: true, message: 'Tweet posted successfully.', ...statusUrl() }; } } return { ok: false, message: 'Tweet submission did not complete before timeout.' }; @@ -194,7 +207,7 @@ cli({ { name: 'text', type: 'string', required: true, positional: true, help: 'The text content of the tweet' }, { name: 'images', type: 'string', required: false, help: 'Image paths, comma-separated, max 4 (jpg/png/gif/webp)' }, ], - columns: ['status', 'message', 'text'], + columns: ['status', 'message', 'text', 'id', 'url'], func: async (page, kwargs) => { if (!page) throw new CommandExecutionError('Browser session required for twitter post'); @@ -231,6 +244,12 @@ cli({ await page.wait(1); const result = await submitTweet(page, text); - return [{ status: result?.ok ? 'success' : 'failed', message: result?.message ?? 'Tweet failed to post.', text }]; + return [{ + status: result?.ok ? 'success' : 'failed', + message: result?.message ?? 'Tweet failed to post.', + text, + ...(result?.id ? { id: result.id } : {}), + ...(result?.url ? { url: result.url } : {}), + }]; } }); diff --git a/clis/twitter/post.test.js b/clis/twitter/post.test.js index 56b7017b3..c0604e1f8 100644 --- a/clis/twitter/post.test.js +++ b/clis/twitter/post.test.js @@ -46,6 +46,11 @@ function makePage(evaluateResults = [], overrides = {}) { describe('twitter post command', () => { const getCommand = () => getRegistry().get('twitter/post'); + it('registers created tweet id/url columns', () => { + const command = getCommand(); + expect(command?.columns).toEqual(['status', 'message', 'text', 'id', 'url']); + }); + it('posts text-only tweet successfully through the current compose route', async () => { const command = getCommand(); const page = makePage([ @@ -63,6 +68,31 @@ describe('twitter post command', () => { expect(page.insertText).toHaveBeenCalledWith('hello world'); }); + it('returns the created tweet URL from the success toast when available', async () => { + const command = getCommand(); + const page = makePage([ + { ok: true }, + { ok: true }, + { ok: true }, + { + ok: true, + message: 'Tweet posted successfully.', + id: '2054239044884693381', + url: 'https://x.com/darthjajaj6z/status/2054239044884693381', + }, + ]); + + const result = await command.func(page, { text: 'with url' }); + + expect(result).toEqual([{ + status: 'success', + message: 'Tweet posted successfully.', + text: 'with url', + id: '2054239044884693381', + url: 'https://x.com/darthjajaj6z/status/2054239044884693381', + }]); + }); + it('returns failed when text area not found', async () => { const command = getCommand(); const page = makePage([ diff --git a/clis/twitter/search.js b/clis/twitter/search.js index 62310f24e..001c1c339 100644 --- a/clis/twitter/search.js +++ b/clis/twitter/search.js @@ -1,7 +1,7 @@ -import { ArgumentError, CommandExecutionError } from '@jackwener/opencli/errors'; +import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors'; import { cli, Strategy } from '@jackwener/opencli/registry'; -import { extractMedia } from './shared.js'; -import { applyTopByEngagement } from './utils.js'; +import { extractMedia, normalizeTwitterGraphqlPayload, resolveTwitterOperationMetadata } from './shared.js'; +import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js'; // ── Public-search operator surface ───────────────────────────────────── // @@ -35,6 +35,67 @@ const PRODUCT_TO_F_PARAM = Object.freeze({ videos: 'video', }); +const PRODUCT_TO_GRAPHQL_PRODUCT = Object.freeze({ + top: 'Top', + live: 'Latest', + photos: 'Photos', + videos: 'Videos', +}); + +const SEARCH_TIMELINE_OPERATION = { + queryId: 'VhUd6vHVmLBcw0uX-6jMLA', + features: { + rweb_video_screen_enabled: true, + rweb_cashtags_enabled: true, + profile_label_improvements_pcf_label_in_post_enabled: true, + responsive_web_profile_redirect_enabled: true, + rweb_tipjar_consumption_enabled: true, + verified_phone_label_enabled: false, + creator_subscriptions_tweet_preview_api_enabled: true, + responsive_web_graphql_timeline_navigation_enabled: true, + responsive_web_graphql_skip_user_profile_image_extensions_enabled: false, + premium_content_api_read_enabled: false, + communities_web_enable_tweet_community_results_fetch: true, + c9s_tweet_anatomy_moderator_badge_enabled: true, + responsive_web_grok_analyze_button_fetch_trends_enabled: false, + responsive_web_grok_analyze_post_followups_enabled: true, + rweb_cashtags_composer_attachment_enabled: true, + responsive_web_jetfuel_frame: true, + responsive_web_grok_share_attachment_enabled: true, + responsive_web_grok_annotations_enabled: true, + articles_preview_enabled: true, + responsive_web_edit_tweet_api_enabled: true, + graphql_is_translatable_rweb_tweet_is_translatable_enabled: true, + view_counts_everywhere_api_enabled: true, + longform_notetweets_consumption_enabled: true, + responsive_web_twitter_article_tweet_consumption_enabled: true, + content_disclosure_indicator_enabled: true, + content_disclosure_ai_generated_indicator_enabled: true, + responsive_web_grok_show_grok_translated_post: false, + responsive_web_grok_analysis_button_from_backend: true, + post_ctas_fetch_enabled: false, + freedom_of_speech_not_reach_fetch_enabled: true, + standardized_nudges_misinfo: true, + tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true, + longform_notetweets_rich_text_read_enabled: true, + longform_notetweets_inline_media_enabled: true, + responsive_web_grok_image_annotation_enabled: true, + responsive_web_grok_imagine_annotation_enabled: true, + responsive_web_grok_community_note_auto_translation_is_enabled: false, + responsive_web_enhance_cards_enabled: false, + }, + fieldToggles: { + withPayments: true, + withAuxiliaryUserLabels: true, + withArticleRichContentState: true, + withArticlePlainText: true, + withArticleSummaryText: true, + withArticleVoiceOver: true, + withGrokAnalyze: true, + withDisallowedReplyControls: true, + }, +}; + const FROM_USER_PATTERN = /^[A-Za-z0-9_]{1,15}$/; const EXCLUDE_TO_OPERATOR = Object.freeze({ @@ -99,125 +160,96 @@ function resolveSearchFParam(kwargs) { return kwargs.filter === 'live' ? 'live' : 'top'; } -/** - * Trigger Twitter search SPA navigation with fallback strategies. - * - * Primary: pushState + popstate (works in most environments). - * Fallback: Type into the search input and press Enter when pushState fails - * intermittently (e.g. due to Twitter A/B tests or timing races — see #690). - * - * Both strategies preserve the JS context so the fetch interceptor stays alive. - * - * @param {object} page - * @param {string} query — final composed query (already merged with operators) - * @param {string} fParam — Twitter URL `f=` value (top|live|image|video) - */ -async function navigateToSearch(page, query, fParam) { - const searchUrl = JSON.stringify(`/search?q=${encodeURIComponent(query)}&f=${fParam}`); - let lastPath = ''; - // Strategy 1 (primary): pushState + popstate with retry - for (let attempt = 1; attempt <= 2; attempt++) { - await page.evaluate(` - (() => { - window.history.pushState({}, '', ${searchUrl}); - window.dispatchEvent(new PopStateEvent('popstate', { state: {} })); - })() - `); - try { - await page.wait({ selector: '[data-testid="primaryColumn"]' }); - } - catch { - // selector timeout — fall through to path check or next attempt - } - lastPath = String(await page.evaluate('() => window.location.pathname') || ''); - if (lastPath.startsWith('/search')) { - return; - } - if (attempt < 2) { - await page.wait(1); - } +function resolveSearchProduct(kwargs) { + const product = kwargs.product || (kwargs.filter === 'live' ? 'live' : 'top'); + return PRODUCT_TO_GRAPHQL_PRODUCT[product] || 'Top'; +} + +function normalizeOperation(operation) { + if (typeof operation === 'string') { + return { + queryId: operation, + features: SEARCH_TIMELINE_OPERATION.features, + fieldToggles: SEARCH_TIMELINE_OPERATION.fieldToggles, + }; } - // Strategy 2 (fallback): Use the search input on /explore. - // The nativeSetter + Enter approach triggers Twitter's own form handler, - // performing SPA navigation without a full page reload. - const queryStr = JSON.stringify(query); - const navResult = await page.evaluate(`(async () => { - try { - const input = document.querySelector('[data-testid="SearchBox_Search_Input"]'); - if (!input) return { ok: false }; + return { + queryId: operation?.queryId || SEARCH_TIMELINE_OPERATION.queryId, + features: operation?.features || SEARCH_TIMELINE_OPERATION.features, + fieldToggles: operation?.fieldToggles || SEARCH_TIMELINE_OPERATION.fieldToggles, + }; +} - input.focus(); - await new Promise(r => setTimeout(r, 300)); +function buildSearchTimelineRequest(operation, rawQuery, product, count, cursor) { + const normalized = normalizeOperation(operation); + const vars = { + rawQuery, + count, + querySource: 'typed_query', + product, + }; + if (cursor) vars.cursor = cursor; + return [ + `/i/api/graphql/${normalized.queryId}/SearchTimeline`, + { + variables: vars, + features: normalized.features, + fieldToggles: normalized.fieldToggles, + }, + ]; +} - const nativeSetter = Object.getOwnPropertyDescriptor( - window.HTMLInputElement.prototype, 'value' - )?.set; - if (!nativeSetter) return { ok: false }; - nativeSetter.call(input, ${queryStr}); - input.dispatchEvent(new Event('input', { bubbles: true })); - input.dispatchEvent(new Event('change', { bubbles: true })); - await new Promise(r => setTimeout(r, 500)); +function unwrapTweetResult(result) { + if (!result) return null; + if (result.__typename === 'TweetWithVisibilityResults' && result.tweet) return result.tweet; + if (result.tweet) return result.tweet; + return result; +} - input.dispatchEvent(new KeyboardEvent('keydown', { - key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true - })); +function tweetToRow(result, seen) { + const tweet = unwrapTweetResult(result); + if (!tweet?.rest_id || seen.has(tweet.rest_id)) return null; + seen.add(tweet.rest_id); + const tweetUser = tweet.core?.user_results?.result; + return { + id: tweet.rest_id, + author: tweetUser?.core?.screen_name || tweetUser?.legacy?.screen_name || 'unknown', + text: tweet.note_tweet?.note_tweet_results?.result?.text || tweet.legacy?.full_text || '', + created_at: tweet.legacy?.created_at || '', + likes: tweet.legacy?.favorite_count || 0, + views: tweet.views?.count || '0', + url: `https://x.com/i/status/${tweet.rest_id}`, + ...extractMedia(tweet.legacy), + }; +} - return { ok: true }; - } catch { - return { ok: false }; - } - })()`); - if (navResult?.ok) { - try { - await page.wait({ selector: '[data-testid="primaryColumn"]' }); +function parseSearchTimeline(data, seen) { + const rows = []; + let nextCursor = null; + const instructions = data?.data?.search_by_raw_query?.search_timeline?.timeline?.instructions || []; + const visit = (value) => { + if (!value || typeof value !== 'object') return; + if (value.tweet_results?.result) { + const row = tweetToRow(value.tweet_results.result, seen); + if (row) rows.push(row); } - catch { - // fall through to path check + if ( + (value.entryType === 'TimelineTimelineCursor' || value.__typename === 'TimelineTimelineCursor') + && (value.cursorType === 'Bottom' || value.cursorType === 'ShowMore') + && value.value + ) { + nextCursor = value.value; } - lastPath = String(await page.evaluate('() => window.location.pathname') || ''); - if (lastPath.startsWith('/search')) { - // The fallback path doesn't carry the f= URL param, so click the - // matching tab to align with the requested product. Only `live` - // currently surfaces a distinct tab label — `image`/`video` tabs - // also need an explicit click, so try them all. - const tabClicked = await clickProductTabIfNeeded(page, fParam); - if (!tabClicked) { - throw new CommandExecutionError(`SPA fallback reached /search but could not select the requested product tab: ${fParam}`); - } + if (Array.isArray(value)) { + for (const item of value) visit(item); return; } - } - throw new CommandExecutionError(`SPA navigation to /search failed. Final path: ${lastPath || '(empty)'}. Twitter may have changed its routing.`); -} - -/** - * After the search-input fallback lands on /search, the f= param is missing - * from the URL. Click the matching tab in the result page header so the - * SearchTimeline call uses the right filter. No-op for fParam=top (default). - */ -async function clickProductTabIfNeeded(page, fParam) { - if (fParam === 'top') return true; - const tabLabels = JSON.stringify({ - live: ['Latest', '最新'], - image: ['Photos', 'Images', '照片', '图片'], - video: ['Videos', '视频'], - }[fParam] || []); - if (tabLabels === '[]') return true; - const clicked = await page.evaluate(`(() => { - const labels = ${tabLabels}; - const tabs = document.querySelectorAll('[role="tab"]'); - for (const tab of tabs) { - const txt = (tab.textContent || '').trim(); - if (labels.some(l => txt.includes(l))) { - tab.click(); - return true; + for (const child of Object.values(value)) { + if (child && typeof child === 'object') visit(child); } - } - return false; - })()`); - if (!clicked) return false; - await page.wait(2); - return true; + }; + visit(instructions); + return { rows, nextCursor }; } cli({ @@ -226,7 +258,7 @@ cli({ access: 'read', description: 'Search Twitter/X for tweets, with optional --from / --has / --exclude / --product filters mapped to X\'s search operators', domain: 'x.com', - strategy: Strategy.INTERCEPT, // Use intercept strategy + strategy: Strategy.COOKIE, browser: true, siteSession: 'persistent', args: [ @@ -248,65 +280,46 @@ cli({ if (!Number.isInteger(Number(kwargs.limit)) || Number(kwargs.limit) <= 0) { throw new ArgumentError('twitter search --limit must be a positive integer', 'Example: opencli twitter search opencli --limit 15'); } - const fParam = resolveSearchFParam(kwargs); - // 1. Navigate to x.com/explore (has a search input at the top) - await page.goto('https://x.com/explore'); - await page.wait(3); - // 2. Install interceptor BEFORE triggering search. - // SPA navigation preserves the JS context, so the monkey-patched - // fetch will capture the SearchTimeline API call. - await page.installInterceptor('SearchTimeline'); - // 3. Trigger SPA navigation to search results via history API. - // pushState + popstate triggers React Router's listener without - // a full page reload, so the interceptor stays alive. - // Note: the previous approach (nativeSetter + Enter keydown on the - // search input) does not reliably trigger Twitter's form submission. - await navigateToSearch(page, finalQuery, fParam); - // 4. Scroll to trigger additional pagination - await page.autoScroll({ times: 3, delayMs: 2000 }); - // 5. Retrieve captured data - const requests = await page.getInterceptedRequests(); - if (!requests || requests.length === 0) - return []; - let results = []; + const cookies = await page.getCookies({ url: 'https://x.com' }); + const ct0 = cookies.find((c) => c.name === 'ct0')?.value || null; + if (!ct0) throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)'); + await page.goto('https://x.com/home', { waitUntil: 'load', settleMs: 1000 }); + const operation = await resolveTwitterOperationMetadata(page, 'SearchTimeline', SEARCH_TIMELINE_OPERATION); + const headers = JSON.stringify({ + 'Authorization': `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`, + 'X-Csrf-Token': ct0, + 'X-Twitter-Auth-Type': 'OAuth2Session', + 'X-Twitter-Active-User': 'yes', + 'Content-Type': 'application/json', + }); + const product = resolveSearchProduct(kwargs); + const results = []; const seen = new Set(); - for (const req of requests) { - try { - const insts = req?.data?.search_by_raw_query?.search_timeline?.timeline?.instructions || []; - const addEntries = insts.find((i) => i.type === 'TimelineAddEntries') - || insts.find((i) => i.entries && Array.isArray(i.entries)); - if (!addEntries?.entries) - continue; - for (const entry of addEntries.entries) { - if (!entry.entryId.startsWith('tweet-')) - continue; - let tweet = entry.content?.itemContent?.tweet_results?.result; - if (!tweet) - continue; - // Handle retweet wrapping - if (tweet.__typename === 'TweetWithVisibilityResults' && tweet.tweet) { - tweet = tweet.tweet; - } - if (!tweet.rest_id || seen.has(tweet.rest_id)) - continue; - seen.add(tweet.rest_id); - // Twitter moved screen_name from legacy to core - const tweetUser = tweet.core?.user_results?.result; - results.push({ - id: tweet.rest_id, - author: tweetUser?.core?.screen_name || tweetUser?.legacy?.screen_name || 'unknown', - text: tweet.note_tweet?.note_tweet_results?.result?.text || tweet.legacy?.full_text || '', - created_at: tweet.legacy?.created_at || '', - likes: tweet.legacy?.favorite_count || 0, - views: tweet.views?.count || '0', - url: `https://x.com/i/status/${tweet.rest_id}`, - ...extractMedia(tweet.legacy), - }); - } - } - catch (e) { - // ignore parsing errors for individual payloads + let cursor = null; + for (let i = 0; i < 5 && results.length < kwargs.limit; i++) { + const fetchCount = Number(kwargs.limit) - results.length + 10; + const [requestUrl, requestPayload] = buildSearchTimelineRequest(operation, finalQuery, product, fetchCount, cursor); + const requestBody = JSON.stringify(requestPayload); + const data = normalizeTwitterGraphqlPayload(await page.evaluate(`async () => { + const options = { + method: 'POST', + headers: ${headers}, + credentials: 'include', + }; + options['body'] = ${JSON.stringify(requestBody)}; + const r = await fetch(${JSON.stringify(requestUrl)}, { + ...options, + }); + return r.ok ? await r.json() : { error: r.status }; + }`)); + if (data?.error) { + if (results.length === 0) throw new CommandExecutionError(`HTTP ${data.error}: SearchTimeline fetch failed — queryId may have expired`); + break; } + const { rows, nextCursor } = parseSearchTimeline(data, seen); + results.push(...rows); + if (!nextCursor || nextCursor === cursor) break; + cursor = nextCursor; } const trimmed = results.slice(0, kwargs.limit); return applyTopByEngagement(trimmed, kwargs['top-by-engagement']); @@ -316,6 +329,9 @@ cli({ export const __test__ = { buildSearchQuery, resolveSearchFParam, + resolveSearchProduct, + buildSearchTimelineRequest, + parseSearchTimeline, HAS_CHOICES, EXCLUDE_CHOICES, PRODUCT_CHOICES, diff --git a/clis/twitter/search.test.js b/clis/twitter/search.test.js index 3041c78e8..645743c4b 100644 --- a/clis/twitter/search.test.js +++ b/clis/twitter/search.test.js @@ -2,71 +2,67 @@ import { describe, expect, it, vi } from 'vitest'; import { getRegistry } from '@jackwener/opencli/registry'; import { __test__ } from './search.js'; -const { buildSearchQuery, resolveSearchFParam, HAS_CHOICES, EXCLUDE_CHOICES, PRODUCT_CHOICES, EXCLUDE_TO_OPERATOR, PRODUCT_TO_F_PARAM, FROM_USER_PATTERN } = __test__; +const { buildSearchQuery, resolveSearchFParam, resolveSearchProduct, buildSearchTimelineRequest, parseSearchTimeline, HAS_CHOICES, EXCLUDE_CHOICES, PRODUCT_CHOICES, EXCLUDE_TO_OPERATOR, PRODUCT_TO_F_PARAM, FROM_USER_PATTERN } = __test__; describe('twitter search command', () => { - it('retries transient SPA navigation failures before giving up', async () => { + function makeSearchPage(data) { + return { + getCookies: vi.fn().mockResolvedValue([{ name: 'ct0', value: 'csrf' }]), + goto: vi.fn().mockResolvedValue(undefined), + evaluate: vi.fn() + .mockResolvedValueOnce(null) // resolveTwitterQueryId fallback + .mockResolvedValueOnce(data), + }; + } + + it('fetches SearchTimeline directly instead of relying on SPA navigation', async () => { const command = getRegistry().get('twitter/search'); expect(command?.func).toBeTypeOf('function'); - const evaluate = vi.fn() - .mockResolvedValueOnce(undefined) - .mockResolvedValueOnce('/explore') - .mockResolvedValueOnce(undefined) - .mockResolvedValueOnce('/search'); - const page = { - goto: vi.fn().mockResolvedValue(undefined), - wait: vi.fn().mockResolvedValue(undefined), - installInterceptor: vi.fn().mockResolvedValue(undefined), - evaluate, - autoScroll: vi.fn().mockResolvedValue(undefined), - getInterceptedRequests: vi.fn().mockResolvedValue([ - { - data: { - search_by_raw_query: { - search_timeline: { - timeline: { - instructions: [ + const page = makeSearchPage({ + data: { + search_by_raw_query: { + search_timeline: { + timeline: { + instructions: [ + { + type: 'TimelineAddEntries', + entries: [ { - type: 'TimelineAddEntries', - entries: [ - { - entryId: 'tweet-1', - content: { - itemContent: { - tweet_results: { - result: { - rest_id: '1', - legacy: { - full_text: 'hello world', - favorite_count: 7, - created_at: 'Thu Mar 26 10:30:00 +0000 2026', - }, - core: { - user_results: { - result: { - core: { - screen_name: 'alice', - }, - }, + entryId: 'tweet-1', + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '1', + legacy: { + full_text: 'hello world', + favorite_count: 7, + created_at: 'Thu Mar 26 10:30:00 +0000 2026', + }, + core: { + user_results: { + result: { + core: { + screen_name: 'alice', }, }, - views: { - count: '12', - }, }, }, + views: { + count: '12', + }, }, }, }, - ], + }, }, ], }, - }, + ], }, }, }, - ]), - }; + }, + }); const result = await command.func(page, { query: 'from:alice', filter: 'top', limit: 5 }); expect(result).toEqual([ { @@ -81,213 +77,19 @@ describe('twitter search command', () => { media_urls: [], }, ]); - expect(page.installInterceptor).toHaveBeenCalledWith('SearchTimeline'); - expect(evaluate).toHaveBeenCalledTimes(4); - }); - it('uses f=live in search URL when filter is live', async () => { - const command = getRegistry().get('twitter/search'); - const evaluate = vi.fn() - .mockResolvedValueOnce(undefined) - .mockResolvedValueOnce('/search'); - const page = { - goto: vi.fn().mockResolvedValue(undefined), - wait: vi.fn().mockResolvedValue(undefined), - installInterceptor: vi.fn().mockResolvedValue(undefined), - evaluate, - autoScroll: vi.fn().mockResolvedValue(undefined), - getInterceptedRequests: vi.fn().mockResolvedValue([]), - }; - await command.func(page, { query: 'breaking news', filter: 'live', limit: 5 }); - const pushStateCall = evaluate.mock.calls[0][0]; - expect(pushStateCall).toContain('f=live'); - expect(pushStateCall).toContain(encodeURIComponent('breaking news')); - }); - it('uses f=top in search URL when filter is top', async () => { - const command = getRegistry().get('twitter/search'); - const evaluate = vi.fn() - .mockResolvedValueOnce(undefined) - .mockResolvedValueOnce('/search'); - const page = { - goto: vi.fn().mockResolvedValue(undefined), - wait: vi.fn().mockResolvedValue(undefined), - installInterceptor: vi.fn().mockResolvedValue(undefined), - evaluate, - autoScroll: vi.fn().mockResolvedValue(undefined), - getInterceptedRequests: vi.fn().mockResolvedValue([]), - }; - await command.func(page, { query: 'test', filter: 'top', limit: 5 }); - const pushStateCall = evaluate.mock.calls[0][0]; - expect(pushStateCall).toContain('f=top'); - }); - it('falls back to top when filter is omitted', async () => { - const command = getRegistry().get('twitter/search'); - const evaluate = vi.fn() - .mockResolvedValueOnce(undefined) - .mockResolvedValueOnce('/search'); - const page = { - goto: vi.fn().mockResolvedValue(undefined), - wait: vi.fn().mockResolvedValue(undefined), - installInterceptor: vi.fn().mockResolvedValue(undefined), - evaluate, - autoScroll: vi.fn().mockResolvedValue(undefined), - getInterceptedRequests: vi.fn().mockResolvedValue([]), - }; - await command.func(page, { query: 'test', limit: 5 }); - const pushStateCall = evaluate.mock.calls[0][0]; - expect(pushStateCall).toContain('f=top'); + expect(page.getCookies).toHaveBeenCalledWith({ url: 'https://x.com' }); + expect(page.goto).toHaveBeenCalledWith('https://x.com/home', { waitUntil: 'load', settleMs: 1000 }); + const searchFetch = page.evaluate.mock.calls[1][0]; + expect(searchFetch).toContain('/SearchTimeline'); + expect(searchFetch).toContain("method: 'POST'"); + expect(searchFetch).toContain('\\"rawQuery\\":\\"from:alice\\"'); }); - it('falls back to search input when pushState fails twice', async () => { - const command = getRegistry().get('twitter/search'); - expect(command?.func).toBeTypeOf('function'); - const evaluate = vi.fn() - .mockResolvedValueOnce(undefined) // pushState attempt 1 - .mockResolvedValueOnce('/explore') // pathname check 1 — not /search - .mockResolvedValueOnce(undefined) // pushState attempt 2 - .mockResolvedValueOnce('/explore') // pathname check 2 — still not /search - .mockResolvedValueOnce({ ok: true }) // search input fallback succeeds - .mockResolvedValueOnce('/search'); // pathname check after fallback - const page = { - goto: vi.fn().mockResolvedValue(undefined), - wait: vi.fn().mockResolvedValue(undefined), - installInterceptor: vi.fn().mockResolvedValue(undefined), - evaluate, - autoScroll: vi.fn().mockResolvedValue(undefined), - getInterceptedRequests: vi.fn().mockResolvedValue([ - { - data: { - search_by_raw_query: { - search_timeline: { - timeline: { - instructions: [ - { - type: 'TimelineAddEntries', - entries: [ - { - entryId: 'tweet-99', - content: { - itemContent: { - tweet_results: { - result: { - rest_id: '99', - legacy: { - full_text: 'fallback works', - favorite_count: 3, - created_at: 'Wed Apr 02 12:00:00 +0000 2026', - }, - core: { - user_results: { - result: { - core: { screen_name: 'bob' }, - }, - }, - }, - views: { count: '5' }, - }, - }, - }, - }, - }, - ], - }, - ], - }, - }, - }, - }, - }, - ]), - }; - const result = await command.func(page, { query: 'test fallback', filter: 'top', limit: 5 }); - expect(result).toEqual([ - { - id: '99', - author: 'bob', - text: 'fallback works', - created_at: 'Wed Apr 02 12:00:00 +0000 2026', - likes: 3, - views: '5', - url: 'https://x.com/i/status/99', - has_media: false, - media_urls: [], - }, - ]); - // 6 evaluate calls: 2x pushState + 2x pathname check + 1x fallback + 1x pathname check - expect(evaluate).toHaveBeenCalledTimes(6); - expect(page.autoScroll).toHaveBeenCalled(); - }); - it('clicks the requested product tab after fallback navigation when f= param is absent', async () => { - const command = getRegistry().get('twitter/search'); - expect(command?.func).toBeTypeOf('function'); - const evaluate = vi.fn() - .mockResolvedValueOnce(undefined) // pushState attempt 1 - .mockResolvedValueOnce('/explore') - .mockResolvedValueOnce(undefined) // pushState attempt 2 - .mockResolvedValueOnce('/explore') - .mockResolvedValueOnce({ ok: true }) // search input fallback - .mockResolvedValueOnce('/search') - .mockResolvedValueOnce(true); // product tab click - const page = { - goto: vi.fn().mockResolvedValue(undefined), - wait: vi.fn().mockResolvedValue(undefined), - installInterceptor: vi.fn().mockResolvedValue(undefined), - evaluate, - autoScroll: vi.fn().mockResolvedValue(undefined), - getInterceptedRequests: vi.fn().mockResolvedValue([]), - }; - const result = await command.func(page, { query: 'cats', product: 'photos', limit: 5 }); - expect(result).toEqual([]); - expect(evaluate).toHaveBeenCalledTimes(7); - expect(evaluate.mock.calls[6][0]).toContain('Photos'); - expect(page.autoScroll).toHaveBeenCalled(); - }); - it('throws when fallback navigation cannot select the requested product tab', async () => { - const command = getRegistry().get('twitter/search'); - expect(command?.func).toBeTypeOf('function'); - const evaluate = vi.fn() - .mockResolvedValueOnce(undefined) // pushState attempt 1 - .mockResolvedValueOnce('/explore') - .mockResolvedValueOnce(undefined) // pushState attempt 2 - .mockResolvedValueOnce('/explore') - .mockResolvedValueOnce({ ok: true }) // search input fallback - .mockResolvedValueOnce('/search') - .mockResolvedValueOnce(false); // requested tab missing - const page = { - goto: vi.fn().mockResolvedValue(undefined), - wait: vi.fn().mockResolvedValue(undefined), - installInterceptor: vi.fn().mockResolvedValue(undefined), - evaluate, - autoScroll: vi.fn().mockResolvedValue(undefined), - getInterceptedRequests: vi.fn(), - }; - await expect(command.func(page, { query: 'cats', product: 'videos', limit: 5 })) - .rejects - .toThrow(/could not select the requested product tab: video/); - expect(page.autoScroll).not.toHaveBeenCalled(); - expect(page.getInterceptedRequests).not.toHaveBeenCalled(); - }); - it('throws with the final path after both attempts fail', async () => { + + it('uses the requested GraphQL product', async () => { const command = getRegistry().get('twitter/search'); - expect(command?.func).toBeTypeOf('function'); - const evaluate = vi.fn() - .mockResolvedValueOnce(undefined) // pushState attempt 1 - .mockResolvedValueOnce('/explore') // pathname check 1 - .mockResolvedValueOnce(undefined) // pushState attempt 2 - .mockResolvedValueOnce('/login') // pathname check 2 - .mockResolvedValueOnce({ ok: false }); // search input fallback - const page = { - goto: vi.fn().mockResolvedValue(undefined), - wait: vi.fn().mockResolvedValue(undefined), - installInterceptor: vi.fn().mockResolvedValue(undefined), - evaluate, - autoScroll: vi.fn().mockResolvedValue(undefined), - getInterceptedRequests: vi.fn(), - }; - await expect(command.func(page, { query: 'from:alice', filter: 'top', limit: 5 })) - .rejects - .toThrow('Final path: /login'); - expect(page.autoScroll).not.toHaveBeenCalled(); - expect(page.getInterceptedRequests).not.toHaveBeenCalled(); - expect(evaluate).toHaveBeenCalledTimes(5); + const page = makeSearchPage({ data: { search_by_raw_query: { search_timeline: { timeline: { instructions: [] } } } } }); + await command.func(page, { query: 'cats', product: 'videos', limit: 5 }); + expect(page.evaluate.mock.calls[1][0]).toContain('\\"product\\":\\"Videos\\"'); }); }); @@ -411,18 +213,15 @@ describe('twitter search filter helpers', () => { }); describe('twitter search end-to-end with new filters', () => { - it('encodes the composed query and product=live into the f= URL param', async () => { + it('encodes the composed query and product=live into the GraphQL request', async () => { const command = getRegistry().get('twitter/search'); const evaluate = vi.fn() - .mockResolvedValueOnce(undefined) - .mockResolvedValueOnce('/search'); + .mockResolvedValueOnce(null) + .mockResolvedValueOnce({ data: { search_by_raw_query: { search_timeline: { timeline: { instructions: [] } } } } }); const page = { + getCookies: vi.fn().mockResolvedValue([{ name: 'ct0', value: 'csrf' }]), goto: vi.fn().mockResolvedValue(undefined), - wait: vi.fn().mockResolvedValue(undefined), - installInterceptor: vi.fn().mockResolvedValue(undefined), evaluate, - autoScroll: vi.fn().mockResolvedValue(undefined), - getInterceptedRequests: vi.fn().mockResolvedValue([]), }; await command.func(page, { query: 'breaking news', @@ -432,37 +231,26 @@ describe('twitter search end-to-end with new filters', () => { product: 'live', limit: 5, }); - const pushStateCall = evaluate.mock.calls[0][0]; - // f=live wins because --product=live trumps the default --filter - expect(pushStateCall).toContain('f=live'); - // composed query should be percent-encoded inside the URL - const encoded = encodeURIComponent('breaking news from:alice filter:images -filter:nativeretweets'); - expect(pushStateCall).toContain(encoded); + const searchFetch = evaluate.mock.calls[1][0]; + expect(searchFetch).toContain('\\"product\\":\\"Latest\\"'); + expect(searchFetch).toContain('\\"rawQuery\\":\\"breaking news from:alice filter:images -filter:nativeretweets\\"'); }); it('throws ArgumentError when query and all filters are empty', async () => { const command = getRegistry().get('twitter/search'); const page = { goto: vi.fn().mockResolvedValue(undefined), - wait: vi.fn().mockResolvedValue(undefined), - installInterceptor: vi.fn().mockResolvedValue(undefined), evaluate: vi.fn(), - autoScroll: vi.fn().mockResolvedValue(undefined), - getInterceptedRequests: vi.fn(), }; await expect(command.func(page, { query: ' ', limit: 5 })) .rejects .toThrow(/empty/i); - expect(page.installInterceptor).not.toHaveBeenCalled(); + expect(page.goto).not.toHaveBeenCalled(); }); it('throws ArgumentError for invalid --from before navigation', async () => { const command = getRegistry().get('twitter/search'); const page = { goto: vi.fn(), - wait: vi.fn(), - installInterceptor: vi.fn(), evaluate: vi.fn(), - autoScroll: vi.fn(), - getInterceptedRequests: vi.fn(), }; await expect(command.func(page, { query: 'hi', from: 'alice filter:links', limit: 5 })) .rejects @@ -473,11 +261,7 @@ describe('twitter search end-to-end with new filters', () => { const command = getRegistry().get('twitter/search'); const page = { goto: vi.fn(), - wait: vi.fn(), - installInterceptor: vi.fn(), evaluate: vi.fn(), - autoScroll: vi.fn(), - getInterceptedRequests: vi.fn(), }; await expect(command.func(page, { query: 'hi', limit: 0 })) .rejects @@ -487,19 +271,16 @@ describe('twitter search end-to-end with new filters', () => { it('runs with only filters set (empty )', async () => { const command = getRegistry().get('twitter/search'); const evaluate = vi.fn() - .mockResolvedValueOnce(undefined) - .mockResolvedValueOnce('/search'); + .mockResolvedValueOnce(null) + .mockResolvedValueOnce({ data: { search_by_raw_query: { search_timeline: { timeline: { instructions: [] } } } } }); const page = { + getCookies: vi.fn().mockResolvedValue([{ name: 'ct0', value: 'csrf' }]), goto: vi.fn().mockResolvedValue(undefined), - wait: vi.fn().mockResolvedValue(undefined), - installInterceptor: vi.fn().mockResolvedValue(undefined), evaluate, - autoScroll: vi.fn().mockResolvedValue(undefined), - getInterceptedRequests: vi.fn().mockResolvedValue([]), }; const result = await command.func(page, { query: '', from: 'alice', limit: 5 }); expect(result).toEqual([]); - const pushStateCall = evaluate.mock.calls[0][0]; - expect(pushStateCall).toContain(encodeURIComponent('from:alice')); + const searchFetch = evaluate.mock.calls[1][0]; + expect(searchFetch).toContain('\\"rawQuery\\":\\"from:alice\\"'); }); }); diff --git a/clis/twitter/shared.js b/clis/twitter/shared.js index 414ce050d..3ef2359ef 100644 --- a/clis/twitter/shared.js +++ b/clis/twitter/shared.js @@ -81,9 +81,101 @@ export function buildTwitterArticleScopeSource(tweetId) { export function sanitizeQueryId(resolved, fallbackId) { return typeof resolved === 'string' && QUERY_ID_PATTERN.test(resolved) ? resolved : fallbackId; } -export async function resolveTwitterQueryId(page, operationName, fallbackId) { + +function keysToFlags(keys) { + if (!Array.isArray(keys)) return {}; + return Object.fromEntries(keys.filter((key) => typeof key === 'string' && key).map((key) => [key, true])); +} + +function normalizeOperationFallback(fallback) { + if (typeof fallback === 'string') return { queryId: fallback, features: {}, fieldToggles: {} }; + return { + queryId: fallback?.queryId || null, + features: fallback?.features || {}, + fieldToggles: fallback?.fieldToggles || {}, + }; +} + +export function unwrapBrowserResult(value) { + if ( + value + && typeof value === 'object' + && typeof value.session === 'string' + && Object.prototype.hasOwnProperty.call(value, 'data') + ) { + return value.data; + } + return value; +} + +export function normalizeTwitterGraphqlPayload(value) { + const unwrapped = unwrapBrowserResult(value); + if (unwrapped?.data && typeof unwrapped.data === 'object') return unwrapped; + if ( + unwrapped + && typeof unwrapped === 'object' + && ( + Object.prototype.hasOwnProperty.call(unwrapped, 'user') + || Object.prototype.hasOwnProperty.call(unwrapped, 'search_by_raw_query') + ) + ) { + return { data: unwrapped }; + } + return unwrapped; +} + +export function sanitizeTwitterOperationMetadata(resolved, fallback) { + const value = unwrapBrowserResult(resolved); + const normalizedFallback = normalizeOperationFallback(fallback); + return { + queryId: sanitizeQueryId(value?.queryId, normalizedFallback.queryId), + features: value?.features && typeof value.features === 'object' + ? value.features + : normalizedFallback.features, + fieldToggles: value?.fieldToggles && typeof value.fieldToggles === 'object' + ? value.fieldToggles + : normalizedFallback.fieldToggles, + }; +} + +export async function resolveTwitterOperationMetadata(page, operationName, fallback) { const resolved = await page.evaluate(`async () => { const operationName = ${JSON.stringify(operationName)}; + const keysToFlags = (keys) => Object.fromEntries((keys || []).map((key) => [key, true])); + const quotedKeys = (source) => source + ? Array.from(source.matchAll(/"([^"]+)"/g)).map((match) => match[1]) + : []; + const parseOperation = (text) => { + const marker = 'operationName:"' + operationName + '"'; + const index = text.indexOf(marker); + if (index < 0) return null; + const start = Math.max(0, text.lastIndexOf('e.exports=', index)); + const endMarker = text.indexOf('}}}', index); + const snippet = text.slice(start, endMarker > index ? endMarker + 3 : index + 2500); + const queryId = snippet.match(/queryId:"([A-Za-z0-9_-]+)"/)?.[1] || null; + if (!queryId) return null; + return { + queryId, + features: keysToFlags(quotedKeys(snippet.match(/featureSwitches:\\[([^\\]]*)\\]/)?.[1])), + fieldToggles: keysToFlags(quotedKeys(snippet.match(/fieldToggles:\\[([^\\]]*)\\]/)?.[1])), + }; + }; + try { + const scripts = Array.from(document.scripts) + .map(s => s.src) + .filter(Boolean) + .concat(performance.getEntriesByType('resource') + .map(r => r.name) + .filter(r => r.includes('client-web') && r.endsWith('.js'))); + const uniqueScripts = Array.from(new Set(scripts)); + for (const scriptUrl of uniqueScripts.slice(-30)) { + try { + const text = await (await fetch(scriptUrl)).text(); + const operation = parseOperation(text); + if (operation) return operation; + } catch {} + } + } catch {} const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 5000); try { @@ -92,27 +184,25 @@ export async function resolveTwitterQueryId(page, operationName, fallbackId) { if (ghResp.ok) { const data = await ghResp.json(); const entry = data?.[operationName]; - if (entry && entry.queryId) return entry.queryId; + if (entry && entry.queryId) { + return { + queryId: entry.queryId, + features: keysToFlags(entry.featureSwitches), + fieldToggles: keysToFlags(entry.fieldToggles), + }; + } } } catch { clearTimeout(timeout); } - try { - const scripts = performance.getEntriesByType('resource') - .filter(r => r.name.includes('client-web') && r.name.endsWith('.js')) - .map(r => r.name); - for (const scriptUrl of scripts.slice(0, 15)) { - try { - const text = await (await fetch(scriptUrl)).text(); - const re = new RegExp('queryId:"([A-Za-z0-9_-]+)"[^}]{0,200}operationName:"' + operationName + '"'); - const match = text.match(re); - if (match) return match[1]; - } catch {} - } - } catch {} return null; }`); - return sanitizeQueryId(resolved, fallbackId); + return sanitizeTwitterOperationMetadata(resolved, fallback); +} + +export async function resolveTwitterQueryId(page, operationName, fallbackId) { + const operation = await resolveTwitterOperationMetadata(page, operationName, fallbackId); + return operation.queryId; } /** * Extract media flags and URLs from a tweet's `legacy` object. @@ -143,6 +233,9 @@ export function extractMedia(legacy) { } export const __test__ = { sanitizeQueryId, + sanitizeTwitterOperationMetadata, + unwrapBrowserResult, + normalizeTwitterGraphqlPayload, extractMedia, parseTweetUrl, buildTwitterArticleScopeSource, diff --git a/clis/twitter/shared.test.js b/clis/twitter/shared.test.js index 1ed8395ee..a4bc2ae46 100644 --- a/clis/twitter/shared.test.js +++ b/clis/twitter/shared.test.js @@ -3,7 +3,40 @@ import { JSDOM } from 'jsdom'; import { __test__ } from './shared.js'; import { ArgumentError } from '@jackwener/opencli/errors'; -const { extractMedia, parseTweetUrl, buildTwitterArticleScopeSource } = __test__; +const { extractMedia, parseTweetUrl, buildTwitterArticleScopeSource, unwrapBrowserResult, normalizeTwitterGraphqlPayload, sanitizeTwitterOperationMetadata } = __test__; + +describe('twitter browser result helpers', () => { + it('unwraps Browser Bridge exec envelopes', () => { + expect(unwrapBrowserResult({ session: 'site:twitter', data: '123' })).toBe('123'); + expect(unwrapBrowserResult({ data: { user: true } })).toEqual({ data: { user: true } }); + }); + + it('sanitizes operation metadata after unwrapping Browser Bridge envelopes', () => { + const result = sanitizeTwitterOperationMetadata({ + session: 'site:twitter', + data: { + queryId: 'abc_123', + features: { feature: true }, + fieldToggles: { field: true }, + }, + }, { queryId: 'fallback', features: {}, fieldToggles: {} }); + expect(result).toEqual({ + queryId: 'abc_123', + features: { feature: true }, + fieldToggles: { field: true }, + }); + }); + + it('normalizes GraphQL payloads when the bridge strips the top-level data key', () => { + expect(normalizeTwitterGraphqlPayload({ user: { result: {} } })).toEqual({ + data: { user: { result: {} } }, + }); + expect(normalizeTwitterGraphqlPayload({ search_by_raw_query: { search_timeline: {} } })).toEqual({ + data: { search_by_raw_query: { search_timeline: {} } }, + }); + expect(normalizeTwitterGraphqlPayload({ data: { user: {} } })).toEqual({ data: { user: {} } }); + }); +}); describe('twitter parseTweetUrl', () => { it('accepts exact Twitter/X tweet URLs and preserves query parameters', () => { diff --git a/clis/twitter/tweets.js b/clis/twitter/tweets.js index 753632268..c59395739 100644 --- a/clis/twitter/tweets.js +++ b/clis/twitter/tweets.js @@ -1,15 +1,17 @@ import { cli, Strategy } from '@jackwener/opencli/registry'; import { AuthRequiredError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors'; -import { resolveTwitterQueryId, sanitizeQueryId, extractMedia } from './shared.js'; +import { resolveTwitterOperationMetadata, sanitizeQueryId, extractMedia, normalizeTwitterGraphqlPayload, unwrapBrowserResult } from './shared.js'; import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js'; -const USER_TWEETS_QUERY_ID = '6fWQaBPK51aGyC_VC7t9GQ'; +const USER_TWEETS_QUERY_ID = 'lrMzG9qPQHpqJdP3AbM-bQ'; const USER_BY_SCREEN_NAME_QUERY_ID = 'IGgvgiOx4QZndDHuD3x9TQ'; const USER_TWEETS_FEATURES = { - rweb_video_screen_enabled: false, + rweb_video_screen_enabled: true, + rweb_cashtags_enabled: true, payments_enabled: false, profile_label_improvements_pcf_label_in_post_enabled: true, + responsive_web_profile_redirect_enabled: true, rweb_tipjar_consumption_enabled: true, verified_phone_label_enabled: false, creator_subscriptions_tweet_preview_api_enabled: true, @@ -20,6 +22,7 @@ const USER_TWEETS_FEATURES = { c9s_tweet_anatomy_moderator_badge_enabled: true, responsive_web_grok_analyze_button_fetch_trends_enabled: false, responsive_web_grok_analyze_post_followups_enabled: true, + rweb_cashtags_composer_attachment_enabled: true, responsive_web_jetfuel_frame: true, responsive_web_grok_share_attachment_enabled: true, responsive_web_grok_annotations_enabled: true, @@ -46,8 +49,21 @@ const USER_TWEETS_FEATURES = { responsive_web_enhance_cards_enabled: false, }; +const USER_TWEETS_FIELD_TOGGLES = { + withPayments: true, + withAuxiliaryUserLabels: true, + withArticleRichContentState: true, + withArticlePlainText: true, + withArticleSummaryText: true, + withArticleVoiceOver: true, + withGrokAnalyze: true, + withDisallowedReplyControls: true, +}; + const USER_BY_SCREEN_NAME_FEATURES = { hidden_profile_subscriptions_enabled: true, + profile_label_improvements_pcf_label_in_post_enabled: true, + responsive_web_profile_redirect_enabled: true, rweb_tipjar_consumption_enabled: true, responsive_web_graphql_exclude_directive_enabled: true, verified_phone_label_enabled: false, @@ -61,7 +77,59 @@ const USER_BY_SCREEN_NAME_FEATURES = { responsive_web_graphql_timeline_navigation_enabled: true, }; -function buildUserTweetsUrl(queryId, userId, count, cursor) { +const USER_BY_SCREEN_NAME_FIELD_TOGGLES = { + withPayments: true, + withAuxiliaryUserLabels: true, +}; + +const USER_TWEETS_OPERATION = { + queryId: USER_TWEETS_QUERY_ID, + features: USER_TWEETS_FEATURES, + fieldToggles: USER_TWEETS_FIELD_TOGGLES, +}; + +const USER_BY_SCREEN_NAME_OPERATION = { + queryId: USER_BY_SCREEN_NAME_QUERY_ID, + features: USER_BY_SCREEN_NAME_FEATURES, + fieldToggles: USER_BY_SCREEN_NAME_FIELD_TOGGLES, +}; + +function normalizeUserTweetsOperation(operation) { + if (typeof operation === 'string') { + return { queryId: operation, features: USER_TWEETS_FEATURES, fieldToggles: USER_TWEETS_FIELD_TOGGLES }; + } + return { + queryId: operation?.queryId || USER_TWEETS_QUERY_ID, + features: operation?.features || USER_TWEETS_FEATURES, + fieldToggles: operation?.fieldToggles || USER_TWEETS_FIELD_TOGGLES, + }; +} + +function normalizeUserByScreenNameOperation(operation) { + if (typeof operation === 'string') { + return { queryId: operation, features: USER_BY_SCREEN_NAME_FEATURES, fieldToggles: USER_BY_SCREEN_NAME_FIELD_TOGGLES }; + } + return { + queryId: operation?.queryId || USER_BY_SCREEN_NAME_QUERY_ID, + features: operation?.features || USER_BY_SCREEN_NAME_FEATURES, + fieldToggles: operation?.fieldToggles || USER_BY_SCREEN_NAME_FIELD_TOGGLES, + }; +} + +function appendGraphqlParams(path, variables, operation) { + const fieldToggles = operation.fieldToggles || {}; + const params = [ + `variables=${encodeURIComponent(JSON.stringify(variables))}`, + `features=${encodeURIComponent(JSON.stringify(operation.features || {}))}`, + ]; + if (Object.keys(fieldToggles).length > 0) { + params.push(`fieldToggles=${encodeURIComponent(JSON.stringify(fieldToggles))}`); + } + return `${path}?${params.join('&')}`; +} + +function buildUserTweetsUrl(operation, userId, count, cursor) { + const normalized = normalizeUserTweetsOperation(operation); const vars = { userId, count, @@ -70,21 +138,20 @@ function buildUserTweetsUrl(queryId, userId, count, cursor) { withVoice: true, }; if (cursor) vars.cursor = cursor; - return `/i/api/graphql/${queryId}/UserTweets` - + `?variables=${encodeURIComponent(JSON.stringify(vars))}` - + `&features=${encodeURIComponent(JSON.stringify(USER_TWEETS_FEATURES))}`; + return appendGraphqlParams(`/i/api/graphql/${normalized.queryId}/UserTweets`, vars, normalized); } -function buildUserByScreenNameUrl(queryId, screenName) { +function buildUserByScreenNameUrl(operation, screenName) { + const normalized = normalizeUserByScreenNameOperation(operation); const vars = { screen_name: screenName, withSafetyModeUserFields: true }; - return `/i/api/graphql/${queryId}/UserByScreenName` - + `?variables=${encodeURIComponent(JSON.stringify(vars))}` - + `&features=${encodeURIComponent(JSON.stringify(USER_BY_SCREEN_NAME_FEATURES))}`; + return appendGraphqlParams(`/i/api/graphql/${normalized.queryId}/UserByScreenName`, vars, normalized); } function extractTweet(result, seen) { if (!result) return null; - const tw = result.tweet || result; + const tw = result.__typename === 'TweetWithVisibilityResults' && result.tweet + ? result.tweet + : (result.tweet || result); const legacy = tw.legacy || {}; if (!tw.rest_id || seen.has(tw.rest_id)) return null; seen.add(tw.rest_id); @@ -112,32 +179,35 @@ function extractTweet(result, seen) { function parseUserTweets(data, seen) { const tweets = []; let nextCursor = null; - const instructions = data?.data?.user?.result?.timeline_v2?.timeline?.instructions - || data?.data?.user?.result?.timeline?.timeline?.instructions - || []; - for (const inst of instructions) { - if (inst.type === 'TimelinePinEntry') continue; - for (const entry of inst.entries || []) { - const content = entry.content; - if (content?.entryType === 'TimelineTimelineCursor' || content?.__typename === 'TimelineTimelineCursor') { - if (content.cursorType === 'Bottom' || content.cursorType === 'ShowMore') nextCursor = content.value; - continue; - } - if (entry.entryId?.startsWith('cursor-bottom-') || entry.entryId?.startsWith('cursor-showMore-')) { - nextCursor = content?.value || content?.itemContent?.value || nextCursor; - continue; - } - const direct = extractTweet(content?.itemContent?.tweet_results?.result, seen); - if (direct) { - tweets.push(direct); - continue; - } - for (const item of content?.items || []) { - const nested = extractTweet(item.item?.itemContent?.tweet_results?.result, seen); - if (nested) tweets.push(nested); - } + const result = data?.data?.user?.result || {}; + const instructionSets = [ + result.timeline_v2?.timeline?.instructions, + result.timeline?.timeline?.instructions, + ].filter(Array.isArray); + const instructions = instructionSets.flat(); + const visit = (value) => { + if (!value || typeof value !== 'object') return; + if (value.type === 'TimelinePinEntry') return; + if (value.tweet_results?.result) { + const tweet = extractTweet(value.tweet_results.result, seen); + if (tweet) tweets.push(tweet); } - } + if ( + (value.entryType === 'TimelineTimelineCursor' || value.__typename === 'TimelineTimelineCursor') + && (value.cursorType === 'Bottom' || value.cursorType === 'ShowMore') + && value.value + ) { + nextCursor = value.value; + } + if (Array.isArray(value)) { + for (const item of value) visit(item); + return; + } + for (const child of Object.values(value)) { + if (child && typeof child === 'object') visit(child); + } + }; + visit(instructions); return { tweets, nextCursor }; } @@ -165,8 +235,8 @@ cli({ const ct0 = cookies.find((c) => c.name === 'ct0')?.value || null; if (!ct0) throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)'); - const userTweetsQueryId = await resolveTwitterQueryId(page, 'UserTweets', USER_TWEETS_QUERY_ID); - const userByScreenNameQueryId = await resolveTwitterQueryId(page, 'UserByScreenName', USER_BY_SCREEN_NAME_QUERY_ID); + const userTweetsOperation = await resolveTwitterOperationMetadata(page, 'UserTweets', USER_TWEETS_OPERATION); + const userByScreenNameOperation = await resolveTwitterOperationMetadata(page, 'UserByScreenName', USER_BY_SCREEN_NAME_OPERATION); const headers = JSON.stringify({ 'Authorization': `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`, @@ -175,13 +245,13 @@ cli({ 'X-Twitter-Active-User': 'yes', }); - const ubsUrl = buildUserByScreenNameUrl(userByScreenNameQueryId, username); - const userId = await page.evaluate(`async () => { + const ubsUrl = buildUserByScreenNameUrl(userByScreenNameOperation, username); + const userId = unwrapBrowserResult(await page.evaluate(`async () => { const resp = await fetch("${ubsUrl}", { headers: ${headers}, credentials: 'include' }); if (!resp.ok) return null; const d = await resp.json(); return d?.data?.user?.result?.rest_id || null; - }`); + }`)); if (!userId) throw new CommandExecutionError(`Could not resolve @${username}`); const seen = new Set(); @@ -189,11 +259,11 @@ cli({ let cursor = null; for (let i = 0; i < 5 && all.length < limit; i++) { const fetchCount = Math.min(100, limit - all.length + 10); - const url = buildUserTweetsUrl(userTweetsQueryId, userId, fetchCount, cursor); - const data = await page.evaluate(`async () => { + const url = buildUserTweetsUrl(userTweetsOperation, userId, fetchCount, cursor); + const data = normalizeTwitterGraphqlPayload(await page.evaluate(`async () => { const r = await fetch("${url}", { headers: ${headers}, credentials: 'include' }); return r.ok ? await r.json() : { error: r.status }; - }`); + }`)); if (data?.error) { if (all.length === 0) throw new CommandExecutionError(`HTTP ${data.error}: UserTweets fetch failed — queryId may have expired`); break; diff --git a/clis/twitter/tweets.test.js b/clis/twitter/tweets.test.js index 442ddc52b..9eeb08431 100644 --- a/clis/twitter/tweets.test.js +++ b/clis/twitter/tweets.test.js @@ -60,6 +60,18 @@ describe('twitter tweets helpers', () => { expect(b.is_retweet).toBe(true); }); + it('unwraps TweetWithVisibilityResults', () => { + const tweet = __test__.extractTweet({ + __typename: 'TweetWithVisibilityResults', + tweet: { + rest_id: '42', + legacy: { full_text: 'visible post', favorite_count: 2, retweet_count: 0, reply_count: 0, created_at: 'now' }, + core: { user_results: { result: { legacy: { screen_name: 'alice', name: 'Alice' } } } }, + }, + }, new Set()); + expect(tweet).toMatchObject({ id: '42', author: 'alice', text: 'visible post' }); + }); + it('parses chronological tweets and skips pinned instruction', () => { const chronEntry = { entryId: 'tweet-1', @@ -122,4 +134,94 @@ describe('twitter tweets helpers', () => { url: 'https://x.com/alice/status/1', }); }); + + it('recursively parses tweets nested in timeline modules', () => { + const payload = { + data: { + user: { + result: { + timeline_v2: { + timeline: { + instructions: [ + { + type: 'TimelineAddEntries', + entries: [ + { + entryId: 'profile-conversation-1', + content: { + entryType: 'TimelineTimelineModule', + items: [ + { + item: { + itemContent: { + tweet_results: { + result: { + rest_id: '2', + legacy: { full_text: 'nested post', favorite_count: 1, retweet_count: 0, reply_count: 0, created_at: 'now' }, + core: { user_results: { result: { legacy: { screen_name: 'alice', name: 'Alice' } } } }, + }, + }, + }, + }, + }, + ], + }, + }, + { + entryId: 'cursor-bottom-2', + content: { entryType: 'TimelineTimelineCursor', cursorType: 'Bottom', value: 'next' }, + }, + ], + }, + ], + }, + }, + }, + }, + }, + }; + const result = __test__.parseUserTweets(payload, new Set()); + expect(result.nextCursor).toBe('next'); + expect(result.tweets).toHaveLength(1); + expect(result.tweets[0]).toMatchObject({ id: '2', text: 'nested post' }); + }); + + it('uses populated timeline instructions when timeline_v2 is present but empty', () => { + const payload = { + data: { + user: { + result: { + timeline_v2: { timeline: { instructions: [] } }, + timeline: { + timeline: { + instructions: [ + { + type: 'TimelineAddEntries', + entries: [ + { + content: { + itemContent: { + tweet_results: { + result: { + rest_id: '3', + legacy: { full_text: 'fallback timeline post', favorite_count: 0, retweet_count: 0, reply_count: 0, created_at: 'now' }, + core: { user_results: { result: { legacy: { screen_name: 'alice', name: 'Alice' } } } }, + }, + }, + }, + }, + }, + ], + }, + ], + }, + }, + }, + }, + }, + }; + const result = __test__.parseUserTweets(payload, new Set()); + expect(result.tweets).toHaveLength(1); + expect(result.tweets[0]).toMatchObject({ id: '3', text: 'fallback timeline post' }); + }); }); From 05f2604888525690b02469f0a67eaaf9992444a9 Mon Sep 17 00:00:00 2001 From: jackwener Date: Wed, 13 May 2026 18:09:13 +0800 Subject: [PATCH 2/2] fix(twitter): prefer baked operation features when bundle parse returns empty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bundle parser in resolveTwitterOperationMetadata locates the queryId via `queryId:"..."` inside a ~2500-char snippet around the operationName marker, then independently extracts `featureSwitches:[...]` and `fieldToggles:[...]` via separate regexes. When minification rearranges the snippet (or the snippet window truncates before the array), either regex can miss while queryId still resolves; keysToFlags(undefined) then returns {}. sanitizeTwitterOperationMetadata previously accepted any object as features / fieldToggles, including {}. Twitter's GraphQL endpoint rejects SearchTimeline / UserTweets requests with empty features (HTTP 400), surfacing a misleading "queryId may have expired" error — the queryId is fresh; only the feature flags are missing. Guard against this by deferring to the baked fallback whenever the resolved map is empty. Adds a JSDOM-free unit test that, reverse-validated, fails on the un-fixed code with the exact silent-fallback shape. Refs PR #1512 --- clis/twitter/shared.js | 14 ++++++++++++-- clis/twitter/shared.test.js | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/clis/twitter/shared.js b/clis/twitter/shared.js index 3ef2359ef..48f2cb685 100644 --- a/clis/twitter/shared.js +++ b/clis/twitter/shared.js @@ -127,12 +127,22 @@ export function normalizeTwitterGraphqlPayload(value) { export function sanitizeTwitterOperationMetadata(resolved, fallback) { const value = unwrapBrowserResult(resolved); const normalizedFallback = normalizeOperationFallback(fallback); + // Empty resolved features / fieldToggles must defer to the baked fallback. + // The bundle parser can find a queryId but miss `featureSwitches:[...]` (e.g. + // a minification change, or the 2500-char snippet window truncating before + // the array). When that happens, keysToFlags(undefined) returns {}; if we + // kept it, Twitter would receive an empty `features` map and respond 400, + // surfacing a misleading "queryId expired" error. return { queryId: sanitizeQueryId(value?.queryId, normalizedFallback.queryId), - features: value?.features && typeof value.features === 'object' + features: value?.features + && typeof value.features === 'object' + && Object.keys(value.features).length > 0 ? value.features : normalizedFallback.features, - fieldToggles: value?.fieldToggles && typeof value.fieldToggles === 'object' + fieldToggles: value?.fieldToggles + && typeof value.fieldToggles === 'object' + && Object.keys(value.fieldToggles).length > 0 ? value.fieldToggles : normalizedFallback.fieldToggles, }; diff --git a/clis/twitter/shared.test.js b/clis/twitter/shared.test.js index a4bc2ae46..fe6786fd5 100644 --- a/clis/twitter/shared.test.js +++ b/clis/twitter/shared.test.js @@ -27,6 +27,43 @@ describe('twitter browser result helpers', () => { }); }); + it('falls back to baked features / fieldToggles when the bundle parser returns empty maps', () => { + // Regression guard: resolveTwitterOperationMetadata's bundle parser can + // find a queryId but miss `featureSwitches:[...]` (e.g. minification + // change, or the 2500-char snippet window truncating before the array). + // In that case keysToFlags(undefined) returns {}; if sanitize kept the + // empty map, Twitter would receive a request with no features and reply + // 400, surfacing a misleading "queryId expired" error. + const result = sanitizeTwitterOperationMetadata({ + queryId: 'newQueryId', + features: {}, + fieldToggles: {}, + }, { + queryId: 'fallback', + features: { fallback_feature: true }, + fieldToggles: { fallback_field: true }, + }); + expect(result).toEqual({ + queryId: 'newQueryId', + features: { fallback_feature: true }, + fieldToggles: { fallback_field: true }, + }); + }); + + it('falls back when resolved features are non-object falsy values', () => { + const result = sanitizeTwitterOperationMetadata({ + queryId: 'newQueryId', + features: null, + fieldToggles: undefined, + }, { + queryId: 'fallback', + features: { fallback_feature: true }, + fieldToggles: { fallback_field: true }, + }); + expect(result.features).toEqual({ fallback_feature: true }); + expect(result.fieldToggles).toEqual({ fallback_field: true }); + }); + it('normalizes GraphQL payloads when the bridge strips the top-level data key', () => { expect(normalizeTwitterGraphqlPayload({ user: { result: {} } })).toEqual({ data: { user: { result: {} } },