news.js

// Native crypto-news aggregation — the engine behind /api/news/feed, the
// related-news rail on /coin/:id, the Oracle's narrative pillar, pump-intel's
// news-meme matcher, and the /markets surfaces. three.ws fetches the source
// RSS/Atom feeds directly; no third-party news API sits in this path.
//
// Design, at the scale of the full publisher registry (news-sources.js):
//
//   * Per-source in-memory cache (5 min TTL) with serve-stale-on-error, so one
//     slow or dead feed never blanks the page and repeat requests inside the
//     TTL cost zero upstream calls.
//   * A bounded worker pool (GLOBAL_CONCURRENCY) fronted by a per-domain
//     semaphore (DOMAIN_CONCURRENCY). Shared hosts carry many feeds — a naive
//     Promise.all over the registry earns an instant 429 from them and starves
//     slow feeds of their timeout budget.
//   * A refresh deadline: a request never blocks on a cold registry. It awaits
//     refreshes for REFRESH_DEADLINE_MS, returns whatever is cached, and lets
//     the stragglers land in cache for the next caller. Sources are refreshed
//     in tier order, so the highest-credibility outlets land first.
//   * Exponential backoff per source: a feed that 404s is not re-fetched every
//     five minutes forever. Failures push nextRetryAt out to MAX_BACKOFF_MS.
//
// Every article is real, parsed from the publisher's own feed; nothing is
// fabricated.

import { createHash } from 'node:crypto';
import { XMLParser } from 'fast-xml-parser';
import { NEWS_SOURCES, sourcesForCategory, sourcesForLanguage, sourcePriority, isFeaturedSource } from './news-sources.js';
import { isSuppressed, excerptText } from './news-rights.js';
import { isDisplayable } from './news-curation.js';

const FEED_TIMEOUT_MS = 7000;
const FRESH_MS = 300_000; // refetch a source after 5 min
const STALE_OK_MS = 24 * 3600_000; // serve a failed source's last-good copy up to 24h
const MAX_BACKOFF_MS = 6 * 3600_000; // a persistently dead feed (404/410) retries at most every 6h
const SOFT_BACKOFF_MS = 30 * 60_000; // a rate-limited or 5xx feed retries at most every 30 min
const REFRESH_DEADLINE_MS = 2500; // how long a request will wait on cold sources
const GLOBAL_CONCURRENCY = 16; // outbound feed fetches in flight, all domains
const DOMAIN_CONCURRENCY = 3; // outbound feed fetches in flight, per domain
const MAX_ARTICLES_PER_SOURCE = 40; // newest-N per feed keeps the working set bounded
const NARROW_QUERY_SOURCES = 8; // at or below this, wait for the feeds rather than return an empty page

// key → { articles, fetchedAt, ok, failures, nextRetryAt }
const sourceCache = new Map();
// de-duplicated in-flight refreshes so concurrent requests share one fetch
const inflight = new Map();

const str = (v) => (typeof v === 'string' && v.trim() ? v.trim() : null);
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

export function stripHtml(s) {
	return String(s || '')
		.replace(/<[^>]+>/g, ' ')
		.replace(/&nbsp;/g, ' ')
		.replace(/&amp;/g, '&')
		.replace(/&lt;/g, '<')
		.replace(/&gt;/g, '>')
		.replace(/&quot;/g, '"')
		.replace(/&#0?39;|&apos;/g, "'")
		.replace(/&#8217;|&rsquo;/g, "'")
		.replace(/&#8220;|&#8221;|&ldquo;|&rdquo;/g, '"')
		.replace(/&#8211;|&ndash;|&#8212;|&mdash;/g, '—')
		.replace(/\s+/g, ' ')
		.trim();
}

// Same 16-hex content-addressed ID scheme as the historical archive, so live
// and archived records share an identity space.
export function articleId(link) {
	return createHash('sha256').update(String(link)).digest('hex').slice(0, 16);
}

// A publisher-stamped date meaningfully in the future is bogus (CoinPaper has
// shipped stories dated seven weeks ahead) and poisons everything keyed on the
// publication month: the permalink month the browser mints, the archive month
// file the cron writes, and the story-page lookup — a future-dated article got
// a permalink that 404'd while the story was live on the feed. Clamp to fetch
// time; honest timezone skew stays inside the 24h allowance.
export function clampFuturePubDate(iso) {
	if (!iso) return iso;
	const t = Date.parse(iso);
	if (!Number.isFinite(t) || t <= Date.now() + 24 * 3_600_000) return iso;
	return new Date().toISOString();
}

// WordPress and its plugins append syndication boilerplate to every excerpt:
// "The post <title> appeared first on <site>.", "Continue reading…",
// "Read more on …", trailing "[…]". None of it is article text — strip it
// before the excerpt reaches a card, a digest summary, or an LLM prompt.
export function stripFeedBoilerplate(text) {
	return String(text || '')
		.replace(/\bThe post .*?appeared first on .*$/is, '')
		.replace(/\bThis (?:post|article) (?:was )?(?:first )?(?:published|appeared).*$/is, '')
		.replace(/\b(?:Continue reading|Read more|Read the full (?:story|article)|The post)\b[^.]*$/i, '')
		.replace(/\[[…\.]+\]\s*$/, '')
		.replace(/\s*(?:…|\.\.\.)\s*$/, '')
		.replace(/\s+/g, ' ')
		.trim();
}

// Cut at a word boundary and mark the elision, so an excerpt never ends
// mid-word ("…the game appeared f").
export function truncateWords(text, max) {
	const t = String(text || '').trim();
	if (!t) return null;
	if (t.length <= max) return t;
	const cut = t.slice(0, max);
	const lastSpace = cut.lastIndexOf(' ');
	return `${(lastSpace > max * 0.6 ? cut.slice(0, lastSpace) : cut).replace(/[\s,;:—–-]+$/, '')}…`;
}

// Models on the free tiers often wrap JSON in a ```json fence despite being
// told not to. Unwrap before JSON.parse rather than failing the completion.
export function stripJsonFence(text) {
	const t = String(text || '').trim();
	const fenced = t.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/);
	return (fenced ? fenced[1] : t).trim();
}

// ── Sentiment (lexicon) ──────────────────────────────────────────────────────
// Fast keyword sentiment matching the archive's { score, label, confidence }
// shape. Used for live articles, which arrive without the offline enrichment
// the 662k archived records carry.

const POSITIVE = [
	'surge', 'surges', 'soar', 'soars', 'rally', 'rallies', 'jump', 'jumps', 'gain', 'gains',
	'all-time high', 'ath', 'record high', 'breakout', 'bullish', 'adoption', 'approval',
	'approve', 'approved', 'partnership', 'integration', 'launch', 'launches', 'upgrade',
	'milestone', 'inflow', 'inflows', 'accumulate', 'accumulation', 'outperform', 'recovery',
	'rebound', 'green', 'wins', 'settle', 'settlement approved',
];
const NEGATIVE = [
	'crash', 'crashes', 'plunge', 'plunges', 'dump', 'dumps', 'selloff', 'sell-off', 'slump',
	'hack', 'hacked', 'exploit', 'exploited', 'breach', 'stolen', 'scam', 'fraud', 'lawsuit',
	'sues', 'sued', 'charges', 'ban', 'bans', 'banned', 'crackdown', 'bearish', 'liquidation',
	'liquidations', 'outflow', 'outflows', 'bankruptcy', 'insolvent', 'collapse', 'warning',
	'red', 'drop', 'drops', 'falls', 'tumble', 'tumbles', 'rug pull', 'delist', 'delisting',
];

export function lexiconSentiment(text) {
	const t = ` ${String(text || '').toLowerCase()} `;
	let pos = 0;
	let neg = 0;
	for (const w of POSITIVE) if (t.includes(w)) pos++;
	for (const w of NEGATIVE) if (t.includes(w)) neg++;
	const hits = pos + neg;
	if (!hits) return { score: 0, label: 'neutral', confidence: 0.5 };
	const score = Math.max(-1, Math.min(1, (pos - neg) / Math.max(2, hits)));
	const label =
		score > 0.5 ? 'very_positive' : score > 0.1 ? 'positive' : score < -0.5 ? 'very_negative' : score < -0.1 ? 'negative' : 'neutral';
	return { score: Number(score.toFixed(2)), label, confidence: Math.min(0.9, 0.5 + hits * 0.1) };
}

// ── Ticker extraction ────────────────────────────────────────────────────────
// $SYMBOL mentions plus a wordlist of majors whose bare names/symbols are
// unambiguous in crypto headlines.

const TICKER_WORDS = new Map([
	['bitcoin', 'BTC'], ['btc', 'BTC'], ['ethereum', 'ETH'], ['eth', 'ETH'], ['ether', 'ETH'],
	['solana', 'SOL'], ['sol', 'SOL'], ['xrp', 'XRP'], ['ripple', 'XRP'], ['bnb', 'BNB'],
	['dogecoin', 'DOGE'], ['doge', 'DOGE'], ['cardano', 'ADA'], ['ada', 'ADA'],
	['tether', 'USDT'], ['usdt', 'USDT'], ['usdc', 'USDC'], ['avalanche', 'AVAX'],
	['avax', 'AVAX'], ['polkadot', 'DOT'], ['chainlink', 'LINK'], ['litecoin', 'LTC'],
	['polygon', 'MATIC'], ['tron', 'TRX'], ['shiba inu', 'SHIB'], ['shib', 'SHIB'],
	['sui', 'SUI'], ['aptos', 'APT'], ['near', 'NEAR'], ['arbitrum', 'ARB'], ['optimism', 'OP'],
	['pepe', 'PEPE'], ['bonk', 'BONK'], ['aave', 'AAVE'], ['uniswap', 'UNI'], ['maker', 'MKR'],
	['stellar', 'XLM'], ['monero', 'XMR'], ['cosmos', 'ATOM'], ['filecoin', 'FIL'],
	['hedera', 'HBAR'], ['injective', 'INJ'], ['celestia', 'TIA'], ['jito', 'JTO'],
	['jupiter', 'JUP'], ['worldcoin', 'WLD'], ['toncoin', 'TON'], ['hyperliquid', 'HYPE'],
]);

export function extractTickers(text) {
	const found = new Set();
	const t = String(text || '');
	for (const m of t.matchAll(/\$([A-Z][A-Z0-9]{1,9})\b/g)) found.add(m[1]);
	const lower = ` ${t.toLowerCase().replace(/[^a-z0-9$ ]/g, ' ')} `;
	for (const [word, sym] of TICKER_WORDS) {
		if (lower.includes(` ${word} `)) found.add(sym);
	}
	return [...found].slice(0, 8);
}

// ── Feed parsing ─────────────────────────────────────────────────────────────

function firstImage(html) {
	const m = String(html || '').match(/<img[^>]+src=["']([^"']+)["']/i);
	return m ? m[1] : null;
}

/**
 * Feeds ship junk in their image slots: data:-URI placeholders (ZeroHedge sends
 * an inline SVG icon as media:content), 1×1 tracking pixels, protocol-relative
 * links, and plain-http URLs that an https page can't render (mixed content is
 * blocked, not downgraded). Only a real, loadable https URL survives this —
 * anything else becomes null so the card falls back cleanly instead of showing
 * a broken or junk preview.
 */
export function cleanImageUrl(raw) {
	let url = str(raw);
	if (!url) return null;
	if (url.startsWith('//')) url = `https:${url}`;
	if (/^http:\/\//i.test(url)) url = `https://${url.slice(7)}`;
	if (!/^https:\/\//i.test(url)) return null; // data:, blob:, ftp:, relative paths
	if (/(?:^|[/_.-])(?:1x1|pixel|spacer|blank|transparent)(?:[/_.-]|\.(?:gif|png)\b)/i.test(url)) return null;
	if (/feeds\.feedburner\.com\/~/i.test(url)) return null; // FeedBurner impression beacons
	return url;
}

/** First <meta property|name="…" content="…"> match, attribute order agnostic. */
export function metaContent(html, patterns) {
	for (const name of patterns) {
		const re = new RegExp(
			`<meta[^>]+(?:property|name)=["']${name}["'][^>]+content=["']([^"']+)["']|<meta[^>]+content=["']([^"']+)["'][^>]+(?:property|name)=["']${name}["']`,
			'i',
		);
		const m = String(html || '').match(re);
		if (m) return (m[1] || m[2] || '').trim() || null;
	}
	return null;
}

/**
 * The article page's own preview image — what the publisher would show on a
 * social share. Backs /api/news/image for articles whose feed carries no image.
 */
export function extractOgImage(html) {
	return cleanImageUrl(
		metaContent(html, ['og:image:secure_url', 'og:image', 'twitter:image', 'twitter:image:src']),
	);
}

function normalizeLink(it) {
	// RSS: <link>url</link>. Atom: <link href="..."/> possibly an array with
	// rel variants — prefer rel="alternate", else the first href.
	if (typeof it?.link === 'string') return str(it.link);
	if (it?.link?.['@href']) return str(it.link['@href']);
	if (Array.isArray(it?.link)) {
		const alt = it.link.find((l) => l?.['@rel'] === 'alternate' && l?.['@href']);
		const any = it.link.find((l) => l?.['@href']);
		return str(alt?.['@href']) || str(any?.['@href']);
	}
	return null;
}

export function parseFeed(xml, sourceKey) {
	const src = NEWS_SOURCES[sourceKey];
	const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: '@' });
	const doc = parser.parse(xml);
	const items = doc?.rss?.channel?.item || doc?.feed?.entry || doc?.['rdf:RDF']?.item || [];
	return (Array.isArray(items) ? items : [items])
		.map((it) => {
			const link = normalizeLink(it);
			const title = stripHtml(str(it?.title?.['#text']) || str(it?.title) || '');
			if (!link || !title) return null;
			const rawDesc =
				str(it?.description) ||
				str(it?.summary?.['#text']) ||
				str(it?.summary) ||
				str(it?.['content:encoded']) ||
				str(it?.content?.['#text']) ||
				'';
			// First candidate that survives hygiene wins — a junk media:content
			// (ZeroHedge ships a data:-URI icon there) must not mask a real
			// thumbnail further down the chain.
			const image =
				[
					str(it?.['media:content']?.['@url']),
					Array.isArray(it?.['media:content']) ? str(it['media:content'][0]?.['@url']) : null,
					str(it?.['media:thumbnail']?.['@url']),
					str(it?.enclosure?.['@url']),
					firstImage(it?.['content:encoded'] || rawDesc),
				]
					.map(cleanImageUrl)
					.find(Boolean) || null;
			const pubDate = str(it?.pubDate) || str(it?.published) || str(it?.updated) || str(it?.['dc:date']);
			const iso = clampFuturePubDate(
				pubDate && !Number.isNaN(Date.parse(pubDate)) ? new Date(pubDate).toISOString() : null,
			);
			const author =
				stripHtml(str(it?.['dc:creator']) || str(it?.author?.name) || str(it?.author) || '') || null;
			// Some feeds (e.g. Bitcoin Magazine) prefix descriptions with their own
			// name and/or repeat the headline verbatim — strip the echo so cards
			// don't read "Source Title Title …".
			let descText = stripHtml(rawDesc);
			const srcName = src?.name || '';
			if (srcName && descText.toLowerCase().startsWith(srcName.toLowerCase())) {
				descText = descText.slice(srcName.length).trimStart();
			}
			if (title && descText.toLowerCase().startsWith(title.toLowerCase())) {
				descText = descText.slice(title.length).replace(/^[\s—–:-]+/, '');
			}
			const description = truncateWords(stripFeedBoilerplate(descText), 320);
			// Full body many feeds ship (WordPress content:encoded, Atom content).
			// Server-side only — the reader uses it when the publisher's site
			// blocks direct fetches; the feed API strips it from list payloads.
			const fullText = stripHtml(str(it?.['content:encoded']) || str(it?.content?.['#text']) || '');
			return {
				id: articleId(link),
				title,
				link,
				description,
				image,
				author,
				source: src?.name || sourceKey,
				source_key: sourceKey,
				category: src?.category || 'general',
				pub_date: iso,
				tickers: extractTickers(`${title} ${description || ''}`),
				sentiment: lexiconSentiment(`${title} ${description || ''}`),
				content_text: fullText.length > (description || '').length + 80 ? fullText.slice(0, 8000) : null,
			};
		})
		.filter(Boolean);
}

// ── Non-RSS sources ──────────────────────────────────────────────────────────
// A source with `kind: 'json'` is shaped by an adapter here instead of by
// parseFeed. The bar for adding one is high: free, keyless, and reachable from
// a datacenter IP (which is what Cloud Run is). Three of cryptocurrency.cv's
// four JSON sources no longer clear it and are deliberately absent —
// CryptoCompare now answers 401, DeFiLlama's /raises answers 402, and Reddit
// answers 403 to cloud egress. Shipping them would mean four sources that fail
// forever.

const JSON_ADAPTERS = {
	// Exchange listing/delisting notices. Market-moving, and published to no RSS
	// feed anywhere. The list endpoint returns title + code + releaseDate only;
	// the canonical permalink is /support/announcement/detail/<code>.
	binance_announcements(data, key) {
		const src = NEWS_SOURCES[key];
		const articles = data?.data?.catalogs?.[0]?.articles;
		if (!Array.isArray(articles)) throw new Error(`${key} unexpected payload`);
		return articles
			.map((a) => {
				const title = stripHtml(a?.title);
				if (!title || !a?.code) return null;
				const link = `https://www.binance.com/en/support/announcement/detail/${a.code}`;
				const iso = clampFuturePubDate(Number.isFinite(a.releaseDate) ? new Date(a.releaseDate).toISOString() : null);
				return {
					id: articleId(link),
					title,
					link,
					description: null,
					image: null,
					author: null,
					source: src.name,
					source_key: key,
					category: src.category,
					pub_date: iso,
					tickers: extractTickers(title),
					sentiment: lexiconSentiment(title),
					content_text: null,
				};
			})
			.filter(Boolean);
	},
};

// ── Concurrency control ──────────────────────────────────────────────────────
// Feeds cluster on shared hosts, so cap outbound fetches globally *and* per
// domain: a naive Promise.all over the registry earns an instant 429 from
// exactly those hosts and starves slow feeds of their timeout budget.

function feedDomain(url) {
	try {
		return new URL(url).hostname;
	} catch {
		return 'invalid';
	}
}

/**
 * Counting semaphore. `release()` hands the slot directly to the next waiter
 * rather than decrementing, so a slot is never double-issued.
 */
function semaphore(limit) {
	let active = 0;
	const waiters = [];
	return {
		async acquire() {
			if (active < limit) {
				active++;
				return;
			}
			await new Promise((resolve) => waiters.push(resolve));
		},
		release() {
			const next = waiters.shift();
			if (next) next();
			else active--;
		},
	};
}

const globalGate = semaphore(GLOBAL_CONCURRENCY);
const domainGates = new Map();

function domainGate(domain) {
	if (!domainGates.has(domain)) domainGates.set(domain, semaphore(DOMAIN_CONCURRENCY));
	return domainGates.get(domain);
}

// ── Per-source fetch ─────────────────────────────────────────────────────────

async function fetchSource(key) {
	const src = NEWS_SOURCES[key];
	const gate = domainGate(feedDomain(src.url));
	// Domain gate first, then the global gate: a request queued behind a busy
	// domain must not occupy a global slot that another domain's feed could use.
	await gate.acquire();
	await globalGate.acquire();
	try {
		const json = src.kind === 'json';
		const resp = await fetch(src.url, {
			headers: {
				accept: json
					? 'application/json'
					: 'application/rss+xml, application/atom+xml, application/xml, text/xml;q=0.9, */*;q=0.8',
				// A polite, identifying bot UA gets through more publisher WAFs than a
				// spoofed browser UA does — a fake Chrome string without the matching
				// TLS/header fingerprint reads as a scraper and earns a 403.
				'user-agent': 'Mozilla/5.0 (compatible; three.ws-news/1.0; +https://three.ws)',
				'accept-language': 'en-US,en;q=0.9',
			},
			signal: AbortSignal.timeout(FEED_TIMEOUT_MS),
			redirect: 'follow',
		});
		if (!resp.ok) {
			const err = new Error(`${key} ${resp.status}`);
			err.status = resp.status;
			throw err;
		}
		if (json) return JSON_ADAPTERS[key](await resp.json(), key);
		return parseFeed(await resp.text(), key);
	} finally {
		globalGate.release();
		gate.release();
	}
}

/** Exponential backoff so a permanently dead feed stops costing us a request every 5 min. */
function backoffFor(failures, status) {
	// A 429/408/5xx (or a timeout, which arrives with no status) means "later",
	// not "gone" — park those under a soft ceiling so a rate-limited publisher
	// recovers within the hour. A 404/410/403 is a real verdict: hard ceiling.
	const transient = !status || status === 429 || status === 408 || (status >= 500 && status < 600);
	const ceiling = transient ? SOFT_BACKOFF_MS : MAX_BACKOFF_MS;
	return Math.min(FRESH_MS * 2 ** Math.max(0, failures - 1), ceiling);
}

function refreshSource(key) {
	if (inflight.has(key)) return inflight.get(key);
	const p = fetchSource(key)
		.then((articles) => {
			const trimmed = articles
				.sort((a, b) => new Date(b.pub_date || 0) - new Date(a.pub_date || 0))
				.slice(0, MAX_ARTICLES_PER_SOURCE);
			sourceCache.set(key, {
				articles: trimmed,
				fetchedAt: Date.now(),
				ok: true,
				failures: 0,
				nextRetryAt: Date.now() + FRESH_MS,
			});
			return trimmed;
		})
		.catch((err) => {
			const prev = sourceCache.get(key);
			const failures = (prev?.failures || 0) + 1;
			// keep the last-good copy; push the next attempt out exponentially
			sourceCache.set(key, {
				articles: prev?.articles || [],
				fetchedAt: prev?.ok ? prev.fetchedAt : Date.now(),
				lastFailAt: Date.now(),
				lastStatus: err?.status || null,
				ok: false,
				failures,
				nextRetryAt: Date.now() + backoffFor(failures, err?.status),
			});
			return prev?.articles || [];
		})
		.finally(() => inflight.delete(key));
	inflight.set(key, p);
	return p;
}

/**
 * Bring `keys` as close to fresh as `deadlineMs` allows, then return every
 * article currently cached for them. Refreshes that miss the deadline keep
 * running and populate the cache for the next request — a cold start degrades
 * to "fewer sources this round", never to a hung request.
 */
async function ensureSources(keys, deadlineMs = REFRESH_DEADLINE_MS) {
	const now = Date.now();
	const stale = keys
		.filter((key) => {
			if (inflight.has(key)) return false;
			const hit = sourceCache.get(key);
			if (!hit) return true;
			if (now < (hit.nextRetryAt || 0)) return false; // backing off
			return now - hit.fetchedAt >= FRESH_MS;
		})
		// highest-credibility sources first, so a deadline-truncated round still
		// returns the outlets that matter most
		.sort((a, b) => sourcePriority(a) - sourcePriority(b));

	if (stale.length) {
		// refreshSource never rejects and self-throttles on the domain + global
		// gates, so kicking them all off here queues rather than stampedes.
		const started = stale.map((key) => refreshSource(key));
		await Promise.race([Promise.all(started), sleep(deadlineMs)]);
	}

	return keys.flatMap((key) => {
		const hit = sourceCache.get(key);
		if (!hit) return [];
		// drop sources whose last-good copy is ancient — stale beyond 24h reads as fake-live
		const age = Date.now() - hit.fetchedAt;
		if (!hit.ok && age > STALE_OK_MS) return [];
		return hit.articles;
	});
}

function dedupe(articles) {
	const seen = new Set();
	const out = [];
	for (const a of articles) {
		const titleKey = a.title.toLowerCase().replace(/[^a-z0-9]/g, '').slice(0, 80);
		if (seen.has(a.id) || (titleKey && seen.has(titleKey))) continue;
		seen.add(a.id);
		if (titleKey) seen.add(titleKey);
		out.push(a);
	}
	return out;
}

/**
 * Aggregate live news across the registry.
 *
 * `lang` defaults to 'en'. The registry carries international feeds in 17
 * languages, and folding them into the default view would interleave Korean and
 * Chinese headlines into an English feed. They are opt-in: pass a language code
 * for one, or 'all' for the whole registry.
 *
 * `featured` narrows the fan-out to the majors (tier1/tier2 or credibility
 * ≥ 0.85) — the "Featured" tab on /markets/news.
 *
 * `curated` applies the editorial display gate (crypto-relevant + above the
 * quality floor, see api/_lib/news-curation.js). Off by default: ingestion and
 * agent-signal callers get the full firehose; human-facing feeds opt in.
 *
 * @param {object} opts { category, source, lang, q, limit, offset, featured, curated }
 * @returns {{ articles: Array, total: number, sources_ok: number, sources_total: number }}
 */
export async function getNews({ category, source, lang = 'en', q, limit = 30, offset = 0, featured = false, curated = false } = {}) {
	let keys;
	if (source && NEWS_SOURCES[source]) keys = [source];
	else if (source) return { articles: [], total: 0, sources_ok: 0, sources_total: 0 };
	else {
		keys = sourcesForCategory(category);
		if (lang && lang !== 'all') {
			const inLang = new Set(sourcesForLanguage(lang));
			keys = keys.filter((k) => inLang.has(k));
		}
		if (featured) keys = keys.filter(isFeaturedSource);
	}

	// A narrow selection — one source, or a small language/category slice — has
	// no other content to fall back on: truncating its refresh at the fan-out
	// deadline returns an empty page rather than a partial one. Give it the full
	// feed timeout. A broad query keeps the short deadline; it has plenty to show
	// while the stragglers land in cache for the next caller.
	const deadline = keys.length <= NARROW_QUERY_SOURCES ? FEED_TIMEOUT_MS + 500 : REFRESH_DEADLINE_MS;
	const all = await ensureSources(keys, deadline);
	// Rights filter, applied at the fan-out's single choke point: everything
	// downstream of getNews (the /markets/news feed, the RSS mirror, related
	// coverage, the coin rails, and the hourly archive-append cron) inherits it,
	// so a withdrawn publisher can neither be displayed nor newly archived.
	let articles = dedupe(all)
		.filter((a) => !isSuppressed(a))
		.sort((a, b) => new Date(b.pub_date || 0) - new Date(a.pub_date || 0));
	// Editorial display gate (opt-in): crypto-relevant + above the quality
	// floor. OFF by default so ingestion (the archive cron) and agent-signal
	// callers keep the full firehose; the human-facing feeds pass curated:true.
	// An explicit single-source request is exempt — the caller asked for that
	// outlet by name, so honour it.
	if (curated && !source) {
		articles = articles.filter((a) => isDisplayable(a));
	}
	if (q) {
		const needle = q.toLowerCase();
		articles = articles.filter((a) =>
			`${a.title} ${a.description || ''} ${a.tickers.join(' ')}`.toLowerCase().includes(needle),
		);
	}
	const total = articles.length;
	const sources_ok = keys.filter((k) => sourceCache.get(k)?.ok).length;
	// content_text is a server-side field for the article reader; list payloads
	// stay light.
	// `description` is bounded here for the same reason content_text is dropped:
	// a content:encoded feed puts the publisher's whole article in it, and this
	// payload is what the feed UI, the RSS mirror and the coin rails render.
	const page = articles
		.slice(offset, offset + limit)
		.map(({ content_text, ...a }) => ({ ...a, description: excerptText(a.description) }));
	return {
		articles: page,
		total,
		sources_ok,
		sources_total: keys.length,
	};
}

/** Search all sources — used by the /coin/:id related-news rail. */
export async function searchNews(q, limit = 8) {
	return getNews({ q, limit });
}

/**
 * Locate one article (with its feed-provided full text, when the publisher
 * ships one) by link or 16-hex id across every cached source. Used by the
 * reader endpoint as the trusted fallback when a publisher blocks direct
 * page fetches — the content still comes from the publisher's own feed.
 */
export async function findArticle({ link, id }) {
	const keys = sourcesForCategory('all');
	const all = await ensureSources(keys);
	const wantId = id || (link ? articleId(link) : null);
	if (!wantId && !link) return null;
	const hit = all.find((a) => a.id === wantId || (link && a.link === link)) || null;
	// A withdrawn story is "not found" to every caller, including the reader's
	// feed-body fallback — otherwise a blocked page fetch would route straight
	// around the rights filter and serve the publisher's feed copy instead.
	return hit && isSuppressed(hit) ? null : hit;
}