famousPerson in wiki.py

# pip install Wikipedia-API openpyxl requests

import openpyxl
import requests
import time
import wikipediaapi
from datetime import datetime
from urllib.request import getproxies_environment

try:
    from urllib.request import getproxies_registry
except ImportError:
    getproxies_registry = None

# 维基百科要求带上可联系的User-Agent，否则更容易被限流
USER_AGENT = "WIKI-mining/1.0 (hongsuwang@fas.harvard.edu)"


def _normalize_proxy(proxy):
    if not proxy:
        return None
    proxy = proxy.strip()
    if not proxy:
        return None
    # 环境变量里常见"host:port"这种不带scheme的写法，httpx要求必须带scheme
    if "://" not in proxy:
        return "http://" + proxy
    return proxy


def _detect_proxy():
    # 优先读环境变量；环境变量里只有NO_PROXY等无关设置时不能算"已配置"，需继续查Windows系统代理
    # （urllib.request.getproxies()自身在这种情况下会直接短路返回，不会再查注册表，因此这里不用它）
    env_proxies = getproxies_environment()
    proxy = env_proxies.get("https") or env_proxies.get("http")
    if proxy:
        return _normalize_proxy(proxy)
    if getproxies_registry:
        registry_proxies = getproxies_registry()
        proxy = registry_proxies.get("https") or registry_proxies.get("http")
        return _normalize_proxy(proxy)
    return None


# 自动探测代理：环境变量优先，其次Windows系统代理设置；都没配就不用代理
PROXY = _detect_proxy()

wiki = wikipediaapi.Wikipedia(
    user_agent=USER_AGENT,
    language="zh",
    extract_format=wikipediaapi.ExtractFormat.WIKI,
    max_retries=5,
    retry_wait=2.0,  # 内置指数退避，遇到429/5xx会自动重试
    proxy=PROXY,
    timeout=20.0,
)

REQUEST_INTERVAL = 0.4  # 每次请求后的固定间隔（秒），防止请求过快被限流


def is_disambiguation(page):
    return any("消歧义" in title for title in page.categories.keys())


def get_disambiguation_options(page):
    # 消歧义页本身没有正文，用页面内的链接作为候选词条（排除带命名空间前缀的非正文页面，如File:/Category:等）
    return [title for title in page.links.keys() if ":" not in title]


REQUEST_PROXIES = {"http": PROXY, "https": PROXY} if PROXY else None


def get_wiki_status(keyword):
    url = "https://zh.wikipedia.org/wiki/" + keyword
    response = requests.head(url, headers={"User-Agent": USER_AGENT}, proxies=REQUEST_PROXIES, timeout=20.0)
    time.sleep(0.1)
    return "1" if response.status_code != 404 else "0"


def write_row(sheet, row, idx, name, status, content, brief_content):
    sheet.cell(row=row, column=1).value = idx
    sheet.cell(row=row, column=2).value = name
    sheet.cell(row=row, column=3).value = status
    sheet.cell(row=row, column=4).value = content
    sheet.cell(row=row, column=5).value = brief_content
    sheet.cell(row=row, column=6).value = datetime.now().strftime("%Y-%m-%d %H:%M:%S")


# 读取关键词文件
with open("input.txt", "r", encoding="utf-8-sig") as file:
    keywords = file.read().splitlines()

total = len(keywords)
start_time = datetime.now()
print(f"[{start_time.strftime('%Y-%m-%d %H:%M:%S')}] 开始处理，共 {total} 条关键词")

# 创建Excel文件
wb = openpyxl.Workbook()
sheet = wb.active

# 写入表头
sheet["A1"] = "id"
sheet["B1"] = "name"
sheet["C1"] = "exist_status"
sheet["D1"] = "content"
sheet["E1"] = "brief_content"
sheet["F1"] = "timestamp"

# 遍历关键词列表
row = 2
count = 0
count_threshold = 10
try:
    for i, keyword in enumerate(keywords):
        count += 1

        status = get_wiki_status(keyword)

        if status != "1":
            write_row(sheet, row, i + 1, keyword, status, None, None)
            row += 1
        else:
            page = wiki.page(keyword)
            time.sleep(REQUEST_INTERVAL)

            if is_disambiguation(page):
                # 消歧义词条，将每个候选选项作为单独的行输出
                for j, option in enumerate(get_disambiguation_options(page)):
                    option_page = wiki.page(option)
                    time.sleep(REQUEST_INTERVAL)
                    if not option_page.exists():
                        continue
                    write_row(
                        sheet,
                        row,
                        i + 1,
                        keyword + " - " + str(j + 1),
                        status,
                        option_page.text,
                        option_page.summary,
                    )
                    row += 1
            else:
                write_row(sheet, row, i + 1, keyword, status, page.text, page.summary)
                row += 1

        if count % count_threshold == 0 or count == total:
            print(f"进度 {count}/{total} 已完成")
finally:
    # 保存Excel文件；即使中途出错也保留已处理的结果，并打印结束时间
    wb.save("wiki_result.xlsx")
    end_time = datetime.now()
    print(f"[{end_time.strftime('%Y-%m-%d %H:%M:%S')}] 结果已保存为wiki_result.xlsx，共耗时 {end_time - start_time}")