movies.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
豆瓣影视记录爬虫 → Letterboxd CSV

爬取用户在豆瓣标记的「在看/看过」影视列表，
生成 Letterboxd 可导入的 CSV 文件。

Letterboxd CSV 列：LetterboxdURI, tmdbID, imdbID, Title, Year, Directors,
    Rating, Rating10, WatchedDate, Rewatch, Tags, Review
本工具默认输出：Title(空白), Year, Directors, Rating, Rating10, WatchedDate, imdbID
（中文标题/评论/标签默认不输出，ID 是唯一可靠的匹配方式）
"""
import os
import sys
import time
import re
import json
import logging
import argparse

import requests
from bs4 import BeautifulSoup

from shared import (
    OUTPUT_DIR, WORK_DIR, USER_ID,
    MOVIES_CSV, LETTERBOXD_CSV, LETTERBOXD_PARTIAL,
    CHECKPOINT_MOVIES, CHECKPOINT_LETTERBOXD,
    DELAY, DELAY_SPREAD, MAX_RETRIES,
    delay, request_with_retry,
    get_default_headers,
    logger,
    read_csv, write_csv,
)

BASE_URL = "https://movie.douban.com"

# Letterboxd CSV 列名
LB_COLS = [
    "Title", "Year", "Directors",
    "Rating", "Rating10", "WatchedDate",
    "imdbID", "Tags", "Review",
]

MOVIE_FIELDS = ["title", "url", "rating", "date", "comment"]

# Douban 1-5 星 → Letterboxd Rating10 1-10（×2）
DOUBAN_TO_RATING10 = 2

IMDB_RE = re.compile(r'''<span\s+class=['"]pl['"]>IMDb:</span>\s*([^\s<]+)''', re.IGNORECASE)
YEAR_RE = re.compile(r'^(\d{4})')
DATEPUBLISHED_RE = re.compile(r'"datePublished"\s*:\s*"(\d{4}-\d{2}-\d{2})"', re.IGNORECASE)

# Column for missing-ID summary files
MISSING_COLS = ["Title", "URL", "Date", "MissingID", "Suggestion"]


# ==================== 解析函数 ====================

def _parse_movie_item(item):
    """从单个 <div class="item comment-item"> 中解析出一部电影的信息"""
    try:
        title_li = item.select_one("li.title")
        if not title_li:
            return None
        title_link = title_li.select_one("a")
        url = title_link.get("href", "").strip() if title_link else ""
        em = title_li.select_one("em")
        title = em.get_text(strip=True) if em else ""
        if not title and title_link:
            title = title_link.get("title", "").strip()
        rating_el = item.select_one('span[class^="rating"]')
        rating_text = rating_el.get("class", [""])[0] if rating_el else ""
        rating_match = re.search(r"rating(\d+)", rating_text)
        rating = int(rating_match.group(1)) if rating_match else ""
        date_el = item.select_one("span.date")
        date = date_el.get_text(strip=True) if date_el else ""
        comment_li = item.select_one("li.comment")
        comment = comment_li.get_text(strip=True) if comment_li else ""
        return {
            "title": title, "url": url,
            "rating": rating, "date": date, "comment": comment,
        }
    except Exception as e:
        logger.warning(f"解析影视条目时出错：{e}")
        return None


def _parse_movie_list(html):
    soup = BeautifulSoup(html, "lxml")
    items = soup.select("div.item.comment-item")
    return [m for m in (_parse_movie_item(i) for i in items) if m]


def _get_next_page_url(html):
    soup = BeautifulSoup(html, "lxml")
    next_link = soup.select_one('link[rel="next"]')
    if next_link:
        path = next_link.get("href", "").replace("&amp;", "&")
        return f"{BASE_URL}{path}" if path.startswith("/") else path
    return None


# ==================== Letterboxd 字段抓取 ====================

def _extract_year_from_detail(text):
    """
    从详情页 HTML 的 JSON-LD 中提取 datePublished，
    取其前 4 位作为 Year。
    永远不从列表页 intro 提取（intro 第一个日期不可靠）。
    """
    m = DATEPUBLISHED_RE.search(text)
    return m.group(1)[:4] if m else ""


def _extract_directors(text):
    """从详情页 HTML 中提取导演名字列表"""
    try:
        soup = BeautifulSoup(text, "lxml")
        directors = soup.select('a[rel="v:directedBy"]')
        return ", ".join(d.get_text(strip=True) for d in directors)
    except Exception:
        return ""


def _extract_original_title(html):
    """
    从详情页 HTML 的 <meta name="keywords"> 中提取原始语言片名。
    keywords 格式：中文名,外文原名,中文名影评,...
    第二项为原始语言片名；对于中文电影，前两项相同，仍返回第二项。
    """
    soup = BeautifulSoup(html, "lxml")
    meta = soup.select_one('meta[name="keywords"]')
    if not meta:
        return ""
    parts = [p.strip() for p in meta.get("content", "").split(",")]
    return parts[1] if len(parts) >= 2 else (parts[0] if parts else "")


def _fetch_letterboxd_fields(url):
    """
    从豆瓣电影详情页抓取 Letterboxd 所需的额外字段。
    返回 (imdb_id, directors, year, title)。
    - imdb_id: 来自 <span class="pl">IMDb:</span> tt{N}
    - directors: 来自 <a rel="v:directedBy"> 标签
    - year: 来自 JSON-LD datePublished 的前 4 位，绝不从 intro 提取
    - title: 原始语言片名，来自 <meta name="keywords"> 第二项
    """
    resp = request_with_retry(
        "GET", url,
        headers=get_default_headers(),  # 需要 Cookie 才能访问详情页
        dump_label="letterboxd_detail",
    )
    if resp is None:
        return "", "", "", ""
    m = IMDB_RE.search(resp.text)
    imdb_id = re.sub(r'<[^>]+>', '', m.group(1)).strip() if m else ""
    directors = _extract_directors(resp.text)
    year = _extract_year_from_detail(resp.text)
    title = _extract_original_title(resp.text)
    return imdb_id, directors, year, title


# ==================== Letterboxd CSV 行构建 ====================

def _build_letterboxd_row(movie, imdb_id, directors, year, title, import_review, include_directors=False):
    """构建一条 Letterboxd CSV 行"""
    rating = movie.get("rating", "")
    lb_rating = int(rating) if rating not in ("", None) else ""
    lb_rating10 = int(rating) * DOUBAN_TO_RATING10 if rating not in ("", None) else ""
    return {
        "Title": title,
        "Year": year,
        "Directors": directors if include_directors else "",  # 中文导演名默认不输出
        "Rating": lb_rating,
        "Rating10": lb_rating10,
        "WatchedDate": movie.get("date", ""),
        "imdbID": imdb_id,
        "Tags": "",       # Douban 标签为中文，默认不输出
        "Review": movie.get("comment", "") if import_review else "",
    }


def _blank_row():
    return {col: "" for col in LB_COLS}


def _missing_row(title, url, date, missing_id):
    return {
        "Title": title,
        "URL": url,
        "Date": date,
        "MissingID": missing_id,
        "Suggestion": "请在 Letterboxd 手动搜索片名添加",
    }


# ==================== 断点 ====================

def _load_checkpoint():
    if not os.path.exists(CHECKPOINT_MOVIES):
        return 0
    try:
        with open(CHECKPOINT_MOVIES, "r", encoding="utf-8") as f:
            data = json.load(f)
        page = data.get("last_page", 0)
        total = data.get("total_movies", 0)
        logger.info(f"发现断点记录：已爬 {page} 页，共 {total} 部")
        return page
    except (json.JSONDecodeError, IOError) as e:
        logger.warning(f"断点文件读取失败：{e}")
        return 0


def _save_checkpoint(page, total_movies, movies):
    os.makedirs(WORK_DIR, exist_ok=True)
    with open(CHECKPOINT_MOVIES, "w", encoding="utf-8") as f:
        json.dump({
            "last_page": page,
            "total_movies": total_movies,
            "movies": movies[-50:],
        }, f, ensure_ascii=False, indent=2)
    logger.debug(f"断点已保存：第 {page} 页，共 {total_movies} 部")


def _clear_checkpoint():
    if os.path.exists(CHECKPOINT_MOVIES):
        os.remove(CHECKPOINT_MOVIES)
        logger.info("断点文件已清除")


# ==================== 列表爬取 ====================

def scrape(start_page=0, max_pages=None):
    """
    爬取豆瓣影视列表，返回 list[dict]

    Args:
        start_page: 从第几页开始（0 = 从第一页，支持断点续爬）
        max_pages: 最多爬取多少页，None 表示不限制
    """
    all_movies = []
    current_page = 1
    url = f"{BASE_URL}/people/{USER_ID}/collect?sort=time&type=all&filter=all&mode=grid"

    if start_page > 1:
        logger.info(f"正在跳转至第 {start_page} 页...")
        for _ in range(start_page - 1):
            resp = request_with_retry("GET", url, headers=get_default_headers())
            if resp is None:
                return None
            url = _get_next_page_url(resp.text)
            if not url:
                logger.warning("无法跳转到指定页，列表页数不足")
                return None
            delay()
        current_page = start_page
        logger.info(f"已跳转至第 {current_page} 页")

    while True:
        if max_pages and current_page > max_pages:
            logger.info(f"已达到指定页数上限（{max_pages}），停止爬取")
            break

        logger.info(f"--- 第 {current_page} 页：{url}")
        resp = request_with_retry(
            "GET", url,
            headers=get_default_headers(),
            dump_label=f"list_p{current_page}",
        )
        if resp is None:
            return None

        movies = _parse_movie_list(resp.text)
        if not movies:
            logger.warning("本页未找到任何影视条目，可能页面结构已变")
        else:
            logger.info(f"本页获取 {len(movies)} 部")
            all_movies.extend(movies)

        _save_checkpoint(current_page, len(all_movies), all_movies)

        next_url = _get_next_page_url(resp.text)
        if not next_url:
            logger.info("已到达最后一页，爬取完成")
            break

        url = next_url
        current_page += 1
        delay()

    write_csv(MOVIES_CSV, all_movies, MOVIE_FIELDS)
    _clear_checkpoint()
    logger.info(f"列表爬取完成！共 {len(all_movies)} 部，已保存至 {MOVIES_CSV}")
    return all_movies


# ==================== Letterboxd CSV 生成 ====================

def build_letterboxd_csv(
    input_csv=MOVIES_CSV,
    output_csv=None,
    resume=False,
    import_review=True,
    records=None,
):
    """
    抓取 IMDB ID、导演和原始片名，生成 Letterboxd CSV

    Args:
        records: 直接传入抓取结果（list[dict]），为 None 时从 input_csv 读取
        import_review: 是否包含中文评论
    """
    if output_csv is None:
        output_csv = LETTERBOXD_CSV
    os.makedirs(OUTPUT_DIR, exist_ok=True)

    cp_file = CHECKPOINT_LETTERBOXD
    start_index = 0
    if resume and os.path.exists(cp_file):
        try:
            with open(cp_file, "r", encoding="utf-8") as f:
                cp = json.load(f)
            start_index = cp.get("processed_index", 0)
        except (json.JSONDecodeError, IOError):
            pass

    # 恢复模式必须从文件读取（内存数据不含已处理行的字段）
    if records is None or (resume and start_index > 0):
        if not os.path.exists(input_csv):
            logger.error(f"找不到文件：{input_csv}")
            sys.exit(1)
        records = read_csv(input_csv)

    total = len(records)
    if total == 0:
        logger.error("没有任何记录")
        sys.exit(1)

    logger.info(f"共有 {total} 部影视，开始构建 Letterboxd CSV ...")
    if start_index:
        logger.info(f"从第 {start_index + 1} 条继续")

    rows = []
    missing_rows = []

    for i in range(start_index, total):
        movie = records[i]
        title = movie.get("title", "")
        url = movie.get("url", "")

        if not url:
            logger.warning(f"第 {i+1} 条 URL 为空，跳过：{title}")
            rows.append(_blank_row())
            missing_rows.append(_missing_row(title, url, movie.get("date", ""), "IMDB ID"))
            continue

        logger.info(f"[{i+1}/{total}] 《{title}》")
        imdb_id, directors, year, orig_title = _fetch_letterboxd_fields(url)
        row = _build_letterboxd_row(movie, imdb_id, directors, year, orig_title, import_review)
        rows.append(row)

        if imdb_id:
            logger.info(f"  IMDB: {imdb_id} | 导演: {directors[:40]}...")
        else:
            logger.warning(f"  未找到 IMDB ID")
            missing_rows.append(_missing_row(title, url, movie.get("date", ""), "IMDB ID"))

        if (i + 1) % 10 == 0 or i == total - 1:
            os.makedirs(WORK_DIR, exist_ok=True)
            with open(cp_file, "w", encoding="utf-8") as f:
                json.dump({"processed_index": i + 1, "total": total}, f)
            write_csv(LETTERBOXD_PARTIAL, rows, LB_COLS)
            logger.debug(f"断点已保存：{i+1}/{total}")

        if i < total - 1:
            delay()

    write_csv(output_csv, rows, LB_COLS)

    if os.path.exists(cp_file):
        os.remove(cp_file)
    if os.path.exists(LETTERBOXD_PARTIAL):
        os.remove(LETTERBOXD_PARTIAL)

    total_imdb = sum(1 for r in rows if r.get("imdbID"))
    total_missing = total - total_imdb
    logger.info(
        f"完成！共处理 {total} 部，"
        f"找到 IMDB ID {total_imdb} 部，缺失 {total_missing} 部。"
        f"\n结果已保存至 {output_csv}"
    )
    if missing_rows:
        missing_csv = output_csv.replace(".csv", ".missing_imdb.csv")
        write_csv(missing_csv, missing_rows, MISSING_COLS)
        logger.info(f"缺失 IMDB ID 的记录已保存至 {missing_csv}")


# ==================== 入口（独立运行） ====================

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="豆瓣影视 → Letterboxd CSV")
    sub = parser.add_subparsers(dest="command", required=True)

    p_list = sub.add_parser("list", help="爬取豆瓣影视列表（movies.csv）")
    p_list.add_argument("--pages", "-p", type=int, default=None)
    p_list.add_argument("--resume", "-r", action="store_true")

    p_csv = sub.add_parser("csv", help="生成 Letterboxd CSV")
    p_csv.add_argument("--input", "-i", default=MOVIES_CSV)
    p_csv.add_argument("--output", "-o", default="letterboxd_movies.csv")
    p_csv.add_argument("--resume", "-r", action="store_true")
    p_csv.add_argument("--no-import-review", dest="import_review", action="store_false", help="不在 CSV 中包含评论（默认输出评论）")
    p_csv.set_defaults(import_review=True)

    args = parser.parse_args()

    if args.command == "list":
        start = _load_checkpoint() if args.resume else 0
        if start == 0:
            _clear_checkpoint()
        scrape(start_page=start, max_pages=args.pages)
    elif args.command == "csv":
        build_letterboxd_csv(
            input_csv=args.input,
            output_csv=args.output,
            resume=args.resume,
            import_review=args.import_review,
        )