pyproject.toml

[build-system]
requires = ["hatchling>=1.26.3"]
build-backend = "hatchling.build"

[project]
name = "edgartools"
description = 'Python library to access and analyze SEC Edgar filings, XBRL financial statements, 10-K, 10-Q, and 8-K reports'
readme = "README.md"
requires-python = ">=3.10"
license = "MIT"
keywords = [
    "sec", "edgar", "filings", "sec filings", "edgar filings",
    "10-K", "10-Q", "8-K", "xbrl", "financial statements",
    "sec api", "edgar api", "python", "finance", "financial data",
    "stock filings", "company filings", "annual report", "quarterly report",
    "institutional holdings", "13F", "insider trading", "form 4"
]
authors = [
  { name = "Dwight Gunning", email = "dgunning@gmail.com" },
]
classifiers = [
  "Development Status :: 4 - Beta",
  "Programming Language :: Python",
  "Programming Language :: Python :: 3.10",
  "Programming Language :: Python :: 3.11",
  "Programming Language :: Python :: 3.12",
  "Programming Language :: Python :: 3.13",
  "Programming Language :: Python :: 3.14",
  "Programming Language :: Python :: Implementation :: CPython",
  "Programming Language :: Python :: Implementation :: PyPy",
  "License :: OSI Approved :: MIT License",
  "Intended Audience :: Financial and Insurance Industry",
  "Intended Audience :: Developers",
  "Intended Audience :: Science/Research",
  "Topic :: Office/Business :: Financial",
  "Topic :: Office/Business :: Financial :: Investment",
  "Topic :: Scientific/Engineering :: Information Analysis",
]
dependencies = [
  # Upper cap is deliberate: httpx upstream is dormant (last release 0.28.1,
  # Dec 2024; issues and discussions closed Feb 2026). Any future release would
  # be unexpected — do not auto-adopt it. The migration path is httpx2
  # (edgartools-q2iz); lift or retarget this cap there, not here.
  "httpx>=0.25.0,<0.29",
  "pandas>=2.0.0",
  "tabulate>=0.9.0",
  "pyarrow>=17.0.0",
  "beautifulsoup4>=4.10.0",
  "lxml>=4.4",
  "rich>=13.8.0",
  "humanize>=4.0.0",
  "stamina>=24.2.0",
  "orjson>=3.6.0",
  "textdistance>=4.5.0",
  "rank_bm25>=0.2.1",
  "rapidfuzz>=3.5.0",
  "unidecode>=1.2.0",
  "pydantic>=2.0.0",
  "tqdm>=4.62.0",
  "nest-asyncio>=1.5.1",
  "jinja2>=3.1.0",
  "pyrate-limiter>=3.0.0",
  # The [httpx] extra is required, not decorative: from 0.6.0 httpxthrottlecache
  # declares httpx and httpx2 as optional extras and installs neither by default.
  # 0.5.0 briefly hard-required the httpx2 fork — which is why this was capped
  # below it — and 0.6.0 reversed that, so the cap is no longer needed to stay on
  # plain httpx. Migrating to httpx2 is tracked separately (edgartools-q2iz).
  # The 0.6.1 floor is load-bearing: caching data.sec.gov reaches an age check
  # that raised uncaught on a machine whose clock trails the origin's Date header
  # (paultiq/httpxthrottlecache#43), taking down the whole get_facts() call.
  # 0.6.1 clamps it.
  "httpxthrottlecache[httpx]>=0.6.1",
  "truststore>=0.9.0",
]
dynamic = ["version"]

[project.optional-dependencies]
ai = [
    # Cap below 2.0.0: that release removed the decorator-based Server API
    # (@app.list_tools() and friends) that edgar/ai/mcp/server.py binds at
    # import time, so an unpinned resolve breaks `edgartools-mcp` outright and
    # fails test collection. Lift the cap once server.py is ported to the 2.0
    # API (add_request_handler / FastMCP).
    "mcp>=1.12.3,<2.0.0",   # Model Context Protocol
    "tiktoken>=0.10.0",     # Token counting for OpenAI models
    "starlette>=0.36.0",    # ASGI framework for HTTP transport
    "uvicorn>=0.27.0",      # ASGI server for HTTP transport
]

ai-dev = [
    "pytest-mock>=3.12.0",       # For mocking AI responses
    "responses>=0.24.0",         # HTTP response mocking
]

data = [
    "duckdb>=1.0.0",            # SQL interface for company datasets
    "zstandard>=0.20.0",        # Decompress datamule tar content
]

test-harness = [
    "click>=8.0.0",              # CLI interface for test harness
]

# Cloud storage support via fsspec
cloud = [
    "fsspec>=2023.1.0",          # Core filesystem abstraction
]

s3 = [
    "fsspec>=2023.1.0",
    "s3fs>=2023.1.0",            # AWS S3, Cloudflare R2, MinIO
]

gcs = [
    "fsspec>=2023.1.0",
    "gcsfs>=2023.1.0",           # Google Cloud Storage
]

azure = [
    "fsspec>=2023.1.0",
    "adlfs>=2023.1.0",           # Azure Blob Storage
]

all-cloud = [
    "fsspec>=2023.1.0",
    "s3fs>=2023.1.0",
    "gcsfs>=2023.1.0",
    "adlfs>=2023.1.0",
]

[project.urls]
Homepage = "https://github.com/dgunning/edgartools"
Documentation = "https://dgunning.github.io/edgartools/"
Issues = "https://github.com/dgunning/edgartools/issues"
Source = "https://github.com/dgunning/edgartools"
Changelog = "https://github.com/dgunning/edgartools/releases"

[project.scripts]
edgartools-mcp = "edgar.ai.mcp.server:main"
edgar-test = "tests.harness.cli:cli"

[tool.hatch.version]
path = "edgar/__about__.py"

[tool.hatch.build]
include = [
  "edgar/**/*.py",
  # PEP 561 marker. This list is an allowlist and no other pattern here would
  # match an extensionless file, so dropping this line ships an untyped wheel
  # while the repo still looks correct — which is what 5.47.0 did.
  "edgar/py.typed",
  "edgar/**/templates/*.html",
  "edgar/**/docs/*.md",
  "edgar/ai/skills/**/*.yaml",
  "edgar/ai/skills/*.yaml",
  "edgar/ai/skills/core/*.md",
  "edgar/reference/data/*",
  "edgar/entity/data/*",
  "edgar/xbrl/standardization/*",
  "LICENSE.txt"
]
artifacts = [
  "LICENSE.txt"  # And this line
]

[tool.hatch.envs.default]
dependencies = [
  "pytest",
  "pytest-cov",
  "pytest-env",
  "pytest-xdist",
  "pytest-asyncio",
  "pytest-retry",
  "pytest-vcr",
  # Pin <8.2: vcrpy 8.2.0 reintroduced an httpx stub whose response
  # deserializer crashes (`'NoneType' object has no attribute 'encode'`)
  # on cassettes with a null reason phrase (e.g. SEC S3 404 responses).
  # 8.1.1 intercepts httpx via httpcore_stubs.py and replays correctly.
  "vcrpy<8.2",
  "filelock",
  "pyinstrument",
  "pyright",
  "ruff",
  "tqdm",
  "xlsxwriter",
  "openpyxl",
  "jupyter",
  "freezegun==1.5.1",
  "mkdocs",
  "mkdocstrings[python]",
  "mkdocs-material"
]

features = [
    "ai",
    "ai-dev"
]

[tool.hatch.envs.default.scripts]
cov = "pytest --cov-report=term-missing --cov-config=pyproject.toml --cov=edgar --cov=tests --ignore=tests/legacy {args}"
no-cov = "cov --no-cov {args}"
lint = "ruff check edgar"
smoke-filings = "python scripts/batch/batch_filings.py {args}"
# Run before testing a branch you didn't write: vcrpy loads cassettes with
# PyYAML's unsafe loader, so a contributed cassette executes code on your
# machine the moment a test touches it. CI gates on this too.
check-cassettes = "python scripts/check_cassettes.py {args}"
check-regression-skips = "python scripts/check_regression_skips.py {args}"
check-regression-provenance = "python scripts/check_regression_provenance.py {args}"
# Test categorization commands (sequential execution)
test-fast = "pytest -m 'fast' {args}"
test-slow = "pytest -m 'slow' {args}"
test-network = "pytest -m 'network' {args}"
test-core = "pytest -m 'not (slow or network or performance or batch)' --ignore=tests/perf {args}"

# Measure which tests actually need the SEC, rather than guessing from filenames.
# Blocks outbound sockets and clears edgar's caches between tests, so the answer
# does not depend on ordering. Takes a path: `hatch run test-offline-audit
# tests/issues/regression/`. Anything that fails here must be re-run WITHOUT the
# plugin before being marked `network` — failing offline and being broken are
# different facts. See tests/_offline_harness.py.
# Two flags that both cost more than they look.
#
# `python -m pytest`, not `pytest`: -p imports the plugin before collection puts
# the rootdir on sys.path, and the console script never adds cwd, so the bare
# form cannot find tests._offline_harness at all.
#
# `-p no:pytest-retry` because the suite retries twice with a 4s delay, and a
# blocked socket never succeeds on the second attempt — it just multiplies the
# cost of exactly the failures the audit is looking for. On one 16-test file
# that is 24.9s against 0.35s. The plugin registers as `pytest-retry`, not
# `retry`, and `-p no:retry` is accepted and silently does nothing; `--retries 0`
# does not disable it either, since the ini value still reads as set. Disabling
# it orphans its ini keys, so the run warns `Unknown config option:
# cumulative_timing` — expected here, and not a sign of a broken config.
test-offline-audit = "python -m pytest -p tests._offline_harness -p no:pytest-retry {args}"

# Parallel test commands (selective parallelization for SEC rate limit safety)
# - Fast tests: Full parallelization (no network calls)
# - Network/slow: Sequential (respects SEC rate limits)
# - Core: Limited parallelization (mixed tests)
test-fast-parallel = "pytest -n auto -m 'fast' {args}"
test-core-parallel = "pytest -n 2 -m 'not (slow or network or performance or batch)' --ignore=tests/perf {args}"
test-parallel-safe = "pytest -n auto -m 'fast' {args}"

# Parallel CI test strategy
#
# SELECTIVE PARALLELIZATION STRATEGY:
# 1. Marker-based: 'not regression' keeps the network/slow regression tests out
#    of the PR jobs — they run in the Regression Tests workflow, which is the
#    only place allowed to hammer the SEC endpoint.
# 2. Path-based: --ignore=tests/issues/regression excludes entire folder
# 3. Auto-marking: conftest.py automatically marks tests as fast/network based on filename
# 4. Selective parallel: Only parallelize tests safe from SEC rate limits
# 5. Combined coverage: CI combines coverage from fast/network/slow groups for 65% threshold
#
# test-ci-fast deliberately does NOT exclude regression. conftest.py classifies
# every regression test as fast or network by measurement, and the offline ones
# belong in the pull-request gate — before this they ran only after merge
# (bead edgartools-07lk.21).
#
# Note: "core" group removed from CI - auto-marking leaves 0 tests without markers

test-ci-fast = "pytest -n auto --cov --cov-report=xml -m 'fast' {args}"
test-ci-network = "pytest --cov --cov-report=xml -m 'network and not slow and not regression' {args}"
test-ci-slow = "pytest --cov --cov-report=xml -m 'slow and not regression' {args}"
test-ci-all = "pytest --cov --cov-report=xml -m 'not regression' --ignore=tests/perf --ignore=tests/issues/regression {args}"

# Regression tests (run separately/on-demand for comprehensive bug prevention)
#
# No `--ignore=tests/issues/reproductions` here, deliberately, even though that
# tree feeding this selection is what bead edgartools-07lk.24 Tier 2 set out to
# stop. `-m regression` selects on the marker rather than the path, so a marked
# test in reproductions/ ran in this lane while sitting outside the provenance,
# no-skip and fast/network gates that apply to tests/issues/regression/ — 32
# tests across 6 files were in that state on 2026-08-10.
#
# The fix is a collection hook in tests/issues/reproductions/conftest.py that
# fails on the marker outright, which makes the boundary hold by construction
# rather than by whoever last edited this line. Adding a path exclusion here
# would be worse than redundant: pytest would stop loading that conftest on
# exactly the run the check is meant to police, so the guard would be off in
# the one lane that needs it.
test-regression = "pytest --cov --cov-report=xml -m regression {args}"

# Other test categories  
test-eval = "pytest tests/test_evaluation_offline.py {args}"
test-batch = "pytest scripts/batch/ -m 'batch' {args}"
test-reproduction = "pytest tests/issues/reproductions/ -m 'reproduction' {args}"
test-full = "pytest --ignore=tests/perf {args}"

[tool.hatch.envs.test]
# Keep this list in step with what CI installs (see the pip install lines in
# .github/workflows/python-hatch-workflow.yml) and with the default env's pins.
# A plugin missing here does not fail — it changes what the tests mean. Without
# pytest-vcr and vcrpy, `@pytest.mark.vcr` is an unregistered marker, no cassette
# is opened, and the request goes to the network or the local HTTP cache while
# the run still reports passed. That is the "a cassette existing is not replay"
# failure one level further down: the cassette can be right and the runner still
# never open it.
dependencies = [
  "pytest",
  "pytest-cov",
  "pytest-env",
  "pytest-xdist",
  "pytest-asyncio",
  "pytest-retry",
  "pytest-mock",
  "pytest-vcr",
  # Pin <8.2 for the same reason as the default env: vcrpy 8.2.0's httpx stub
  # crashes deserializing a cassette with a null reason phrase.
  "vcrpy<8.2",
  "filelock",
  "tqdm",
  "responses",
  "freezegun==1.5.1"
]

[[tool.hatch.envs.test.matrix]]
python = ["39", "310", "311", "312"]

[tool.coverage.run]
branch = true
parallel = true
source = ["edgar"]
omit = [
  "edgar/__about__.py",
  # Example and demo files (not part of core library)
  "edgar/**/examples.py",
  "edgar/**/examples/*.py",
  "edgar/display/demo.py",
  "edgar/thirteenf/demo_comparison.py",
  # __main__-only data download stubs (not library code)
  "edgar/reference/financials.py",
  # Training scripts (ML training utilities, not runtime code)
  "edgar/entity/training/*",
  # Migration utilities (one-time migration helpers)
  "edgar/documents/migration.py",
  # CLI entry points (tested via integration, not unit tests)
  "edgar/**/__main__.py",
  # MCP server (optional external integration, requires MCP runtime)
  "edgar/ai/mcp/*",
  # AI evaluation framework (dev tooling: LLM-as-judge, A/B skill testing)
  "edgar/ai/evaluation/*",
  # AI exporters (CLI tooling for skill packaging)
  "edgar/ai/exporters/*",
  # Experimental AI features
  "edgar/ai/formats.py",
  "edgar/ai/helpers.py",
  # SSL diagnostics (CLI tool, not library code)
  "edgar/diagnose_ssl/*",
]

[tool.coverage.report]
exclude_lines = [
  "no cov",
  "if __name__ == .__main__.:",
  "if TYPE_CHECKING:",
]
# Coverage threshold moved to CI combined coverage check
# Individual test groups don't meet 65% alone, only combined


[tool.ruff]
line-length = 150
exclude = [
  ".git",
  "__pycache__",
  "build",
  "dist",
  "docs/source/conf.py",
  "tests",
]
extend-include = ["*.ipynb"]

[tool.ruff.lint]
select = ["F", "E", "W", "S", "B", "G", "N", "I001", "T", "PD", "C90"]
ignore = [
  "W291",
  "E501",
  "S608",
  "PD901",
  "C901",
  "S101"
]

[tool.ruff.lint.per-file-ignores]
"__init__.py" = ["F401"]

[dependency-groups]
dev = [
    "pre-commit>=4.3.0",
    "pytest-retry>=1.7.0",
]

[tool.pytest.ini_options]
testpaths = ["tests"]
env = [
    "EDGAR_IDENTITY=Dev Gunning developer-gunning@gmail.com",
]
asyncio_mode = "auto"
asyncio_default_fixture_loop_scope = "function"
retries = 2
retry_delay = 4
cumulative_timing = false
retry_outcome = "rerun"

# An xfail that passes is a failure, not a quiet success.
#
# Without this, xfail-pass and xfail-fail are BOTH green, so an xfail-marked
# test has no outcome that can fail and stops being a test. Three in
# test_issue_583_equity_labels.py sat that way for months reporting XPASS in
# every CI run: they were marked "not yet implemented" and the feature had
# shipped, which the file's own docstring recorded (edgartools-07lk.24
# finding 2). Strict mode turns the next one of those into a failure the day
# the behaviour changes, which is the only moment the information is useful.
#
# The four remaining xfails are genuinely still failing and stay xfail --
# strict does not touch those. If you mark a test xfail, it must actually fail;
# when it starts passing, delete the marker rather than widen this setting.
xfail_strict = true

# Pytest markers for test categorization
markers = [
    "fast: Fast tests that run quickly (< 0.1s each)",
    "slow: Slow tests that take significant time (> 1s each)",
    "network: Tests that require internet connectivity",
    "regression: Regression tests for specific GitHub issues",
    "batch: Batch processing tests",
    "performance: Performance and benchmarking tests",
    "reproduction: Tests that reproduce specific issues",
    "integration: Integration tests",
    "data_quality: Data quality validation tests",
    "vcr: Tests using VCR cassettes for recorded HTTP responses",
    "manual: Manual-only tests, excluded from CI (slow evaluations, benchmarks)",
    "viewer_corpus: SEC R*.htm viewer verification corpus (beads edgartools-doup)",
]

[tool.pyright]
# Suppress false positives for C extension modules
reportAttributeAccessIssue = "none"
# PyArrow is a C extension - type stubs are incomplete
ignoreExternal = true

[tool.ty]
# Suppress unresolved-attribute errors for pyarrow.compute (C extension with incomplete stubs)
# These are false positives - the methods exist at runtime
[[tool.ty.overrides]]
include = ["edgar/_filings.py"]
rules = { unresolved-attribute = "ignore" }