diff --git a/constraints-dev.txt b/constraints-dev.txt index c2625ab9..28150b85 100644 --- a/constraints-dev.txt +++ b/constraints-dev.txt @@ -1,7 +1,8 @@ +absl-py==2.3.0 # via tensorboard accelerate==1.0.1 # via peft, trl, -r requirements-cuda.txt, -r requirements-rocm.txt aiofiles==24.1.0 # via -r requirements.txt aiohappyeyeballs==2.6.1 # via aiohttp -aiohttp==3.12.4 # via fsspec +aiohttp==3.12.6 # via fsspec aiosignal==1.3.2 # via aiohttp annotated-types==0.7.0 # via pydantic anyio==4.9.0 # via httpx, jupyter-server @@ -17,11 +18,12 @@ beautifulsoup4==4.13.4 # via nbconvert bitsandbytes==0.46.0 # via -r requirements-cuda.txt bleach==6.2.0 # via nbconvert cachetools==6.0.0 # via tox -certifi==2025.4.26 # via httpcore, httpx, requests +certifi==2025.4.26 # via httpcore, httpx, requests, sentry-sdk cffi==1.17.1 # via argon2-cffi-bindings cfgv==3.4.0 # via pre-commit chardet==5.2.0 # via tox charset-normalizer==3.4.2 # via requests +click==8.2.1 # via wandb colorama==0.4.6 # via tox comm==0.2.2 # via ipykernel, ipywidgets contourpy==1.3.2 # via matplotlib @@ -33,6 +35,7 @@ deepspeed==0.16.9 # via -r requirements-deepspeed.txt defusedxml==0.7.1 # via nbconvert dill==0.3.8 # via datasets, multiprocess, pylint distlib==0.3.9 # via virtualenv +docker-pycreds==0.4.0 # via wandb einops==0.8.1 # via deepspeed, flash-attn executing==2.2.0 # via stack-data fastjsonschema==2.21.1 # via nbformat @@ -42,6 +45,9 @@ fonttools==4.58.1 # via matplotlib fqdn==1.5.1 # via jsonschema frozenlist==1.6.0 # via aiohttp, aiosignal fsspec==2025.3.0 # via datasets, huggingface-hub, torch +gitdb==4.0.12 # via gitpython +gitpython==3.1.44 # via wandb +grpcio==1.71.0 # via tensorboard h11==0.16.0 # via httpcore hf-xet==1.1.2 # via huggingface-hub hjson==3.1.0 # via deepspeed @@ -50,9 +56,10 @@ httpx==0.28.1 # via jupyterlab huggingface-hub==0.32.3 # via accelerate, datasets, peft, tokenizers, transformers, -r requirements-dev.txt identify==2.6.12 # via pre-commit idna==3.10 # via anyio, httpx, jsonschema, requests, yarl +iniconfig==2.1.0 # via pytest instructlab-dolomite==0.2.0 # via -r requirements.txt ipykernel==6.29.5 # via jupyter, jupyter-console, jupyterlab, -r requirements-dev.txt -ipython==9.2.0 # via ipykernel, ipywidgets, jupyter-console, -r requirements-dev.txt +ipython==9.3.0 # via ipykernel, ipywidgets, jupyter-console, -r requirements-dev.txt ipython-pygments-lexers==1.1.1 # via ipython ipywidgets==8.1.7 # via jupyter isoduration==20.11.0 # via jsonschema @@ -78,8 +85,9 @@ jupyterlab-widgets==3.0.15 # via ipywidgets kiwisolver==1.4.8 # via matplotlib liger-kernel==0.5.10 # via -r requirements-cuda.txt llvmlite==0.44.0 # via numba +markdown==3.8 # via tensorboard markdown-it-py==3.0.0 # via rich -markupsafe==3.0.2 # via jinja2, nbconvert +markupsafe==3.0.2 # via jinja2, nbconvert, werkzeug matplotlib==3.10.3 # via -r requirements-dev.txt matplotlib-inline==0.1.7 # via ipykernel, ipython mccabe==0.7.0 # via pylint @@ -89,6 +97,8 @@ mpmath==1.3.0 # via sympy msgpack==1.1.0 # via deepspeed multidict==6.4.4 # via aiohttp, yarl multiprocess==0.70.16 # via datasets +mypy==1.16.0 # via -r requirements-dev.txt +mypy-extensions==1.1.0 # via mypy nbclient==0.10.2 # via nbconvert nbconvert==7.16.6 # via jupyter, jupyter-server nbformat==5.10.4 # via jupyter-server, nbclient, nbconvert @@ -99,7 +109,7 @@ nodeenv==1.9.1 # via pre-commit notebook==7.4.3 # via jupyter notebook-shim==0.2.4 # via jupyterlab, notebook numba==0.61.2 # via -r requirements.txt -numpy==1.26.4 # via accelerate, bitsandbytes, contourpy, datasets, deepspeed, matplotlib, numba, pandas, peft, transformers, -r requirements-dev.txt, -r requirements.txt +numpy==1.26.4 # via accelerate, bitsandbytes, contourpy, datasets, deepspeed, matplotlib, numba, pandas, peft, tensorboard, transformers, -r requirements-dev.txt, -r requirements.txt nvidia-cublas-cu12==12.4.5.8 # via nvidia-cudnn-cu12, nvidia-cusolver-cu12, torch nvidia-cuda-cupti-cu12==12.4.127 # via torch nvidia-cuda-nvrtc-cu12==12.4.127 # via torch @@ -114,26 +124,28 @@ nvidia-nccl-cu12==2.21.5 # via torch nvidia-nvjitlink-cu12==12.4.127 # via nvidia-cusolver-cu12, nvidia-cusparse-cu12, torch nvidia-nvtx-cu12==12.4.127 # via torch overrides==7.7.0 # via jupyter-server -packaging==25.0 # via accelerate, datasets, deepspeed, huggingface-hub, ipykernel, jupyter-events, jupyter-server, jupyterlab, jupyterlab-server, matplotlib, nbconvert, peft, pyproject-api, tox, transformers, -r requirements.txt +packaging==25.0 # via accelerate, datasets, deepspeed, huggingface-hub, ipykernel, jupyter-events, jupyter-server, jupyterlab, jupyterlab-server, matplotlib, nbconvert, peft, pyproject-api, pytest, tensorboard, tox, transformers, -r requirements.txt pandas==2.2.3 # via datasets pandocfilters==1.5.1 # via nbconvert parso==0.8.4 # via jedi +pathspec==0.12.1 # via mypy peft==0.15.2 # via -r requirements.txt pexpect==4.9.0 # via ipython pillow==11.2.1 # via matplotlib -platformdirs==4.3.8 # via jupyter-core, pylint, tox, virtualenv -pluggy==1.6.0 # via tox +platformdirs==4.3.8 # via jupyter-core, pylint, tox, virtualenv, wandb +pluggy==1.6.0 # via pytest, tox pre-commit==4.2.0 # via -r requirements-dev.txt prometheus-client==0.22.0 # via jupyter-server prompt-toolkit==3.0.51 # via ipython, jupyter-console propcache==0.3.1 # via aiohttp, yarl -psutil==7.0.0 # via accelerate, deepspeed, ipykernel, peft +protobuf==6.31.1 # via tensorboard, wandb +psutil==7.0.0 # via accelerate, deepspeed, ipykernel, peft, wandb ptyprocess==0.7.0 # via pexpect, terminado pure-eval==0.2.3 # via stack-data py-cpuinfo==9.0.0 # via deepspeed, -r requirements.txt pyarrow==20.0.0 # via datasets pycparser==2.22 # via cffi -pydantic==2.11.5 # via deepspeed, pylint-pydantic, -r requirements.txt +pydantic==2.11.5 # via deepspeed, pylint-pydantic, wandb, -r requirements.txt pydantic-core==2.33.2 # via pydantic pygments==2.19.1 # via ipython, ipython-pygments-lexers, jupyter-console, nbconvert, rich pylint==3.3.7 # via pylint-plugin-utils, pylint-pydantic, -r requirements-dev.txt @@ -141,14 +153,15 @@ pylint-plugin-utils==0.8.2 # via pylint-pydantic pylint-pydantic==0.3.5 # via -r requirements-dev.txt pyparsing==3.2.3 # via matplotlib pyproject-api==1.9.1 # via tox +pytest==8.3.5 # via -r requirements-dev.txt python-dateutil==2.9.0.post0 # via arrow, jupyter-client, matplotlib, pandas python-json-logger==3.3.0 # via jupyter-events pytz==2025.2 # via pandas -pyyaml==6.0.2 # via accelerate, datasets, huggingface-hub, jupyter-events, peft, pre-commit, transformers, -r requirements.txt +pyyaml==6.0.2 # via accelerate, datasets, huggingface-hub, jupyter-events, peft, pre-commit, transformers, wandb, -r requirements.txt pyzmq==26.4.0 # via ipykernel, jupyter-client, jupyter-console, jupyter-server referencing==0.36.2 # via jsonschema, jsonschema-specifications, jupyter-events regex==2024.11.6 # via transformers -requests==2.32.3 # via datasets, huggingface-hub, jupyterlab-server, transformers +requests==2.32.3 # via datasets, huggingface-hub, jupyterlab-server, transformers, wandb rfc3339-validator==0.1.4 # via jsonschema, jupyter-events rfc3986-validator==0.1.1 # via jsonschema, jupyter-events rich==14.0.0 # via -r requirements.txt @@ -156,12 +169,17 @@ rpds-py==0.25.1 # via jsonschema, referencing ruff==0.11.12 # via -r requirements-dev.txt safetensors==0.5.3 # via accelerate, instructlab-dolomite, peft, transformers send2trash==1.8.3 # via jupyter-server -setuptools==80.9.0 # via jupyterlab -six==1.17.0 # via python-dateutil, rfc3339-validator +sentry-sdk==2.29.1 # via wandb +setproctitle==1.3.6 # via wandb +setuptools==80.9.0 # via jupyterlab, tensorboard, wandb +six==1.17.0 # via docker-pycreds, python-dateutil, rfc3339-validator, tensorboard +smmap==5.0.2 # via gitdb sniffio==1.3.1 # via anyio soupsieve==2.7 # via beautifulsoup4 stack-data==0.6.3 # via ipython sympy==1.13.1 # via torch +tensorboard==2.19.0 # via -r requirements-dev.txt +tensorboard-data-server==0.7.2 # via tensorboard terminado==0.18.1 # via jupyter-server, jupyter-server-terminals tinycss2==1.4.0 # via bleach tokenizers==0.21.1 # via transformers @@ -176,16 +194,21 @@ transformers==4.52.4 # via instructlab-dolomite, peft, trl, -r requirements triton==3.2.0 # via liger-kernel, torch trl==0.18.1 # via -r requirements.txt types-python-dateutil==2.9.0.20250516 # via arrow -typing-extensions==4.13.2 # via anyio, beautifulsoup4, huggingface-hub, ipython, pydantic, pydantic-core, referencing, torch, typing-inspection +types-pyyaml==6.0.12.20250516 # via -r requirements-dev.txt +types-requests==2.32.0.20250515 # via types-tqdm +types-tqdm==4.67.0.20250516 # via -r requirements-dev.txt +typing-extensions==4.13.2 # via anyio, beautifulsoup4, huggingface-hub, ipython, mypy, pydantic, pydantic-core, referencing, torch, typing-inspection, wandb typing-inspection==0.4.1 # via pydantic tzdata==2025.2 # via pandas uri-template==1.3.0 # via jsonschema -urllib3==2.4.0 # via requests +urllib3==2.4.0 # via requests, sentry-sdk, types-requests virtualenv==20.31.2 # via pre-commit, tox +wandb==0.19.11 # via -r requirements-dev.txt wcwidth==0.2.13 # via prompt-toolkit webcolors==24.11.1 # via jsonschema webencodings==0.5.1 # via bleach, tinycss2 websocket-client==1.8.0 # via jupyter-server +werkzeug==3.1.3 # via tensorboard wheel==0.45.1 # via -r requirements.txt widgetsnbextension==4.0.14 # via ipywidgets xxhash==3.5.0 # via datasets diff --git a/requirements-dev.txt b/requirements-dev.txt index 0f789f18..3fb79e43 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,12 +8,20 @@ numpy pre-commit>=3.0.4 pylint>=2.16.2 pylint-pydantic +pytest ruff tox>=4.4.2 tox-current-env +mypy>=1.10.0 +types-tqdm +types-PyYAML + ipython ipykernel jupyter huggingface_hub + +wandb +tensorboard diff --git a/tox.ini b/tox.ini index e1200b71..fb55428e 100644 --- a/tox.ini +++ b/tox.ini @@ -23,9 +23,6 @@ passenv = HF_HOME INSTRUCTLAB_NCCL_TIMEOUT_MS deps = - pytest - wandb - tensorboard -r requirements-dev.txt commands = {envpython} -m pytest tests/unit {posargs} # NOTE: {posargs} is a placeholder for input positional arguments @@ -39,7 +36,6 @@ passenv = HF_HOME INSTRUCTLAB_NCCL_TIMEOUT_MS deps = - pytest -r requirements-dev.txt -r requirements-cuda.txt commands = {envpython} -m pytest tests/smoke {posargs} @@ -97,7 +93,7 @@ basepython = {[testenv:py3]basepython} skip_install = true skipsdist = true deps = - mypy>=1.10.0,<2.0 + mypy types-tqdm types-PyYAML pytest