LLM-Finetuning-Lab/requirements.txt at master · Synthora-AI/LLM-Finetuning-Lab · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
# Core ML/DL Frameworks
torch>=2.0.0
torchvision>=0.15.0
transformers>=4.35.0
accelerate>=0.24.0
peft>=0.7.0
bitsandbytes>=0.41.0
deepspeed>=0.11.0

# Training & Optimization
datasets>=2.14.0
evaluate>=0.4.0
wandb>=0.16.0
tensorboard>=2.14.0
optuna>=3.4.0
mlflow>=2.8.0

# Data Processing
pandas>=2.0.0
numpy>=1.24.0
scipy>=1.10.0
scikit-learn>=1.3.0

# NLP Tools
nltk>=3.8.0
spacy>=3.7.0
sentencepiece>=0.1.99
rouge-score>=0.1.2
bert-score>=0.3.13
langdetect>=1.0.9

# API & Integration
fastapi>=0.104.0
uvicorn[standard]>=0.24.0
pydantic>=2.4.0
pydantic-settings>=2.0.0
requests>=2.31.0
python-dotenv>=1.0.0
slowapi>=0.1.9

# MongoDB Integration
pymongo>=4.5.0
motor>=3.3.0

# Vector Database
pinecone-client>=2.2.0
faiss-cpu>=1.7.4

# Monitoring & Observability
prometheus-client>=0.18.0
prometheus-fastapi-instrumentator>=6.1.0
psutil>=5.9.0

# Utilities
tqdm>=4.66.0
colorlog>=6.7.0
pyyaml>=6.0.0
python-json-logger>=2.0.0

# Visualization
matplotlib>=3.8.0
seaborn>=0.13.0
plotly>=5.18.0

# Testing
pytest>=7.4.0
pytest-cov>=4.1.0
pytest-asyncio>=0.21.0
pytest-xdist>=3.5.0
pytest-benchmark>=4.0.0

# Code Quality
black>=23.10.0
flake8>=6.1.0
isort>=5.12.0
mypy>=1.6.0
pylint>=3.0.0
bandit[toml]>=1.7.5

# Documentation
mkdocs>=1.5.0
mkdocs-material>=9.4.0

# LangChain Integration
langchain>=0.1.0
langchain-community>=0.0.10
langgraph>=0.0.20

# Cloud Storage
boto3>=1.28.0  # AWS S3
google-cloud-storage>=2.10.0  # Google Cloud Storage
azure-storage-blob>=12.19.0  # Azure Blob Storage

# Model Serving & Optimization
onnx>=1.15.0
onnxruntime>=1.16.0
onnxruntime-gpu>=1.16.0

# Jupyter
jupyter>=1.0.0
ipykernel>=6.26.0
ipywidgets>=8.1.0

# Caching & Performance
redis>=5.0.0
hiredis>=2.2.3

# Rate Limiting & Retry
tenacity>=8.2.0
slowapi>=0.1.9

# Environment Management
python-decouple>=3.8

# Security
python-jose[cryptography]>=3.3.0
passlib[bcrypt]>=1.7.4
cryptography>=41.0.0

# Distributed Computing
ray[default]>=2.8.0

# Profiling & Debugging
py-spy>=0.3.14
memory-profiler>=0.61.0

# Content Moderation
better-profanity>=0.7.0

# API Documentation
httpx>=0.25.0

# Configuration Validation
cerberus>=1.3.5