Update master branch

Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.11
+
+WORKDIR /sd
+
+# install dependencies
+COPY requirements.txt .
+#RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install -r requirements.txt
+
+# Create a writable cache directory inside the container
+RUN mkdir -p /model/cache 
+
+# copy source code
+COPY . .
+
+EXPOSE 5000
+
+# Setup an app user so the container doesn't run as the root user
+RUN useradd -m sd
+USER sd
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"]

docker-compose.yml

@@ -0,0 +1,8 @@
+services:
+  web:
+    build: .
+    ports:
+      - "8000:8000"
+    volumes:
+      - .:/sd                                       # Live-sync local project to container
+      - ./logs:/sd/logs                             # Persist logs to host machine

logs/log_configs.py

@@ -0,0 +1,13 @@
+import logging
+
+#logging setup
+logging.basicConfig(
+    level=logging.INFO,  # Use DEBUG for verbose logs
+    format="%(asctime)s [%(levelname)s] %(message)s",
+    handlers=[
+        logging.FileHandler("logs/app.log"),      # Log to file
+        logging.StreamHandler()                   # Log to console
+    ]
+)
+
+logger = logging.getLogger(__name__)

main.py

@@ -9,6 +9,11 @@
 from fastapi.responses import HTMLResponse, JSONResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
+from fastapi.middleware.cors import CORSMiddleware
+from slowapi import Limiter, _rate_limit_exceeded_handler
+from slowapi.util import get_remote_address
+from slowapi.errors import RateLimitExceeded
+from slowapi.middleware import SlowAPIMiddleware
 from supabase import create_client, Client
 import os
 from os.path import join, dirname
@@ -21,14 +26,11 @@
 from collections import Counter
 import jwt
 from functools import wraps
-import logging
+#import logger
+from logs.log_configs import logger
 import tensorflow as tf
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
 
-# Configure logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
 tf_model_path_english = "deep_learning_models/final_model_english"
 tf_model_path_korean = "deep_learning_models/final_model_korean"
 tf_model_path_german = "deep_learning_models/final_model_german"
@@ -51,14 +53,14 @@ def _load_tf_model_and_tokenizer(lang_code, model_path, tokenizer_path):
             tf_models[lang_code] = tf.saved_model.load(model_path)
             logger.info(f"TensorFlow {lang_code.upper()} model loaded successfully from {model_path}")
         except Exception as e:
-            logging.error(f"Failed to load TensorFlow {lang_code.upper()} model from {model_path}: {e}")
+            logger.error(f"Failed to load TensorFlow {lang_code.upper()} model from {model_path}: {e}")
             tf_models[lang_code] = None
     if lang_code not in tf_tokenizers:
         try:
             tf_tokenizers[lang_code] = AutoTokenizer.from_pretrained(tokenizer_path)
-            logging.info(f"{lang_code.upper()} Tokenizer loaded successfully from {tokenizer_path}.")
+            logger.info(f"{lang_code.upper()} Tokenizer loaded successfully from {tokenizer_path}.")
         except Exception as e:
-            logging.error(f"Failed to load {lang_code.upper()} tokenizer from {tokenizer_path}: {e}")
+            logger.error(f"Failed to load {lang_code.upper()} tokenizer from {tokenizer_path}: {e}")
             tf_tokenizers[lang_code] = None
     return tf_models.get(lang_code), tf_tokenizers.get(lang_code)
 
@@ -80,7 +82,7 @@ def _load_spacy_nlp_pipeline(lang_code):
             spacy_nlp_pipelines[lang_code] = spacy.load(spacy_model_name)
             logger.info(f"SpaCy '{spacy_model_name}' loaded successfully for {lang_code.upper()}.")
         except Exception as e:
-            logging.error(f"Failed to load SpaCy model '{spacy_model_name}' for '{lang_code}': {e}. Please ensure it is installed (e.g., 'python -m spacy download {spacy_model_name}')")
+            logger.error(f"Failed to load SpaCy model '{spacy_model_name}' for '{lang_code}': {e}. Please ensure it is installed (e.g., 'python -m spacy download {spacy_model_name}')")
             spacy_nlp_pipelines[lang_code] = None
     return spacy_nlp_pipelines.get(lang_code)
 
@@ -94,7 +96,7 @@ def _load_m2m_translator():
             m2m_translator_model = AutoModelForSeq2SeqLM.from_pretrained(m2m_model_name)
             logger.info(f"M2M100 Translator model loaded successfully: {m2m_model_name}")
         except Exception as e:
-            logging.error(f"Failed to load M2M100 Translator model: {e}")
+            logger.error(f"Failed to load M2M100 Translator model: {e}")
             m2m_translator_tokenizer = None
             m2m_translator_model = None
     return m2m_translator_tokenizer, m2m_translator_model
@@ -112,14 +114,34 @@ def _load_m2m_translator():
 nltk.download('punkt', quiet=False, force=True)
 nltk.download('punkt_tab', quiet=False, force=True)
 
+# Initialize the limiter
+limiter = Limiter(key_func=get_remote_address)
 
 app = FastAPI()
+
+# CORS settings
+origins = [
+    "http://localhost:8000",      
+    "http://127.0.0.1:8000",      
+    # Add production domain here when deployed
+]
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
 app.mount("/static", StaticFiles(directory="static"), name="static")
 templates = Jinja2Templates(directory="templates")
 security = HTTPBearer(auto_error=False)
 
-from langdetect import detect
+# Connect rate limiter
+app.state.limiter = limiter
+app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
 
+from langdetect import detect, detect_langs
 
 MAX_CHUNK_LENGTH = 512
 def auto_translate_to_english(text: str):
@@ -143,9 +165,9 @@ def auto_translate_to_english(text: str):
         if spacy_for_segmentation and spacy_for_segmentation.has_pipe('sentencizer'): # Check for sentencizer component
             doc = spacy_for_segmentation(text)
             sentences = [sent.text for sent in doc.sents]
-            logging.info(f"SpaCy successfully segmented text for language: {lang}. Sentences found: {len(sentences)}")
+            logger.info(f"SpaCy successfully segmented text for language: {lang}. Sentences found: {len(sentences)}")
         else:
-            logging.warning(f"SpaCy pipeline for '{lang}' not available or lacks sentencizer. Falling back to basic regex segmentation for translation.")
+            logger.warning(f"SpaCy pipeline for '{lang}' not available or lacks sentencizer. Falling back to basic regex segmentation for translation.")
             import re
             sentences = re.split(r'(?<=[.!?؟\u061F\u06D4])\s+', text)
             sentences = [s.strip() for s in sentences if s.strip()]
@@ -635,10 +657,15 @@ def predict_with_tf_model(text: str, model_tf, tokenizer_tf):
 
     return predicted_ilr, probabilities.tolist()
 
+
 @app.post("/predict")
-async def predict_ilr(request: TextRequest, background_tasks: BackgroundTasks, current_user: dict = Depends(get_current_user_required)):
+@limiter.limit("5/minute")
+async def predict_ilr(Text_Request: TextRequest, request: Request, background_tasks: BackgroundTasks, current_user: dict = Depends(get_current_user_required)):
     logger.info(f"Predict request from user: {current_user.get('email', 'unknown')}")
-    raw_text = request.text
+    ip = request.client.host
+    logger.info(f"Received prediction request from {ip}")
+
+    raw_text = Text_Request.text
     logger.info(f"Processing text of length: {len(raw_text)}")
 
     predicted_ilr = 2
@@ -720,7 +747,6 @@ async def predict_ilr(request: TextRequest, background_tasks: BackgroundTasks, c
         user_id=current_user["user_id"],
         features_dict=features_dict # Pass complete features_dict (includes probabilities)
     )
-
     # --- IMMEDIATE RESPONSE TO FRONTEND ---
     return {
         "predicted_ilr": int(predicted_ilr),
@@ -732,11 +758,15 @@ async def predict_ilr(request: TextRequest, background_tasks: BackgroundTasks, c
     }
 
 @app.post("/predict-batch")
-def predict_batch(request: BatchTextRequest, background_tasks: BackgroundTasks, current_user: dict = Depends(get_current_user_required)):
+@limiter.limit("5/minute")
+def predict_batch(Batch_Text_Request: BatchTextRequest, request: Request, background_tasks: BackgroundTasks, current_user: dict = Depends(get_current_user_required)):
+    ip = request.client.host
+    logger.info(f"Received prediction request from {ip}")
+
     try:
         results = []
 
-        for i, text in enumerate(request.texts):
+        for i, text in enumerate(Batch_Text_Request.texts):
             if not text.strip():
                 continue
 
@@ -805,6 +835,7 @@ def predict_batch(request: BatchTextRequest, background_tasks: BackgroundTasks,
                 normalized_features_for_item = {}
 
             # --- Background Storage for item ---
+
             background_tasks.add_task(
                 store_prediction_background,
                 raw_text=text,
@@ -814,7 +845,7 @@ def predict_batch(request: BatchTextRequest, background_tasks: BackgroundTasks,
                 user_id=current_user["user_id"],
                 features_dict=features_dict_for_item
             )
-
+            
             result = {
                 "index": i,
                 "text": text,
@@ -858,4 +889,4 @@ async def upload_files(files: List[UploadFile] = File(...), current_user: dict =
 
 if __name__ == "__main__":
     import uvicorn
-    uvicorn.run(app, host="0.0.0.0", port=8000)
+    uvicorn.run(app, host="0.0.0.0", port=8000)

requirements.txt

@@ -1,8 +1,10 @@
+absl-py==2.3.1
 aiohappyeyeballs==2.6.1
 aiohttp==3.11.13
 aiosignal==1.3.2
 annotated-types==0.7.0
 anyio==4.8.0
+astunparse==1.6.3
 attrs==25.3.0
 blinker==1.9.0
 blis==1.2.0
@@ -16,62 +18,88 @@ confection==0.1.5
 contourpy==1.3.2
 cycler==0.12.1
 cymem==2.0.11
+de_core_news_sm @ https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-3.8.0/de_core_news_sm-3.8.0-py3-none-any.whl#sha256=fec69fec52b1780f2d269d5af7582a5e28028738bd3190532459aeb473bfa3e7
+Deprecated==1.2.18
+deprecation==2.1.0
 distro==1.9.0
 dotenv==0.9.9
 en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl#sha256=1932429db727d4bff3deed6b34cfc05df17794f4a52eeb26cf8928f7c1a0fb85
 fastapi==0.78.0
 filelock==3.18.0
 Flask==3.1.0
+flatbuffers==25.2.10
 fonttools==4.58.0
 fr_core_news_sm @ https://github.com/explosion/spacy-models/releases/download/fr_core_news_sm-3.8.0/fr_core_news_sm-3.8.0-py3-none-any.whl#sha256=7d6ad14cd5078e53147bfbf70fb9d433c6a3865b695fda2657140bbc59a27e29
 frozenlist==1.5.0
 fsspec==2025.3.0
+gast==0.6.0
+google-pasta==0.2.0
+gotrue==2.12.3
+grpcio==1.73.1
 h11==0.14.0
+h2==4.2.0
+h5py==3.14.0
+hpack==4.1.0
 httpcore==1.0.7
 httpx==0.28.1
 huggingface-hub==0.29.3
+hyperframe==6.1.0
 idna==3.10
 itsdangerous==2.2.0
 jieba==0.42.1
 Jinja2==3.1.5
 jiter==0.9.0
 joblib==1.5.0
+keras==3.10.0
 kiwisolver==1.4.8
+ko_core_news_sm @ https://github.com/explosion/spacy-models/releases/download/ko_core_news_sm-3.8.0/ko_core_news_sm-3.8.0-py3-none-any.whl#sha256=15486b93a22ec8ee43f787390afdb4e2086851573eb12b69dd2c4f2a95b3e0a4
 langcodes==3.5.0
 langdetect==1.0.9
 language_data==1.3.0
+libclang==18.1.1
+limits==5.4.0
 marisa-trie==1.2.1
+Markdown==3.8.2
 markdown-it-py==3.0.0
 MarkupSafe==3.0.2
 matplotlib==3.10.3
 mdurl==0.1.2
+ml_dtypes==0.5.1
 mpmath==1.3.0
 multidict==6.1.0
 murmurhash==1.0.12
+namex==0.1.0
 networkx==3.4.2
 nltk==3.9.1
-numpy==2.2.2
+numpy==2.1.3
 openai==0.28.0
+opt_einsum==3.4.0
+optree==0.16.0
 opuscleaner==0.4.4
 opustools==1.7.1
 opustools-pkg==0.0.52
 packaging==24.2
 pandas==2.2.3
 pillow==11.2.1
+postgrest==1.1.1
 preshed==3.0.9
 propcache==0.3.0
+protobuf==5.29.5
 psycopg2==2.9.10
 py==1.11.0
 pydantic==1.10.21
 pydantic_core==2.27.2
 PyDirectInput==1.0.4
 Pygments==2.19.1
+PyJWT==2.10.1
 pynput==1.7.7
 pyparsing==3.2.3
 python-dateutil==2.9.0.post0
 python-dotenv==1.0.1
+python-multipart==0.0.20
 pytz==2025.1
 PyYAML==6.0.2
+realtime==2.5.3
 regex==2024.11.6
 requests==2.32.3
 rich==13.9.4
@@ -80,31 +108,44 @@ ruamel.yaml.clib==0.2.12
 safetensors==0.5.3
 scikit-learn==1.6.1
 scipy==1.15.3
+sentence-transformers==5.0.0
 sentencepiece==0.2.0
 setuptools==75.8.0
 shellingham==1.5.4
 six==1.17.0
+slowapi==0.1.9
 smart-open==7.1.0
 sniffio==1.3.1
 spacy==3.8.4
 spacy-legacy==3.0.12
 spacy-loggers==1.0.5
 srsly==2.5.1
 starlette==0.19.1
+storage3==0.12.0
+StrEnum==0.4.15
+supabase==2.16.0
+supafunc==0.10.1
 sympy==1.13.1
+tensorboard==2.19.0
+tensorboard-data-server==0.7.2
+tensorflow==2.19.0
+termcolor==3.1.0
+tf_keras==2.19.0
 thinc==8.3.4
 threadpoolctl==3.6.0
 tokenizers==0.21.1
 torch==2.6.0
 tqdm==4.67.1
 transformers==4.49.0
 typer==0.15.1
-typing_extensions==4.12.2
+typing-inspection==0.4.1
+typing_extensions==4.14.1
 tzdata==2025.1
 urllib3==2.3.0
 uvicorn==0.20.0
 wasabi==1.1.3
 weasel==0.4.1
+websockets==15.0.1
 Werkzeug==3.1.3
 wheel==0.45.1
 wrapt==1.17.2