Merge pull request #91 from DL4DS/dev_branch

Merge to main after dev_branch code restructure

Author: trgardos

.gitignore

@@ -169,5 +169,18 @@ code/.chainlit/translations/
 storage/logs/*
 vectorstores/*
 
-*/.files/*
+**/.files/*
 code/storage/models/
+
+**/translations/en-US.json
+**/translations/zh-CN.json
+
+
+**/vectorstores/*
+
+**/private/students.json
+
+**/apps/*/storage/logs/*
+**/apps/*/private/*
+
+.idea/

Dockerfile

@@ -3,10 +3,12 @@ FROM python:3.11
 WORKDIR /code
 
 COPY ./requirements.txt /code/requirements.txt
+COPY ./setup.py /code/setup.py
 
 RUN pip install --upgrade pip
 
 RUN pip install --no-cache-dir -r /code/requirements.txt
+RUN pip install -e .
 
 COPY . /code
 
@@ -17,12 +19,15 @@ RUN ls -R /code
 RUN chmod -R 777 /code
 
 # Create a logs directory and set permissions
-RUN mkdir /code/logs && chmod 777 /code/logs
+RUN mkdir /code/apps/ai_tutor/logs && chmod 777 /code/apps/ai_tutor/logs
 
 # Create a cache directory within the application's working directory
 RUN mkdir /.cache && chmod -R 777 /.cache
 
-WORKDIR /code/code
+WORKDIR /code/apps/ai_tutor
+
+# Expose the port the app runs on
+EXPOSE 7860
 
 RUN --mount=type=secret,id=HUGGINGFACEHUB_API_TOKEN,mode=0444,required=true 
 RUN --mount=type=secret,id=OPENAI_API_KEY,mode=0444,required=true 
@@ -35,4 +40,4 @@ RUN --mount=type=secret,id=LITERAL_API_KEY_LOGGING,mode=0444,required=true
 RUN --mount=type=secret,id=CHAINLIT_AUTH_SECRET,mode=0444,required=true 
 
 # Default command to run the application
-CMD ["sh", "-c", "python -m modules.vectorstore.store_manager && uvicorn app:app --host 0.0.0.0 --port 7860"]
+CMD python -m modules.vectorstore.store_manager --config_file config/config.yml --project_config_file config/project_config.yml && python -m uvicorn app:app --host 0.0.0.0 --port 7860

Dockerfile.dev

@@ -3,29 +3,36 @@ FROM python:3.11
 WORKDIR /code
 
 COPY ./requirements.txt /code/requirements.txt
+COPY ./setup.py /code/setup.py
 
 RUN pip install --upgrade pip
 
 RUN pip install --no-cache-dir -r /code/requirements.txt
+RUN pip install -e .
 
 COPY . /code
 
+# Copy .env file to the application directory
+COPY .env /code/apps/ai_tutor/.env
+
 # List the contents of the /code directory to verify files are copied correctly
 RUN ls -R /code
 
 # Change permissions to allow writing to the directory
 RUN chmod -R 777 /code
 
 # Create a logs directory and set permissions
-RUN mkdir /code/logs && chmod 777 /code/logs
+RUN mkdir /code/apps/ai_tutor/logs && chmod 777 /code/apps/ai_tutor/logs
 
 # Create a cache directory within the application's working directory
 RUN mkdir /.cache && chmod -R 777 /.cache
 
-WORKDIR /code/code
+WORKDIR /code/apps/ai_tutor
+
+RUN ls -R /code
 
 # Expose the port the app runs on
-EXPOSE 8000
+EXPOSE 7860
 
 # Default command to run the application
-CMD ["sh", "-c", "python -m modules.vectorstore.store_manager && chainlit run main.py --host 0.0.0.0 --port 8000"]
+CMD python -m modules.vectorstore.store_manager --config_file config/config.yml --project_config_file config/project_config.yml && python -m uvicorn app:app --host 0.0.0.0 --port 7860

README.md

@@ -9,6 +9,12 @@ app_port: 7860
 ---
 # DL4DS Tutor 🏃
 
+![Build Status](https://github.com/DL4DS/dl4ds_tutor/actions/workflows/push_to_hf_space.yml/badge.svg)
+![License](https://img.shields.io/github/license/DL4DS/dl4ds_tutor)
+![GitHub stars](https://img.shields.io/github/stars/DL4DS/dl4ds_tutor)
+![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)
+
+
 Check out the configuration reference at [Hugging Face Spaces Config Reference](https://huggingface.co/docs/hub/spaces-config-reference).
 
 You can find a "production" implementation of the Tutor running live at [DL4DS Tutor](https://dl4ds-dl4ds-tutor.hf.space/)  from the
@@ -31,26 +37,31 @@ Please visit [setup](https://dl4ds.github.io/dl4ds_tutor/guide/setup/) for more
    git clone https://github.com/DL4DS/dl4ds_tutor
    ```
 
-2. **Put your data under the `storage/data` directory**
+2. Create your app in the apps folder. (An example is the `apps/ai_tutor` app)
+   ```
+   cd apps
+   mkdir your_app
+   ```
+
+2. **Put your data under the `apps/your_app/storage/data` directory**
    - Add URLs in the `urls.txt` file.
-   - Add other PDF files in the `storage/data` directory.
+   - Add other PDF files in the `apps/your_app/storage/data` directory.
 
 3. **To test Data Loading (Optional)**
    ```bash
-   cd code
-   python -m modules.dataloader.data_loader --links "your_pdf_link"
+   cd apps/your_app
+   python -m modules.dataloader.data_loader --links "your_pdf_link" --config_file config/config.yml --project_config_file config/project_config.yml
    ```
 
 4. **Create the Vector Database**
    ```bash
-   cd code
-   python -m modules.vectorstore.store_manager
+   cd apps/your_app
+   python -m modules.vectorstore.store_manager --config_file config/config.yml --project_config_file config/project_config.yml
    ```
-   - Note: You need to run the above command when you add new data to the `storage/data` directory, or if the `storage/data/urls.txt` file is updated.
 
 6. **Run the FastAPI App**
    ```bash
-   cd code
+   cd apps/your_app
    uvicorn app:app --port 7860 
    ```
 
@@ -65,7 +76,7 @@ The HuggingFace Space is built using the `Dockerfile` in the repository. To run
 
 ```bash
 docker build --tag dev  -f Dockerfile.dev .
-docker run -it --rm -p 8000:8000 dev
+docker run -it --rm -p 7860:7860 dev
 ```
 
 ## Contributing

code/.chainlit/config.toml renamed to apps/ai_tutor/.chainlit/config.toml

@@ -69,7 +69,7 @@ github = "https://github.com/DL4DS/dl4ds_tutor"
 
 # Specify a CSS file that can be used to customize the user interface.
 # The CSS file can be served from the public directory or via an external link.
-custom_css = "/public/test.css"
+custom_css = "/public/files/test.css"
 
 # Specify a Javascript file that can be used to customize the user interface.
 # The Javascript file can be served from the public directory.

apps/ai_tutor/README.md

@@ -0,0 +1,12 @@
+# WIP
+
+
+## Run the encrypt_students script
+
+- If you don't want the emails to be public, run this script to encrypt the emails of the students.
+- This will create a new file in the public/files/ directory.
+- Place your file with the students' emails in the private/ directory (do not commit this file to the repository).
+
+```bash
+python encrypt_students.py --students-file private/students.json --encrypted-students-file public/files/students_encrypted.json
+```

code/app.py renamed to apps/ai_tutor/app.py

@@ -8,24 +8,33 @@
 import secrets
 import json
 import base64
-from modules.config.constants import (
+from config.constants import (
     OAUTH_GOOGLE_CLIENT_ID,
     OAUTH_GOOGLE_CLIENT_SECRET,
     CHAINLIT_URL,
-    GITHUB_REPO,
-    DOCS_WEBSITE,
-    ALL_TIME_TOKENS_ALLOCATED,
-    TOKENS_LEFT,
+    EMAIL_ENCRYPTION_KEY,
 )
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
-from modules.chat_processor.helpers import (
-    get_user_details,
+from helpers import (
     get_time,
     reset_tokens_for_user,
     check_user_cooldown,
-    update_user_info,
 )
+from modules.chat_processor.helpers import get_user_details, update_user_info
+from config.config_manager import config_manager
+import hashlib
+
+# set config
+config = config_manager.get_config().dict()
+
+# set constants
+GITHUB_REPO = config["misc"]["github_repo"]
+DOCS_WEBSITE = config["misc"]["docs_website"]
+ALL_TIME_TOKENS_ALLOCATED = config["token_config"]["all_time_tokens_allocated"]
+TOKENS_LEFT = config["token_config"]["tokens_left"]
+COOLDOWN_TIME = config["token_config"]["cooldown_time"]
+REGEN_TIME = config["token_config"]["regen_time"]
 
 GOOGLE_CLIENT_ID = OAUTH_GOOGLE_CLIENT_ID
 GOOGLE_CLIENT_SECRET = OAUTH_GOOGLE_CLIENT_SECRET
@@ -46,13 +55,8 @@
 CHAINLIT_PATH = "/chainlit_tutor"
 
 # only admin is given any additional permissions for now -- no limits on tokens
-USER_ROLES = {
-    "[email protected]": ["instructor", "bu"],
-    "[email protected]": ["admin", "instructor", "bu"],
-    "[email protected]": ["instructor", "bu"],
-    "[email protected]": ["guest"],
-    # Add more users and roles as needed
-}
+with open("public/files/students_encrypted.json", "r") as file:
+    USER_ROLES = json.load(file)
 
 # Create a Google OAuth flow
 flow = Flow.from_client_config(
@@ -80,7 +84,20 @@
 
 
 def get_user_role(username: str):
-    return USER_ROLES.get(username, ["guest"])  # Default to "guest" role
+
+    # Function to deterministically hash emails
+    def deterministic_hash(email, salt):
+        return hashlib.pbkdf2_hmac("sha256", email.encode(), salt, 100000).hex()
+
+    # encrypt email (#FIXME: this is not the best way to do this, not really encryption, more like a hash)
+    encryption_salt = EMAIL_ENCRYPTION_KEY.encode()
+    encrypted_email = deterministic_hash(username, encryption_salt)
+    role = USER_ROLES.get(encrypted_email, ["guest"])
+
+    if "guest" in role:
+        return "unauthorized"
+
+    return role
 
 
 async def get_user_info_from_cookie(request: Request):
@@ -146,6 +163,11 @@ async def login_page(request: Request):
 #     return response
 
 
+@app.get("/unauthorized", response_class=HTMLResponse)
+async def unauthorized(request: Request):
+    return templates.TemplateResponse("unauthorized.html", {"request": request})
+
+
 @app.get("/login/google")
 async def login_google(request: Request):
     # Clear any existing session cookies to avoid conflicts with guest sessions
@@ -176,6 +198,9 @@ async def auth_google(request: Request):
         profile_image = user_info.get("picture", "")
         role = get_user_role(email)
 
+        if role == "unauthorized":
+            return RedirectResponse("/unauthorized")
+
         session_token = secrets.token_hex(16)
         session_store[session_token] = {
             "email": email,
@@ -210,7 +235,7 @@ async def cooldown(request: Request):
     user_details = await get_user_details(user_info["email"])
     current_datetime = get_time()
     cooldown, cooldown_end_time = await check_user_cooldown(
-        user_details, current_datetime
+        user_details, current_datetime, COOLDOWN_TIME, TOKENS_LEFT, REGEN_TIME
     )
     print(f"User in cooldown: {cooldown}")
     print(f"Cooldown end time: {cooldown_end_time}")
@@ -228,7 +253,11 @@ async def cooldown(request: Request):
     else:
         user_details.metadata["in_cooldown"] = False
         await update_user_info(user_details)
-        await reset_tokens_for_user(user_details)
+        await reset_tokens_for_user(
+            user_details,
+            config["token_config"]["tokens_left"],
+            config["token_config"]["regen_time"],
+        )
         return RedirectResponse("/post-signin")
 
 
@@ -256,13 +285,19 @@ async def post_signin(request: Request):
     if "last_message_time" in user_details.metadata and "admin" not in get_user_role(
         user_info["email"]
     ):
-        cooldown, _ = await check_user_cooldown(user_details, current_datetime)
+        cooldown, _ = await check_user_cooldown(
+            user_details, current_datetime, COOLDOWN_TIME, TOKENS_LEFT, REGEN_TIME
+        )
         if cooldown:
             user_details.metadata["in_cooldown"] = True
             return RedirectResponse("/cooldown")
         else:
             user_details.metadata["in_cooldown"] = False
-            await reset_tokens_for_user(user_details)
+            await reset_tokens_for_user(
+                user_details,
+                config["token_config"]["tokens_left"],
+                config["token_config"]["regen_time"],
+            )
 
     if user_info:
         username = user_info["email"]
@@ -335,15 +370,19 @@ async def get_tokens_left(request: Request):
     try:
         user_info = await get_user_info_from_cookie(request)
         user_details = await get_user_details(user_info["email"])
-        await reset_tokens_for_user(user_details)
+        await reset_tokens_for_user(
+            user_details,
+            config["token_config"]["tokens_left"],
+            config["token_config"]["regen_time"],
+        )
         tokens_left = user_details.metadata["tokens_left"]
         return {"tokens_left": tokens_left}
     except Exception as e:
         print(f"Error getting tokens left: {e}")
         return {"tokens_left": 0}
 
 
-mount_chainlit(app=app, target="main.py", path=CHAINLIT_PATH)
+mount_chainlit(app=app, target="chainlit_app.py", path=CHAINLIT_PATH)
 
 if __name__ == "__main__":
     import uvicorn