Release/v1.5.0

.devcontainer/Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1.3
-ARG UV_VERSION=0.5.10
+ARG UV_VERSION=latest
 ARG CORE_IMAGE=ubuntu:noble
 
 FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS astral-uv-source

.devcontainer/devcontainer.json

@@ -4,11 +4,14 @@
 	"name": "aymurai",
 	// "initializeCommand": "make core-build",
 	"dockerComposeFile": "docker-compose.yml",
-	"service": "aymurai-devcontainer-gpu",
+	"service": "aymurai-devcontainer",
 	"runServices": [
-		"aymurai-devcontainer-gpu"
+		"aymurai-devcontainer"
 	],
 	"workspaceFolder": "/workspace",
+	"mounts": [
+		"source=codex-data,target=/home/ubuntu/.codex,type=volume"
+	],
 	"customizations": {
 		"vscode": {
 			"settings": {
@@ -47,9 +50,11 @@
 				"cweijan.vscode-database-client2",
 				"christian-kohler.path-intellisense",
 				"github.vscode-github-actions",
-				"seatonjiang.gitmoji-vscode"
+				"seatonjiang.gitmoji-vscode",
+				"openai.chatgpt"
 			]
 		}
 	},
-	"postCreateCommand": "bash /home/ubuntu/entrypoint.sh"
+	"postCreateCommand": "bash /home/ubuntu/entrypoint.sh",
+	"postStartCommand": "sudo chown -R ubuntu:ubuntu /home/ubuntu/.codex && sudo chmod -R u+rwX /home/ubuntu/.codex"
 }

.devcontainer/docker-compose.yml

@@ -5,7 +5,7 @@ x-template: &template
     - ..:/workspace:cached
     - ../notebooks/:/notebooks
     - ../resources/:/resources
-    - ../test/:/test
+    - ../tests/:/tests
     - $HOME/.ssh/:/home/ubuntu/.ssh
     - /var/run/docker.sock:/var/run/docker.sock
   env_file:
@@ -27,7 +27,9 @@ services:
       resources:
         reservations:
           devices:
-            - capabilities: [ gpu ]
+            - driver: nvidia
+              count: all
+              capabilities: [ gpu ]
 
   aymurai-devcontainer:
     <<: *template

.devcontainer/entrypoint.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # install dependencies
-uv sync --frozen --all-extras
+uv sync --frozen --all-extras --all-groups
 
 # configure precommit
 uv run pre-commit install

.env.common

@@ -1,31 +1,8 @@
-DATASETS_BASEPATH=/resources/datasets
-MODELS_BASEPATH=/resources/models
-
+CACHE_PATH=/resources/cache
 AYMURAI_CACHE_BASEPATH=/resources/cache/aymurai
 
-CACHE_PATH=/resources/cache
-HF_DATASETS_CACHE=/resources/cache/huggingface/cache
-TRANSFORMERS_CACHE=/resources/cache/huggingface/transformers
+HF_HOME=/resources/cache/huggingface
 TOKENIZERS_PARALLELISM=1
 
-TESSDATA_PREFIX=/usr/local/share/tessdata
-
 AYMURAI_RESTRICTED_DOCUMENT_PDFS_PATH="/resources/data/restricted/ar-juz-pcyf-10/RESOLUCIONES DEL JUZGADO-pdf"
 AYMURAI_RESTRICTED_DOCUMENT_DOCS_PATH="/resources/data/restricted/ar-juz-pcyf-10/RESOLUCIONES DEL JUZGADO"
-
-TF_CPP_MIN_LOG_LEVEL=3
-TFHUB_CACHE_DIR=/resources/cache/tfhub_modules
-
-TORCH_VERSION=2.0.1
-CUDA_VERSION=cu118
-
-CORE_IMAGE_CUDA=registry.gitlab.com/collective.ai/datagenero-public/aymurai-core
-CORE_IMAGE_CPU=registry.gitlab.com/collective.ai/datagenero-public/aymurai-core-cpu
-API_IMAGE=registry.gitlab.com/collective.ai/datagenero-public/aymurai-api
-
-API_HOST=0.0.0.0
-API_PORT=8899
-
-SRC_VOLUME_MOUNT=src:/src
-RESOURCES_VOLUME_MOUNT=resources:/resources
-NOTEBOOKS_VOLUME_MOUNT=notebooks:/notebooks

.github/workflows/pytest.yml

@@ -0,0 +1,86 @@
+name: pytest
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - ready_for_review
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: pr-tests-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pytest:
+    if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
+    name: pytest (${{ matrix.os }}, py${{ matrix.python-version }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+          - windows-latest
+        python-version:
+          - "3.10"
+          # - "3.11"
+          # - "3.12"
+          # - "3.13"
+          # - "3.14"
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: latest
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Configure es_AR locale (Ubuntu)
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install --yes locales
+          sudo locale-gen es_AR.UTF-8
+          sudo update-locale LANG=es_AR.UTF-8 LC_ALL=es_AR.UTF-8
+          echo "LANG=es_AR.UTF-8" >> "$GITHUB_ENV"
+          echo "LC_ALL=es_AR.UTF-8" >> "$GITHUB_ENV"
+          locale -a
+
+      - name: Install dependencies
+        run: |
+          uv sync --frozen --python python --no-dev --no-python-downloads --group tests
+
+      - name: Run api tests
+        env:
+          DISKCACHE_ROOT: /tmp
+        run: uv run --no-sync pytest -q --tb=short --disable-warnings --color=yes --maxfail=5 tests/api
+
+      - name: Download pipelines data
+        env:
+          DISKCACHE_ROOT: /tmp
+          RESOURCES_BASEPATH: resources
+          AYMURAI_CACHE_BASEPATH: resources/cache/aymurai
+        run: uv run --no-sync pipeline-download
+
+      - name: Run pipeline tests
+        env:
+          DISKCACHE_ROOT: /tmp
+          RESOURCES_BASEPATH: resources
+          AYMURAI_CACHE_BASEPATH: resources/cache/aymurai
+        run: uv run --no-sync pytest -q --tb=short --disable-warnings --color=yes --maxfail=5 tests/integration/pipelines

.gitignore

@@ -128,3 +128,10 @@ resources
 .venv
 
 aymurai/version.py
+
+.agents
+.opencode.json
+.sisyphus
+notebooks/**
+!notebooks/**/
+!notebooks/**/*.ipynb

.pre-commit-config.yaml

@@ -1,16 +1,11 @@
 repos:
-    - repo: https://github.com/ambv/black
-      rev: 22.6.0
+    - repo: https://github.com/astral-sh/ruff-pre-commit
+      rev: v0.7.2
       hooks:
-          - id: black
-            language_version: python3.10
+          - id: ruff
+          - id: ruff-format
 
     - repo: https://github.com/kynan/nbstripout
       rev: 0.6.0
       hooks:
-          - id: nbstripout
-
-    # - repo: https://github.com/pre-commit/pre-commit-hooks
-    #   rev: v2.20.0
-    #   hooks:
-    #       - id: flake8
+          - id: nbstripout

.vscode/settings.json

@@ -0,0 +1,28 @@
+{
+    "python.testing.pytestArgs": [
+        "tests"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true,
+    "python-envs.defaultEnvManager": "ms-python.python:venv",
+    "python-envs.pythonProjects": [],
+    "files.associations": {
+        ".csv": "csv",
+        ".env*": "dotenv",
+        ".json": "json",
+        ".jsonc": "jsonc",
+        ".jsonl": "jsonl",
+        ".md": "markdown"
+    },
+    "github.copilot.enable": {
+        "*": true,
+        "dotenv": false,
+        "csv": false,
+        "json": false,
+        "jsonc": false,
+        "jsonl": false,
+        "markdown": false,
+        "plaintext": false,
+        "scminput": false
+    }
+}

Makefile

@@ -3,20 +3,37 @@ export $(shell sed 's/=.*//' .env)
 include .env.common
 export $(shell sed 's/=.*//' .env.common)
 
+# Select which API service to control (override with API_SERVICE=aymurai-api-gpu)
+API_SERVICE ?= aymurai-api
+# Select which full API service to control (override with API_FULL_SERVICE=aymurai-api-full-gpu)
+API_FULL_SERVICE ?= aymurai-api-full
+
 
 api-build:
-	docker compose build aymurai-api
+	docker compose build $(API_SERVICE)
 api-run:
-	docker compose run --service-ports aymurai-api
+	docker compose run --service-ports $(API_SERVICE)
+api-up:
+	docker compose up -d $(API_SERVICE)
+api-stop:
+	docker compose stop $(API_SERVICE)
+api-logs:
+	docker compose logs -f $(API_SERVICE)
 api-pull:
-	docker compose pull aymurai-api
+	docker compose pull $(API_SERVICE)
 
 api-full-build:
-	docker compose build aymurai-api-full
+	docker compose build $(API_FULL_SERVICE)
 api-full-run:
-	docker compose run --service-ports aymurai-api-full
+	docker compose run --service-ports $(API_FULL_SERVICE)
+api-full-up:
+	docker compose up -d $(API_FULL_SERVICE)
+api-full-stop:
+	docker compose stop $(API_FULL_SERVICE)
+api-full-logs:
+	docker compose logs -f $(API_FULL_SERVICE)
 api-full-pull:
-	docker compose pull aymurai-api-full
+	docker compose pull $(API_FULL_SERVICE)
 
 stress-test:
 	locust -f locustfile.py --host http://localhost:8899