Basic API docs (IVS-319)

.env

@@ -14,6 +14,7 @@ DJANGO_SECRET_KEY = django-insecure-um7-^+&jbk_=80*xcc9uf4nh$4koida7)ja&6!vb*$8@
 DJANGO_ALLOWED_HOSTS = localhost
 DJANGO_TRUSTED_ORIGINS = http://localhost:3000 http://localhost http://localhost:8000
 DJANGO_LOG_LEVEL = INFO
+GHERKIN_LOG_FOLDER = /gherkin_logs
 DJANGO_GUNICORN_WORKERS = 3
 DJANGO_GUNICORN_THREADS_PER_WORKER = 4
 
@@ -31,6 +32,7 @@ CELERY_TASK_SOFT_TIME_LIMIT = 3600
 CELERY_TASK_TIME_LIMIT = 4000
 TASK_TIMEOUT_LIMIT = 3600
 DJANGO_DB_USER_CONTEXT = SYSTEM
+DJANGO_DB_BULK_CREATE_BATCH_SIZE = 1000
 CELERY_CONCURRENCY = 4
 
 # Email
@@ -46,4 +48,4 @@ B2C_CLIENT_ID = <B2C_CLIENT_ID>
 B2C_CLIENT_SECRET = <B2C_CLIENT_SECRET>
 B2C_AUTHORITY = <B2C_AUTHORITY>
 B2C_USER_FLOW = <B2C_USER_FLOW>
-USE_WHITELIST = False
+USE_WHITELIST = False

.github/workflows/ci_cd.yml

@@ -10,7 +10,6 @@ on:
       - development
     paths-ignore:
       - 'README.md'
-      - '.github/**'
   pull_request:
     branches:
       - development
@@ -58,6 +57,8 @@ jobs:
 
       - name: Fetch submodules
         run: |
+          git clean -fdx -e ${{ vars.ENV_FILE }} -e 'redeploy.*.sh' -e 'docker/'
+          git submodule foreach --recursive 'git clean -fdx'
           cd backend
           cd apps
           git submodule update --init --recursive
@@ -79,7 +80,16 @@ jobs:
           source venv/bin/activate
           pip install --upgrade pip
           pip install -r requirements.txt
-          pip install ifcopenshell # TEMP workaround
+
+      - name: Install ifcopenshell package (temp)
+        run: |
+          cd backend
+          source venv/bin/activate
+          # use version of ifcopenshell with desired schema parsing
+          # TODO: revert to pyPI when schema parsing is published in the future
+          wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.3-260bc80-linux64.zip"
+          mkdir -p venv/lib/python3.11/site-packages
+          unzip -d venv/lib/python3.11/site-packages /tmp/ifcopenshell_python.zip
 
       - name: Check Django config
         run: |
@@ -91,7 +101,12 @@ jobs:
         run: |
           cd backend
           source venv/bin/activate
-          python3 manage.py test
+          MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps/ifc_validation_models --settings apps.ifc_validation_models.test_settings --debug-mode --verbosity 3
+          MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation.tests.tests_header_validation_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
+          MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation.tests.tests_header_syntax_validation_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
+          MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation.tests.tests_syntax_validation_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
+          MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation.tests.tests_schema_validation_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
+          MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation.tests.tests_bsdd_validation_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
 
   deploy:
 
@@ -125,10 +140,11 @@ jobs:
           sshpass ssh -o StrictHostKeyChecking=no ${{ secrets.SSH_USERNAME }}@${{ secrets.SSH_HOST }} bash <<'ENDSSH'
           cd ${{ vars.REPO_CLONE_PATH }}
           git checkout -q ${{ vars.BRANCH_NAME }} && git pull
+          git clean -fdx -e ${{ vars.ENV_FILE }} -e 'redeploy.*.sh' -e 'docker/'
+          git submodule foreach --recursive 'git clean -fdx'
           sudo make fetch-modules
           cd ${{ vars.REPO_CLONE_PATH }}/backend/apps/ifc_validation/checks/ifc_gherkin_rules && git checkout -q ${{ vars.BRANCH_NAME }} && git pull
           cd ./ifc_validation_models && git checkout -q ${{ vars.BRANCH_NAME }} && git pull
-          cd ${{ vars.REPO_CLONE_PATH }}/backend/apps/ifc_validation/checks/step_file_parser && git checkout -q master && git pull
           cd ${{ vars.REPO_CLONE_PATH }}/backend/apps/ifc_validation_models && git checkout -q ${{ vars.BRANCH_NAME }} && git pull
           cd ${{ vars.REPO_CLONE_PATH }}
           ./check-submodules.sh

.gitignore

@@ -8,6 +8,9 @@
 .env.*
 redeploy.*
 .VERSION
+tmp*
+django_static
+local
 
 ### VisualStudioCode ###
 .vscode/*

.gitmodules

@@ -2,11 +2,10 @@
 	path = backend/apps/ifc_validation/checks/ifc_gherkin_rules
 	url = https://github.com/buildingSMART/ifc-gherkin-rules
 	branch = development
-[submodule "backend/apps/ifc_validation/checks/step_file_parser"]
-	path = backend/apps/ifc_validation/checks/step_file_parser
-	url = https://github.com/IfcOpenShell/step-file-parser
-	branch = master
 [submodule "backend/apps/ifc_validation_models"]
 	path = backend/apps/ifc_validation_models
 	url = https://github.com/buildingSMART/ifc-validation-data-model
 	branch = development
+[submodule "backend/apps/ifc_validation/checks/signatures/store"]
+	path = backend/apps/ifc_validation/checks/signatures/store
+	url = https://github.com/buildingsmart-certificates/validation-service-vendor-certificates

Makefile

@@ -71,4 +71,6 @@ clean-all:
 
 fetch-modules:
 	git submodule update --init --recursive
-	git submodule update --remote
+	git submodule foreach git clean -f .
+	git submodule foreach git reset --hard
+	git submodule update --remote --recursive

README.md

@@ -5,14 +5,13 @@
 
 # Application Structure
 
-The application consists of three main submodules, each hosted in separate GitHub repositories. Docker Compose is configured to automatically bind the correct submodule versions for local deployment.
+The application consists of two main submodules, each hosted in separate GitHub repositories. Docker Compose is configured to automatically bind the correct submodule versions for local deployment.
 
 ### Submodules
 
 Documentation of the seperate functionalities can be found within each submodule.
 
-1. **File Parser**: A module within IfcOpenShell, dedicated to parsing files. https://github.com/IfcOpenShell/step-file-parser
-2. **Gherkin Rules**: Contains the rules for validation. It can be run independently by cloning the repository and executing:
+1. **Gherkin Rules**: Contains the rules for validation. It can be run independently by cloning the repository and executing:
 https://github.com/buildingSMART/ifc-gherkin-rules
 
    ```
@@ -27,7 +26,7 @@ https://github.com/buildingSMART/ifc-gherkin-rules
    python test/test_main.py path_to_separate_file.py # For a separate file
    ``````
 
-3. **Shared DataModel**: This module includes Django data models shared between the main repository and the Gherkin repository, serving as a submodule for both.
+2. **Shared DataModel**: This module includes Django data models shared between the main repository and the Gherkin repository, serving as a submodule for both.
 https://github.com/buildingSMART/ifc-validation-data-model
 
 ## Running Validation Checks
@@ -144,7 +143,7 @@ flower   - Celery flower dashboard
 
 ```shell
 cd backend
-make install
+make install (or make install-macos/install-macos-m1)
 make start-django
 ```
 

backend/.env

@@ -10,6 +10,7 @@
 DEBUG = True
 ENV = Development
 PUBLIC_URL = http://localhost:3000
+GHERKIN_LOG_FOLDER=.dev/gherkin_logs
 
 # Django
 MEDIA_ROOT = .dev/files_storage
@@ -34,6 +35,7 @@ CELERY_TASK_SOFT_TIME_LIMIT = 3600
 CELERY_TASK_TIME_LIMIT = 4000
 TASK_TIMEOUT_LIMIT = 3600
 DJANGO_DB_USER_CONTEXT = SYSTEM
+DJANGO_DB_BULK_CREATE_BATCH_SIZE = 1000
 
 # Email
 MAILGUN_API_URL = <MG_API_URL>
@@ -48,4 +50,4 @@ B2C_CLIENT_ID = <B2C_CLIENT_ID>
 B2C_CLIENT_SECRET = <B2C_CLIENT_SECRET>
 B2C_AUTHORITY = <B2C_AUTHORITY>
 B2C_USER_FLOW = <B2C_USER_FLOW>
-USE_WHITELIST = False
+USE_WHITELIST = False

backend/Makefile

@@ -1,6 +1,8 @@
 .ONESHELL:
 
 VIRTUAL_ENV = .dev/venv
+PYTHON = $(VIRTUAL_ENV)/bin/python
+PIP = $(VIRTUAL_ENV)/bin/pip
 
 none:
 	@echo "MAKE: Enter at least one target (venv, install, install-dev, start-backend, start-worker, clean)"
@@ -10,53 +12,74 @@ venv:
 	test -d $(VIRTUAL_ENV) || python3.11 -m venv $(VIRTUAL_ENV)
 
 install: venv
-	. $(VIRTUAL_ENV)/bin/activate && \
-	pip install --upgrade pip && \
-	pip install -r requirements.txt && \
-    wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.7.9-c18e4ea-linux64.zip" && \
-    mkdir -p $(VIRTUAL_ENV)/lib/python3.11/site-packages && \
-    unzip -f -d $(VIRTUAL_ENV)/lib/python3.11/site-packages /tmp/ifcopenshell_python.zip && \
+	$(PIP) install --upgrade pip
+	$(PIP) install -r requirements.txt
+	wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.3-260bc80-linux64.zip"
+	mkdir -p $(VIRTUAL_ENV)/lib/python3.11/site-packages
+	unzip -f -d $(VIRTUAL_ENV)/lib/python3.11/site-packages /tmp/ifcopenshell_python.zip
+	rm /tmp/ifcopenshell_python.zip
+
+install-macos: venv
+	$(PIP) install --upgrade pip
+	$(PIP) install -r requirements.txt
+	wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.3-260bc80-macos64.zip"
+	mkdir -p $(VIRTUAL_ENV)/lib/python3.11/site-packages
+	unzip /tmp/ifcopenshell_python.zip -d $(VIRTUAL_ENV)/lib/python3.11/site-packages
+	rm /tmp/ifcopenshell_python.zip
+
+install-macos-m1: venv
+	$(PIP) install --upgrade pip
+	$(PIP) install -r requirements.txt
+	wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.3-260bc80-macosm164.zip"
+	mkdir -p $(VIRTUAL_ENV)/lib/python3.11/site-packages
+	unzip /tmp/ifcopenshell_python.zip -d $(VIRTUAL_ENV)/lib/python3.11/site-packages
 	rm /tmp/ifcopenshell_python.zip
 
 fetch-modules:
-	cd ./apps && \
-	git submodule update --init --recursive
+	cd ./apps && git submodule update --init --recursive
 
 start-django: start-backend
 
 start-backend:
-	. $(VIRTUAL_ENV)/bin/activate && \
-	python3 manage.py makemigrations && \
-	python3 manage.py migrate && \
-	python3 manage.py runserver
+	$(PYTHON) manage.py makemigrations
+	$(PYTHON) manage.py migrate
+	$(PYTHON) manage.py collectstatic --noinput
+	$(PYTHON) manage.py runserver
 
 start-worker:
-	. $(VIRTUAL_ENV)/bin/activate && \
-	celery --app=core worker --loglevel=DEBUG --concurrency 2 --task-events --hostname=worker@%n
+	$(PYTHON) -m celery --app=core worker --loglevel=DEBUG --concurrency 2 --task-events --hostname=worker@%n
 
 start-worker2:
-	. $(VIRTUAL_ENV)/bin/activate && \
-	celery --app=core worker --loglevel=DEBUG --concurrency 2 --task-events --hostname=worker2@%n
+	$(PYTHON) -m celery --app=core worker --loglevel=DEBUG --concurrency 2 --task-events --hostname=worker2@%n
 
 start-worker3:
-	. $(VIRTUAL_ENV)/bin/activate && \
-	celery --app=core worker --loglevel=DEBUG --concurrency 2 --task-events --hostname=worker3@%n
+	$(PYTHON) -m celery --app=core worker --loglevel=DEBUG --concurrency 2 --task-events --hostname=worker3@%n
 
 start-worker4:
-	. $(VIRTUAL_ENV)/bin/activate && \
-	celery --app=core worker --loglevel=DEBUG --concurrency 2 --task-events --hostname=worker4@%n
+	$(PYTHON) -m celery --app=core worker --loglevel=DEBUG --concurrency 2 --task-events --hostname=worker4@%n
 
 start-worker-scheduler:
-	. $(VIRTUAL_ENV)/bin/activate && \
-	celery --app=core worker --loglevel=DEBUG --concurrency 5 --task-events --beat
+	$(PYTHON) -m celery --app=core worker --loglevel=DEBUG --concurrency 5 --task-events --beat
+
+test: test-models test-bsdd-task test-header-validation-task test-syntax-task test-syntax-header-validation-task test-schema-task
 
 test-models:
-	. $(VIRTUAL_ENV)/bin/activate && \
-	python3 manage.py test apps/ifc_validation_models --settings apps.ifc_validation_models.test_settings --debug-mode --verbosity 3
+	MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps/ifc_validation_models --settings apps.ifc_validation_models.test_settings --debug-mode --verbosity 3
+
+test-bsdd-task:
+	MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation.tests.tests_bsdd_validation_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
+
+test-syntax-header-validation-task:
+	MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation.tests.tests_header_syntax_validation_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
+
+test-header-validation-task:
+	MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation.tests.tests_header_validation_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
+
+test-syntax-task:
+	MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation.tests.tests_header_validation_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
 
-test-tasks:
-	. $(VIRTUAL_ENV)/bin/activate && \
-	MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation.tests_tasks --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
+test-schema-task:
+	MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation.tests.tests_schema_validation_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
 
 clean:
 	rm -rf .dev
@@ -66,12 +89,8 @@ clean:
 	find . -type d -name __pycache__  -prune -exec rm -rf {} \;
 
 init-db:
-	. $(VIRTUAL_ENV)/bin/activate && \
-	PGPASSWORD=postgres psql -h localhost -U postgres --dbname postgres -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;" && \
-	rm ./apps/ifc_validation_models/migrations/0001_initial.bak || true && \
-	mv ./apps/ifc_validation_models/migrations/0001_initial.py ./apps/ifc_validation_models/migrations/0001_initial.bak || true && \
-	rm ./apps/ifc_validation_models/migrations/0001_initial.bak || true && \
-	python3 manage.py makemigrations && \
-	python3 manage.py migrate && \
-	DJANGO_SUPERUSER_USERNAME=root DJANGO_SUPERUSER_PASSWORD=root DJANGO_SUPERUSER_EMAIL=root@localhost python3 manage.py createsuperuser --noinput && \
-	DJANGO_SUPERUSER_USERNAME=SYSTEM DJANGO_SUPERUSER_PASSWORD=system DJANGO_SUPERUSER_EMAIL=system@localhost python3 manage.py createsuperuser --noinput
+	PGPASSWORD=postgres psql -h localhost -U postgres --dbname postgres -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;"
+	$(PYTHON) manage.py makemigrations
+	$(PYTHON) manage.py migrate
+	DJANGO_SUPERUSER_USERNAME=root DJANGO_SUPERUSER_PASSWORD=root DJANGO_SUPERUSER_EMAIL=root@localhost $(PYTHON) manage.py createsuperuser --noinput
+	DJANGO_SUPERUSER_USERNAME=SYSTEM DJANGO_SUPERUSER_PASSWORD=system DJANGO_SUPERUSER_EMAIL=system@localhost $(PYTHON) manage.py createsuperuser --noinput

backend/README.md

@@ -5,7 +5,7 @@
 1. Syntax Validation
 
 ```shell
-python3 apps/ifc_validation/checks/step_file_parser/main.py --json filename 
+python3 -m ifcopenshell.simple_spf --json filename 
 ```
 
 2. Parse Info