split unit & integration tests

Signed-off-by: Panos Vagenas <[email protected]>

.github/workflows/checks.yml

@@ -14,3 +14,5 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Run checks
         run: poetry run pre-commit run --all-files
+      - name: Run integration tests
+        run: poetry run pytest tests/integration

.github/workflows/ci.yml

@@ -25,4 +25,3 @@ jobs:
   #     - uses: ./.github/actions/setup-poetry
   #     - name: Build docs
   #       run: poetry run mkdocs build --verbose --clean
-

.pre-commit-config.yaml

@@ -31,7 +31,7 @@ repos:
         files: '\.py$'
       - id: pytest
         name: Pytest
-        entry: poetry run pytest tests
+        entry: poetry run pytest tests/unit
         pass_filenames: false
         language: system
         files: '\.py$'

tests/test_retrieval.py → tests/integration/test_retrieval.py

@@ -44,6 +44,6 @@ def test_retrieval():
         )
         retr_res = retriever.retrieve(QUERY)
         act_data = dict(root=[n.text for n in retr_res])
-        with open("tests/data/2_out_retrieval_results.json") as f:
+        with open("tests/integration/data/0_out_retrieval_results.json") as f:
             exp_data = json.load(fp=f)
         assert exp_data == act_data

tests/test_hierarchical_chunker.py → tests/unit/test_hierarchical_chunker.py

@@ -11,24 +11,24 @@
 
 
 def test_chunk_without_metadata():
-    with open("tests/data/0_inp_dl_doc.json") as f:
+    with open("tests/unit/data/0_inp_dl_doc.json") as f:
         data_json = f.read()
     dl_doc = DLDocument.model_validate_json(data_json)
     chunker = HierarchicalChunker(include_metadata=False)
     chunks = chunker.chunk(dl_doc=dl_doc)
     act_data = dict(root=[n.model_dump() for n in chunks])
-    with open("tests/data/0_out_chunks_wout_meta.json") as f:
+    with open("tests/unit/data/0_out_chunks_wout_meta.json") as f:
         exp_data = json.load(fp=f)
     assert exp_data == act_data
 
 
 def test_chunk_with_metadata():
-    with open("tests/data/0_inp_dl_doc.json") as f:
+    with open("tests/unit/data/0_inp_dl_doc.json") as f:
         data_json = f.read()
     dl_doc = DLDocument.model_validate_json(data_json)
     chunker = HierarchicalChunker(include_metadata=True)
     chunks = chunker.chunk(dl_doc=dl_doc)
     act_data = dict(root=[n.model_dump() for n in chunks])
-    with open("tests/data/0_out_chunks_with_meta.json") as f:
+    with open("tests/unit/data/0_out_chunks_with_meta.json") as f:
         exp_data = json.load(fp=f)
     assert exp_data == act_data

tests/test_li_hierarchical_node_parser.py → tests/unit/test_li_hierarchical_node_parser.py

@@ -11,12 +11,12 @@
 
 
 def test_node_parse():
-    with open("tests/data/1_inp_li_doc.json") as f:
+    with open("tests/unit/data/1_inp_li_doc.json") as f:
         data_json = f.read()
     li_doc = LIDocument.from_json(data_json)
     node_parser = HierarchicalNodeParser(id_gen_seed=42)
     nodes = node_parser._parse_nodes(nodes=[li_doc])
     act_data = dict(root=[n.dict() for n in nodes])
-    with open("tests/data/1_out_nodes.json") as f:
+    with open("tests/unit/data/1_out_nodes.json") as f:
         exp_data = json.load(fp=f)
     assert exp_data == act_data