move chunker into new docling_core.transforms package

Signed-off-by: Panos Vagenas <[email protected]>

docling_core/transforms/__init__.py

@@ -0,0 +1,6 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Data transformations package."""

docling_core/transforms/chunker/__init__.py

@@ -0,0 +1,15 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Define the chunker types."""
+
+from docling_core.transforms.chunker.base import (  # noqa
+    BaseChunker,
+    Chunk,
+    ChunkWithMetadata,
+)
+from docling_core.transforms.chunker.hierarchical_chunker import (  # noqa
+    HierarchicalChunker,
+)

docling_core/chunker/hierarchical_chunker.py → docling_core/transforms/chunker/hierarchical_chunker.py

@@ -14,7 +14,7 @@
 import pandas as pd
 from pydantic import BaseModel, PositiveInt
 
-from docling_core.chunker.base import BaseChunker, Chunk, ChunkWithMetadata
+from docling_core.transforms.chunker import BaseChunker, Chunk, ChunkWithMetadata
 from docling_core.types import BaseText
 from docling_core.types import Document as DLDocument
 from docling_core.types import Ref, Table
@@ -260,12 +260,7 @@ def _build_chunk_impl(
                 )
             else:
                 # if root, augment with title (if available and different)
-                return (
-                    text_entries
-                    # ([doc_map.glob.title.text] + texts)
-                    # if doc_map.glob.title and [doc_map.glob.title.text] != texts
-                    # else texts
-                )
+                return text_entries
         else:
             return []
 

test/test_hierarchical_chunker.py

@@ -5,7 +5,7 @@
 
 import json
 
-from docling_core.chunker import HierarchicalChunker
+from docling_core.transforms.chunker import HierarchicalChunker
 from docling_core.types import Document as DLDocument