[Docs] Use jinja template in documentation generation (#606)

* Remove generated files from git

* Use jinja template for generating the mint json file

test/website/test_process_api_reference.py

@@ -17,7 +17,8 @@
 @pytest.fixture
 def template_content():
     """Fixture providing the template JSON content."""
-    return {
+    template = """
+    {
         "name": "AG2",
         "logo": {"dark": "/logo/ag2-white.svg", "light": "/logo/ag2.svg"},
         "navigation": [
@@ -27,20 +28,14 @@ def template_content():
                 "pages": [
                     "docs/installation/Installation",
                     "docs/installation/Docker",
-                    "docs/installation/Optional-Dependencies",
-                ],
+                    "docs/installation/Optional-Dependencies"
+                ]
             },
-            {"group": "API Reference", "pages": ["PLACEHOLDER"]},
-            # {
-            #     "group": "AutoGen Studio",
-            #     "pages": [
-            #         "docs/autogen-studio/getting-started",
-            #         "docs/autogen-studio/usage",
-            #         "docs/autogen-studio/faqs",
-            #     ],
-            # },
-        ],
+            {"group": "API Reference", "pages": ["PLACEHOLDER"]}
+        ]
     }
+    """
+    return template
 
 
 @pytest.fixture
@@ -53,9 +48,9 @@ def temp_dir():
 @pytest.fixture
 def template_file(temp_dir, template_content):
     """Fixture creating a template file in a temporary directory."""
-    template_path = temp_dir / "mint-json-template.json"
+    template_path = temp_dir / "mint-json-template.json.jinja"
     with open(template_path, "w") as f:
-        json.dump(template_content, f, indent=2)
+        f.write(template_content)
     return template_path
 
 
@@ -75,9 +70,10 @@ def test_generate_mint_json_from_template(template_file, target_file, template_c
 
     # Verify the contents
     with open(target_file) as f:
-        generated_content = json.load(f)
+        actual = json.load(f)
 
-    assert generated_content == template_content
+    expected = json.loads(template_content)
+    assert actual == expected
 
 
 def test_generate_mint_json_existing_file(template_file, target_file, template_content):
@@ -92,9 +88,10 @@ def test_generate_mint_json_existing_file(template_file, target_file, template_c
 
     # Verify the contents were overwritten
     with open(target_file) as f:
-        generated_content = json.load(f)
+        actual = json.load(f)
 
-    assert generated_content == template_content
+    expected = json.loads(template_content)
+    assert actual == expected
 
 
 def test_generate_mint_json_missing_template(target_file):

test/website/test_process_notebooks.py

@@ -14,6 +14,8 @@
 sys.path.append(str(Path(__file__).resolve().parent.parent.parent / "website"))
 from process_notebooks import (
     add_authors_and_social_img_to_blog_posts,
+    add_front_matter_to_metadata_mdx,
+    cleanup_tmp_dirs_if_no_metadata,
     convert_callout_blocks,
     ensure_mint_json_exists,
     extract_example_group,
@@ -34,6 +36,171 @@ def test_ensure_mint_json():
         ensure_mint_json_exists(tmp_path)  # Should not raise any exception
 
 
+def test_cleanup_tmp_dirs_if_no_metadata():
+    # Test without the tmp_dir / "snippets" / "data" / "NotebooksMetadata.mdx"
+    # the tmp_dir / "notebooks" should be removed.
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        tmp_path = Path(tmp_dir)
+        notebooks_dir = tmp_path / "notebooks"
+        notebooks_dir.mkdir(parents=True, exist_ok=True)
+        (notebooks_dir / "example-1.mdx").touch()
+        (notebooks_dir / "example-2.mdx").touch()
+
+        cleanup_tmp_dirs_if_no_metadata(tmp_path)
+        assert not notebooks_dir.exists()
+
+    # Test with the tmp_dir / "snippets" / "data" / "NotebooksMetadata.mdx"
+    # the tmp_dir / "notebooks" should not be removed.
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        tmp_path = Path(tmp_dir)
+
+        notebooks_dir = tmp_path / "notebooks"
+        notebooks_dir.mkdir(parents=True, exist_ok=True)
+        (notebooks_dir / "example-1.mdx").touch()
+        (notebooks_dir / "example-2.mdx").touch()
+
+        metadata_dir = tmp_path / "snippets" / "data"
+        metadata_dir.mkdir(parents=True, exist_ok=True)
+        (metadata_dir / "NotebooksMetadata.mdx").touch()
+
+        cleanup_tmp_dirs_if_no_metadata(tmp_path)
+        assert notebooks_dir.exists()
+
+
+class TestAddFrontMatterToMetadataMdx:
+    def test_without_metadata_mdx(self):
+        front_matter_dict = {
+            "title": "some title",
+            "link": "/notebooks/some-title",
+            "description": "some description",
+            "image": "some image",
+            "tags": ["tag1", "tag2"],
+            "source_notebook": "/notebook/some-title.ipynb",
+        }
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            tmp_path = Path(tmp_dir)
+
+            metadata_dir = tmp_path / "snippets" / "data"
+            metadata_dir.mkdir(parents=True, exist_ok=True)
+
+            rendered_mdx = tmp_path / "source"
+            rendered_mdx.mkdir(parents=True, exist_ok=True)
+            (rendered_mdx / "some-title.mdx").touch()
+
+            # when the metadata file is not present, the file should be created and the front matter should be added
+            add_front_matter_to_metadata_mdx(front_matter_dict, tmp_path, rendered_mdx)
+
+            assert (metadata_dir / "NotebooksMetadata.mdx").exists()
+
+            with open(metadata_dir / "NotebooksMetadata.mdx", "r") as f:
+                actual = f.read()
+
+            assert (
+                actual
+                == """{/*
+Auto-generated file - DO NOT EDIT
+Please edit the add_front_matter_to_metadata_mdx function in process_notebooks.py
+*/}
+
+export const notebooksMetadata = [
+    {
+        "title": "some title",
+        "link": "/notebooks/source",
+        "description": "some description",
+        "image": "some image",
+        "tags": [
+            "tag1",
+            "tag2"
+        ],
+        "source": "/notebook/some-title.ipynb"
+    }
+];
+"""
+            )
+
+    def test_with_metadata_mdx(self):
+        front_matter_dict = {
+            "title": "some title",
+            "link": "/notebooks/some-title",
+            "description": "some description",
+            "image": "some image",
+            "tags": ["tag1", "tag2"],
+            "source_notebook": "/notebook/some-title.ipynb",
+        }
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            tmp_path = Path(tmp_dir)
+
+            metadata_dir = tmp_path / "snippets" / "data"
+            metadata_dir.mkdir(parents=True, exist_ok=True)
+            metadata_content = """{/*
+Auto-generated file - DO NOT EDIT
+Please edit the add_front_matter_to_metadata_mdx function in process_notebooks.py
+*/}
+
+export const notebooksMetadata = [
+    {
+        "title": "some other title",
+        "link": "/notebooks/some-other-title",
+        "description": "some other description",
+        "image": "some other image",
+        "tags": [
+            "tag3",
+            "tag4"
+        ],
+        "source": "/notebook/some-other-title.ipynb"
+    }
+];
+"""
+            with open(metadata_dir / "NotebooksMetadata.mdx", "w") as f:
+                f.write(metadata_content)
+
+            rendered_mdx = tmp_path / "source"
+            rendered_mdx.mkdir(parents=True, exist_ok=True)
+            (rendered_mdx / "some-title.mdx").touch()
+
+            # when the metadata file is present, the front matter should be added to the existing metadata
+            add_front_matter_to_metadata_mdx(front_matter_dict, tmp_path, rendered_mdx)
+
+            assert (metadata_dir / "NotebooksMetadata.mdx").exists()
+
+            with open(metadata_dir / "NotebooksMetadata.mdx", "r") as f:
+                actual = f.read()
+
+            assert (
+                actual
+                == """{/*
+Auto-generated file - DO NOT EDIT
+Please edit the add_front_matter_to_metadata_mdx function in process_notebooks.py
+*/}
+
+export const notebooksMetadata = [
+    {
+        "title": "some other title",
+        "link": "/notebooks/some-other-title",
+        "description": "some other description",
+        "image": "some other image",
+        "tags": [
+            "tag3",
+            "tag4"
+        ],
+        "source": "/notebook/some-other-title.ipynb"
+    },
+    {
+        "title": "some title",
+        "link": "/notebooks/source",
+        "description": "some description",
+        "image": "some image",
+        "tags": [
+            "tag1",
+            "tag2"
+        ],
+        "source": "/notebook/some-title.ipynb"
+    }
+];
+"""
+            )
+
+
 class TestAddBlogsToNavigation:
     @pytest.fixture
     def test_dir(self):

website/.gitignore

@@ -11,6 +11,7 @@ docs/reference
 /notebooks
 /blog
 mint.json
+/snippets/data/NotebooksMetadata.mdx
 
 docs/tutorial/*.mdx
 docs/tutorial/**/*.png

website/process_api_reference.py

@@ -16,6 +16,8 @@
 from pathlib import Path
 from typing import Any
 
+from jinja2 import Template
+
 
 def run_pydoc_markdown(config_file: Path) -> None:
     """Run pydoc-markdown with the specified config file.
@@ -179,10 +181,15 @@ def generate_mint_json_from_template(mint_json_template_path: Path, mint_json_pa
         os.remove(mint_json_path)
 
     # Copy the template file to mint.json
-    mint_json_template_content = read_file_content(mint_json_template_path)
+    contents = read_file_content(mint_json_template_path)
+    mint_json_template_content = Template(contents).render()
+
+    # Parse the rendered template content as JSON
+    mint_json_data = json.loads(mint_json_template_content)
 
     # Write content to mint.json
-    write_file_content(mint_json_path, mint_json_template_content)
+    with open(mint_json_path, "w") as f:
+        json.dump(mint_json_data, f, indent=2)
 
 
 def main() -> None:
@@ -208,7 +215,7 @@ def main() -> None:
     convert_md_to_mdx(args.api_dir)
 
     # Create mint.json from the template file
-    mint_json_template_path = script_dir / "mint-json-template.json"
+    mint_json_template_path = script_dir / "mint-json-template.json.jinja"
     mint_json_path = script_dir / "mint.json"
 
     print("Generating mint.json from template...")

website/process_notebooks.py

@@ -321,15 +321,21 @@ def add_front_matter_to_metadata_mdx(
 
     metadata_mdx = website_dir / "snippets" / "data" / "NotebooksMetadata.mdx"
 
+    if not metadata_mdx.exists():
+        with open(metadata_mdx, "w", encoding="utf-8") as f:
+            f.write(
+                "{/*\nAuto-generated file - DO NOT EDIT\nPlease edit the add_front_matter_to_metadata_mdx function in process_notebooks.py\n*/}\n\n"
+            )
+            f.write("export const notebooksMetadata = [];\n")
+
     metadata = []
-    if metadata_mdx.exists():
-        with open(metadata_mdx, encoding="utf-8") as f:
-            content = f.read()
-            if content:
-                start = content.find("export const notebooksMetadata = [")
-                end = content.rfind("]")
-                if start != -1 and end != -1:
-                    metadata = json.loads(content[start + 32 : end + 1])
+    with open(metadata_mdx, encoding="utf-8") as f:
+        content = f.read()
+        if content:
+            start = content.find("export const notebooksMetadata = [")
+            end = content.rfind("]")
+            if start != -1 and end != -1:
+                metadata = json.loads(content[start + 32 : end + 1])
 
     # Create new entry for current notebook
     entry = {
@@ -899,6 +905,21 @@ def ensure_mint_json_exists(website_dir: Path) -> None:
         sys.exit(1)
 
 
+def cleanup_tmp_dirs_if_no_metadata(website_dir: Path) -> None:
+    """Remove the temporary notebooks directory if NotebooksMetadata.mdx is not found.
+
+    This is to ensure a clean build and generate the metadata file as well as to
+    update the navigation with correct entries.
+    """
+    metadata_mdx = website_dir / "snippets" / "data" / "NotebooksMetadata.mdx"
+    if not metadata_mdx.exists():
+        print(f"NotebooksMetadata.mdx not found at {metadata_mdx}")
+
+        notebooks_dir = website_dir / "notebooks"
+        print(f"Removing the {notebooks_dir} and to ensure a clean build.")
+        shutil.rmtree(notebooks_dir, ignore_errors=True)
+
+
 def main() -> None:
     script_dir = Path(__file__).parent.absolute()
     parser = argparse.ArgumentParser()
@@ -929,6 +950,7 @@ def main() -> None:
         sys.exit(1)
 
     ensure_mint_json_exists(args.website_directory)
+    cleanup_tmp_dirs_if_no_metadata(args.website_directory)
 
     if args.notebooks:
         collected_notebooks = args.notebooks