feat: added sitemap script to update-gh-pages.yml (#745)

* feat: added sitemap script to update-gh-pages.yml

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: moe-ad

.github/workflows/update-gh-pages.yml

@@ -1,17 +1,41 @@
-
-name: Update index.html in GH Pages
+name: Custom Github pages post-processing
 
 on:
   workflow_dispatch:
   workflow_call:
+  push:
+    tags:
+      - "*"
+
+env:
+  MAIN_PYTHON_VERSION: '3.12'
 
 jobs:
   update-gh-pages:
     runs-on: ubuntu-latest
 
     steps:
+    - name: Install Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ env.MAIN_PYTHON_VERSION }}
+
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install requests
+
     - name: Checkout repository
       uses: actions/checkout@v4
+
+    - name: Copy tools folder to /tmp and run sitemap script
+      working-directory: /tmp
+      run: |
+        cp -r /home/runner/work/pyansys/pyansys/tools/ .
+        python ./tools/catsitemap.py
+
+    - name: Checkout repository gh-pages branch
+      uses: actions/checkout@v4
       with:
         ref: gh-pages
 
@@ -24,8 +48,12 @@ jobs:
         # Replace "version/stable" with "version/dev" in the sitemap.xml
         sed -i 's/version\/stable/version\/dev/g' sitemap.xml
 
+    - name: Move sitemaps/ to gh-pages root
+      run: |
+        rm -rf sitemaps/ && mv /tmp/sitemaps/ .
+
     - name: "Commit changes"
       uses: EndBug/add-and-commit@v9
       with:
         default_author: github_actions
-        message: "update index.html"
+        message: "update index.html and sitemaps folder"

tools/catsitemap.py

@@ -0,0 +1,118 @@
+"""Script for automatic generation/download of sitemaps for pyansys projects.
+
+Intended for the pyansys project and to be used with the update-gh-pages
+workflow.
+
+"""
+
+from pathlib import Path
+from xml.dom import minidom
+import xml.etree.ElementTree as ET
+
+from links import LINKS
+import requests
+
+
+def download_file(url: str, dest_path: Path) -> None:
+    """Given a sitemap url, this function downloads the file into (dest_path).
+
+    Parameters
+    ----------
+    url : str
+        The url of the sitemap file to be downloaded
+    dest_path : Path
+        The destination path to save the downloaded file
+
+    Raises
+    ------
+    requests.exceptions.Timeout
+        Raises this exception when accessing a link takes too long
+    """
+    # Send the request
+    response = requests.get(url, stream=True, timeout=30)
+
+    # Write the file content to the specified location
+    with dest_path.open(mode="w", encoding="utf-8") as file:
+        file.write(response.text)
+
+
+def extract_urls_and_headers(links_dict: dict) -> tuple:
+    """Extract valid project names and sitemap urls from metadata dictionary.
+
+    Parameters
+    ----------
+    links_dict : dict
+        Dictionary containing metadata of projects
+
+    Returns
+    -------
+    tuple
+        Contains the list of project names and the list of sitemap urls
+    """
+    valid_project_names = []
+    valid_urls = []
+    for project_name, url in links_dict.items():
+        # The form of url is "https://subdomain.docs.pyansys.com/version/stable
+        # where subdomain may contain nested subdomains depending on the project
+        # see links.py from which LINKS was imported for examples
+        if url is None:
+            continue
+        # url is changed to "https://subdomain.docs.pyansys.com/sitemap.xml
+        # this is general form of the link to the sitemap file of each project
+        updated_url = url.split("docs.pyansys.com")[0] + "docs.pyansys.com/sitemap.xml"
+        if requests.get(url).status_code == 404:
+            continue
+        else:
+            valid_project_names.append(project_name)
+            valid_urls.append(updated_url)
+
+    return valid_project_names, valid_urls
+
+
+def generate_sitemap_index(project_names: list, dest_path: Path) -> None:
+    """Generate the global sitemap file which will point to all other sitemaps.
+
+    Parameters
+    ----------
+    project_names : list
+        List of project names with a downloadable sitemap file
+    dest_path : Path
+        The destination path to save the generated sitemap file
+    """
+    # Create the root element with namespace
+    sitemap_index = ET.Element("sitemapindex", xmlns="http://www.sitemaps.org/schemas/sitemap/0.9")
+
+    # Create sitemap elements for each URL
+    for project in project_names:
+        # Modify the url to point to the correct gh-pages directory
+        modified_url = f"https://docs.pyansys.com/sitemap/{project}_sitemap.xml"
+
+        sitemap = ET.SubElement(sitemap_index, "sitemap")
+        loc = ET.SubElement(sitemap, "loc")
+        loc.text = modified_url
+
+    # Format XML with indentation
+    rough_string = ET.tostring(sitemap_index, "utf-8")
+    reparsed = minidom.parseString(rough_string)
+    pretty_xml = reparsed.toprettyxml(indent="  ")
+
+    # Create the tree and write to XML file
+    with dest_path.open(mode="w", encoding="utf-8") as file:
+        file.write(pretty_xml)
+
+
+if __name__ == "__main__":
+    # Create path
+    folder_path = Path(".") / "sitemaps"
+    folder_path.mkdir(parents=True, exist_ok=True)
+
+    # Get actual valid URLS and corresponding project names
+    project_names, project_urls = extract_urls_and_headers(LINKS)
+
+    # Generate global sitemap file
+    file_path = folder_path / "globalsitemap.xml"
+    generate_sitemap_index(project_names, file_path)
+
+    for ith_url, url in enumerate(project_urls):
+        file_path = folder_path / (project_names[ith_url] + "_sitemap.xml")
+        download_file(url, file_path)