Add basic mine cran workflow

Signed-off-by: ziad hany <[email protected]>

Author: ziadhany

minecode_pipelines/miners/cran.py

@@ -0,0 +1,28 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+from pathlib import Path
+import requests
+
+
+def get_cran_db(output_file="cran_db.json") -> Path:
+    """
+    Download the CRAN package database (~250MB JSON) in a memory-efficient way.
+    Saves it to a file instead of loading everything into memory.
+    """
+
+    url = "https://crandb.r-pkg.org/-/all"
+    output_path = Path(output_file)
+
+    with requests.get(url, stream=True) as response:
+        response.raise_for_status()
+        with output_path.open("wb") as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                f.write(chunk)
+
+    return output_path

minecode_pipelines/pipelines/mine_cran.py

@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+import os
+from scanpipe.pipelines import Pipeline
+from scanpipe.pipes import federatedcode
+
+from minecode_pipelines.miners.cran import get_cran_db
+from minecode_pipelines.pipes import cran
+
+FEDERATEDCODE_CRAN_GIT_URL = os.environ.get(
+    "FEDERATEDCODE_CRAN_GIT_URL", ""
+)
+
+
+class MineandPublishCRANPURLs(Pipeline):
+    """
+    Mine all packageURLs from a CRAN R index and publish them to a FederatedCode repo.
+    """
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.check_federatedcode_eligibility,
+            cls.setup_federatedcode_cran,
+            cls.mine_and_publish_cran_packageurls,
+        )
+
+    def check_federatedcode_eligibility(self):
+        """
+        Check if the project fulfills the following criteria for
+        pushing the project result to FederatedCode.
+        """
+        federatedcode.check_federatedcode_configured_and_available()
+
+    def setup_federatedcode_cran(self):
+        """
+        Clone the FederatedCode CRAN repository and download the CRAN DB JSON file.
+        """
+        self.fed_repo = federatedcode.clone_repository(repo_url=FEDERATEDCODE_CRAN_GIT_URL)
+        self.db_path = get_cran_db()
+
+    def mine_and_publish_cran_packageurls(self):
+        """Get cran packageURLs for all mined cran package names."""
+        cran.mine_and_publish_cran_packageurls(fed_repo=self.fed_repo, db_path=self.db_path)

minecode_pipelines/pipes/cran.py

@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+import json
+from pathlib import Path
+from packageurl import PackageURL
+from aboutcode.hashid import get_package_purls_yml_file_path
+from minecode_pipelines.utils import git_stage_purls, commit_and_push_changes
+
+
+def mine_and_publish_cran_packageurls(fed_repo, db_path):
+    for purls in extract_cran_packages(db_path):
+        if not purls:
+            continue
+
+        first_purl = purls[0]
+        purl_yaml_path = get_package_purls_yml_file_path(first_purl)
+        git_stage_purls(purls, fed_repo, purl_yaml_path)
+
+    commit_and_push_changes(fed_repo)
+
+
+def extract_cran_packages(json_file_path: str) -> list:
+    """
+    Extract package names and their versions from a CRAN DB JSON file.
+    """
+    db_path = Path(json_file_path)
+    if not db_path.exists():
+        raise FileNotFoundError(f"File not found: {db_path}")
+
+    with open(db_path, encoding="utf-8") as f:
+        data = json.load(f)
+
+    for pkg_name, pkg_data in data.items():
+        versions = list(pkg_data.get("versions", {}).keys())
+        purls = []
+        for version in versions:
+            purl = PackageURL(
+                type="cran",
+                name=pkg_name,
+                version=version,
+            )
+            purls.append(purl.to_string())
+        yield purls

minecode_pipelines/tests/pipes/test_cran.py

@@ -0,0 +1,8 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#

minecode_pipelines/utils.py

@@ -9,8 +9,17 @@
 
 import os
 import tempfile
+import textwrap
+from pathlib import Path
 
 from commoncode.fileutils import create_dir
+from minecode_pipelines.miners import write_data_to_file
+
+VERSION = os.environ.get("VERSION", "")
+PURLDB_ALLOWED_HOST = os.environ.get("FEDERATEDCODE_GIT_ALLOWED_HOST", "")
+author_name = os.environ.get("FEDERATEDCODE_GIT_SERVICE_NAME", "")
+author_email = os.environ.get("FEDERATEDCODE_GIT_SERVICE_EMAIL", "")
+remote_name = os.environ.get("FEDERATEDCODE_GIT_REMOTE_NAME", "origin")
 
 
 def system_temp_dir(temp_dir=os.getenv("MINECODE_TMP")):
@@ -49,3 +58,33 @@ def get_temp_file(file_name="data", extension=".file", dir_name=""):
     temp_dir = get_temp_dir(dir_name)
     location = os.path.join(temp_dir, file_name)
     return location
+
+
+def git_stage_purls(purls, repo, purls_file):
+    """Write package URLs to a file and stage it in the local Git repository."""
+    relative_purl_file_path = Path(purls_file)
+
+    write_to = Path(repo.working_dir) / relative_purl_file_path
+
+    write_data_to_file(path=write_to, data=purls)
+
+    repo.index.add([relative_purl_file_path])
+    return relative_purl_file_path
+
+
+def commit_and_push_changes(repo):
+    """
+    Commit staged changes to the local repository and push them
+    to the remote on the current active branch.
+    """
+
+    commit_message = f"""\
+    Add/Update list of available package versions
+    Tool: pkg:github/aboutcode-org/purldb@v{VERSION}
+    Reference: https://{PURLDB_ALLOWED_HOST}/
+    Signed-off-by: {author_name} <{author_email}>
+    """
+
+    default_branch = repo.active_branch.name
+    repo.index.commit(textwrap.dedent(commit_message))
+    repo.git.push(remote_name, default_branch, "--no-verify")