Baby steps to make registry talk to catalog

datalad_registry/overview.py

@@ -3,10 +3,24 @@
 
 import logging
 
-from flask import Blueprint, render_template, request
-from sqlalchemy import nullslast, select
+import datalad.api as dl
+from flask import (
+    Blueprint,
+    render_template,
+    request,
+    send_from_directory,
+)
+from sqlalchemy import (
+    nullslast,
+    select,
+)
 
-from datalad_registry.models import RepoUrl, db
+from datalad_registry.blueprints.api.url_metadata import URLMetadataModel
+from datalad_registry.models import (
+    RepoUrl,
+    URLMetadata,
+    db,
+)
 from datalad_registry.search import parse_query
 
 lgr = logging.getLogger(__name__)
@@ -69,3 +83,52 @@ def overview():  # No type hints due to mypy#7187.
         search_query=query,
         search_error=search_error,
     )
+
+
+# @bp.route('/catalog/', defaults={'path': ''})
+# TODO: move from placing dataset identifier within path -- place into query
+# TODO: do not use ID may be but use URL, or allow for both -- that would make it
+#  possible to make those URLs pointing to datasets easier to create/digest for humans
+@bp.route("/catalog/<int:id_>/<path:path>")
+def send_report(id_, path):
+    # ds_id = request.args.get("id", None, type=int)
+    if not path:
+        path = "index.html"
+    if path == "index.html":
+        lgr.warning(f"PATH: {path}  id: {id_}")
+        # let's get metadata for the ds_id
+        repo_url_row = db.session.execute(
+            db.select(RepoUrl).filter_by(id=id_)
+        ).one_or_none()
+        if repo_url_row:
+            repo_url_row = repo_url_row[0]
+            metadatas = {}
+            for mr in repo_url_row.metadata_:
+                if mr.extractor_name not in {
+                    "metalad_core",
+                    "bids_dataset",
+                    "metalad_studyminimeta",
+                }:
+                    continue
+                # TODO: here metadta record had only @context and @graph and no other
+                #  fields figure out if enough....
+                m = URLMetadataModel.from_orm(mr).dict()
+                # lgr.warning(f"ROW: {m}")
+                m["type"] = "dataset"
+                m["dataset_id"] = repo_url_row.ds_id
+                # Didn't want to translate yet
+                lgr.warning(f"Translating record with keys {m.keys()}")
+                m_translated = dl.catalog_translate(m)[0]["translated_metadata"]
+                metadatas[mr.extractor_name] = m_translated
+
+            if "metalad_studyminimeta" not in metadatas:
+                metadatas["metalad_core"]["name"] = repo_url_row.url
+
+            for m in metadatas.values():
+                m["name"] = repo_url_row.url
+                lgr.warning(f"URL: {repo_url_row.url!r} {type(repo_url_row.url)}")
+                dl.catalog_add("/app-catalog", metadata=m)
+    # TODO: figure out how to pass all the metadata goodness to the catalog
+    # f'/app-catalog/dataset/{repo_url_row.ds_id}/'
+    #                     f'{metadatas['metalad_core']['dataset_version']}'
+    return send_from_directory("/app-catalog", path)

datalad_registry/resources/catalog-config.json

@@ -0,0 +1,80 @@
+{
+  "catalog_name": "DataCat of Registry",
+  "logo_path": "",
+  "link_color": "#fba304",
+  "link_hover_color": "#af7714",
+  "social_links": {
+    "about": null,
+    "documentation": "https://docs.datalad.org/projects/catalog/en/latest/",
+    "github": "https://github.com/datalad/datalad-catalog",
+    "mastodon": "https://fosstodon.org/@datalad",
+    "x": "https://x.com/datalad"
+  },
+  "dataset_options": {
+    "include_metadata_export": true
+  },
+  "property_sources": {
+    "dataset": {
+      "dataset_id": {
+        "rule": "single",
+        "source": "metalad_core"
+      },
+      "dataset_version": {
+        "rule": "single",
+        "source": "metalad_core"
+      },
+      "type": {
+        "rule": "single",
+        "source": "metalad_core"
+      },
+      "children": {
+        "rule": "merge",
+        "source": "any"
+      },
+      "short_name": {},
+      "description": {
+        "rule": "priority",
+        "source": [
+          "catalog_readme",
+          "metalad_studyminimeta",
+          "datacite_gin",
+          "bids_dataset"
+        ]
+      },
+      "doi": {},
+      "url": {
+        "rule": "merge",
+        "source": "any"
+      },
+      "authors": {
+        "rule": "merge",
+        "source": "any"
+      },
+      "keywords": {
+        "rule": "merge",
+        "source": "any"
+      },
+      "license": {},
+      "funding": {
+        "rule": "merge",
+        "source": "any"
+      },
+      "publications": {
+        "rule": "merge",
+        "source": "any"
+      },
+      "subdatasets": {
+        "rule": "merge",
+        "source": "any"
+      },
+      "additional_display": {
+        "rule": "merge",
+        "source": "any"
+      },
+      "top_display": {
+        "rule": "merge",
+        "source": "any"
+      }
+    }
+  }
+}

datalad_registry/templates/overview.html

@@ -150,7 +150,10 @@ <h2>Search query syntax</h2>
       </tr>
       {%- for i in pagination -%}
         <tr>
-          <td><a href="{{ i.url }}">{{ i.url }}</a></td>
+          <td>
+            <a href="{{ i.url }}">{{ i.url }}</a>
+            <a href="/overview/catalog/{{ i.id }}/index.html">C</a>
+          </td>
           <td class="mono">
 			{% if i.ds_id is not none %}
 			<a href="{{ url_for('.overview', query='ds_id:' + i.ds_id) }}">{{ i.ds_id }}</a>

docker-compose.dev.override.yml

@@ -11,10 +11,12 @@ services:
     command: [
       "/sbin/my_init", "--",
       "bash", "-c",
-      "git config --global --add safe.directory /app && pip3 install -U -e . && flask init-db && exec flask run --host=0.0.0.0 --debug"
+      "git config --global --add safe.directory /app && pip3 install -U -e . && pip install -e /metalad && pip install -e /catalog && flask init-db && datalad catalog-create -c /app-catalog -F /app/datalad_registry/resources/catalog-config.json && exec flask run --host=0.0.0.0 --debug"
     ]
     volumes:
       - ./:/app
+      - ../datalad-catalog:/catalog
+      - ../datalad-metalad:/metalad
       - ./instance:/app/instance
 
   worker:

pyproject.toml

@@ -2,6 +2,12 @@
 requires = ["setuptools >= 46.4.0", "versioningit ~= 3.0", "wheel ~= 0.32"]
 build-backend = "setuptools.build_meta"
 
+[tool.isort]
+force_grid_wrap = 2
+include_trailing_comma = true
+multi_line_output = 3
+combine_as_imports = true
+
 [tool.versioningit]
 
 [tool.pytest.ini_options]