diff --git a/docs/releases/v14/migrate_thesis_to_dissertation.py b/docs/releases/v14/migrate_thesis_to_dissertation.py new file mode 100644 index 00000000..d62e0761 --- /dev/null +++ b/docs/releases/v14/migrate_thesis_to_dissertation.py @@ -0,0 +1,323 @@ +# SPDX-FileCopyrightText: 2025 CERN. +# SPDX-FileCopyrightText: 2026 Graz University of Technology. +# SPDX-License-Identifier: MIT + +"""Reference helpers for the "Thesis" / "Dissertation" resource type change (v14). + +These functions are reference material, not a runnable script. They are here so +you can copy and adapt the parts that fit your instance when carrying out the +*optional* resource type change described in the v14 upgrade guide. Read them, +lift what you need into your own `invenio shell` snippet or script, and always +test against a copy of your data first. + +Two independent operations are shown: + +- `run_update_for_resource_type` rewrites every published record and unpublished + draft with resource type publication-thesis to publication-dissertation (in + metadata.resource_type and in any metadata.related_identifiers), going through + the service layer so DataCite DOI metadata is re-registered and records are + re-indexed. + +- `run_update_doi_metadata_for_resource_type` re-registers the DataCite DOI + metadata of every published record of a resource type, without changing the + record. Use it when you keep your own resource type id but changed its + props.datacite_general (e.g. to "Dissertation") in your vocabulary: the DataCite + serializer reads props.datacite_general live from the vocabulary, so only a DOI + update is needed. + +The resource type rewrite has been tested for the following scenarios: +1. Draft with resource type publication-thesis +2. Record with resource type publication-thesis with a DOI and no draft +3. Record with resource type publication-thesis with no DOI and an existing draft +4. Records with multiple versions +""" + +from click import secho +from invenio_access.permissions import system_identity +from invenio_db import db +from invenio_drafts_resources.resources.records.errors import DraftNotCreatedError +from invenio_rdm_records.proxies import current_rdm_records_service as records_service +from invenio_rdm_records.services.errors import RecordDeletedException +from invenio_search.api import RecordsSearchV2 + + +def run_upgrade(migrate_record, migrate_draft): + """Run upgrade on selected records and drafts. + + Args: + migrate_record (callable): Function to migrate a record. + migrate_draft (callable): Function to migrate a draft. + """ + errored_record_ids = [] + errored_draft_ids = [] + + # Handle published records + published_records = ( + RecordsSearchV2(index=records_service.record_cls.index._name) + .filter( + "query_string", + query="metadata.resource_type.id:publication-thesis OR metadata.related_identifiers.resource_type.id:publication-thesis", + ) + .source(["id"]) + .scan() + ) # Only need to fetch the record IDs to make the query faster + # Convert the search results to a list to avoid keeping the scroll context open, as it errors out after 15 minutes + published_record_ids = [result["id"] for result in published_records] + for record_id in published_record_ids: + try: + migrate_record(record_id) + except Exception as error: + secho(f"> Error {repr(error)}", fg="red") + secho(f"Record {record_id} failed to update", fg="red") + errored_record_ids.append((record_id, error)) + + # Handle draft records + draft_records = ( + RecordsSearchV2(index=records_service.draft_cls.index._name) + .filter("term", has_draft=False) + .filter( + "query_string", + query="metadata.resource_type.id:publication-thesis OR metadata.related_identifiers.resource_type.id:publication-thesis", + ) + .source(["id"]) + .scan() + ) + # Convert the search results to a list to avoid keeping the scroll context open, as it errors out after 15 minutes + draft_record_ids = [result["id"] for result in draft_records] + for draft_id in draft_record_ids: + try: + migrate_draft(draft_id) + except Exception as error: + secho(f"> Error {repr(error)}", fg="red") + secho(f"Draft {draft_id} failed to update", fg="red") + errored_draft_ids.append((draft_id, error)) + + if len(errored_record_ids) > 0: + secho(f"Errored record IDs: {errored_record_ids}", fg="red") + else: + secho("records have been updated successfully", fg="green") + + if len(errored_draft_ids) > 0: + secho(f"Errored draft IDs: {errored_draft_ids}", fg="red") + else: + secho("drafts have been updated successfully", fg="green") + + +def run_update_for_resource_type(): + """Run update for resource type.""" + + def migrate_resource_type_in_record(record_id): + """ + Update resource type from publication-thesis to publication-dissertation. + + We go through the service layer to automatically trigger the DOI update and re-indexing. + """ + secho(f"Updating resource type for record {record_id}", fg="yellow") + record = records_service.read(system_identity, record_id, include_deleted=True) + if record.data["metadata"]["resource_type"][ + "id" + ] != "publication-thesis" and not any( + related_identifier.get("resource_type", {}).get("id") + == "publication-thesis" + for related_identifier in record.data["metadata"].get( + "related_identifiers", [] + ) + ): + secho( + f"Skipping record <{record.id}> because it doesn't have resource-type 'publication-thesis'!", + fg="yellow", + ) + return + + try: + draft = records_service.read_draft(system_identity, record.id) + # Step 1: Update the resource type in the record via low-level API + # We need to make sure we don't publish the record with different metadata + secho( + f"Record <{record.id}> has an existing draft <{draft.id}>! Updating record via low-level API.", + fg="yellow", + ) + # Update the record directly without affecting the draft + if ( + record._record["metadata"]["resource_type"]["id"] + == "publication-thesis" + ): + record._record["metadata"]["resource_type"][ + "id" + ] = "publication-dissertation" + for related_identifier in record._record["metadata"].get( + "related_identifiers", [] + ): + if ( + related_identifier.get("resource_type", {}).get("id") + == "publication-thesis" + ): + related_identifier["resource_type"][ + "id" + ] = "publication-dissertation" + # Save the record changes and reindex + secho( + f"Record <{record.id}> has been updated... committing changes.", + fg="green", + ) + record._record.commit() + # Step 2: Update the resource type in the draft + if draft._record["metadata"]["resource_type"]["id"] == "publication-thesis": + draft._record["metadata"]["resource_type"][ + "id" + ] = "publication-dissertation" + for related_identifier in draft._record["metadata"].get( + "related_identifiers", [] + ): + if ( + related_identifier.get("resource_type", {}).get("id") + == "publication-thesis" + ): + related_identifier["resource_type"][ + "id" + ] = "publication-dissertation" + # After updating the record, update the draft's fork_version_id to match the record's new version_id, to avoid conflicts when publishing + draft._record.fork_version_id = record._record.revision_id + draft._record.commit() + # Commit the changes for both the record and the draft in one transaction + db.session.commit() + records_service.indexer.index(record._record) + records_service.draft_indexer.index(draft._record) + secho(f"Draft <{draft.id}> has been updated successfully.", fg="green") + # Update DOI metadata if record has DOI + if (record._record.get("pids") or {}).get("doi"): + records_service.pids.register_or_update( + system_identity, record.id, "doi", parent=False + ) + secho( + f"DOI metadata for record {record.id} has been updated successfully.", + fg="green", + ) + except DraftNotCreatedError: + # If the draft didn't exist, we simply edit and publish the record + draft = records_service.edit(system_identity, record.id) + if draft.data["metadata"]["resource_type"]["id"] == "publication-thesis": + draft.data["metadata"]["resource_type"][ + "id" + ] = "publication-dissertation" + for related_identifier in draft.data["metadata"].get( + "related_identifiers", [] + ): + if ( + related_identifier.get("resource_type", {}).get("id") + == "publication-thesis" + ): + related_identifier["resource_type"][ + "id" + ] = "publication-dissertation" + updated_draft = records_service.update_draft( + system_identity, draft.id, draft.data + ) + record = records_service.publish(system_identity, updated_draft.id) + except RecordDeletedException: + # If the draft was deleted, we ignore it + # In the future, we should add include_deleted to read_draft and update the draft metadata in these cases + secho(f"Draft <{draft.id}> has been deleted, skipping...", fg="yellow") + + secho(f"Record <{record.id}> has been updated successfully.", fg="green") + + def migrate_resource_type_in_draft(draft_id): + """ + Update resource type from publication-thesis to publication-dissertation. + + We go through the service layer to automatically trigger the DOI update and re-indexing. + """ + secho(f"Updating resource type for draft {draft_id}", fg="yellow") + draft = records_service.edit(system_identity, draft_id) + if draft.data["metadata"]["resource_type"][ + "id" + ] != "publication-thesis" and not any( + related_identifier.get("resource_type", {}).get("id") + == "publication-thesis" + for related_identifier in draft.data["metadata"].get( + "related_identifiers", [] + ) + ): + secho( + f"Skipping draft <{draft.id}> because it doesn't have resource-type 'publication-thesis'!", + fg="yellow", + ) + return + + if draft.data["metadata"]["resource_type"]["id"] == "publication-thesis": + draft.data["metadata"]["resource_type"]["id"] = "publication-dissertation" + for related_identifier in draft.data["metadata"].get("related_identifiers", []): + if ( + related_identifier.get("resource_type", {}).get("id") + == "publication-thesis" + ): + related_identifier["resource_type"]["id"] = "publication-dissertation" + updated_draft = records_service.update_draft( + system_identity, draft.id, draft.data + ) + secho(f"Draft <{updated_draft.id}> has been updated successfully.", fg="green") + + secho("Resource type update has started.", fg="green") + + run_upgrade( + migrate_resource_type_in_record, + migrate_resource_type_in_draft, + ) + + secho("Resource type update has finished.", fg="green") + + +def run_update_doi_metadata_for_resource_type(resource_type_id="publication-thesis"): + """Re-register DataCite DOI metadata for published records of a resource type. + + Use this after changing the resource type's props.datacite_general in your + vocabulary and reloading the fixture. The DataCite serializer reads + props.datacite_general live from the vocabulary, so re-registering each DOI + pushes the new value to DataCite; the records are not otherwise changed. Only + version DOIs are refreshed here (pass parent=True to also update the concept + DOI). + """ + secho( + f"DOI metadata update for resource type {resource_type_id} has started.", + fg="green", + ) + + errored_record_ids = [] + + published_records = ( + RecordsSearchV2(index=records_service.record_cls.index._name) + .filter( + "query_string", + query=f"metadata.resource_type.id:{resource_type_id} OR metadata.related_identifiers.resource_type.id:{resource_type_id}", + ) + .source(["id"]) + .scan() + ) + # Convert the search results to a list to avoid keeping the scroll context open, as it errors out after 15 minutes + published_record_ids = [result["id"] for result in published_records] + for record_id in published_record_ids: + try: + record = records_service.read(system_identity, record_id) + # Only records that already have a DOI can have their metadata updated + if not (record._record.get("pids") or {}).get("doi"): + secho( + f"Skipping record <{record_id}> because it has no DOI!", + fg="yellow", + ) + continue + records_service.pids.register_or_update( + system_identity, record_id, "doi", parent=False + ) + secho( + f"DOI metadata for record {record_id} has been updated successfully.", + fg="green", + ) + except Exception as error: + secho(f"> Error {repr(error)}", fg="red") + secho(f"Record {record_id} failed to update", fg="red") + errored_record_ids.append((record_id, error)) + + if len(errored_record_ids) > 0: + secho(f"Errored record IDs: {errored_record_ids}", fg="red") + else: + secho("DOI metadata has been updated successfully", fg="green") diff --git a/docs/releases/v14/upgrade-v14.0.md b/docs/releases/v14/upgrade-v14.0.md index 687acfdc..d06fbe80 100644 --- a/docs/releases/v14/upgrade-v14.0.md +++ b/docs/releases/v14/upgrade-v14.0.md @@ -241,7 +241,6 @@ invenio shell $(find $(dirname $(dirname $(uv python find)))/lib/*/site-packages # if using pipenv invenio shell $(find $(pipenv --venv)/lib/*/site-packages/invenio_app_rdm -name migrate_13_to_14.py) ``` -TODO: create migrate_13_to_14.py script #### OAuth client changes @@ -249,11 +248,11 @@ The `extra_data` column of the `oauthclient_remoteaccount` table, storing remote This gives significant performance improvements when running certain queries. An automated Alembic migration is included and has been executed when you ran the [database migration](#apply-database-migrations) step above. -However, if your `oauthclient_remoteaccount` table has more than ~50k rows and you are unable to take the system offline offline for an update, this operation could overload your database and create a lock lasting several minutes, due to the need to individually transform every row. +However, if your `oauthclient_remoteaccount` table has more than ~50k rows and you are unable to take the system offline for an update, this operation could overload your database and create a lock lasting several minutes, due to the need to individually transform every row. To avoid issues in such cases, we recommend instead running the migration manually. Please follow [the upgrade guide](https://invenio-oauthclient.readthedocs.io/en/latest/upgrading.html#v6-0-0). -### Update document engine mappings and content +### Update search engine mappings and content Many mappings have been updated in this release. You can perform granular changes if you want to avoid downtimes or simply discard and rebuild the indices in one go. @@ -611,3 +610,74 @@ you want to keep the existing data. See also the [documentation on how to configure the new module](../../operate/customize/code_archival.md). That's it, you have upgraded to InvenioRDM v14! + +## Align "Thesis" and "Dissertation" resource types — optional + +Your upgrade is complete. This last section describes an **entirely optional** change that is not part of the upgrade. Nothing here affects your instance unless you choose to run it, and you can do so at any later time. Because resource types are a highly visible and commonly customized vocabulary, we suggest rather than impose this change. Decide together with your instance's stakeholders (librarians, curators) whether it fits your data before applying it. + +### What changed and why + +With InvenioRDM v14, the default resource types "Publication / Thesis" (id `publication-thesis`) and "Publication / Dissertation" (id `publication-dissertation`) were merged into one: "Publication / Thesis" (id `publication-dissertation`). This resource type maps to Datacite's `resourceTypeGeneral` "Dissertation" (`datacite_general: Dissertation` in the InvenioRDM's default resource type YAML file). The separate `publication-thesis` entry is dropped. + +InvenioRDM interprets [Datacite's Dissertation](https://datacite-metadata-schema.readthedocs.io/en/4.7/appendices/appendix-1/resourceTypeGeneral/#dissertation) as covering both former entries, so a single entry mapping to "Dissertation" was more accurate. Datacite's `resourceTypeGeneral` "Text" is not as precise in this context. Staying close to the DataCite schema as a default is a core goal of InvenioRDM. + +If you deliberately want to keep both types (for example `publication-dissertation` for PhD work and `publication-thesis` for the rest), or you have customized their DataCite mappings, you may prefer to keep your current setup or apply only part of this change. Skipping this section does not affect your InvenioRDM installation. + +### Applying the change + +If you decide to go ahead, follow whichever of the two options below matches your instance. + +Both rely on the [`migrate_thesis_to_dissertation.py`](./migrate_thesis_to_dissertation.py) helper. This is a set of **reference functions**, not a ready-to-run tool: read it, and copy or adapt the parts that fit your case. One convenient way to use it as-is is to download it and load its functions into an `invenio shell`: + +```bash +curl -LsSf https://raw.githubusercontent.com/inveniosoftware/docs-invenio-rdm/master/docs/releases/v14/migrate_thesis_to_dissertation.py -o /tmp/migrate_thesis_to_dissertation.py +``` + +```python +# in `invenio shell` +exec(open("/tmp/migrate_thesis_to_dissertation.py").read()) +``` + +Always test against a copy of your data first. + +#### Option A: adopt the new default `publication-dissertation` + +Use this if you want to switch your `publication-thesis` records over to the new default `publication-dissertation` resource type, id and all. It rewrites your records and drafts, re-registers their DataCite DOIs, and re-indexes them. + +1. If you are using a customized list of resource types in `/app_data/vocabularies/resource_types.yaml`, then: + 1. Set `title.` to "Thesis" (in the appropriate language) for the entry with `id` equal or equivalent to `publication-dissertation`. + 2. Remove the entry with `id` equal or equivalent to `publication-thesis`. + +2. Apply the resource types change: + - `invenio rdm-records add-to-fixture resourcetypes` + - Note that this changes the vocabulary, but does not delete `publication-thesis` from your data stores. Deletion is done in step 4. + +3. Rewrite every existing record and draft from `publication-thesis` to `publication-dissertation`. The `run_update_for_resource_type` function rewrites the resource type (including in related identifiers), re-registers DataCite DOI metadata, and re-indexes the affected records: + + ```python + # in `invenio shell`, with the helper loaded (see above) + run_update_for_resource_type() + ``` + +4. Delete the `publication-thesis` vocabulary entry via `invenio shell`: + + ```python + from invenio_vocabularies.proxies import current_service as vocabulary_service + vocabulary_service.delete(system_identity, ('resourcetypes', 'publication-thesis')) + ``` + +#### Option B: keep your resource type, map it to DataCite's "Dissertation" + +Use this if you want to keep your own resource type (for example your existing `publication-thesis`, or any custom type) but have its DataCite DOIs use `Dissertation` as a value instead of `Text`. Your records are not changed, only the DataCite metadata of their DOIs is updated. + +1. In `/app_data/vocabularies/resource_types.yaml`, set `props.datacite_general` to `Dissertation` for your resource type entry (and adjust `props.datacite_type` if you set one). + +2. Apply the vocabulary change: + - `invenio rdm-records add-to-fixture resourcetypes` + +3. Re-register the DataCite DOI metadata of every published record of that resource type, so DataCite reflects the new value. The `run_update_doi_metadata_for_resource_type` function does this without changing the records (replace `publication-thesis` with your resource type id): + + ```python + # in `invenio shell`, with the helper loaded (see above) + run_update_doi_metadata_for_resource_type("publication-thesis") + ``` diff --git a/docs/releases/v14/version-v14.0.md b/docs/releases/v14/version-v14.0.md index d64bc97a..07516f5d 100644 --- a/docs/releases/v14/version-v14.0.md +++ b/docs/releases/v14/version-v14.0.md @@ -213,6 +213,7 @@ Here is a quick summary of the other improvements in this release: - Temporarily pinned `bcrypt<5.0.0` due to compatibility issues ([flask-security-fork#82](https://github.com/inveniosoftware/flask-security-fork/pull/82)). Will be lifted in a future release. - A new configuration variable, `RDM_RECORDS_RELATED_IDENTIFIERS_SCHEMES`, enables configuring identifier schemes specifically for related identifiers, defaulting to `RDM_RECORDS_IDENTIFIERS_SCHEMES` when not defined. - Deposit form: "Creators" label was changed to "Authors" to clarify that they appear in citations. +- Resource types: the default vocabulary now labels `publication-dissertation` as "Thesis" and drops the separate `publication-thesis` entry, so DataCite DOIs use the standard `Dissertation` value instead of a custom `Text` one. Migrating your existing records to this change is entirely optional; see [aligning the "Thesis" and "Dissertation" resource types](./upgrade-v14.0.md#align-thesis-and-dissertation-resource-types-optional) in the upgrade guide. - A new configuration variable, `RDM_RECORDS_REQUIRE_SECRET_LINKS_EXPIRATION`, controls whether an expiration date must be set for access links and secret links. Defaults to `FALSE` when not defined. - Addition of support for Wikidata identifiers (QIDs) for creators and contributors of records and their affiliations. - Addition of an HTTP User-Agent helper (`invenio_user_agent`) for outbound HTTP requests in `invenio-vocabularies` datastreams.