diff --git a/_data/documentation-dev.yml b/_data/documentation-dev.yml index 5e6730c..7ade27a 100644 --- a/_data/documentation-dev.yml +++ b/_data/documentation-dev.yml @@ -1,9 +1,4 @@ - id: 1 title: OVERVIEW description: Overview - path: /documentation/dev/overview - -- id: 2 - title: OPENAPI MODEL GENERATION - description: OpenAPI Model Generation (E.g. for accessing Ingestor API) - path: /documentation/dev/openapigen \ No newline at end of file + path: /documentation/dev/overview \ No newline at end of file diff --git a/_data/documentation-op.yml b/_data/documentation-op.yml index 704bcb7..ab97f9d 100644 --- a/_data/documentation-op.yml +++ b/_data/documentation-op.yml @@ -14,15 +14,16 @@ path: /documentation/admin/req-infrastructure - id: 4 - title: INSTALLATION - description: Installation + title: INGESTOR INSTALLATION + description: Ingestor Installation and Configuration # Note trailing slash needed since installation has subpages - path: /documentation/admin/installation/ + path: /documentation/admin/installation/ingestor - id: 5 - title: ADMINISTRATION - description: Administration & Maintenance - path: /documentation/admin/admin-maintenance + title: GLOBUS CONNECT SERVER INSTALLATION + description: Globus Connect Server Installation and Configuration + # Note trailing slash needed since installation has subpages + path: /documentation/admin/installation/globus - id: 6 title: SUPPORT diff --git a/_data/facilities.yml b/_data/facilities.yml index 0dff927..470f1cc 100644 --- a/_data/facilities.yml +++ b/_data/facilities.yml @@ -24,7 +24,7 @@ url: https://www.psi.ch/en/emf contacts: - name: Data Catalog Admins - email: scicat-help@l.psi.ch + email: scicat-help@list.psi.ch facilityBackend: https://emf-ingestor.psi.ch/ - name: Dubochet Center for Imaging Lausanne diff --git a/_data/installation.yml b/_data/installation.yml deleted file mode 100644 index 9a753dd..0000000 --- a/_data/installation.yml +++ /dev/null @@ -1,9 +0,0 @@ -# Navigation for the installation section -- id: 1 - title: Globus - description: Set up Globus for data transfer - path: /documentation/admin/installation/globus -- id: 2 - title: Ingestor - description: How to install and configure the Ingestor-Backend - path: /documentation/admin/installation/ingestor diff --git a/about.md b/about.md index b8af101..1e55c5b 100644 --- a/about.md +++ b/about.md @@ -31,7 +31,7 @@ OpenEM will target both researchers producing EM data and consumers of open data Our primary goal is to strengthen co-operation and innovation in the field of electron microscopy. -OpenEM enables collaboration and data exchange in specific groups by providing a platform for data exchange and streamlining data collection processes. OpenEM ensures integration with globally recognised platforms and closes gaps in functionality and application. +OpenEM enables collaboration and data exchange in specific groups by providing a platform for data exchange and streamlining data collection processes. OpenEM ensures integration with globally recognized platforms and closes gaps in functionality and application. ### Partners & Collaborations diff --git a/assets/files/ext_transfer.patch b/assets/files/ext_transfer.patch deleted file mode 100644 index e39b9ca..0000000 --- a/assets/files/ext_transfer.patch +++ /dev/null @@ -1,373 +0,0 @@ -diff --git a/.env b/.env -index bb07f22..5236b59 100644 ---- a/.env -+++ b/.env -@@ -2,7 +2,7 @@ - BE_VERSION=v4 - - ## Enable JOBS feature (disable if required or set in command line). To later disable, either unset or set to an empty value --# JOBS_ENABLED=true -+JOBS_ENABLED=true - - ## Enable v4 ELASTIC feature (disable if required or set in command line). To later disable, either unset or set to an empty value - # ELASTIC_ENABLED=true -@@ -11,7 +11,7 @@ BE_VERSION=v4 - # LDAP_ENABLED=true - - ## Enable OIDC authentication backend (disable if required or set in command line). To later disable, either unset or set to an empty value --# OIDC_ENABLED=true -+OIDC_ENABLED=true - - ## Run SciCat services in DEV mode - # DEV=true -diff --git a/services/backend/services/keycloak/config/facility-realm.json b/services/backend/services/keycloak/config/facility-realm.json -index 19a5f28..8e40106 100644 ---- a/services/backend/services/keycloak/config/facility-realm.json -+++ b/services/backend/services/keycloak/config/facility-realm.json -@@ -38,6 +38,7 @@ - "bruteForceProtected" : false, - "permanentLockout" : false, - "maxTemporaryLockouts" : 0, -+ "bruteForceStrategy" : "MULTIPLE", - "maxFailureWaitSeconds" : 900, - "minimumQuickLoginWaitSeconds" : 60, - "waitIncrementSeconds" : 60, -@@ -245,6 +246,31 @@ - "containerId" : "ef4efef1-7641-4041-a394-865527c807a7", - "attributes" : { } - } ], -+ "ingestor" : [ { -+ "id" : "ed44a221-60cc-4e15-847a-b971e9a10694", -+ "name" : "FAC_ingestor_write", -+ "description" : "", -+ "composite" : false, -+ "clientRole" : true, -+ "containerId" : "9c2a2827-d35c-4351-9d30-c293f0659d37", -+ "attributes" : { } -+ }, { -+ "id" : "962794ac-4013-4b45-9c4f-b9db03dcb1d5", -+ "name" : "FAC_ingestor_read", -+ "description" : "", -+ "composite" : false, -+ "clientRole" : true, -+ "containerId" : "9c2a2827-d35c-4351-9d30-c293f0659d37", -+ "attributes" : { } -+ }, { -+ "id" : "7890bf0e-ac1e-4c5b-a8d7-2fef74e21ecc", -+ "name" : "FAC_ingestor_admin", -+ "description" : "", -+ "composite" : false, -+ "clientRole" : true, -+ "containerId" : "9c2a2827-d35c-4351-9d30-c293f0659d37", -+ "attributes" : { } -+ } ], - "security-admin-console" : [ ], - "admin-cli" : [ ], - "account-console" : [ ], -@@ -336,6 +362,22 @@ - } - }, - "groups" : [ { -+ "id" : "ad1aaa78-dc22-495c-898f-5a2a80442e18", -+ "name" : "DST-FAC-2", -+ "path" : "/DST-FAC-2", -+ "subGroups" : [ ], -+ "attributes" : { }, -+ "realmRoles" : [ ], -+ "clientRoles" : { } -+ }, { -+ "id" : "fe34e5a5-eea1-4a63-b5ce-bd6600b7cba0", -+ "name" : "SRC-FAC-1", -+ "path" : "/SRC-FAC-1", -+ "subGroups" : [ ], -+ "attributes" : { }, -+ "realmRoles" : [ ], -+ "clientRoles" : { } -+ }, { - "id" : "397a50f3-8f47-4d05-8bbc-054e8160bb26", - "name" : "group1", - "path" : "/group1", -@@ -343,6 +385,14 @@ - "attributes" : { }, - "realmRoles" : [ ], - "clientRoles" : { } -+ }, { -+ "id" : "c0acd68c-ede0-4be2-aa17-4aa60933c881", -+ "name" : "ingestor", -+ "path" : "/ingestor", -+ "subGroups" : [ ], -+ "attributes" : { }, -+ "realmRoles" : [ ], -+ "clientRoles" : { } - } ], - "defaultRole" : { - "id" : "cd9651d7-d1b6-4d39-a1ee-9187d687fd8c", -@@ -405,8 +455,11 @@ - "disableableCredentialTypes" : [ ], - "requiredActions" : [ ], - "realmRoles" : [ "default-roles-facility" ], -+ "clientRoles" : { -+ "ingestor" : [ "FAC_ingestor_write", "FAC_ingestor_read" ] -+ }, - "notBefore" : 0, -- "groups" : [ "/group1" ] -+ "groups" : [ "/DST-FAC-2", "/SRC-FAC-1", "/group1", "/ingestor" ] - } ], - "scopeMappings" : [ { - "clientScope" : "offline_access", -@@ -441,6 +494,7 @@ - "frontchannelLogout" : false, - "protocol" : "openid-connect", - "attributes" : { -+ "realm_client" : "false", - "post.logout.redirect.uris" : "+" - }, - "authenticationFlowBindingOverrides" : { }, -@@ -471,6 +525,7 @@ - "frontchannelLogout" : false, - "protocol" : "openid-connect", - "attributes" : { -+ "realm_client" : "false", - "post.logout.redirect.uris" : "+", - "pkce.code.challenge.method" : "S256" - }, -@@ -508,10 +563,12 @@ - "frontchannelLogout" : false, - "protocol" : "openid-connect", - "attributes" : { -+ "realm_client" : "false", -+ "client.use.lightweight.access.token.enabled" : "true", - "post.logout.redirect.uris" : "+" - }, - "authenticationFlowBindingOverrides" : { }, -- "fullScopeAllowed" : false, -+ "fullScopeAllowed" : true, - "nodeReRegistrationTimeout" : 0, - "defaultClientScopes" : [ "web-origins", "acr", "roles", "profile", "basic", "email" ], - "optionalClientScopes" : [ "address", "phone", "offline_access", "microprofile-jwt" ] -@@ -536,6 +593,7 @@ - "frontchannelLogout" : false, - "protocol" : "openid-connect", - "attributes" : { -+ "realm_client" : "true", - "post.logout.redirect.uris" : "+" - }, - "authenticationFlowBindingOverrides" : { }, -@@ -543,6 +601,60 @@ - "nodeReRegistrationTimeout" : 0, - "defaultClientScopes" : [ "web-origins", "acr", "roles", "profile", "email" ], - "optionalClientScopes" : [ "address", "phone", "offline_access", "microprofile-jwt" ] -+ }, { -+ "id" : "9c2a2827-d35c-4351-9d30-c293f0659d37", -+ "clientId" : "ingestor", -+ "name" : "Ingestor", -+ "description" : "", -+ "rootUrl" : "http://localhost:8888/", -+ "adminUrl" : "http://localhost:8888/", -+ "baseUrl" : "http://localhost:8888/", -+ "surrogateAuthRequired" : false, -+ "enabled" : true, -+ "alwaysDisplayInConsole" : false, -+ "clientAuthenticatorType" : "client-secret", -+ "redirectUris" : [ "http://localhost:8888/callback" ], -+ "webOrigins" : [ "http://localhost:8888" ], -+ "notBefore" : 0, -+ "bearerOnly" : false, -+ "consentRequired" : false, -+ "standardFlowEnabled" : true, -+ "implicitFlowEnabled" : false, -+ "directAccessGrantsEnabled" : false, -+ "serviceAccountsEnabled" : false, -+ "publicClient" : true, -+ "frontchannelLogout" : true, -+ "protocol" : "openid-connect", -+ "attributes" : { -+ "realm_client" : "false", -+ "oidc.ciba.grant.enabled" : "false", -+ "backchannel.logout.session.required" : "true", -+ "standard.token.exchange.enabled" : "false", -+ "post.logout.redirect.uris" : "http://localhost:8888/", -+ "oauth2.device.authorization.grant.enabled" : "false", -+ "backchannel.logout.revoke.offline.tokens" : "false" -+ }, -+ "authenticationFlowBindingOverrides" : { }, -+ "fullScopeAllowed" : true, -+ "nodeReRegistrationTimeout" : -1, -+ "protocolMappers" : [ { -+ "id" : "cf327e1e-5bef-4bb6-8d69-e7be2b4601e6", -+ "name" : "accessGroups", -+ "protocol" : "openid-connect", -+ "protocolMapper" : "oidc-group-membership-mapper", -+ "consentRequired" : false, -+ "config" : { -+ "full.path" : "false", -+ "introspection.token.claim" : "true", -+ "userinfo.token.claim" : "true", -+ "id.token.claim" : "true", -+ "lightweight.claim" : "false", -+ "access.token.claim" : "true", -+ "claim.name" : "accessGroups" -+ } -+ } ], -+ "defaultClientScopes" : [ "web-origins", "acr", "roles", "profile", "basic", "email" ], -+ "optionalClientScopes" : [ "address", "phone", "offline_access", "microprofile-jwt" ] - }, { - "id" : "ef4efef1-7641-4041-a394-865527c807a7", - "clientId" : "realm-management", -@@ -564,6 +676,7 @@ - "frontchannelLogout" : false, - "protocol" : "openid-connect", - "attributes" : { -+ "realm_client" : "true", - "post.logout.redirect.uris" : "+" - }, - "authenticationFlowBindingOverrides" : { }, -@@ -602,6 +715,7 @@ - "oauth2.device.authorization.grant.enabled" : "false", - "backchannel.logout.revoke.offline.tokens" : "false", - "use.refresh.tokens" : "true", -+ "realm_client" : "false", - "oidc.ciba.grant.enabled" : "false", - "client.use.lightweight.access.token.enabled" : "false", - "backchannel.logout.session.required" : "true", -@@ -656,11 +770,13 @@ - "frontchannelLogout" : false, - "protocol" : "openid-connect", - "attributes" : { -+ "realm_client" : "false", -+ "client.use.lightweight.access.token.enabled" : "true", - "post.logout.redirect.uris" : "+", - "pkce.code.challenge.method" : "S256" - }, - "authenticationFlowBindingOverrides" : { }, -- "fullScopeAllowed" : false, -+ "fullScopeAllowed" : true, - "nodeReRegistrationTimeout" : 0, - "protocolMappers" : [ { - "id" : "e2e3fab6-4daa-4942-9948-a9924c270ee9", -@@ -850,6 +966,58 @@ - "access.token.claim" : "true" - } - } ] -+ }, { -+ "id" : "e7478055-3f3f-4a84-b57f-2329d1e3a14c", -+ "name" : "service_account", -+ "description" : "Specific scope for a client enabled for service accounts", -+ "protocol" : "openid-connect", -+ "attributes" : { -+ "include.in.token.scope" : "false", -+ "display.on.consent.screen" : "false" -+ }, -+ "protocolMappers" : [ { -+ "id" : "f6cfd980-2549-463a-9ac0-b2a6c683fdd7", -+ "name" : "Client ID", -+ "protocol" : "openid-connect", -+ "protocolMapper" : "oidc-usersessionmodel-note-mapper", -+ "consentRequired" : false, -+ "config" : { -+ "user.session.note" : "client_id", -+ "id.token.claim" : "true", -+ "introspection.token.claim" : "true", -+ "access.token.claim" : "true", -+ "claim.name" : "client_id", -+ "jsonType.label" : "String" -+ } -+ }, { -+ "id" : "07d92653-5f7b-4d10-bb8a-1dd7ea33819f", -+ "name" : "Client IP Address", -+ "protocol" : "openid-connect", -+ "protocolMapper" : "oidc-usersessionmodel-note-mapper", -+ "consentRequired" : false, -+ "config" : { -+ "user.session.note" : "clientAddress", -+ "id.token.claim" : "true", -+ "introspection.token.claim" : "true", -+ "access.token.claim" : "true", -+ "claim.name" : "clientAddress", -+ "jsonType.label" : "String" -+ } -+ }, { -+ "id" : "080e6b22-d987-4b54-acc3-1793a4c6be3a", -+ "name" : "Client Host", -+ "protocol" : "openid-connect", -+ "protocolMapper" : "oidc-usersessionmodel-note-mapper", -+ "consentRequired" : false, -+ "config" : { -+ "user.session.note" : "clientHost", -+ "id.token.claim" : "true", -+ "introspection.token.claim" : "true", -+ "access.token.claim" : "true", -+ "claim.name" : "clientHost", -+ "jsonType.label" : "String" -+ } -+ } ] - }, { - "id" : "f06e1efe-76e1-4374-bb95-4d9ce49ac7c9", - "name" : "offline_access", -@@ -1247,7 +1415,7 @@ - "subType" : "authenticated", - "subComponents" : { }, - "config" : { -- "allowed-protocol-mapper-types" : [ "oidc-address-mapper", "saml-role-list-mapper", "oidc-usermodel-property-mapper", "oidc-full-name-mapper", "oidc-sha256-pairwise-sub-mapper", "saml-user-attribute-mapper", "saml-user-property-mapper", "oidc-usermodel-attribute-mapper" ] -+ "allowed-protocol-mapper-types" : [ "saml-role-list-mapper", "oidc-usermodel-attribute-mapper", "oidc-usermodel-property-mapper", "oidc-address-mapper", "oidc-full-name-mapper", "oidc-sha256-pairwise-sub-mapper", "saml-user-attribute-mapper", "saml-user-property-mapper" ] - } - }, { - "id" : "19ebf56a-47c4-4238-84e7-1bd737c4ca4a", -@@ -1263,7 +1431,7 @@ - "subType" : "anonymous", - "subComponents" : { }, - "config" : { -- "allowed-protocol-mapper-types" : [ "saml-role-list-mapper", "saml-user-property-mapper", "oidc-address-mapper", "oidc-full-name-mapper", "oidc-usermodel-attribute-mapper", "oidc-sha256-pairwise-sub-mapper", "oidc-usermodel-property-mapper", "saml-user-attribute-mapper" ] -+ "allowed-protocol-mapper-types" : [ "saml-user-attribute-mapper", "oidc-usermodel-property-mapper", "saml-role-list-mapper", "oidc-full-name-mapper", "oidc-usermodel-attribute-mapper", "oidc-sha256-pairwise-sub-mapper", "saml-user-property-mapper", "oidc-address-mapper" ] - } - }, { - "id" : "8b2ef3e0-29c0-4760-858c-9cc8978be203", -@@ -1899,22 +2067,23 @@ - "firstBrokerLoginFlow" : "first broker login", - "attributes" : { - "cibaBackchannelTokenDeliveryMode" : "poll", -+ "cibaExpiresIn" : "120", - "cibaAuthRequestedUserHint" : "login_hint", -+ "oauth2DeviceCodeLifespan" : "600", - "clientOfflineSessionMaxLifespan" : "0", - "oauth2DevicePollingInterval" : "5", - "clientSessionIdleTimeout" : "0", -- "clientOfflineSessionIdleTimeout" : "0", -- "cibaInterval" : "5", -- "realmReusableOtpCode" : "false", -- "cibaExpiresIn" : "120", -- "oauth2DeviceCodeLifespan" : "600", - "parRequestUriLifespan" : "60", - "clientSessionMaxLifespan" : "0", -- "organizationsEnabled" : "false" -+ "clientOfflineSessionIdleTimeout" : "0", -+ "cibaInterval" : "5", -+ "realmReusableOtpCode" : "false" - }, -- "keycloakVersion" : "25.0.0", -+ "keycloakVersion" : "26.2.4", - "userManagedAccessAllowed" : false, - "organizationsEnabled" : false, -+ "verifiableCredentialsEnabled" : false, -+ "adminPermissionsEnabled" : false, - "clientProfiles" : { - "profiles" : [ ] - }, -diff --git a/services/backend/services/v4/config/jobConfig.yaml b/services/backend/services/v4/config/jobConfig.yaml -index 2085a79..379c1a6 100644 ---- a/services/backend/services/v4/config/jobConfig.yaml -+++ b/services/backend/services/v4/config/jobConfig.yaml -@@ -48,3 +48,12 @@ jobs: - actions: - - actionType: log - - *rabbitmq -+ - jobType: globus_transfer_job -+ create: -+ auth: "#datasetOwner" -+ actions: -+ - actionType: log -+ update: -+ auth: "archiveManager" -+ actions: -+ - actionType: log -\ No newline at end of file diff --git a/assets/img/documentation/admin/installation/ingestor/img0.png b/assets/img/documentation/admin/installation/ingestor/img0.png deleted file mode 100644 index 492bda0..0000000 Binary files a/assets/img/documentation/admin/installation/ingestor/img0.png and /dev/null differ diff --git a/assets/img/documentation/admin/installation/ingestor/img1.png b/assets/img/documentation/admin/installation/ingestor/img1.png deleted file mode 100644 index 11cf722..0000000 Binary files a/assets/img/documentation/admin/installation/ingestor/img1.png and /dev/null differ diff --git a/assets/img/documentation/admin/installation/ingestor/img10.png b/assets/img/documentation/admin/installation/ingestor/img10.png deleted file mode 100644 index 2e21b6f..0000000 Binary files a/assets/img/documentation/admin/installation/ingestor/img10.png and /dev/null differ diff --git a/assets/img/documentation/admin/installation/ingestor/img11.png b/assets/img/documentation/admin/installation/ingestor/img11.png deleted file mode 100644 index ab5086d..0000000 Binary files a/assets/img/documentation/admin/installation/ingestor/img11.png and /dev/null differ diff --git a/assets/img/documentation/admin/installation/ingestor/img12.png b/assets/img/documentation/admin/installation/ingestor/img12.png deleted file mode 100644 index 5c3571d..0000000 Binary files a/assets/img/documentation/admin/installation/ingestor/img12.png and /dev/null differ diff --git a/assets/img/documentation/admin/installation/ingestor/img13.png b/assets/img/documentation/admin/installation/ingestor/img13.png deleted file mode 100644 index aedc5f8..0000000 Binary files a/assets/img/documentation/admin/installation/ingestor/img13.png and /dev/null differ diff --git a/assets/img/documentation/admin/installation/ingestor/img2.png b/assets/img/documentation/admin/installation/ingestor/img2.png deleted file mode 100644 index a765335..0000000 Binary files a/assets/img/documentation/admin/installation/ingestor/img2.png and /dev/null differ diff --git a/assets/img/documentation/admin/installation/ingestor/img3.png b/assets/img/documentation/admin/installation/ingestor/img3.png deleted file mode 100644 index 231f6fb..0000000 Binary files a/assets/img/documentation/admin/installation/ingestor/img3.png and /dev/null differ diff --git a/assets/img/documentation/admin/installation/ingestor/img4.png b/assets/img/documentation/admin/installation/ingestor/img4.png deleted file mode 100644 index 8be457f..0000000 Binary files a/assets/img/documentation/admin/installation/ingestor/img4.png and /dev/null differ diff --git a/assets/img/documentation/admin/installation/ingestor/img5.png b/assets/img/documentation/admin/installation/ingestor/img5.png deleted file mode 100644 index 3212267..0000000 Binary files a/assets/img/documentation/admin/installation/ingestor/img5.png and /dev/null differ diff --git a/assets/img/documentation/admin/installation/ingestor/img6.png b/assets/img/documentation/admin/installation/ingestor/img6.png deleted file mode 100644 index 5440849..0000000 Binary files a/assets/img/documentation/admin/installation/ingestor/img6.png and /dev/null differ diff --git a/assets/img/documentation/admin/installation/ingestor/img7.png b/assets/img/documentation/admin/installation/ingestor/img7.png deleted file mode 100644 index 428abef..0000000 Binary files a/assets/img/documentation/admin/installation/ingestor/img7.png and /dev/null differ diff --git a/assets/img/documentation/admin/installation/ingestor/img8.png b/assets/img/documentation/admin/installation/ingestor/img8.png deleted file mode 100644 index d6e80c1..0000000 Binary files a/assets/img/documentation/admin/installation/ingestor/img8.png and /dev/null differ diff --git a/assets/img/documentation/admin/installation/ingestor/img9.png b/assets/img/documentation/admin/installation/ingestor/img9.png deleted file mode 100644 index a8b6050..0000000 Binary files a/assets/img/documentation/admin/installation/ingestor/img9.png and /dev/null differ diff --git a/assets/img/documentation/dev/components.drawio.png b/assets/img/documentation/dev/components.drawio.png new file mode 100644 index 0000000..8fbfec1 Binary files /dev/null and b/assets/img/documentation/dev/components.drawio.png differ diff --git a/documentation.md b/documentation.md index e169a76..fbdfe0e 100644 --- a/documentation.md +++ b/documentation.md @@ -16,10 +16,6 @@ tags: ### Documentation -Here you will find a table of contents of the available documentation. - -Click directly on a chapter to jump to it. - - [**User Manual**](/documentation/user/overview) {%- for item in site.data.documentation-user %} 1. {{ item.description }} diff --git a/documentation/admin/admin-maintenance.md b/documentation/admin/admin-maintenance.md deleted file mode 100644 index 6b5398c..0000000 --- a/documentation/admin/admin-maintenance.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -layout: page -title: Administration & Maintenance -permalink: /documentation/admin/admin-maintenance ---- - - -{% include documentationStepper/stepper.html %} - -## Administration & Maintenance - -In this chapter you will find an explanation of the relationships between Ingestor, Metadata Extractor and Scicat. The explanation is the basis for the configuration options of the Ingestor. - -It is generally possible to customise Ingestor via an admin UI or config files. - -### Interaction between Ingestor, metadata extractor and SciCat - -
-
- Workflow Sequence -
- -### Updating the ingestor - -This section is still under development and will be delivered soon. - -### Install metadata extractors - -This section is still under development and will be delivered soon. - -### Manage transfer orders - -This section is still under development and will be delivered soon. - -### Customise basic configuration - -This section is still under development and will be delivered soon. - - -{% include documentationStepper/forwardBackward.html showBack=true showNext=true %} diff --git a/documentation/admin/general.md b/documentation/admin/general.md index ad05c8a..f8aa437 100644 --- a/documentation/admin/general.md +++ b/documentation/admin/general.md @@ -28,7 +28,7 @@ The long-term storage is managed by CSCS or ETHZ. The data is archived there for ### Development -The maintenance and further development of the components is ensured by PSI and the open source community. Individual further development requests can be realised by the user due to the open source code. +The maintenance and further development of the components is ensured by PSI and the open source community. Individual further development requests can be realized by the user due to the open source code. {% include documentationStepper/forwardBackward.html showBack=true showNext=true %} diff --git a/documentation/admin/installation/globus.md b/documentation/admin/installation/globus.md index ffd3b4d..aa49f09 100644 --- a/documentation/admin/installation/globus.md +++ b/documentation/admin/installation/globus.md @@ -1,19 +1,20 @@ --- layout: page -title: Globus Installation +title: Globus Connect Server Installation permalink: /documentation/admin/installation/globus share-description: Instructions for installing the globus --- + +{% include documentationStepper/stepper.html %} ## Introduction -Globus is the preferred method for transferring data from your facility -to PSI. If you are not transferring data (eg for ETHZ, who use the -[ScopeMArchiver](https://github.com/SwissOpenEM/ScopeMArchiver)), then this step can be -skipped. +OpenEM leverages [Globus](https://www.globus.org) for transferring data from facilities to PSI. -In this step you will install [Globus Connect -Server](https://docs.globus.org/globus-connect-server/v5/) (GCS) on a system with access to +{: .box-note} +If you are not transferring data to PSI (e.g. for ETHZ which uses the [ETHZ Archiving Service](https://github.com/SwissOpenEM/ScopeMArchiver)), please refer to its documentation. + +In this step you will install [Globus Connect Server](https://docs.globus.org/globus-connect-server/v5/) (GCS) on a system with access to your facility data. This can be a transfer server or a VM which mounts the facility data. It should have a good network connection, ideally a 10Gbps connection to both the facility and the SWITCH internet backbone. @@ -26,10 +27,10 @@ docs](https://docs.globus.org/globus-connect-server/v5/). The following TCP ports should be opened in the firewall (see all [firewall rules](/documentation/admin/req-infrastructure#firewall-rules)): -| Port | Direction | IP range | Reason | -|---|---|---|---| -|tcp/443| bidirectional | 54.237.254.192/29 | Globus Control | -|tcp/50000-51000| outgoing | 192.33.126.53 (lx-globus-01.psi.ch)
192.33.126.54 (lx-globus-02.psi.ch) | Globus GridFTP Out | +| Port | Direction | IP range | Reason | +|-----------------|---------------|-----------------------------------------------------------------------------|--------------------| +| tcp/443 | bidirectional | 54.237.254.192/29 | Globus Control | +| tcp/50000-51000 | outgoing | 192.33.126.53 (lx-globus-01.psi.ch)
192.33.126.54 (lx-globus-02.psi.ch) | Globus GridFTP Out | You should assign a domain name for the server (`em-globus.facility.ch` in examples) an provision SSL certificates; see [requirements](/documentation/admin/req-infrastructure#domain-names) @@ -40,8 +41,11 @@ Follow the [Globus Connect Server installation guide](https://docs.globus.org/globus-connect-server/v5/). This will install the Apache web server and the globus. -No subscription features are used by OpenEM. Set up a single Mapped Collection for your -data. +{: .box-note} +No subscription features are used by OpenEM. + +{: .box-note} +Set up a single Mapped Collection for your data. ### Identity Mapping @@ -91,7 +95,7 @@ globus-connect-server storage-gateway update posix \ The PSI globus proxy requires the endpoint to be registered before it will be available for use. Please send the following information to -[scicat-help@l.psi.ch](mailto:scicat-help@l.psi.ch) to register the new endpoint with +[scicat-help@list.psi.ch](mailto:scicat-help@list.psi.ch) to register the new endpoint with OpenEM: - domain name @@ -99,3 +103,5 @@ OpenEM: - facility name The PSI admins will reply with the correct ingestor configuration for data transfer. + +{% include documentationStepper/forwardBackward.html showBack=true showNext=true %} diff --git a/documentation/admin/installation/ingestor.md b/documentation/admin/installation/ingestor.md index de2b499..e2b97fa 100644 --- a/documentation/admin/installation/ingestor.md +++ b/documentation/admin/installation/ingestor.md @@ -4,299 +4,102 @@ title: Ingestor Installation permalink: /documentation/admin/installation/ingestor share-description: Instructions for installing the ingestor for OpenEM --- + +{% include documentationStepper/stepper.html %} ## Installing the ingestor -{: .box-note} -**Note:** This is just an example of installing and running the service. You should adapt this method to your facility's needs. - -### Running through docker - -1. Make sure that docker is up and running -2. Check that you have docker compose -3. Modify the config in the `docker-compose.yaml` file under the `configs.openem-ingestor-config.yaml` section according to your needs (see below). -4. Modify the environment variables according to your needs (remove service user parameters if you are using `ExtGlobus`). -5. Run `docker compose up` when you're done. - -### Running without docker containerization - -1. Install go -2. Create a VM or setup a bare-metal server for the ingestor. You can also run it on your own computer. -3. `git clone git@github.com:SwissOpenEM/Ingestor.git` -4. `cd [REPO_DIR]/cmd/openem-ingestor-service` -5. `go build -o build/ingestor .` -6. setup a config under `build/openem-ingestor-config.yaml` or `~/.config/openem-ingestor/openem-ingestor-config.yaml` -7. set environment variables, if necessary, here -8. `./build/ingestor` launches the ingestor. - -## Create a base configuration for the ingestor +{: .box-warning} +**Prerequisites:** `docker` and `docker compose` installed on target machine; familiarity with `docker` and `docker compose`, see -{: .box-note} -**Note:** Additional information can be found in the [Ingestor documentation](https://github.com/SwissOpenEM/Ingestor/blob/main/configs/ReadMe.md) +### Basic Setup and Configuration -Run the following command: +#### 1. Checkout the deployment repository at ```bash -cp [REPO_DIR]/configs/openem-ingestor-config.yaml [REPO_DIR]/cmd/openem-ingestor-service/build/openem-ingestor-config.yaml +git clone https://github.com/SwissOpenEM/openem-deployment ``` -## Configuring the ingestor - -### SciCat - -The relevant section of the config for Scicat is: - -```yaml -... -Scicat: - Host: "https://scicat.backend/api/v3/" -... -``` - -### Transfer: PSI Globus Transfer Request Service - Recommended - -```yaml -Transfer: - Method: ExtGlobus - ExtGlobus: - TransferServiceUrl: "https://url.at.psi/globus/service" - SrcFacility: "EXAMPLE-FACILITY-1" # "FAC-1" if you're using the default scicatlive setup - DstFacility: "EXAMPLE-FACILITY-2" # "FAC-2" if you're using the default scicatlive setup - CollectionRootPath: "/some/path" # the path at which the Source Globus Collection is mounted (eg. '/home') -``` - -{: .box-note} -**Disable service account check**: using this mode, the `webserver.other.DisableServiceAccountCheck` should be set to `true`, as there's no need for any service account in the Ingestor in this mode. - -### Transfer: Direct Globus Requests - Back-Up Option, not recommended - -```yaml -{%- raw -%} -... -Transfer: - Method: Globus - Globus: - ClientID: "globus-auth-client-id" - ClientSecret: "globus-auth-client-secret[optional]" - RedirectURL: "[insert ingestor frontend url]" - Scopes: - - scope1 - - scope2 - ... - SourceCollectionID: "uuid-of-source-collection" - CollectionRootPath: "/insert/path/here" - DestinationCollectionID: "uuid-of-destination-collection" - DestinationTemplate: "/nacsa/{{ .Username }}/{{ replace .Pid \".\" \"_\" }}/{{ .DatasetFolder }}" -... -{% endraw -%} -``` - -{: .box-note} -**Transfer.Globus.ClientID**: this should be set to the same client-id as the one you'll use in the next paragraph. You need to create a new client on `app.globus.org`, please check out the [globus page]({% link documentation/admin/installation/globus.md %}) for more information - -{: .box-note} -**Scopes**: These will include scopes for accessing the Globus Connect Server endpoints you want to interact with in the name of the user. Usually, you're only required to specify the following scope for each endpoint: `"urn:globus:auth:scope:transfer.api.globus.org:all[*https://auth.globus.org/scopes/[ENDPOINT ID HERE]/data_access]"` where you replace `[ENDPOINT ID HERE]` with the endpoint's UUID. +#### 2. Modify the `.env.example` and save it as `.env` + +The `.env.example` file contains configuration values; some are specific to the PSI endpoints and some are facility specific values which need to be adapted. + +| Parameter | Example Value | Description | Facility Specific | +|-------------------------------------------|---------------------------------|---------------------------------------------------------------------------------------------------------|-------------------| +| `FACILITY` | `myfacility` | Facility name; used in some naming conventions | Yes | +| `INGESTOR_VERSION` | `v1.0.0 or latest` | Version of the Ingestor | Yes | +| `INGESTOR_DOMAIN` | `https://ingestor.facility.com` | URL to the facility's ingestor | Yes | +| `HOST_COLLECTION_PATH` | `/server/data` | Path to the directory on the host system to the data | Yes | +| `HOST_COLLECTION_NAME` | `DataServer` | Name of the data directory that will appear in the UI | Yes | +| `GLOBUS_SOURCE_FACILITY` | `DCIL` | Globus source facility tag, one of DCIL, UNIBAS, UNIGE, UNIBE | Yes | +| `GLOBUS_COLLECTION_ROOT_PATH` | `/server/data` | Path of the collection passed to Globus, needs to match HOST_COLLECTION_PATH | yes | +| `KEYCLOAK_CLIENT_ID` | `openem-ingestor-DCIL` | Keycloak client for this facility's ingestor, one of openem-ingestor-DCIL, openem-ingestor-UNIBAS, etc. | Yes | +| `LIFESCIENCE_EXTRACTOR_ADDITIONAL_PARAMS` | `--cs 2.7` | Optional, additional parameters for the life science metadata extractor | yes | +| `GLOBUS_DESTINATION_FACILITY` | `PSI` | Destination facility for Globus, one of PSI, PSI_QA, PS_DEV | No | +| `SCICAT_BACKEND_URL` | `https://dacat.psi.ch` | URL of Scicat's backend | No | +| `SCICAT_FRONTEND_URL` | `https://discovery.psi.ch` | URL of Scicat's frontend | No | +| `GLOBUS_TRANSFER_PROXY_URL` | `https://globus-proxy..psi.ch` | URL to the Globus Proxy | No | +| `KEYCLOAK_URL` | `https://kc.psi.ch` | URL to the Keycloak instance | No | +| `KEYCLOAK_REALM` | `awi` | URL to the Keycloak realm | No | + +The complete configuration of the ingestor can be found in the docker-compose file of the ingestor, see . {: .box-warning} -**Warning:** The source and destination endpoint scopes are only intended for Globus Connect Server endpoints. For Globus Connect Personal (GCP), just skip specifying the scope made from its `collection-id`. You have to make sure that the GCP collection is owned by the token's user. - -{: .box-note} -**Service account**: using this mode, the `webserver.other.DisableServiceAccountCheck` should be set to `false`, and a service account must be set using the `INGESTOR_SERVICE_USER_NAME` and `INGESTOR_SERVICE_USER_PASS` environment variables. These are the credentials for an internal SciCat user, which has the right to update any dataset. It is needed in order to safely mark any dataset as archivable in this mode. +It is not generally necessary to modify the configuration except for updating the Metadata Extractor, see section *Advanced Configuration*. -### Auth - -The following section in the config file describes the necessary setup for authentication. Only OIDC is supported for SSO, and we don't provide any internal user login system. - -```yaml -... -WebServer: - Auth: - ... - FrontendUrl: "http://frontend.url" # optional value to set a redirect to a frontend. - OAuth2: - ClientID: "ingestor" - RedirectURL: "http://localhost:8888/callback" - Scopes: - - email - OIDC: - IssuerURL: "http://oidc.provider/" - JWT: - UseJWKS: true - JwksURL: "http://[OIDC_URL]/.../certs" - JwksSignatureMethods: - - RS256 - RBAC: - AdminRole: "ingestor-admin" - CreateModifyTasksRole: "ingestor-write" - ViewTasksRole: "ingestor-read" -... -``` +#### 3. Start the service and verify that it is running without errors -Please make sure the following fields are properly set: +Start in detached mode -- **WebServer.Auth.ClientID**: this is the client id of the ingestor. It should be added to the IdP that you want to use with the ingestor. This id shouldn't be shared with other ingestor instances. Look up your IdP's docs for adding a new client. -- **WebServer.Auth.OAuth2.RedirectURL**: The url at which the ingestor would be deployed. This should be known by you. -- **WebServer.Auth.OIDC.IssuerURL**: the url to the OIDC provider. It should conform to the Discovery spec. In case of Keycloak, it usually looks like `http://[KEYCLOAK_URL]/realms/[REALM_NAME]`. -- **WebServer.Auth.JWT.JwksURL**: It is the JwksURL of the OIDC provider. It is used to provide the client with the current set of public keys. It should have the same base url, but the rest of the path depends on the OIDC provider. In case of Keycloak, it should have the following format: `http://[KEYCLOAK_URL]/realms/[REALM_NAME]/protocol/openid-connect/certs`. If your provider does not support Jwks, then you can set the keys manually as follows: - -```yaml -... - JWT: - UseJWKS: false - Key: "[insert public key here]" - KeySignMethod: "[set the key signature method here]" -... +```bash +docker compose up -d ``` -- **WebServer.Auth.RBAC.[X]Role**: this is where you set your expected role names. It's a way to customize role names, but you can leave them as is. If facilities use shared OAuth2 client-id's (shouldn't be the case) then these roles should contain the name of each facility to make. You should also customize these if your IdP of choice can't separate what roles to map to users based on clientid. These roles specifically give permission to interact with the ingestor endpoints, and nothing else. Accessing datasets is determined by the `AccessGroups` of the user on SciCat. - -{: .box-note} -If you're using the supplied example scicatlive config for testing, the roles are named `FAC_ingestor_[function]` where `[function]` can be "admin", "write" or "read". - -{: .box-note} -**Note:** If your IdP isn't keycloak you have to make sure that the roles are mapped to OAuth2 claims in the same way as Keycloak: `[access_token_jwt].resource_access[(client_id)].roles` - -### Paths +This will start a container with the name `openem-ingestor` +and check its logs -```yaml -... -WebServer: - Paths: - CollectionLocations: - location1: "/some/path/location1" - Projects: "/some/other/path/location2" - ExtractorOutputLocation: "(optional)/location/to/output/temp/files" -... +```bash +docker logs openem-ingestor ``` -- It's important configure `CollectionLocation` as that is where the ingestor will look for to find datasets. -- The ExtractorOutputLocation sets a custom path for the temporary extractor files. Normally they're outputted to /tmp. -- Due to the way the config library works, all location keys will be lowercased. +{: box-warning} +There should be no messages with an `ERROR` tag in the logs -### Metadata Extractors +Verify that the container is reachable by opening -Example config: - -```yaml -... -MetadataExtractors: - InstallationPath: ./extractors/ - SchemasLocation: ./schemas/ - DownloadMissingExtractors: true - Timeout: 2m - Extractors: - - Name: LS - GithubOrg: SwissOpenEM - GithubProject: LS_Metadata_reader - Version: v0.2.8 - Executable: LS_Metadata_reader - Checksum: e8a2abc7a0d8759edf4559e27879b7977000a868a2f7d39b7804ff5e5c0d1559 - ChecksumAlg: sha256 - {% raw -%}CommandLineTemplate: "-i '{{.SourceFolder}}' -o '{{.OutputFile}}'"{% endraw %} - Methods: - - Name: Single Particle - Schema: oscem_schemas.schema.json - Url: "http://some.url/" - - Name: Cellular Tomography - Schema: oscem_cellular_tomo.json - Url: "http://some.url/" - - Name: Tomography - Schema: oscem_tomo.json - Url: "http://some.url/" - - Name: EnvironmentalTomography - Schema: oscem_env_tomo.json - Url: "http://some.url/" -... +```bash +https:///version ``` -- **InstallationPath** determines where the extractors should be downloaded/installed. -- **SchemasLocation** determines where the schemas for extractors reside. -- **DownloadMissingExtractors** sets whether to download extractors automatically from github -- **Timeout** sets the maximal time any extractor should run before timing out -- **Extractors** is the list of extractors. - - if using github for downloading, the following link is used: `https://github.com/[GithubOrg]/[GithubProject].git` - - **Version`** is the *release tag* that will be attempted to be used. - - **Executable** is the file that will be executed. - - **Checksum** is used to verify the integrity of the executable - - **ChecksumAlg** is to define the algorithm used for the checksum (only sha256 is used) - - **CommandLineTemplate** is the command template to use with the executable, it appends a formatted list of paramters. - - **Methods** is where you can define a list of methods that can be used with a particular extractor. - - **Name** is the name of the method - - **Schema** is the metadata schema to use for this method (must exist in **SchemasLocation**) - - **Url** is the url for the schema, it will be used when the schema is not found locally to download it. - -### Metadata Extractor Jobs +in a browser. -This section is for configuring the metadata extractor job system. It is a system to process extraction requests in parallel and in order of requests. +### Advanced Configuration -```yaml -WebServer: - MetadataExtJobs: - ConcurrencyLimit: 4 - QueueSize: 200 -``` - -Where the **ConcurrencyLimit** is the max. number of extractions to be executed in parallel, and **QueueSize** is the max queue size which has FIFO order. -If there are more pending requests than **QueueSize** then those requests will be processed randomly. +{: .box-note} +Detailed information about the configuration of the ingestor can be found in its repo -## Adding the Ingestor to Keycloak +#### Caddy Reverse Proxy {: .box-note} -**Note:** This section assumes you're running the Ingestor locally. -Replace all instances of `http://localhost:8888/` if you've deployed it in some other way. +Installing an additional reverse proxy is not needed in general if Globus Connect Server is installed alongside. -### Keycloak Setup +#### Metadata Extractors -You can use this [patch file](/assets/files/ext_transfer.patch) on the `scicatlive` project's main branch ([commit 296eb79](https://github.com/SciCatProject/scicatlive/tree/296eb79e548b0345a6516e6e95f2b144b5a408e6), if the patch became incompatible with the up-to-date main branch), which will do the following steps automatically with default values. +Updating an extractor requires to change its version and the checksum in the configuration within the `docker-compose.yml`. Both can be found +in the respective release pages of the extractor, e.g. -1. Setup keycloak, preferably with Docker -2. [OPTIONAL] Add another realm where you'll have your ingestor client added. - {% include gallery.html alt="adding a realm" image="/assets/img/documentation/admin/installation/ingestor/img0.png" width="60%" %} -3. Add a new client with the following parameters - {% assign images = "/assets/img/documentation/admin/installation/ingestor/img1.png - /assets/img/documentation/admin/installation/ingestor/img2.png - /assets/img/documentation/admin/installation/ingestor/img3.png" | split: " - " %} - {%- include gallery.html images=images caption="Creating the keycloak client" %} -4. Edit your client and add client-specific roles that match the ones from your Ingestor config - ![adding client roles](/assets/img/documentation/admin/installation/ingestor/img5.png){: style="margin-top: 2em; margin-bottom: 2em;"} -5. Under the client's "Client Scopes" tab, click on `ingestor-dedicated` -6. `Add mapper` button -> "By configuration" -> Group Membership -7. The `token claim name` should be "accessGroups" and `Full group path` should be *turned off* - -{: .box-note} -**Note:** In most cases you will be using some external source of users in Keycloak, in which case, you need to map some claim of the incoming user to the roles that were setup in Step 4. This is not covered in this Install guide as it is highly specific to your own setup. If by any chance you're setting up users directly in Keycloak, you can assign them the roles directly within the Keycloak admin menu. +See for a more detailed description. -The next section is useful for developers only. +Updating schemas can be done by restarting the ingestor if the schema URLs are pointing to `latest` and not a specific version. Otherwise, the ingestor +needs to be stopped, the URL adapted and the ingestor started again. -### Testing with authentication enabled locally (Developers only) +#### User Identity -1. Add a new test user. Don't forget to set a password. - {% assign images = "/assets/img/documentation/admin/installation/ingestor/img6.png - /assets/img/documentation/admin/installation/ingestor/img7.png - /assets/img/documentation/admin/installation/ingestor/img8.png - /assets/img/documentation/admin/installation/ingestor/img9.png" | split: " - " %} - {% assign alts = "Step 1,Step 2,Step 3,Step 4" | split: "," %} - {%- include gallery.html images=images caption="Creating a user" alts=alts%} -2. Assign the read and write roles of the ingestor to this user. - {% assign images = "/assets/img/documentation/admin/installation/ingestor/img10.png - /assets/img/documentation/admin/installation/ingestor/img11.png" | split: " - " %} - {%- include gallery.html images=images caption="Assigning roles"%} -3. Go to [http://localhost:8888/login](http://localhost:8888/login) -4. This will open up the keycloak login page. Use your test user for logging in. - {% include gallery.html alt="login page" image="/assets/img/documentation/admin/installation/ingestor/img12.png" width="60%" %} -5. If everything went well, you should be redirected to `RedirectURL`, and you should - see a "user" cookie associated to the `localhost` domain in your browser's debugger. - If you also have a valid `FrontendUrl` your browser will get redirected to your - Ingestor frontend, where you should be able to interact with the ingestor backend - using the cookie. - ![browser debugger with cookie](/assets/img/documentation/admin/installation/ingestor/img13.png) -6. [OPTIONAL] To test the ingestor's auth directly, copy the cookie value from the browser, then you can use the following curl command: +In case the ingestor needs to run using as a specific user, add the following variables to th `.env` file - ```bash - curl -v --cookie "user=[INSERT COOKIE VALUE HERE]" "localhost:8888/transfer?page=1" - ``` +| Parameter | Example Value | Description | Facility Specific | +|-----------|---------------|-------------|-------------------| +| UID | 1001 | User id | yes | +| GID | 1001 | Group id | yes | - If the auth is successful, you should get an empty list as a reply. +{% include documentationStepper/forwardBackward.html showBack=true showNext=true %} diff --git a/documentation/admin/overview.md b/documentation/admin/overview.md index b706257..6c196da 100644 --- a/documentation/admin/overview.md +++ b/documentation/admin/overview.md @@ -13,7 +13,7 @@ In this section of the website you will find the documentation for the installat maintenance of the facility-related components. The instructions are aimed at the administrators of the OpenEM components that are -managed within the facilites. For questions and problems regarding the infrastructure +managed within the facilities. For questions and problems regarding the infrastructure outside the university, please use the corresponding support channels which can be found on the website. diff --git a/documentation/admin/req-infrastructure.md b/documentation/admin/req-infrastructure.md index 88d6504..8744de2 100644 --- a/documentation/admin/req-infrastructure.md +++ b/documentation/admin/req-infrastructure.md @@ -18,7 +18,7 @@ research institutions. This is where the ingestor is installed and the metadata extractors are stored. | Component | Minimum Requirements | Recommended Requirements | -| ------------- | -------------------- | ------------------------ | +|---------------|----------------------|--------------------------| | Memory | 8 GB | 16 GB or more | | CPU | 4 cores | 8 cores or more | | Network | 1 Gbps | 10 Gbps or more | @@ -45,7 +45,7 @@ The cache storage should provide enough capacity to hold datasets until they can archived; typically a minimum of 30 days is recommended. | Component | Minimum Requirements | Recommended Requirements | -| --------- | -------------------- | ------------------------ | +|-----------|----------------------|--------------------------| | Storage | 50 TB | 100 TB or more | ## Software Requirements @@ -54,7 +54,15 @@ Software requirements for server components ### Operating System -Linux is required for Globus Connect Server. +One of the following Linux distributions supported by GCS is required: + +- Red Hat Enterprise Linux 8, 9, 10 and derivatives +- Rocky Linux, AlmaLinux, Oracle Linux, CentOS Stream +- Fedora 41, 42 +- Debian 11, 12, 13 +- Ubuntu 22.04 LTS, 24.04 LTS, 25.04 +- SUSE Linux Enterprise Server 15.6 +- OpenSUSE Leap 15.6 The ingestor software can run on either Linux or Windows. @@ -84,7 +92,7 @@ The following ports should be open for a standard configuration using globus to data to PSI from the same machine that runs the ingestor. | Service | Port | Source | Destination | Reason | -| -------- | --------------- | ------------------------------------- | --------------------------------------------------------------------------------- | ------------------- | +|----------|-----------------|---------------------------------------|-----------------------------------------------------------------------------------|---------------------| | Globus | tcp/443 | ingestor-server | 54.237.254.192/29 | Globus Control Out | | Globus | tcp/443 | 54.237.254.192/29 | ingestor-server | Globus Control In | | Globus | tcp/50000-51000 | ingestor-server | 192.33.126.53 (lx-globus-01.psi.ch)
192.33.126.54 (lx-globus-02.psi.ch) | Globus GridFTP Out | @@ -93,8 +101,8 @@ data to PSI from the same machine that runs the ingestor. | SciCat | tcp/443 | ingestor-server
User workstations | dacat.psi.ch
dacat-qa.psi.ch[^2]
scicat.development.psi.ch[^2] | SciCat backend | | SciCat | tcp/443 | ingestor-server
User workstations | globus-proxy.psi.ch
globus-proxy.development.psi.ch[^2] | OpenEM globus proxy | -[^1]: Configurable -[^2]: Testing only +[^1]: The port at the work station is configurable and independent of Globus +[^2]: URLs for testing purposes only ### Domain names @@ -115,3 +123,7 @@ See [facility overview](facilities.md) for current facility domain names. {% include documentationStepper/forwardBackward.html showBack=true showNext=true %} + +----------- + +#### References diff --git a/documentation/admin/support.md b/documentation/admin/support.md index be4264e..09845b8 100644 --- a/documentation/admin/support.md +++ b/documentation/admin/support.md @@ -9,7 +9,22 @@ permalink: /documentation/admin/support ### Support -_This section is still under development and will be delivered soon._ +#### First Level Support + +The first contact point for any issues and questions regarding components of OpenEM are the local contacts at facilities + +| Facility | Contact | Email | +|----------------------------------------------|---------|-------| +| BioEM and Nanoimaging Lab (UNIBAS) | | | +| PSI Electron Microscopy Facility (PSI) | | | +| Dubochet Center for Imaging Lausanne (DCI-L) | | | +| ScopeM (ETHZ) | | | +| DCI Bern (UNIBE) | | | +| EMPA (EMPA) | | | + +#### Scicat and further support + +In case issues cannot be resolved by the facility, contact [SciCat Support](mailto:scicat-help@lists.psi.ch) {% include documentationStepper/forwardBackward.html showBack=true showNext=false %} diff --git a/documentation/dev/generateOpenApiModels.md b/documentation/dev/generateOpenApiModels.md deleted file mode 100644 index 68c86c1..0000000 --- a/documentation/dev/generateOpenApiModels.md +++ /dev/null @@ -1,78 +0,0 @@ ---- -layout: page -title: OpenAPI Model Generation -permalink: /documentation/dev/openapigen ---- - - -{% include documentationStepper/stepper.html %} - -We generate the objects from the openapi specification so that the ingestor UI can address the ingestor interfaces. The following instructions explain how. - -# OpenAPI Generator - What is that? - -The OpenAPI Generator is an open source tool that provides code generators to generate client libraries, server stubs and API documentation from an OpenAPI specification (e.g. *openapi.yaml* or *open-api.json*). For more information, visit the [OpenAPI Generator website](https://openapi-generator.tech/). - -The tool works as follows: - -* Input: It takes an OpenAPI specification file (e.g. *openapi.yaml* or *openapi.json*) that describes how the API works, including endpoints, parameters, response structures and data models. -* Generator selection: You choose a generator (e.g. for *TypeScript Angular*). -* Output: The generator automatically creates source code that can be used to communicate with the API or as a framework for the API implementation. - -# How to install the OpenAPI Generator - -## Installation via Docker - -```sh -docker pull openapitools/openapi-generator-cli -``` - -# How do I use the OpenAPI Generator for TypeScript Angular? - -## Prerequisites - -* OpenAPI Generator has been installed -* openapi.yaml is available - -## Generation of Typescript Angular objects - -### Using Docker - -```sh -docker run --rm -v ${PWD}:/local openapitools/openapi-generator-cli generate \ - -i /local/openapi.yaml \ - -g typescript-angular \ - -o /local/out -``` - -* -i: Path of the input file (openapi.yaml) -* -g: Generator type (*typescript-angular*) -* -o: target directory - -### Using the OpenAPI Generator CLI - -```sh -openapi-generator-cli generate \ - -i openapi.yaml \ - -g typescript-angular \ - -o /local/out \ -``` - -Optional specify angular version: - -```sh ---additional-properties=ngVersion=16 -``` - -## Integration into an angular project - -* Copy the generated files from the *./local/out/model* folder into your Angular project (model folder). - -# openapi.yaml as input for OpenAPI Generator - -OpenAPI.yaml contains the following things: - -* Endpoints: Which URL routes are available and what they do. -* HTTP methods: GET, POST, PUT, DELETE etc. -* Parameters: Query parameters, body data, etc. -* Responses: status codes and return data (e.g. JSON objects) diff --git a/documentation/dev/overview.md b/documentation/dev/overview.md index fd9d38d..f10cbc5 100644 --- a/documentation/dev/overview.md +++ b/documentation/dev/overview.md @@ -7,18 +7,40 @@ permalink: /documentation/dev/overview {% include documentationStepper/stepper.html %} -## Overview +## High-level Overview -In this section of the website you will find the documentation for the development of the OpenEM components. +The following diagram shows the components involved in OpenEM and a (simplified) view of typical interactions between them. -### External links +![OpenEM Components](/assets/img/documentation/dev/components.drawio.png) +_**Green**: Newly developed components; **Blue**: Modified existing components; **Grey**: Newly deployed third-party components; **Orange**: External components_ -- [SwissOpenEm Projects](/outreach#opensourceprojects) -- [SciCat Development Guide](https://scicatproject.github.io/documentation/Development/) +{: .box-note} +For simplicity, interactions regarding authentication flow are omitted. Please refer to the [Ingestor](https://github.com/SwissOpenEM/Ingestor) for a detailed description. + +{: .box-note} +For a detailed view of the situation at ETHZ, please refer to the documentation of the [ETHZ Archiving Service](https://www.openem.ch/ScopeMArchiver/). + +## Open Source Projects -## Chapters +For detailed instructions and documentation of the individual components, refer to the respective repositories. -{% include toc_table.md docs=site.data.documentation-dev %} +| Project | Link | +|-----------------------------------------|------------------------------------------------------------------------------------------------------------------| +| Ingestor | [https://github.com/SwissOpenEM/Ingestor](https://github.com/SwissOpenEM/Ingestor) | +| Depositor | [https://github.com/SwissOpenEM/Depositor](https://github.com/SwissOpenEM/Depositor) | +| SciCat Frontend | [https://github.com/SwissOpenEM/scicat-frontend](https://github.com/SwissOpenEM/scicat-frontend) | +| SciCat Backend | [https://github.com/SciCatProject/scicat-backend-next](https://github.com/SciCatProject/scicat-backend-next) | +| SciCat CLI | [https://github.com/paulscherrerinstitute/scicat-cli](https://github.com/paulscherrerinstitute/scicat-cli) | +| ETHZ Archiving Services | [https://github.com/SwissOpenEM/ScopeMArchiver](https://github.com/SwissOpenEM/ScopeMArchiver) | +| Golang Globus transfer library | [https://github.com/SwissOpenEM/globus-transfer-request](https://github.com/SwissOpenEM/globus-transfer-request) | +| Metadata Extraction - Life Sciences | [https://github.com/SwissOpenEM/LS_Metadata_reader](https://github.com/SwissOpenEM/LS_Metadata_reader) | +| Metadata Extraction - Material Sciences | [https://github.com/SwissOpenEM/MS_Metadata_reader](https://github.com/SwissOpenEM/MS_Metadata_reader) | +| OSC-EM format converters | [https://github.com/osc-em/converter-JSON-to-mmCIF](https://github.com/osc-em/converter-JSON-to-mmCIF) | +| OSC-EM Schema | [https://github.com/osc-em/OSCEM_Schemas](https://github.com/osc-em/OSCEM_Schemas) | + +### External links + +- [SciCat Development Guide](https://scicatproject.github.io/documentation/Development/) {% include documentationStepper/forwardBackward.html showBack=false showNext=false %} diff --git a/outreach.md b/outreach.md index 339d776..ebe17fe 100644 --- a/outreach.md +++ b/outreach.md @@ -13,7 +13,7 @@ tags: ### Presentations | Event | Date | More information | -| -------------------------------------------------------------------------------------------------------------- | ------------------------------------- | ------------------------------------------------------- | +|----------------------------------------------------------------------------------------------------------------|---------------------------------------|---------------------------------------------------------| | CCPEM Spring Symposium 2023 | 24-26 April 2023 | [Poster](https://zenodo.org/doi/10.5281/zenodo.7845285) | | Agreeing community standards for cryoET data preservation and reuse: data structure, metadata, ontologies etc. | 17-19 Apr 2024, EMBL-EBI, Hinxton, UK | | | CCPEM Doppio Workshop | 16-18 Jan 2024, Oxford, UK | | @@ -25,24 +25,9 @@ tags: ### Conferences & workshops | Event | Date | More information | -| -------------------------------- | -------------- | ------------------------- | +|----------------------------------|----------------|---------------------------| | OSC-EM Standards Workshop at PSI | 22-23 Feb 2024 | 22 in-person participants | -### Open Source Projects - -| Project | Link | -| --------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | -| Ingestor | [https://github.com/SwissOpenEM/Ingestor](https://github.com/SwissOpenEM/Ingestor) | -| Depositor | [https://github.com/SwissOpenEM/Depositor](https://github.com/SwissOpenEM/Depositor) | -| SciCat Backend | [https://github.com/SciCatProject/scicat-backend-next](https://github.com/SciCatProject/scicat-backend-next) | -| SciCat CLI | [https://github.com/paulscherrerinstitute/scicat-cli](https://github.com/paulscherrerinstitute/scicat-cli) | -| ETHZ Archiving Services | [https://github.com/SwissOpenEM/ScopeMArchiver](https://github.com/SwissOpenEM/ScopeMArchiver) | -| Golang Globus transfer library | [https://github.com/SwissOpenEM/globus-transfer-request](https://github.com/SwissOpenEM/globus-transfer-request) | -| Metadata Extraction - Life Sciences | [https://github.com/SwissOpenEM/LS_Metadata_reader](https://github.com/SwissOpenEM/LS_Metadata_reader) | -| Metadata Extraction - Material Sciences | [https://github.com/SwissOpenEM/MS_Metadata_reader](https://github.com/SwissOpenEM/MS_Metadata_reader) | -| OSC-EM format converters | [https://github.com/osc-em/converter-JSON-to-mmCIF](https://github.com/osc-em/converter-JSON-to-mmCIF) | -| OSC-EM Schema | [https://github.com/osc-em/OSCEM_Schemas](https://github.com/osc-em/OSCEM_Schemas) | - ### Further Publications *Publications describing OpenEM and the OSC-EM standard are in preparation.*