Various minor tweaks

* MetadataCollection: terminate ingestion on terminal errors even in error mode pass * File: handle versioning when a repository resource exists but lacks binary (update it with the binary without new version creation then) * CI tuning
acdh-oeaw · Sep 20, 2024 · 7909f60 · 7909f60
1 parent ade6e5e
commit 7909f60
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 6 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -20,8 +20,8 @@ jobs:
     - name: clone repo config
       run: |
         git clone --depth 1 --branch arche https://github.com/acdh-oeaw/arche-docker-config.git config
-        chmod +x config/run.d/xdebug.sh
-        rm -f config/run.d/gui.sh config/run.d/oaipmh.sh config/run.d/resolver.sh config/run.d/fcs.sh config/run.d/openrefine.sh
+        chmod -x config/run.d/*sh
+        chmod +x config/run.d/*xdebug.sh config/run.d/*postgresql.sh config/run.d/*config-yaml.sh
         cp tests/config.yaml config/yaml/local.yaml
         cp .github/workflows/30-fixVocabularies.sh config/initScripts/
     - name: run repo docker

diff --git a/src/acdhOeaw/arche/lib/ingest/File.php b/src/acdhOeaw/arche/lib/ingest/File.php
@@ -162,9 +162,14 @@ private function versioningAsync(): RepoResourcePromise | RepoResource {
                 $newVersion = $locModDate > $modDate;
                 break;
             case Indexer::VERSIONING_DIGEST:
-                $hash       = explode(':', (string) $oldMeta->getObject(new PT($schema->hash)));
-                $locHash    = $this->getHash($hash[0]);
-                $newVersion = $hash[1] !== $locHash;
+                $hash       = (string) $oldMeta->getObject(new PT($schema->hash));
+                if (empty($hash)) {
+                    $newVersion = false;
+                } else {
+                    $hash       = explode(':', $hash);
+                    $locHash    = $this->getHash($hash[0]);
+                    $newVersion = $hash[1] !== $locHash;
+                }
                 break;
             case Indexer::VERSIONING_ALWAYS:
                 $newVersion = true;
@@ -178,7 +183,7 @@ private function versioningAsync(): RepoResourcePromise | RepoResource {
 
         // progress meter
         $upload = $this->withinSizeLimit() ? '+ upload ' : '';
-        echo ProgressMeter::format($this->meterId, $this->n, "Processing " . $this->info->getPathname() . " ({n}/{t} {p}%): new version $upload pidPass ".(int)$this->pidPass."\n");
+        echo ProgressMeter::format($this->meterId, $this->n, "Processing " . $this->info->getPathname() . " ({n}/{t} {p}%): new version $upload pidPass " . (int) $this->pidPass . "\n");
 
         // create the new version
         $repoIdNmsp = $this->repo->getBaseUrl();

diff --git a/src/acdhOeaw/arche/lib/ingest/MetadataCollection.php b/src/acdhOeaw/arche/lib/ingest/MetadataCollection.php
@@ -332,6 +332,10 @@ function (RepoResource $repoRes) use ($meta, $progress) {
                         $msg    = $chunk[$n] . ": " . $msg . "(" . get_class($j) . ")";
                         $errors .= "\t$msg\n";
                         echo self::$debug ? "\tERROR while processing $msg\n" : '';
+                        // terminal errors: transaction doesn't exist, database max connections reached
+                        if (preg_match("/Transaction [0-9]+ doesn't exist|SQLSTATE\[08006\]/", $msg)) {
+                            break 2;
+                        }
                     }
                     if ($j instanceof RepoResource || $errorMode === self::ERRMODE_INCLUDE) {
                         $allRepoRes[] = $j;