Commit triggered by a change on the main branch of helm-charts-dev

EOEPCA · Jul 3, 2023 · 678ecef · 678ecef
1 parent 041d6a2
commit 678ecef
Show file tree

Hide file tree

Showing 2 changed files with 118 additions and 84 deletions.
diff --git a/charts/ades/Chart.yaml b/charts/ades/Chart.yaml
@@ -15,9 +15,9 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 2.0.11
+version: 2.0.18
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: 2.0.13
+appVersion: 2.0.17
diff --git a/charts/ades/values.yaml b/charts/ades/values.yaml
@@ -8,7 +8,7 @@ image:
   repository: eoepca/proc-ades
   pullPolicy: IfNotPresent
   # Overrides the image tag whose default is the chart appVersion.
-  tag: "2.0.13"
+  tag: "2.0.17"
   proxyRepository: eoepca/kubectl-proxy
   proxyTag: "0.9.0"
 
@@ -96,78 +96,106 @@ workflowExecutor:
     APP: ades
 
   main:
-    cwl: ""    
+      cwl: |
+          class: Workflow
+          $namespaces:
+              cwltool: http://commonwl.org/cwltool#
+          doc: Main stage manager
+          id: main
+          label: macro-cwl
+          inputs: {}
+          outputs: {}
+          hints:
+              "cwltool:Secrets":
+                  secrets: []
+          requirements:
+              SubworkflowFeatureRequirement: {}
+              ScatterFeatureRequirement: {}
+              InlineJavascriptRequirement: {}
 
   stageout:
     cwl: |
-      cwlVersion: v1.0
-      baseCommand: Stars
-      doc: "Run Stars for staging results"
-      class: CommandLineTool
-      hints:
-        DockerRequirement:
-          dockerPull: terradue/stars:2.3.0
-        "cwltool:Secrets":
-          secrets:
-            - ADES_STAGEOUT_AWS_SERVICEURL
-            - ADES_STAGEOUT_AWS_REGION
-            - ADES_STAGEOUT_AWS_ACCESS_KEY_ID
-            - ADES_STAGEOUT_AWS_SECRET_ACCESS_KEY
-      id: stars
-      arguments:
-        - copy
-        - -v
-        - -r
-        - '4'
-        - -o
-        - $( inputs.ADES_STAGEOUT_OUTPUT + "/" + inputs.process )
-        - valueFrom: |
-            ${
-                if( !Array.isArray(inputs.wf_outputs) )
-                {
-                    return inputs.wf_outputs.path + "/catalog.json";
-                }
-                var args=[];
-                for (var i = 0; i < inputs.wf_outputs.length; i++)
-                {
-                    args.push(inputs.wf_outputs[i].path + "/catalog.json");
-                }
-                return args;
-            }
-      inputs:
-        ADES_STAGEOUT_AWS_SERVICEURL:
-          type: string?
-        ADES_STAGEOUT_AWS_ACCESS_KEY_ID:
-          type: string?
-        ADES_STAGEOUT_AWS_SECRET_ACCESS_KEY:
-          type: string?
-        ADES_STAGEOUT_OUTPUT:
-          type: string?
-        ADES_STAGEOUT_AWS_REGION:
-          type: string?
-        process:
-          type: string
-      outputs:
-        s3_catalog_output:
-          outputBinding:
-            outputEval: ${ return inputs.ADES_STAGEOUT_OUTPUT + "/" + inputs.process + "/catalog.json"; }
-          type: string
-      requirements:
-        InlineJavascriptRequirement: {}
-        EnvVarRequirement:
-          envDef:
-            PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-            AWS__ServiceURL: $(inputs.ADES_STAGEOUT_AWS_SERVICEURL)
-            AWS__Region: $(inputs.ADES_STAGEOUT_AWS_REGION)
-            AWS__AuthenticationRegion: $(inputs.ADES_STAGEOUT_AWS_REGION)
-            AWS_ACCESS_KEY_ID: $(inputs.ADES_STAGEOUT_AWS_ACCESS_KEY_ID)
-            AWS_SECRET_ACCESS_KEY: $(inputs.ADES_STAGEOUT_AWS_SECRET_ACCESS_KEY)
-        ResourceRequirement: {}
+          cwlVersion: v1.0
+          baseCommand: ['/bin/bash', 'stageout.sh']
+          doc: "Run Stars for staging results"
+          class: CommandLineTool
+          hints:
+            DockerRequirement:
+              dockerPull: terradue/stars:2.3.1
+            "cwltool:Secrets":
+              secrets:
+              - ADES_STAGEOUT_AWS_SERVICEURL
+              - ADES_STAGEOUT_AWS_REGION
+              - ADES_STAGEOUT_AWS_ACCESS_KEY_ID
+              - ADES_STAGEOUT_AWS_SECRET_ACCESS_KEY
+          id: stars
+          arguments:
+            - copy
+            - -v
+            - -r
+            - '4'
+            - -o
+            - $( inputs.ADES_STAGEOUT_OUTPUT + "/" + inputs.process )
+            - -res
+            - $( inputs.process + ".res" )
+            - valueFrom: |
+                      ${
+                          if( !Array.isArray(inputs.wf_outputs) ) 
+                          {
+                              return inputs.wf_outputs.path + "/catalog.json";
+                          }
+                          var args=[];
+                          for (var i = 0; i < inputs.wf_outputs.length; i++) 
+                          {
+                              args.push(inputs.wf_outputs[i].path + "/catalog.json");
+                          }
+                          return args;
+                      }
+          inputs: 
+            ADES_STAGEOUT_AWS_PROFILE:
+              type: string?
+            ADES_STAGEOUT_AWS_SERVICEURL: 
+              type: string?
+            ADES_STAGEOUT_AWS_ACCESS_KEY_ID: 
+              type: string?
+            ADES_STAGEOUT_AWS_SECRET_ACCESS_KEY: 
+              type: string?
+            aws_profiles_location:
+              type: File?
+            ADES_STAGEOUT_OUTPUT:
+              type: string?
+            ADES_STAGEOUT_AWS_REGION:
+              type: string?
+            process:
+              type: string?
+          outputs: 
+            StacCatalogUri:
+              outputBinding:
+                outputEval: ${  return inputs.ADES_STAGEOUT_OUTPUT + "/" + inputs.process + "/catalog.json"; }
+              type: string
+          requirements:
+            InitialWorkDirRequirement:
+              listing:
+              - entryname: stageout.sh
+                entry: |-
+                  #!/bin/bash
+                  export AWS__ServiceURL=$(inputs.ADES_STAGEOUT_AWS_SERVICEURL)
+                  export AWS__Region=$(inputs.ADES_STAGEOUT_AWS_REGION)
+                  export AWS__AuthenticationRegion=$(inputs.ADES_STAGEOUT_AWS_REGION)
+                  export AWS_ACCESS_KEY_ID=$(inputs.ADES_STAGEOUT_AWS_ACCESS_KEY_ID)
+                  export AWS_SECRET_ACCESS_KEY=$(inputs.ADES_STAGEOUT_AWS_SECRET_ACCESS_KEY)
+                  Stars $@
+            InlineJavascriptRequirement: {}
+            EnvVarRequirement:
+              envDef:
+                PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+            ResourceRequirement: {}
+
+
 
   stagein:
     cwl: |
       cwlVersion: v1.0
-      baseCommand: Stars
       doc: "Run Stars for staging input data"
       class: CommandLineTool
       hints:
@@ -179,15 +207,6 @@ workflowExecutor:
           - ADES_STAGEIN_AWS_ACCESS_KEY_ID
           - ADES_STAGEIN_AWS_SECRET_ACCESS_KEY
       id: stars
-      arguments:
-      - copy
-      - -v
-      - -rel
-      - -r
-      - '4'
-      - -o
-      - ./
-      - --harvest
       inputs:
         ADES_STAGEIN_AWS_SERVICEURL:
           type: string?
@@ -196,15 +215,27 @@ workflowExecutor:
         ADES_STAGEIN_AWS_SECRET_ACCESS_KEY:
           type: string?
       outputs: {}
+      baseCommand: ['/bin/bash', 'stagein.sh']
       requirements:
+        InitialWorkDirRequirement:
+          listing:
+          - entryname: stagein.sh
+            entry: |-
+              #!/bin/bash
+              export AWS__ServiceURL=$(inputs.ADES_STAGEIN_AWS_SERVICEURL)
+              export AWS_ACCESS_KEY_ID=$(inputs.ADES_STAGEIN_AWS_ACCESS_KEY_ID)
+              export AWS_SECRET_ACCESS_KEY=$(inputs.ADES_STAGEIN_AWS_SECRET_ACCESS_KEY)
+              url=$1
+              if curl --output /dev/null --silent --head --fail "$url"; then
+                echo "URL: $url"
+              else
+                echo "URL does not exist: $url"
+                exit 1
+              fi
+              Stars copy -v -rel -r 4 -o ./ --harvest $url
         EnvVarRequirement:
           envDef:
             PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-            # AWS__Profile: $(inputs.aws_profile)
-            # AWS__ProfilesLocation: $(inputs.aws_profiles_location.path)
-            AWS__ServiceURL: $(inputs.ADES_STAGEIN_AWS_SERVICEURL)
-            AWS_ACCESS_KEY_ID: $(inputs.ADES_STAGEIN_AWS_ACCESS_KEY_ID)
-            AWS_SECRET_ACCESS_KEY: $(inputs.ADES_STAGEIN_AWS_SECRET_ACCESS_KEY)
         ResourceRequirement: {}
 
   rulez:
@@ -314,19 +345,22 @@ workflowExecutor:
   resourceManagerWorkspacePrefix: "rm-user"
   platformDomain: "https://platform-eoepca.com"
 
+  # path to the username value in the JWT token payload
+  usernameJwtJsonPath: "user_name"
+
   # adds a label to the job namespace
   # jobNamespaceLabels:
   #   app: "ades-app"
 
 
   # custom backoff limit for calrissian job
-  backofflimit: 3
+  backofflimit: 2
 
   # translate k8s error messages
   # available placeholders: $namespace $steps_exit_codes
   errorMessages:
     "Job has reached the specified backoff limit": "Unexpected application error occurred. ( namespace: $namespace , exit codes:  $steps_exit_codes )"
-
+    "namespace not found": "Job namespace $namespace does not exist or has been removed."
 
   # Calrissian image tag
   calrissianImage: "terradue/calrissian:0.12.0"