Merge branch 'release-3.2.x'

opencb · Jul 17, 2024 · b99b99b · b99b99b
2 parents 51f0e85 + 055dcae
commit b99b99b
Show file tree

Hide file tree

Showing 642 changed files with 13,641 additions and 5,212 deletions.
diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
@@ -13,23 +13,24 @@ jobs:
     with:
       maven_opts: -Phdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
 
-  test:
-    uses: ./.github/workflows/test-analysis.yml
-    needs: build
-    with:
-      test_profile: runShortTests,runMediumTests
-    secrets: inherit
+## TASK-4970: remove the test job from the develop.yml workflow because it is not needed anymore.
+#  test:
+#    uses: ./.github/workflows/test-analysis.yml
+#    needs: build
+#    with:
+#      test_profile: runShortTests,runMediumTests
+#    secrets: inherit
 
   deploy-maven:
     uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
-    needs: test
+    needs: build
     with:
       maven_opts: -P hdp3.1 -Dopencga.war.name=opencga
     secrets: inherit
 
   deploy-docker:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
-    needs: test
+    needs: build
     with:
       cli: python3 ./build/cloud/docker/docker-build.py push --images base,init
     secrets: inherit
diff --git a/.github/workflows/docker-aws-emr.yml b/.github/workflows/docker-aws-emr.yml
diff --git a/.github/workflows/long-test-analysis.yml b/.github/workflows/long-test-analysis.yml
@@ -14,8 +14,8 @@ jobs:
       matrix:
         hadoop: [ "hdp3.1", "hdi5.1", "emr6.1", "emr6.13" ]
     uses: ./.github/workflows/test-analysis.yml
-    secrets: inherit
     with:
       test_profile: runShortTests,runMediumTests,runLongTests
       hadoop: ${{ matrix.hadoop }}
+    secrets: inherit
 
diff --git a/.github/workflows/manual-deploy-docker.yml b/.github/workflows/manual-deploy-docker.yml
@@ -51,10 +51,9 @@ jobs:
           name: build-folder
           path: build
 
-
   deploy-docker:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
     needs: build
     with:
       cli: python3 ./build/cloud/docker/docker-build.py push --images base,init --tag ${{ inputs.tag }}
-    secrets: inherit
+    secrets: inherit
diff --git a/.github/workflows/manual-deploy-ext-tools.yml b/.github/workflows/manual-deploy-ext-tools.yml
@@ -46,11 +46,9 @@ jobs:
           name: build-folder
           path: build
 
-
   deploy-docker-ext-tools:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop
     needs: build
     with:
       cli: python3 ./build/cloud/docker/docker-build.py push --images ext-tools --tag ${{ inputs.tag }}
     secrets: inherit
-
diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml
@@ -0,0 +1,19 @@
+name: Pull request approve workflow
+
+on:
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  build:
+    uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop
+    with:
+      maven_opts: -Phdp3.1,RClient -Dopencga.war.name=opencga -Dcheckstyle.skip
+
+  test:
+    name: "Run all tests before merging, ie. short, medium and long tests."
+    uses: ./.github/workflows/test-analysis.yml
+    needs: build
+    with:
+      test_profile: runShortTests,runMediumTests,runLongTests
+    secrets: inherit
diff --git a/.github/workflows/pull-request-merge.yml → .github/workflows/pull-request-merged.yml b/.github/workflows/pull-request-merge.yml → .github/workflows/pull-request-merged.yml
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -24,25 +24,25 @@ jobs:
     secrets: inherit
 
   # Add the deploy-maven and deploy-python jobs they depend on the build-hdp job
-#  deploy-maven:
-#    uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
-#    needs: build-hdp
-#    with:
-#      maven_opts: -P hdp3.1 -Dopencga.war.name=opencga
-#    secrets: inherit
-#
-#  deploy-python:
-#    uses: opencb/java-common-libs/.github/workflows/deploy-python-workflow.yml@develop
-#    needs: build-hdp
-#    with:
-#      cli: ./clients/python/python-build.sh push
-#      artifact: build-folder
-#    secrets: inherit
+  deploy-maven:
+    uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop
+    needs: build-hdp
+    with:
+      maven_opts: -P hdp3.1 -Dopencga.war.name=opencga
+    secrets: inherit
+
+  deploy-python:
+    uses: opencb/java-common-libs/.github/workflows/deploy-python-workflow.yml@develop
+    needs: build-hdp
+    with:
+      cli: ./clients/python/python-build.sh push
+      artifact: build-folder
+    secrets: inherit
 
   # Add the release job that depends on all the previous jobs
   release:
     uses: opencb/java-common-libs/.github/workflows/release-github-workflow.yml@develop
-    needs: [ build-hdp, deploy-docker-hdp]
+    needs: [ build-hdp, deploy-docker-hdp, deploy-maven, deploy-python]
     with:
       artifact: build-folder
       file: |

diff --git a/.github/workflows/task.yml b/.github/workflows/task.yml
@@ -17,9 +17,9 @@ jobs:
   test:
     uses: ./.github/workflows/test-analysis.yml
     needs: build
-    secrets: inherit
     with:
       test_profile: runShortTests
+    secrets: inherit
 
   deploy-docker:
     uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop

diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml
@@ -38,7 +38,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: '0'
+          fetch-depth: '10'
       - name: Set up JDK 11
         uses: actions/setup-java@v4
         with:
@@ -65,7 +65,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: '0'
+          fetch-depth: '10'
       - name: Set up JDK 8
         uses: actions/setup-java@v4
         with:
@@ -90,8 +90,8 @@ jobs:
       - name: Maven build
         run: mvn -B clean install -DskipTests -P ${{ inputs.hadoop }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
       - name: Run Junit tests
-        run: mvn -B verify surefire-report:report --fail-never -f ${{ (inputs.module == '' || inputs.module == 'all') && '.' || inputs.module }} -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
-      - name: Publish Test Report
+        run: mvn -B verify surefire-report:report --fail-never -Dsurefire.testFailureIgnore=true -f ${{ (inputs.module == '' || inputs.module == 'all') && '.' || inputs.module }} -P ${{ inputs.hadoop }},${{ inputs.test_profile }} -Dcheckstyle.skip ${{ inputs.mvn_opts }}
+      - name: Publish Test Report on GitHub
         uses: scacap/action-surefire-report@v1
         env:
           NODE_OPTIONS: '--max_old_space_size=4096'

diff --git a/opencga-analysis/pom.xml b/opencga-analysis/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.opencb.opencga</groupId>
         <artifactId>opencga</artifactId>
-        <version>3.1.0</version>
+        <version>3.2.0</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
 
@@ -357,6 +357,17 @@
     </dependencyManagement>
 
     <build>
+        <testResources>
+            <testResource>
+                <directory>../opencga-core/src/test/resources</directory>
+                <includes>
+                    <include>log4j2-test.xml</include>
+                </includes>
+            </testResource>
+            <testResource>
+                <directory>src/test/resources</directory>
+            </testResource>
+        </testResources>
         <plugins>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>

diff --git a/...alysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java b/...alysis/src/main/java/org/opencb/opencga/analysis/alignment/AlignmentCoverageAnalysis.java
@@ -54,6 +54,10 @@ protected void check() throws Exception {
         super.check();
 
         // Sanity check
+        if (StringUtils.isEmpty(getJobId())) {
+            throw new ToolException("Missing job ID");
+        }
+
         if (StringUtils.isEmpty(getStudy())) {
             throw new ToolException("Missing study when computing alignment coverage");
         }
@@ -168,20 +172,22 @@ protected void run() throws Exception {
                         + ") was not create, please, check log files.");
             }
 
-            // Try to copy the BW file into the BAM file directory
+            // Try to move the BW file into the BAM file directory
+            boolean moveSuccessful = false;
             Path targetPath = Paths.get(bamCatalogFile.getUri()).getParent().resolve(bwPath.getFileName());
             try {
-                Files.move(bwPath, targetPath);
+                Path movedPath = Files.move(bwPath, targetPath);
+                moveSuccessful = targetPath.equals(movedPath);
             } catch (Exception e) {
-                // Do nothing
-                logger.info("Moving from {} to {}: {}", bwPath, targetPath, e.getMessage());
+                // Log message
+                logger.info("Error moving the coverage file into the BAM folder {} to {}", bwPath, targetPath, e);
             }
 
-            if (targetPath.toFile().exists()) {
+            if (moveSuccessful) {
                 bwPath = targetPath;
-                logger.info("Coverage file was copied into the BAM folder: {}", bwPath);
+                logger.info("Coverage file was moved into the BAM folder: {}", bwPath);
             } else {
-                logger.info("Couldn't copy the coverage file into the BAM folder. The coverage file is in the job folder instead: {}",
+                logger.info("Couldn't move the coverage file into the BAM folder. The coverage file is in the job folder instead: {}",
                         bwPath);
             }