CrosscheckFingerprint speedup, extra functionality and NIO enabling (broadinstitute#1086)

Getting CrosscheckFingeprints to run on Paths, and a few more changes that were needed to crosscheck 20K samples.

- several speed-ups to the fingerprinting code, especially Crosscheck
- added option to enable mapping samplenames before crosschecking. This option makes it possible to change what the algorithm considers "expected". It is only available for the case where checking INPUT against SECOND_INPUT. One can map either INPUT or SECOND_INPUT names (or both). Some sanity checks are included.
- modified build to make cloud support more simple (compile cloudJar to get cloud support bundled)

Author: Yossi Farjoun

README.md

@@ -83,9 +83,38 @@ During development in Picard, it is sometimes necessary to build locally against
 
 #### Releasing Picard
 
-Full instructions on how to create a new release of Picard are [here](https://github.com/broadinstitute/picard/wiki/How-to-release-Picard)
+Full instructions on how to create a new release of 
+Picard are [here](https://github.com/broadinstitute/picard/wiki/How-to-release-Picard)
 
-----
+#### Path providers
+
+Picard has limited support for reading from Path providers. 
+Currently only google's api is supported, and only a few tools support this.
+To run with this support you need to compile the cloudJar target with gradle:
+```bash
+./gradlew cloudJar
+
+```
+then run picard as follows:
+
+```bash
+java -jar build/lib/picardcloud.jar <Picard arguments starting from program>
+```
+For example:
+
+```bash 
+java -jar build/lib/picardcloud.jar CrosscheckFingerprints \
+   I=gs://sample1.vcf \
+   I=gs://sample2.vcf \
+   CROSSCHECK_BY=FILE \
+   H=Haplotype_db.txt \
+   O=crosscheck.out
+```
+
+Alternatively, you can run the tool via the [GATK](https://software.broadinstitute.org/gatk/download/) which bundles the Google-Cloud
+jar, and should thus "Just Work".
+
+#### GA4GH API
 
 It's also possible to build a version of Picard that supports reading from
 GA4GH API, e.g. Google Genomics:

build.gradle

@@ -48,26 +48,40 @@ jacoco {
 }
 
 final htsjdkVersion = System.getProperty('htsjdk.version', '2.14.3')
+final googleNioVersion= "0.28.0-alpha:shaded"
 
 // Get the jdk files we need to run javaDoc. We need to use these during compile, testCompile,
 // test execution, and gatkDoc generation, but we don't want them as part of the runtime
 // classpath and we don't want to redistribute them in the uber jar.
 final javadocJDKFiles = files(((URLClassLoader) ToolProvider.getSystemToolClassLoader()).getURLs())
 
+configurations {
+    cloudConfiguration {
+        extendsFrom runtime
+    }
+}
+
 dependencies {
     compile('com.intel.gkl:gkl:0.8.2') {
         exclude module: 'htsjdk'
     }
     compile 'com.google.guava:guava:15.0'
     compile 'com.github.samtools:htsjdk:' + htsjdkVersion
     compile 'org.broadinstitute:barclay:2.0.0'
+    configurations.cloudConfiguration {
+        compile ('com.google.cloud:google-cloud-nio:' + googleNioVersion)
+    }
+
+    compileOnly 'com.google.cloud:google-cloud-nio:' + googleNioVersion
 
     // javadoc utilities; compile/test only to prevent redistribution of sdk jars
     compileOnly(javadocJDKFiles)
     testCompile(javadocJDKFiles)
 
     testCompile 'org.testng:testng:6.9.10'
     testCompile 'org.apache.commons:commons-lang3:3.6'
+
+
 }
 
 configurations.all {
@@ -192,6 +206,15 @@ shadowJar {
     }
 }
 
+
+
+task cloudJar(type: com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar)  {
+    configurations = [project.configurations.cloudConfiguration]
+    from project.sourceSets.main.output
+    archiveName 'picardcloud.jar'
+}
+
+
 // Create picardBarclay.jar, which is a identical to picard.jar, but contains a .properties
 // file that tells Picard to use the Barclay command line parser instead of the Picard
 // command line parser.

src/main/java/picard/cmdline/CommandLineProgram.java

@@ -51,6 +51,7 @@
 import picard.cmdline.argumentcollections.OptionalReferenceArgumentCollection;
 import picard.cmdline.argumentcollections.ReferenceArgumentCollection;
 import picard.cmdline.argumentcollections.RequiredReferenceArgumentCollection;
+import picard.nio.PathHelper;
 import picard.util.PropertyUtils;
 
 import java.io.File;
@@ -243,21 +244,33 @@ public int instanceMain(final String[] argv) {
             System.setProperty("java.io.tmpdir", f.getAbsolutePath()); // in loop so that last one takes effect
         }
 
+        PathHelper.initilizeAll();
+
         if (!QUIET) {
             System.err.println("[" + new Date() + "] " + commandLine);
 
             // Output a one liner about who/where and what software/os we're running on
             try {
+                final StringBuilder pathProvidersBuilder = new StringBuilder();
+
+                for (PathHelper.PathProviders providers : PathHelper.PathProviders.values()) {
+                    pathProvidersBuilder.append(String.format("Provider %s is%s available; ",
+                            providers.name(), providers.isAvailable() ? "" : " not"));
+                }
+                final int lastSpacePos = pathProvidersBuilder.lastIndexOf(" ");
+                pathProvidersBuilder.delete(lastSpacePos, lastSpacePos + 1);
+
                 final boolean usingIntelDeflater = (BlockCompressedOutputStream.getDefaultDeflaterFactory() instanceof IntelDeflaterFactory &&
                         ((IntelDeflaterFactory)BlockCompressedOutputStream.getDefaultDeflaterFactory()).usingIntelDeflater());
                 final boolean usingIntelInflater = (BlockGunzipper.getDefaultInflaterFactory() instanceof IntelInflaterFactory &&
                         ((IntelInflaterFactory)BlockGunzipper.getDefaultInflaterFactory()).usingIntelInflater());
                 final String msg = String.format(
-                    "[%s] Executing as %s@%s on %s %s %s; %s %s; Deflater: %s; Inflater: %s; Picard version: %s",
+                    "[%s] Executing as %s@%s on %s %s %s; %s %s; Deflater: %s; Inflater: %s; %s Picard version: %s",
                     new Date(), System.getProperty("user.name"), InetAddress.getLocalHost().getHostName(),
                     System.getProperty("os.name"), System.getProperty("os.version"), System.getProperty("os.arch"),
                     System.getProperty("java.vm.name"), System.getProperty("java.runtime.version"),
                     usingIntelDeflater ? "Intel" : "Jdk", usingIntelInflater ? "Intel" : "Jdk",
+                    pathProvidersBuilder.toString(),
                     getCommandLineParser().getVersion());
                 System.err.println(msg);
             }

src/main/java/picard/fingerprint/CheckFingerprint.java

@@ -47,6 +47,8 @@
 import picard.cmdline.programgroups.DiagnosticsAndQCProgramGroup;
 
 import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
 import java.util.Collections;
 import java.util.List;
 
@@ -169,7 +171,7 @@ public class CheckFingerprint extends CommandLineProgram {
     @Argument(shortName = StandardOptionDefinitions.INPUT_SHORT_NAME, doc = "Input file SAM/BAM or VCF.  If a VCF is used, " +
             "it must have at least one sample.  If there are more than one samples in the VCF, the parameter OBSERVED_SAMPLE_ALIAS must " +
             "be provided in order to indicate which sample's data to use.  If there are no samples in the VCF, an exception will be thrown.")
-    public File INPUT;
+    public String INPUT;
 
     @Argument(optional = true, doc = "If the input is a VCF, this parameters used to select which sample's data in the VCF to use.")
     public String OBSERVED_SAMPLE_ALIAS;
@@ -187,7 +189,7 @@ public class CheckFingerprint extends CommandLineProgram {
 
     @Argument(shortName = "G", doc = "File of genotypes (VCF) to be used in comparison. May contain " +
             "any number of genotypes; CheckFingerprint will use only those that are usable for fingerprinting.")
-    public File GENOTYPES;
+    public String GENOTYPES;
 
     @Argument(shortName = "SAMPLE_ALIAS", optional = true, doc = "This parameter can be used to specify which sample's genotypes to use from the " +
             "expected VCF file (the GENOTYPES file).  If it is not supplied, the sample name from the input " +
@@ -212,10 +214,8 @@ public class CheckFingerprint extends CommandLineProgram {
     public static final String FINGERPRINT_SUMMARY_FILE_SUFFIX = "fingerprinting_summary_metrics";
     public static final String FINGERPRINT_DETAIL_FILE_SUFFIX = "fingerprinting_detail_metrics";
 
-    // Stock main method
-    public static void main(final String[] args) {
-        new CheckFingerprint().instanceMainWithExit(args);
-    }
+    private Path inputPath;
+    private Path genotypesPath;
 
     @Override
     protected int doWork() {
@@ -229,28 +229,34 @@ protected int doWork() {
             outputSummaryMetricsFile = new File(OUTPUT + FINGERPRINT_SUMMARY_FILE_SUFFIX);
         }
 
-        IOUtil.assertFileIsReadable(INPUT);
+        try {
+            inputPath = IOUtil.getPath(INPUT);
+            genotypesPath = IOUtil.getPath(GENOTYPES);
+        } catch (IOException e) {
+            throw new IllegalArgumentException(e);
+        }
+        IOUtil.assertFileIsReadable(inputPath);
         IOUtil.assertFileIsReadable(HAPLOTYPE_MAP);
-        IOUtil.assertFileIsReadable(GENOTYPES);
+        IOUtil.assertFileIsReadable(genotypesPath);
         IOUtil.assertFileIsWritable(outputDetailMetricsFile);
         IOUtil.assertFileIsWritable(outputSummaryMetricsFile);
 
         final FingerprintChecker checker = new FingerprintChecker(HAPLOTYPE_MAP);
         List<FingerprintResults> results;
 
         String observedSampleAlias = null;
-        final boolean isBamOrSamFile = isBamOrSamFile(INPUT);
+        final boolean isBamOrSamFile = isBamOrSam(inputPath);
         if (isBamOrSamFile) {
-            SequenceUtil.assertSequenceDictionariesEqual(SAMSequenceDictionaryExtractor.extractDictionary(INPUT.toPath()), SAMSequenceDictionaryExtractor.extractDictionary(GENOTYPES.toPath()), true);
-            SequenceUtil.assertSequenceDictionariesEqual(SAMSequenceDictionaryExtractor.extractDictionary(INPUT.toPath()), checker.getHeader().getSequenceDictionary(), true);
+            SequenceUtil.assertSequenceDictionariesEqual(SAMSequenceDictionaryExtractor.extractDictionary(inputPath), SAMSequenceDictionaryExtractor.extractDictionary(genotypesPath), true);
+            SequenceUtil.assertSequenceDictionariesEqual(SAMSequenceDictionaryExtractor.extractDictionary(inputPath), checker.getHeader().getSequenceDictionary(), true);
 
             // Verify that there's only one sample in the SAM/BAM.
-            final SamReader in = SamReaderFactory.makeDefault().referenceSequence(REFERENCE_SEQUENCE).open(INPUT.toPath());
+            final SamReader in = SamReaderFactory.makeDefault().referenceSequence(REFERENCE_SEQUENCE).open(inputPath);
             for (final SAMReadGroupRecord rec : in.getFileHeader().getReadGroups()) {
                 if (observedSampleAlias == null) {
                     observedSampleAlias = rec.getSample();
                 } else if (!observedSampleAlias.equals(rec.getSample())) {
-                    throw new PicardException("INPUT SAM/BAM file must not contain data from multiple samples.");
+                    throw new PicardException("inputPath SAM/BAM file must not contain data from multiple samples.");
                 }
             }
             CloserUtil.close(in);
@@ -261,28 +267,28 @@ protected int doWork() {
             }
 
             results = checker.checkFingerprints(
-                    Collections.singletonList(INPUT),
-                    Collections.singletonList(GENOTYPES),
+                    Collections.singletonList(inputPath),
+                    Collections.singletonList(genotypesPath),
                     EXPECTED_SAMPLE_ALIAS,
                     IGNORE_READ_GROUPS);
         } else {            // Input is a VCF
             // Note that FingerprintChecker.loadFingerprints() verifies that the VCF's Sequence Dictionaries agree with that of the Haplotye Map File
 
             // Verify that there is only one sample in the VCF
-            final VCFFileReader fileReader = new VCFFileReader(INPUT, false);
+            final VCFFileReader fileReader = new VCFFileReader(inputPath, false);
             final VCFHeader fileHeader = fileReader.getFileHeader();
             if (fileHeader.getNGenotypeSamples() < 1) {
-                throw new PicardException("INPUT VCF file must contain at least one sample.");
+                throw new PicardException("inputPath VCF file must contain at least one sample.");
             }
             if ((fileHeader.getNGenotypeSamples() > 1) && (OBSERVED_SAMPLE_ALIAS == null)) {
-                throw new PicardException("INPUT VCF file contains multiple samples and yet the OBSERVED_SAMPLE_ALIAS parameter is not set.");
+                throw new PicardException("inputPath VCF file contains multiple samples and yet the OBSERVED_SAMPLE_ALIAS parameter is not set.");
             }
             // set observedSampleAlias to the parameter, if set.  Otherwise, if here, this must be a single sample VCF, get it's sample
             observedSampleAlias = (OBSERVED_SAMPLE_ALIAS != null) ? OBSERVED_SAMPLE_ALIAS : fileHeader.getGenotypeSamples().get(0);
 
             // Now verify that observedSampleAlias is, in fact, in the VCF
             if (!fileHeader.getGenotypeSamples().contains(observedSampleAlias)) {
-                throw new PicardException("INPUT VCF file does not contain OBSERVED_SAMPLE_ALIAS: " + observedSampleAlias);
+                throw new PicardException("inputPath VCF file does not contain OBSERVED_SAMPLE_ALIAS: " + observedSampleAlias);
             }
 
             if (OBSERVED_SAMPLE_ALIAS == null) {
@@ -295,9 +301,9 @@ protected int doWork() {
                 EXPECTED_SAMPLE_ALIAS = observedSampleAlias;
             }
 
-            results = checker.checkFingerprints(
-                    Collections.singletonList(INPUT),
-                    Collections.singletonList(GENOTYPES),
+            results = checker.checkFingerprintsFromPaths(
+                    Collections.singletonList(inputPath),
+                    Collections.singletonList(genotypesPath),
                     observedSampleAlias,
                     EXPECTED_SAMPLE_ALIAS);
         }
@@ -368,19 +374,26 @@ protected int doWork() {
     }
 
     protected String[] customCommandLineValidation() {
-        IOUtil.assertFileIsReadable(INPUT);
 
-        boolean isBamOrSamFile = isBamOrSamFile(INPUT);
-        if (!isBamOrSamFile && IGNORE_READ_GROUPS) {
-            return new String[]{"The parameter IGNORE_READ_GROUPS can only be used with BAM/SAM inputs."};
-        }
-        if (isBamOrSamFile && OBSERVED_SAMPLE_ALIAS != null) {
-            return new String[]{"The parameter OBSERVED_SAMPLE_ALIAS can only be used with a VCF input."};
+        try {
+            final boolean isBamOrSamFile = isBamOrSam(IOUtil.getPath(INPUT));
+            if (!isBamOrSamFile && IGNORE_READ_GROUPS) {
+                return new String[]{"The parameter IGNORE_READ_GROUPS can only be used with BAM/SAM inputs."};
+            }
+            if (isBamOrSamFile && OBSERVED_SAMPLE_ALIAS != null) {
+                return new String[]{"The parameter OBSERVED_SAMPLE_ALIAS can only be used with a VCF input."};
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
         }
         return super.customCommandLineValidation();
     }
 
-    static boolean isBamOrSamFile(final File f) {
-        return (BamFileIoUtils.isBamFile(f) || f.getName().endsWith(IOUtil.SAM_FILE_EXTENSION));
+    static boolean isBamOrSam(final File f) {
+        return isBamOrSam(f.toPath());
+    }
+
+    static boolean isBamOrSam(final Path p) {
+        return (p.toUri().getRawPath().endsWith(BamFileIoUtils.BAM_FILE_EXTENSION) || p.toUri().getRawPath().endsWith(IOUtil.SAM_FILE_EXTENSION));
     }
 }