From adff40ff589b95d28decd0c3836e9a6e9626eee4 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Tue, 27 Dec 2011 13:16:25 -0500 Subject: [PATCH 001/356] Minor optimizations to avoid extra processing (esp. for reduced reads) --- .../genotyper/DiploidSNPGenotypeLikelihoods.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidSNPGenotypeLikelihoods.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidSNPGenotypeLikelihoods.java index 295cf86884..ae70772305 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidSNPGenotypeLikelihoods.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidSNPGenotypeLikelihoods.java @@ -275,19 +275,22 @@ public int add(ReadBackedPileup pileup, boolean ignoreBadBases, boolean capBaseQ public int add(PileupElement elt, boolean ignoreBadBases, boolean capBaseQualsAtMappingQual, int minBaseQual) { byte obsBase = elt.getBase(); byte qual = qualToUse(elt, ignoreBadBases, capBaseQualsAtMappingQual, minBaseQual); + if ( qual == 0 ) + return 0; if ( elt.isReducedRead() ) { // reduced read representation if ( BaseUtils.isRegularBase( obsBase )) { - add(obsBase, qual, (byte)0, (byte)0, elt.getRepresentativeCount()); // fast calculation of n identical likelihoods - return elt.getRepresentativeCount(); // we added nObs bases here + int representativeCount = elt.getRepresentativeCount(); + add(obsBase, qual, (byte)0, (byte)0, representativeCount); // fast calculation of n identical likelihoods + return representativeCount; // we added nObs bases here } // odd bases or deletions => don't use them return 0; } - return qual > 0 ? add(obsBase, qual, (byte)0, (byte)0, 1) : 0; + return add(obsBase, qual, (byte)0, (byte)0, 1); } public int add(List overlappingPair, boolean ignoreBadBases, boolean capBaseQualsAtMappingQual, int minBaseQual) { @@ -519,7 +522,7 @@ private static byte qualToUse(PileupElement p, boolean ignoreBadBases, boolean c if ( qual > SAMUtils.MAX_PHRED_SCORE ) throw new UserException.MalformedBAM(p.getRead(), String.format("the maximum allowed quality score is %d, but a quality of %d was observed in read %s. Perhaps your BAM incorrectly encodes the quality scores in Sanger format; see http://en.wikipedia.org/wiki/FASTQ_format for more details", SAMUtils.MAX_PHRED_SCORE, qual, p.getRead().getReadName())); if ( capBaseQualsAtMappingQual ) - qual = (byte)Math.min((int)p.getQual(), p.getMappingQual()); + qual = (byte)Math.min((int)qual, p.getMappingQual()); if ( (int)qual < minBaseQual ) qual = (byte)0; From d20a25d68175e14c972980426156fb035d3424d4 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Tue, 27 Dec 2011 16:50:38 -0500 Subject: [PATCH 002/356] A much better way of choosing the alternate allele(s) to genotype in the SNP model of UG: instead of looking at the sum of base qualities (which can and did lead to us over-genotyping esp. when allowing multiple alternate alleles), we look at the likelihoods themselves (free since we are already calculating likelihoods for all 10 genotypes). Now, even if the base quals exceed some arbitrary threshold, we only bother genotyping an alternate allele when there's a sample for which it is more likely than ref/ref (I can generate weird edge cases where this falls apart, but none that model truly variable sites that we actually want to call). This leads to a huge efficiency improvement esp. for exomes (and esp. for many samples) where we almost always were trying to genotype all 3 alternate alleles. Integration tests change only because ref calls have slight QUAL differences (because the best alt allele is still chosen arbitrarily, but differently). --- ...NPGenotypeLikelihoodsCalculationModel.java | 132 +++++++++--------- .../UnifiedGenotyperIntegrationTest.java | 4 +- 2 files changed, 71 insertions(+), 65 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java index 57cc5594a3..eee89674ac 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java @@ -46,8 +46,6 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsCalculationModel { - private static final int MIN_QUAL_SUM_FOR_ALT_ALLELE = 50; - private boolean ALLOW_MULTIPLE_ALLELES; private final boolean useAlleleFromVCF; @@ -56,15 +54,19 @@ protected SNPGenotypeLikelihoodsCalculationModel(UnifiedArgumentCollection UAC, super(UAC, logger); ALLOW_MULTIPLE_ALLELES = UAC.MULTI_ALLELIC; useAlleleFromVCF = UAC.GenotypingMode == GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES; + + // make sure the PL cache has been initialized with enough alleles + if ( UnifiedGenotyperEngine.PLIndexToAlleleIndex == null || UnifiedGenotyperEngine.PLIndexToAlleleIndex.length < 4 ) // +1 for 0 alt alleles + UnifiedGenotyperEngine.calculatePLcache(3); } - public VariantContext getLikelihoods(RefMetaDataTracker tracker, - ReferenceContext ref, - Map contexts, - AlignmentContextUtils.ReadOrientation contextType, - GenotypePriors priors, - Allele alternateAlleleToUse, - boolean useBAQedPileup) { + public VariantContext getLikelihoods(final RefMetaDataTracker tracker, + final ReferenceContext ref, + final Map contexts, + final AlignmentContextUtils.ReadOrientation contextType, + final GenotypePriors priors, + final Allele alternateAlleleToUse, + final boolean useBAQedPileup) { if ( !(priors instanceof DiploidSNPGenotypePriors) ) throw new StingException("Only diploid-based SNP priors are supported in the SNP GL model"); @@ -79,6 +81,20 @@ public VariantContext getLikelihoods(RefMetaDataTracker tracker, alleles.add(Allele.create(refBase, true)); final VariantContextBuilder builder = new VariantContextBuilder("UG_call", loc.getContig(), loc.getStart(), loc.getStop(), alleles); + // calculate the GLs + ArrayList GLs = new ArrayList(contexts.size()); + for ( Map.Entry sample : contexts.entrySet() ) { + ReadBackedPileup pileup = AlignmentContextUtils.stratify(sample.getValue(), contextType).getBasePileup(); + if ( useBAQedPileup ) + pileup = createBAQedPileup( pileup ); + + // create the GenotypeLikelihoods object + final DiploidSNPGenotypeLikelihoods GL = new DiploidSNPGenotypeLikelihoods((DiploidSNPGenotypePriors)priors, UAC.PCR_error); + final int nGoodBases = GL.add(pileup, true, true, UAC.MIN_BASE_QUALTY_SCORE); + if ( nGoodBases > 0 ) + GLs.add(new SampleGenotypeData(sample.getKey(), GL, getFilteredDepth(pileup))); + } + // find the alternate allele(s) that we should be using if ( alternateAlleleToUse != null ) { basesToUse[BaseUtils.simpleBaseToBaseIndex(alternateAlleleToUse.getBases()[0])] = true; @@ -93,7 +109,7 @@ public VariantContext getLikelihoods(RefMetaDataTracker tracker, basesToUse[BaseUtils.simpleBaseToBaseIndex(allele.getBases()[0])] = true; } else { - determineAlternateAlleles(basesToUse, refBase, contexts, useBAQedPileup); + determineAlternateAlleles(basesToUse, refBase, GLs); // how many alternate alleles are we using? int alleleCounter = Utils.countSetBits(basesToUse); @@ -125,22 +141,12 @@ public VariantContext getLikelihoods(RefMetaDataTracker tracker, builder.alleles(alleles); // create the genotypes; no-call everyone for now - GenotypesContext genotypes = GenotypesContext.create(); + final GenotypesContext genotypes = GenotypesContext.create(); final List noCall = new ArrayList(); noCall.add(Allele.NO_CALL); - for ( Map.Entry sample : contexts.entrySet() ) { - ReadBackedPileup pileup = AlignmentContextUtils.stratify(sample.getValue(), contextType).getBasePileup(); - if ( useBAQedPileup ) - pileup = createBAQedPileup( pileup ); - - // create the GenotypeLikelihoods object - final DiploidSNPGenotypeLikelihoods GL = new DiploidSNPGenotypeLikelihoods((DiploidSNPGenotypePriors)priors, UAC.PCR_error); - final int nGoodBases = GL.add(pileup, true, true, UAC.MIN_BASE_QUALTY_SCORE); - if ( nGoodBases == 0 ) - continue; - - final double[] allLikelihoods = GL.getLikelihoods(); + for ( SampleGenotypeData sampleData : GLs ) { + final double[] allLikelihoods = sampleData.GL.getLikelihoods(); final double[] myLikelihoods = new double[numLikelihoods]; int myLikelihoodsIndex = 0; @@ -151,60 +157,48 @@ public VariantContext getLikelihoods(RefMetaDataTracker tracker, } // normalize in log space so that max element is zero. - GenotypeLikelihoods likelihoods = GenotypeLikelihoods.fromLog10Likelihoods(MathUtils.normalizeFromLog10(myLikelihoods, false, true)); + final GenotypeLikelihoods likelihoods = GenotypeLikelihoods.fromLog10Likelihoods(MathUtils.normalizeFromLog10(myLikelihoods, false, true)); - HashMap attributes = new HashMap(); - attributes.put(VCFConstants.DEPTH_KEY, getFilteredDepth(pileup)); + final HashMap attributes = new HashMap(); + attributes.put(VCFConstants.DEPTH_KEY, sampleData.depth); attributes.put(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY, likelihoods); - genotypes.add(new Genotype(sample.getKey(), noCall, Genotype.NO_LOG10_PERROR, null, attributes, false)); + genotypes.add(new Genotype(sampleData.name, noCall, Genotype.NO_LOG10_PERROR, null, attributes, false)); } return builder.genotypes(genotypes).make(); } // fills in the allelesToUse array - protected void determineAlternateAlleles(boolean[] allelesToUse, byte ref, Map contexts, boolean useBAQedPileup) { - int[] qualCounts = new int[4]; - - for ( Map.Entry sample : contexts.entrySet() ) { - // calculate the sum of quality scores for each base - ReadBackedPileup pileup = useBAQedPileup ? createBAQedPileup( sample.getValue().getBasePileup() ) : sample.getValue().getBasePileup(); - for ( PileupElement p : pileup ) { - // ignore deletions - if ( p.isDeletion() || (!p.isReducedRead() && p.getQual() < UAC.MIN_BASE_QUALTY_SCORE) ) - continue; - - final int index = BaseUtils.simpleBaseToBaseIndex(p.getBase()); - if ( index >= 0 ) { - qualCounts[index] += p.getQual(); - } + protected void determineAlternateAlleles(final boolean[] allelesToUse, final byte ref, final List sampleDataList) { + + final int baseIndexOfRef = BaseUtils.simpleBaseToBaseIndex(ref); + final int PLindexOfRef = DiploidGenotype.createDiploidGenotype(ref, ref).ordinal(); + final double[] likelihoodCounts = new double[4]; + + // based on the GLs, find the alternate alleles with the most probability + for ( SampleGenotypeData sampleData : sampleDataList ) { + final double[] likelihoods = sampleData.GL.getLikelihoods(); + final int PLindexOfBestGL = MathUtils.maxElementIndex(likelihoods); + if ( PLindexOfBestGL != PLindexOfRef ) { + int[] alleles = UnifiedGenotyperEngine.PLIndexToAlleleIndex[3][PLindexOfBestGL]; + if ( alleles[0] != baseIndexOfRef ) + likelihoodCounts[alleles[0]] += likelihoods[PLindexOfBestGL] - likelihoods[PLindexOfRef]; + // don't double-count it + if ( alleles[1] != baseIndexOfRef && alleles[1] != alleles[0] ) + likelihoodCounts[alleles[1]] += likelihoods[PLindexOfBestGL] - likelihoods[PLindexOfRef]; } } if ( ALLOW_MULTIPLE_ALLELES ) { - for ( byte altAllele : BaseUtils.BASES ) { - if ( altAllele == ref ) - continue; - int index = BaseUtils.simpleBaseToBaseIndex(altAllele); - if ( qualCounts[index] >= MIN_QUAL_SUM_FOR_ALT_ALLELE ) { - allelesToUse[index] = true; + for ( int i = 0; i < 4; i++ ) { + if ( likelihoodCounts[i] > 0.0 ) { + allelesToUse[i] = true; } } } else { - // set the non-ref base which has the maximum quality score sum - int maxCount = 0; - int indexOfMax = 0; - for ( byte altAllele : BaseUtils.BASES ) { - if ( altAllele == ref ) - continue; - int index = BaseUtils.simpleBaseToBaseIndex(altAllele); - if ( qualCounts[index] > maxCount ) { - maxCount = qualCounts[index]; - indexOfMax = index; - } - } - - if ( maxCount > 0 ) + // set the non-ref base which has the maximum sum of non-ref GLs + final int indexOfMax = MathUtils.maxElementIndex(likelihoodCounts); + if ( likelihoodCounts[indexOfMax] > 0.0 ) allelesToUse[indexOfMax] = true; } } @@ -227,4 +221,16 @@ public BAQedPileupElement( final PileupElement PE ) { public byte getQual( final int offset ) { return BAQ.calcBAQFromTag(getRead(), offset, true); } } -} \ No newline at end of file + private static class SampleGenotypeData { + + public final String name; + public final DiploidSNPGenotypeLikelihoods GL; + public final int depth; + + public SampleGenotypeData(final String name, final DiploidSNPGenotypeLikelihoods GL, final int depth) { + this.name = name; + this.GL = GL; + this.depth = depth; + } + } +} diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java index 3c6131d6c5..d4518078be 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java @@ -129,8 +129,8 @@ public void testCallingParameters() { public void testOutputParameter() { HashMap e = new HashMap(); e.put( "-sites_only", "44f3b5b40e6ad44486cddfdb7e0bfcd8" ); - e.put( "--output_mode EMIT_ALL_CONFIDENT_SITES", "42e4ea7878ef8d96215accb3ba4e97b7" ); - e.put( "--output_mode EMIT_ALL_SITES", "e0443c720149647469f2a2f3fb73942f" ); + e.put( "--output_mode EMIT_ALL_CONFIDENT_SITES", "553f6b4cbf380885bec9dd634cf68742" ); + e.put( "--output_mode EMIT_ALL_SITES", "6d8624e45ad9dae5803ac705b39e4ffa" ); for ( Map.Entry entry : e.entrySet() ) { WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec( From e45ca8bfa2d02252933c4f9977c7ac4496f69100 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Sun, 1 Jan 2012 19:12:48 -0500 Subject: [PATCH 003/356] Protect against too many alternate alleles in the haplotype caller. From d05f0c2318ebe475e7b0d8cd59b11488485a0972 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Mon, 2 Jan 2012 09:58:46 -0500 Subject: [PATCH 004/356] GATKPerformanceOverTime script update -- Automatic detection of most recent version of GATK release (just tell the script now to use 1.2, 1.3, and 1.4) -- Uses 1.4 now -- By default we do 9 runs of each non-parallel test -- In PathUtils added convenience utility to find most recent release GATK jar with a specific release number --- .../broadinstitute/sting/utils/PathUtils.java | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/PathUtils.java b/public/java/src/org/broadinstitute/sting/utils/PathUtils.java index 822d04dfd5..db655d25c3 100755 --- a/public/java/src/org/broadinstitute/sting/utils/PathUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/PathUtils.java @@ -1,10 +1,14 @@ package org.broadinstitute.sting.utils; +import org.apache.commons.io.comparator.LastModifiedFileComparator; +import org.apache.log4j.Logger; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import java.io.File; import java.io.FilenameFilter; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.List; /** @@ -17,6 +21,8 @@ * A set of static utility methods for common operations on paths. */ public class PathUtils { + private static Logger logger = Logger.getLogger(PathUtils.class); + /** * Constructor access disallowed...static utility methods only! */ @@ -36,7 +42,7 @@ public static List findFilesInPath(final File basePath, final String rel List filesInPath = new ArrayList(); FilenameFilter filter = new OrFilenameFilter(new DirectoryFilter(), - new ExtensionFilter(extension)); + new ExtensionFilter(extension)); File[] contents = basePath.listFiles( filter ); for (File content : contents) { String relativeFileName = relativePrefix.trim().length() != 0 ? @@ -118,4 +124,47 @@ public static void refreshVolume(File file) { } } -} + + /** + * Walk over the GATK released directories to find the most recent JAR files corresponding + * to the version prefix. For example, providing input "1.2" will + * return the full path to the most recent GenomeAnalysisTK.jar in the GATK_RELEASE_DIR + * in directories that match gatkReleaseDir/GenomeAnalysisTK-1.2* + * + * @param gatkReleaseDir Path to directory containing GATK release binaries (e.g., /humgen/gsa-hpprojects/GATK/bin/) + * @param releaseVersionNumber Desired GATK version number (e.g., 1.2) + * @return A file pointing to the most recent GATK file in the release directory with GATK release number + */ + public static File findMostRecentGATKVersion(final File gatkReleaseDir, final String releaseVersionNumber) { + final String versionString = "GenomeAnalysisTK-" + releaseVersionNumber; + + final List gatkJars = new ArrayList(); + for ( final String path : gatkReleaseDir.list(new isGATKVersion(versionString)) ) { + gatkJars.add(new File(gatkReleaseDir.getAbsolutePath() + "/" + path + "/GenomeAnalysisTK.jar")); + } + + if ( gatkJars.isEmpty() ) + return null; + else { + Collections.sort(gatkJars, LastModifiedFileComparator.LASTMODIFIED_REVERSE); + //for ( File jar : gatkJars ) logger.info(String.format("%s => %d", jar, jar.lastModified())); + final File last = gatkJars.get(0); + logger.debug(String.format("findMostRecentGATKVersion: Found %d jars for %s, keeping last one %s", + gatkJars.size(), releaseVersionNumber, last)); + return last; + } + } + + private final static class isGATKVersion implements FilenameFilter { + private final String versionString; + + private isGATKVersion(final String versionString) { + this.versionString = versionString; + } + + @Override + public boolean accept(final File file, final String s) { + return s.contains(versionString); + } + } +} \ No newline at end of file From 188bd481397fd460131f967a1d5a1b242e7d29e8 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Mon, 2 Jan 2012 10:39:05 -0500 Subject: [PATCH 005/356] runGATKReport only archives and shows errors for last days runs From b3e613647aeda71328f02eea786e876a82412071 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Mon, 2 Jan 2012 13:56:44 -0500 Subject: [PATCH 006/356] GATKPerformanceOverTime bug fixes -- Don't try to do nt 16, it's just too painful as the threading doesn't work well and it consumes a large chunk of our available slots on gsa4 -- bugfix: only do multi-threaded test for each iteration, not expanding by subiterations, so we no longer try to do 3x3 nt 16 runs From 94791a2a754730c4cbe3e076cf4868d8db87e698 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Wed, 28 Dec 2011 12:15:53 -0500 Subject: [PATCH 007/356] Add support for reads starting with insertion * Modified cleanCigarShift to allow insertions in the beginning and end of the read * Allowed cigars starting/ending in insertions in the systematic ReadClipper tests * Updated all ReadClipper unit tests * ReduceReads does not hard clip leading insertions by default anymore * SlidingWindow adjusts start location if read starts with insertion * SlidingWindow creates an empty element with insertions to the right * Fixed all potential divide by zero with totalCount() (from BaseCounts) * Updated all Integration tests * Added new integration test for multiple interval reducing --- .../sting/utils/clipping/ClippingOp.java | 54 ++-- .../sting/utils/clipping/ReadClipper.java | 33 ++- .../sting/utils/sam/ArtificialSAMUtils.java | 2 +- .../sting/utils/sam/GATKSAMRecord.java | 7 +- .../sting/utils/sam/ReadUtils.java | 185 +++++++------ .../utils/clipping/ReadClipperTestUtils.java | 19 +- .../utils/clipping/ReadClipperUnitTest.java | 244 +++++++++--------- 7 files changed, 301 insertions(+), 243 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/clipping/ClippingOp.java b/public/java/src/org/broadinstitute/sting/utils/clipping/ClippingOp.java index 921a0a599b..fb133d902f 100644 --- a/public/java/src/org/broadinstitute/sting/utils/clipping/ClippingOp.java +++ b/public/java/src/org/broadinstitute/sting/utils/clipping/ClippingOp.java @@ -70,27 +70,27 @@ public GATKSAMRecord apply(ClippingRepresentation algorithm, GATKSAMRecord read) break; case SOFTCLIP_BASES: - if ( read.getReadUnmappedFlag() ) { + if (read.getReadUnmappedFlag()) { // we can't process unmapped reads throw new UserException("Read Clipper cannot soft clip unmapped reads"); } //System.out.printf("%d %d %d%n", stop, start, read.getReadLength()); int myStop = stop; - if ( (stop + 1 - start) == read.getReadLength() ) { + if ((stop + 1 - start) == read.getReadLength()) { // BAM representation issue -- we can't SOFTCLIP away all bases in a read, just leave it alone //Walker.logger.info(String.format("Warning, read %s has all bases clip but this can't be represented with SOFTCLIP_BASES, just leaving it alone", read.getReadName())); //break; myStop--; // just decrement stop } - if ( start > 0 && myStop != read.getReadLength() - 1 ) + if (start > 0 && myStop != read.getReadLength() - 1) throw new RuntimeException(String.format("Cannot apply soft clipping operator to the middle of a read: %s to be clipped at %d-%d", read.getReadName(), start, myStop)); Cigar oldCigar = read.getCigar(); int scLeft = 0, scRight = read.getReadLength(); - if ( start == 0 ) + if (start == 0) scLeft = myStop + 1; else scRight = start; @@ -134,8 +134,7 @@ else if (matchesCount > 0) { unclippedCigar.add(new CigarElement(matchesCount, CigarOperator.MATCH_OR_MISMATCH)); matchesCount = 0; unclippedCigar.add(element); - } - else + } else unclippedCigar.add(element); } if (matchesCount > 0) @@ -284,10 +283,9 @@ private Cigar softClip(final Cigar __cigar, final int __startClipEnd, final int } @Requires({"start <= stop", "start == 0 || stop == read.getReadLength() - 1"}) - private GATKSAMRecord hardClip (GATKSAMRecord read, int start, int stop) { + private GATKSAMRecord hardClip(GATKSAMRecord read, int start, int stop) { if (start == 0 && stop == read.getReadLength() - 1) return GATKSAMRecord.emptyRead(read); -// return new GATKSAMRecord(read.getHeader()); // If the read is unmapped there is no Cigar string and neither should we create a new cigar string @@ -296,8 +294,8 @@ private GATKSAMRecord hardClip (GATKSAMRecord read, int start, int stop) { // the cigar may force a shift left or right (or both) in case we are left with insertions // starting or ending the read after applying the hard clip on start/stop. int newLength = read.getReadLength() - (stop - start + 1) - cigarShift.shiftFromStart - cigarShift.shiftFromEnd; - byte [] newBases = new byte[newLength]; - byte [] newQuals = new byte[newLength]; + byte[] newBases = new byte[newLength]; + byte[] newQuals = new byte[newLength]; int copyStart = (start == 0) ? stop + 1 + cigarShift.shiftFromStart : cigarShift.shiftFromStart; System.arraycopy(read.getReadBases(), copyStart, newBases, 0, newLength); @@ -321,11 +319,11 @@ private GATKSAMRecord hardClip (GATKSAMRecord read, int start, int stop) { } @Requires({"!cigar.isEmpty()"}) - private CigarShift hardClipCigar (Cigar cigar, int start, int stop) { + private CigarShift hardClipCigar(Cigar cigar, int start, int stop) { Cigar newCigar = new Cigar(); int index = 0; int totalHardClipCount = stop - start + 1; - int alignmentShift = 0; // caused by hard clipping insertions or deletions + int alignmentShift = 0; // caused by hard clipping deletions // hard clip the beginning of the cigar string if (start == 0) { @@ -353,7 +351,7 @@ private CigarShift hardClipCigar (Cigar cigar, int start, int stop) { // element goes beyond what we need to clip else if (index + shift > stop + 1) { int elementLengthAfterChopping = cigarElement.getLength() - (stop - index + 1); - alignmentShift += calculateHardClippingAlignmentShift(cigarElement, stop-index+1); + alignmentShift += calculateHardClippingAlignmentShift(cigarElement, stop - index + 1); newCigar.add(new CigarElement(totalHardClipCount + alignmentShift, CigarOperator.HARD_CLIP)); newCigar.add(new CigarElement(elementLengthAfterChopping, cigarElement.getOperator())); } @@ -388,7 +386,7 @@ else if (index + shift > stop + 1) { if (index + shift < start) newCigar.add(new CigarElement(cigarElement.getLength(), cigarElement.getOperator())); - // element goes beyond our clip starting position + // element goes beyond our clip starting position else { int elementLengthAfterChopping = start - index; alignmentShift += calculateHardClippingAlignmentShift(cigarElement, cigarElement.getLength() - (start - index)); @@ -396,7 +394,7 @@ else if (index + shift > stop + 1) { // if this last element is a HARD CLIP operator, just merge it with our hard clip operator to be added later if (cigarElement.getOperator() == CigarOperator.HARD_CLIP) totalHardClipCount += elementLengthAfterChopping; - // otherwise, maintain what's left of this last operator + // otherwise, maintain what's left of this last operator else newCigar.add(new CigarElement(elementLengthAfterChopping, cigarElement.getOperator())); } @@ -408,7 +406,7 @@ else if (index + shift > stop + 1) { } // check if we are hard clipping indels - while(cigarElementIterator.hasNext()) { + while (cigarElementIterator.hasNext()) { cigarElement = cigarElementIterator.next(); alignmentShift += calculateHardClippingAlignmentShift(cigarElement, cigarElement.getLength()); @@ -444,34 +442,30 @@ private CigarShift cleanHardClippedCigar(Cigar cigar) { boolean readHasStarted = false; boolean addedHardClips = false; - while(!cigarStack.empty()) { + while (!cigarStack.empty()) { CigarElement cigarElement = cigarStack.pop(); - if ( !readHasStarted && - cigarElement.getOperator() != CigarOperator.INSERTION && + if (!readHasStarted && +// cigarElement.getOperator() != CigarOperator.INSERTION && cigarElement.getOperator() != CigarOperator.DELETION && cigarElement.getOperator() != CigarOperator.HARD_CLIP) readHasStarted = true; - else if ( !readHasStarted && cigarElement.getOperator() == CigarOperator.HARD_CLIP) + else if (!readHasStarted && cigarElement.getOperator() == CigarOperator.HARD_CLIP) totalHardClip += cigarElement.getLength(); - else if ( !readHasStarted && cigarElement.getOperator() == CigarOperator.INSERTION) - shift += cigarElement.getLength(); - - else if ( !readHasStarted && cigarElement.getOperator() == CigarOperator.DELETION) + else if (!readHasStarted && cigarElement.getOperator() == CigarOperator.DELETION) totalHardClip += cigarElement.getLength(); if (readHasStarted) { - if (i==1) { + if (i == 1) { if (!addedHardClips) { if (totalHardClip > 0) inverseCigarStack.push(new CigarElement(totalHardClip, CigarOperator.HARD_CLIP)); addedHardClips = true; } inverseCigarStack.push(cigarElement); - } - else { + } else { if (!addedHardClips) { if (totalHardClip > 0) cleanCigar.add(new CigarElement(totalHardClip, CigarOperator.HARD_CLIP)); @@ -498,7 +492,7 @@ private int calculateAlignmentStartShift(Cigar oldCigar, Cigar newCigar) { int newShift = 0; int oldShift = 0; - boolean readHasStarted = false; // if the new cigar is composed of S and H only, we have to traverse the entire old cigar to calculate the shift + boolean readHasStarted = false; // if the new cigar is composed of S and H only, we have to traverse the entire old cigar to calculate the shift for (CigarElement cigarElement : newCigar.getCigarElements()) { if (cigarElement.getOperator() == CigarOperator.HARD_CLIP || cigarElement.getOperator() == CigarOperator.SOFT_CLIP) newShift += cigarElement.getLength(); @@ -509,7 +503,7 @@ private int calculateAlignmentStartShift(Cigar oldCigar, Cigar newCigar) { } for (CigarElement cigarElement : oldCigar.getCigarElements()) { - if (cigarElement.getOperator() == CigarOperator.HARD_CLIP || cigarElement.getOperator() == CigarOperator.SOFT_CLIP ) + if (cigarElement.getOperator() == CigarOperator.HARD_CLIP || cigarElement.getOperator() == CigarOperator.SOFT_CLIP) oldShift += cigarElement.getLength(); else if (readHasStarted) break; @@ -522,7 +516,7 @@ private int calculateHardClippingAlignmentShift(CigarElement cigarElement, int c if (cigarElement.getOperator() == CigarOperator.INSERTION) return -clippedLength; - // Deletions should be added to the total hard clip count + // Deletions should be added to the total hard clip count else if (cigarElement.getOperator() == CigarOperator.DELETION) return cigarElement.getLength(); diff --git a/public/java/src/org/broadinstitute/sting/utils/clipping/ReadClipper.java b/public/java/src/org/broadinstitute/sting/utils/clipping/ReadClipper.java index afe7fa9753..7a664bd616 100644 --- a/public/java/src/org/broadinstitute/sting/utils/clipping/ReadClipper.java +++ b/public/java/src/org/broadinstitute/sting/utils/clipping/ReadClipper.java @@ -374,24 +374,43 @@ public static GATKSAMRecord revertSoftClippedBases(GATKSAMRecord read) { * Generic functionality to hard clip a read, used internally by hardClipByReferenceCoordinatesLeftTail * and hardClipByReferenceCoordinatesRightTail. Should not be used directly. * + * Note, it REQUIRES you to give the directionality of your hard clip (i.e. whether you're clipping the + * left of right tail) by specifying either refStart < 0 or refStop < 0. + * * @param refStart first base to clip (inclusive) * @param refStop last base to clip (inclusive) * @return a new read, without the clipped bases */ - @Requires("!read.getReadUnmappedFlag()") // can't handle unmapped reads, as we're using reference coordinates to clip + @Requires({"!read.getReadUnmappedFlag()", "refStart < 0 || refStop < 0"}) // can't handle unmapped reads, as we're using reference coordinates to clip protected GATKSAMRecord hardClipByReferenceCoordinates(int refStart, int refStop) { - int start = (refStart < 0) ? 0 : ReadUtils.getReadCoordinateForReferenceCoordinate(read, refStart, ReadUtils.ClippingTail.RIGHT_TAIL); - int stop = (refStop < 0) ? read.getReadLength() - 1 : ReadUtils.getReadCoordinateForReferenceCoordinate(read, refStop, ReadUtils.ClippingTail.LEFT_TAIL); + if (read.isEmpty()) + return read; - if (read.isEmpty() || (start == 0 && stop == read.getReadLength() - 1)) - return GATKSAMRecord.emptyRead(read); -// return new GATKSAMRecord(read.getHeader()); + int start; + int stop; + + // Determine the read coordinate to start and stop hard clipping + if (refStart < 0) { + if (refStop < 0) + throw new ReviewedStingException("Only one of refStart or refStop must be < 0, not both (" + refStart + ", " + refStop + ")"); + start = 0; + stop = ReadUtils.getReadCoordinateForReferenceCoordinate(read, refStop, ReadUtils.ClippingTail.LEFT_TAIL); + } + else { + if (refStop >= 0) + throw new ReviewedStingException("Either refStart or refStop must be < 0 (" + refStart + ", " + refStop + ")"); + start = ReadUtils.getReadCoordinateForReferenceCoordinate(read, refStart, ReadUtils.ClippingTail.RIGHT_TAIL); + stop = read.getReadLength() - 1; + } + +// if ((start == 0 && stop == read.getReadLength() - 1)) +// return GATKSAMRecord.emptyRead(read); if (start < 0 || stop > read.getReadLength() - 1) throw new ReviewedStingException("Trying to clip before the start or after the end of a read"); if ( start > stop ) - throw new ReviewedStingException("START > STOP -- this should never happen -- call Mauricio!"); + throw new ReviewedStingException(String.format("START (%d) > (%d) STOP -- this should never happen -- call Mauricio!", start, stop)); if ( start > 0 && stop < read.getReadLength() - 1) throw new ReviewedStingException(String.format("Trying to clip the middle of the read: start %d, stop %d, cigar: %s", start, stop, read.getCigarString())); diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/ArtificialSAMUtils.java b/public/java/src/org/broadinstitute/sting/utils/sam/ArtificialSAMUtils.java index cedd56bdfb..542adea775 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/ArtificialSAMUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/ArtificialSAMUtils.java @@ -238,7 +238,7 @@ public static GATKSAMRecord createArtificialRead( SAMFileHeader header, String n */ public static GATKSAMRecord createArtificialRead( byte[] bases, byte[] qual, String cigar ) { SAMFileHeader header = ArtificialSAMUtils.createArtificialSamHeader(1, 1, 1000000); - return ArtificialSAMUtils.createArtificialRead(header, "default_read", 0, 1, bases, qual, cigar); + return ArtificialSAMUtils.createArtificialRead(header, "default_read", 0, 10000, bases, qual, cigar); } diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java index 96713edc26..5e0802fa60 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java @@ -24,7 +24,6 @@ package org.broadinstitute.sting.utils.sam; -import com.google.java.contract.Ensures; import net.sf.samtools.*; import org.broadinstitute.sting.utils.NGSPlatform; @@ -277,7 +276,6 @@ public void simplify () { * * @return the unclipped start of the read taking soft clips (but not hard clips) into account */ - @Ensures({"result >= getUnclippedStart()", "result <= getUnclippedEnd() || ReadUtils.readIsEntirelyInsertion(this)"}) public int getSoftStart() { int start = this.getUnclippedStart(); for (CigarElement cigarElement : this.getCigar().getCigarElements()) { @@ -286,17 +284,17 @@ public int getSoftStart() { else break; } + return start; } /** * Calculates the reference coordinate for the end of the read taking into account soft clips but not hard clips. * - * Note: getUnclippedStart() adds soft and hard clips, this function only adds soft clips. + * Note: getUnclippedEnd() adds soft and hard clips, this function only adds soft clips. * * @return the unclipped end of the read taking soft clips (but not hard clips) into account */ - @Ensures({"result >= getUnclippedStart()", "result <= getUnclippedEnd() || ReadUtils.readIsEntirelyInsertion(this)"}) public int getSoftEnd() { int stop = this.getUnclippedStart(); @@ -313,6 +311,7 @@ public int getSoftEnd() { else shift = 0; } + return (lastOperator == CigarOperator.HARD_CLIP) ? stop-1 : stop+shift-1 ; } diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java b/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java index f2e54713f3..d52814ef7c 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java @@ -58,7 +58,7 @@ public enum ClippingTail { /** * A HashMap of the SAM spec read flag names - *

+ * * Note: This is not being used right now, but can be useful in the future */ private static final Map readFlagNames = new HashMap(); @@ -79,49 +79,47 @@ public enum ClippingTail { /** * This enum represents all the different ways in which a read can overlap an interval. - *

+ * * NO_OVERLAP_CONTIG: * read and interval are in different contigs. - *

+ * * NO_OVERLAP_LEFT: * the read does not overlap the interval. - *

- * |----------------| (interval) - * <----------------> (read) - *

+ * + * |----------------| (interval) + * <----------------> (read) + * * NO_OVERLAP_RIGHT: * the read does not overlap the interval. - *

- * |----------------| (interval) - * <----------------> (read) - *

+ * + * |----------------| (interval) + * <----------------> (read) + * * OVERLAP_LEFT: * the read starts before the beginning of the interval but ends inside of it - *

- * |----------------| (interval) - * <----------------> (read) - *

+ * + * |----------------| (interval) + * <----------------> (read) + * * OVERLAP_RIGHT: * the read starts inside the interval but ends outside of it - *

- * |----------------| (interval) - * <----------------> (read) - *

+ * + * |----------------| (interval) + * <----------------> (read) + * * OVERLAP_LEFT_AND_RIGHT: * the read starts before the interval and ends after the interval - *

- * |-----------| (interval) - * <-------------------> (read) - *

+ * + * |-----------| (interval) + * <-------------------> (read) + * * OVERLAP_CONTAINED: * the read starts and ends inside the interval - *

- * |----------------| (interval) - * <--------> (read) + * + * |----------------| (interval) + * <--------> (read) */ - public enum ReadAndIntervalOverlap { - NO_OVERLAP_CONTIG, NO_OVERLAP_LEFT, NO_OVERLAP_RIGHT, NO_OVERLAP_HARDCLIPPED_LEFT, NO_OVERLAP_HARDCLIPPED_RIGHT, OVERLAP_LEFT, OVERLAP_RIGHT, OVERLAP_LEFT_AND_RIGHT, OVERLAP_CONTAINED - } + public enum ReadAndIntervalOverlap {NO_OVERLAP_CONTIG, NO_OVERLAP_LEFT, NO_OVERLAP_RIGHT, NO_OVERLAP_HARDCLIPPED_LEFT, NO_OVERLAP_HARDCLIPPED_RIGHT, OVERLAP_LEFT, OVERLAP_RIGHT, OVERLAP_LEFT_AND_RIGHT, OVERLAP_CONTAINED} /** * Creates a SAMFileWriter with the given compression level if you request a bam file. Creates a regular @@ -141,15 +139,15 @@ public static SAMFileWriter createSAMFileWriterWithCompression(SAMFileHeader hea /** * is this base inside the adaptor of the read? - *

+ * * There are two cases to treat here: - *

+ * * 1) Read is in the negative strand => Adaptor boundary is on the left tail * 2) Read is in the positive strand => Adaptor boundary is on the right tail - *

+ * * Note: We return false to all reads that are UNMAPPED or have an weird big insert size (probably due to mismapping or bigger event) * - * @param read the read to test + * @param read the read to test * @param basePos base position in REFERENCE coordinates (not read coordinates) * @return whether or not the base is in the adaptor */ @@ -166,22 +164,22 @@ public static boolean isBaseInsideAdaptor(final GATKSAMRecord read, long basePos * the read boundary. If the read is in the positive strand, this is the first base after the end of the * fragment (Picard calls it 'insert'), if the read is in the negative strand, this is the first base before the * beginning of the fragment. - *

+ * * There are two cases we need to treat here: - *

+ * * 1) Our read is in the reverse strand : - *

- * <----------------------| * - * |---------------------> - *

- * in these cases, the adaptor boundary is at the mate start (minus one) - *

+ * + * <----------------------| * + * |---------------------> + * + * in these cases, the adaptor boundary is at the mate start (minus one) + * * 2) Our read is in the forward strand : - *

- * |----------------------> * - * <----------------------| - *

- * in these cases the adaptor boundary is at the start of the read plus the inferred insert size (plus one) + * + * |----------------------> * + * <----------------------| + * + * in these cases the adaptor boundary is at the start of the read plus the inferred insert size (plus one) * * @param read the read being tested for the adaptor boundary * @return the reference coordinate for the adaptor boundary (effectively the first base IN the adaptor, closest to the read. NULL if the read is unmapped or the mate is mapped to another contig. @@ -264,7 +262,7 @@ public final static List sortReadsByCoordinate(List + * * Warning: If the read has Hard or Soft clips before the insertion this function will return 0. * * @param read @@ -272,7 +270,7 @@ public final static List sortReadsByCoordinate(List + * * Warning: If the read has Hard or Soft clips after the insertion this function will return 0. * * @param read @@ -288,7 +286,7 @@ public final static int getFirstInsertionOffset(SAMRecord read) { */ public final static int getLastInsertionOffset(SAMRecord read) { CigarElement e = read.getCigar().getCigarElement(read.getCigarLength() - 1); - if (e.getOperator() == CigarOperator.I) + if ( e.getOperator() == CigarOperator.I ) return e.getLength(); else return 0; @@ -297,8 +295,7 @@ public final static int getLastInsertionOffset(SAMRecord read) { /** * Determines what is the position of the read in relation to the interval. * Note: This function uses the UNCLIPPED ENDS of the reads for the comparison. - * - * @param read the read + * @param read the read * @param interval the interval * @return the overlap type as described by ReadAndIntervalOverlap enum (see above) */ @@ -309,30 +306,30 @@ public static ReadAndIntervalOverlap getReadAndIntervalOverlapType(GATKSAMRecord int uStart = read.getUnclippedStart(); int uStop = read.getUnclippedEnd(); - if (!read.getReferenceName().equals(interval.getContig())) + if ( !read.getReferenceName().equals(interval.getContig()) ) return ReadAndIntervalOverlap.NO_OVERLAP_CONTIG; - else if (uStop < interval.getStart()) + else if ( uStop < interval.getStart() ) return ReadAndIntervalOverlap.NO_OVERLAP_LEFT; - else if (uStart > interval.getStop()) + else if ( uStart > interval.getStop() ) return ReadAndIntervalOverlap.NO_OVERLAP_RIGHT; - else if (sStop < interval.getStart()) + else if ( sStop < interval.getStart() ) return ReadAndIntervalOverlap.NO_OVERLAP_HARDCLIPPED_LEFT; - else if (sStart > interval.getStop()) + else if ( sStart > interval.getStop() ) return ReadAndIntervalOverlap.NO_OVERLAP_HARDCLIPPED_RIGHT; - else if ((sStart >= interval.getStart()) && - (sStop <= interval.getStop())) + else if ( (sStart >= interval.getStart()) && + (sStop <= interval.getStop()) ) return ReadAndIntervalOverlap.OVERLAP_CONTAINED; - else if ((sStart < interval.getStart()) && - (sStop > interval.getStop())) + else if ( (sStart < interval.getStart()) && + (sStop > interval.getStop()) ) return ReadAndIntervalOverlap.OVERLAP_LEFT_AND_RIGHT; - else if ((sStart < interval.getStart())) + else if ( (sStart < interval.getStart()) ) return ReadAndIntervalOverlap.OVERLAP_LEFT; else @@ -340,36 +337,52 @@ else if ((sStart < interval.getStart())) } /** - * Pre-processes the results of getReadCoordinateForReferenceCoordinate(GATKSAMRecord, int) in case it falls in - * a deletion following the typical clipping needs. If clipping the left tail (beginning of the read) returns - * the base prior to the deletion. If clipping the right tail (end of the read) returns the base after the - * deletion. + * Pre-processes the results of getReadCoordinateForReferenceCoordinate(GATKSAMRecord, int) to take care of + * two corner cases: + * + * 1. If clipping the right tail (end of the read) getReadCoordinateForReferenceCoordinate and fall inside + * a deletion return the base after the deletion. If clipping the left tail (beginning of the read) it + * doesn't matter because it already returns the previous base by default. + * + * 2. If clipping the left tail (beginning of the read) getReadCoordinateForReferenceCoordinate and the + * read starts with an insertion, and you're requesting the first read based coordinate, it will skip + * the leading insertion (because it has the same reference coordinate as the following base). * * @param read * @param refCoord * @param tail * @return the read coordinate corresponding to the requested reference coordinate for clipping. */ - @Requires({"refCoord >= read.getUnclippedStart()", "refCoord <= read.getUnclippedEnd()"}) + @Requires({"refCoord >= read.getUnclippedStart()", "refCoord <= read.getUnclippedEnd() || (read.getUnclippedEnd() < read.getUnclippedStart())"}) @Ensures({"result >= 0", "result < read.getReadLength()"}) public static int getReadCoordinateForReferenceCoordinate(GATKSAMRecord read, int refCoord, ClippingTail tail) { Pair result = getReadCoordinateForReferenceCoordinate(read, refCoord); int readCoord = result.getFirst(); + // Corner case one: clipping the right tail and falls on deletion, move to the next + // read coordinate. It is not a problem for the left tail because the default answer + // from getReadCoordinateForReferenceCoordinate is to give the previous read coordinate. if (result.getSecond() && tail == ClippingTail.RIGHT_TAIL) readCoord++; + // clipping the left tail and first base is insertion, go to the next read coordinate + // with the same reference coordinate. Advance to the next cigar element, or to the + // end of the read if there is no next element. + Pair firstElementIsInsertion = readStartsWithInsertion(read); + if (readCoord == 0 && tail == ClippingTail.LEFT_TAIL && firstElementIsInsertion.getFirst()) + readCoord = Math.min(firstElementIsInsertion.getSecond().getLength(), read.getReadLength() - 1); + return readCoord; } /** * Returns the read coordinate corresponding to the requested reference coordinate. - *

+ * * WARNING: if the requested reference coordinate happens to fall inside a deletion in the read, this function * will return the last read base before the deletion. This function returns a * Pair(int readCoord, boolean fallsInsideDeletion) so you can choose which readCoordinate to use when faced with * a deletion. - *

+ * * SUGGESTION: Use getReadCoordinateForReferenceCoordinate(GATKSAMRecord, int, ClippingTail) instead to get a * pre-processed result according to normal clipping needs. Or you can use this function and tailor the * behavior to your needs. @@ -421,7 +434,7 @@ public static Pair getReadCoordinateForReferenceCoordinate(GAT if (endsWithinCigar) fallsInsideDeletion = cigarElement.getOperator() == CigarOperator.DELETION; - // if we end outside the current cigar element, we need to check if the next element is an insertion or deletion. + // if we end outside the current cigar element, we need to check if the next element is an insertion or deletion. else { nextCigarElement = cigarElementIterator.next(); @@ -442,13 +455,13 @@ public static Pair getReadCoordinateForReferenceCoordinate(GAT if (!fallsInsideDeletion && cigarElement.getOperator().consumesReadBases()) readBases += shift; - // If we reached our goal inside a deletion, but the deletion is the next cigar element then we need - // to add the shift of the current cigar element but go back to it's last element to return the last - // base before the deletion (see warning in function contracts) + // If we reached our goal inside a deletion, but the deletion is the next cigar element then we need + // to add the shift of the current cigar element but go back to it's last element to return the last + // base before the deletion (see warning in function contracts) else if (fallsInsideDeletion && !endsWithinCigar) readBases += shift - 1; - // If we reached our goal inside a deletion then we must backtrack to the last base before the deletion + // If we reached our goal inside a deletion then we must backtrack to the last base before the deletion else if (fallsInsideDeletion && endsWithinCigar) readBases--; } @@ -457,7 +470,6 @@ else if (fallsInsideDeletion && endsWithinCigar) if (!goalReached) throw new ReviewedStingException("Somehow the requested coordinate is not covered by the read. Too many deletions?"); - return new Pair(readBases, fallsInsideDeletion); } @@ -465,12 +477,11 @@ else if (fallsInsideDeletion && endsWithinCigar) * Compares two SAMRecords only the basis on alignment start. Note that * comparisons are performed ONLY on the basis of alignment start; any * two SAM records with the same alignment start will be considered equal. - *

+ * * Unmapped alignments will all be considered equal. */ @Requires({"read1 != null", "read2 != null"}) - @Ensures("result == 0 || result == 1 || result == -1") public static int compareSAMRecords(GATKSAMRecord read1, GATKSAMRecord read2) { AlignmentStartComparator comp = new AlignmentStartComparator(); return comp.compare(read1, read2); @@ -479,7 +490,7 @@ public static int compareSAMRecords(GATKSAMRecord read1, GATKSAMRecord read2) { /** * Is a base inside a read? * - * @param read the read to evaluate + * @param read the read to evaluate * @param referenceCoordinate the reference coordinate of the base to test * @return true if it is inside the read, false otherwise. */ @@ -502,4 +513,22 @@ public static boolean readIsEntirelyInsertion(GATKSAMRecord read) { } + /** + * Checks if a read starts with an insertion. It looks beyond Hard and Soft clips + * if there are any. + * + * @param read + * @return A pair with the answer (true/false) and the element or null if it doesn't exist + */ + public static Pair readStartsWithInsertion(GATKSAMRecord read) { + for (CigarElement cigarElement : read.getCigar().getCigarElements()) { + if (cigarElement.getOperator() == CigarOperator.INSERTION) + return new Pair(true, cigarElement); + + else if (cigarElement.getOperator() != CigarOperator.HARD_CLIP && cigarElement.getOperator() != CigarOperator.SOFT_CLIP) + break; + } + return new Pair(false, null); + } + } diff --git a/public/java/test/org/broadinstitute/sting/utils/clipping/ReadClipperTestUtils.java b/public/java/test/org/broadinstitute/sting/utils/clipping/ReadClipperTestUtils.java index 18108e0a10..16b141bc34 100644 --- a/public/java/test/org/broadinstitute/sting/utils/clipping/ReadClipperTestUtils.java +++ b/public/java/test/org/broadinstitute/sting/utils/clipping/ReadClipperTestUtils.java @@ -112,8 +112,9 @@ private static boolean isCigarValid(Cigar cigar) { } } - if (startingOp != CigarOperator.DELETION && endingOp != CigarOperator.DELETION && startingOp != CigarOperator.INSERTION && endingOp != CigarOperator.INSERTION) - return true; // we don't accept reads starting or ending in deletions (add any other constraint here) +// if (startingOp != CigarOperator.DELETION && endingOp != CigarOperator.DELETION && startingOp != CigarOperator.INSERTION && endingOp != CigarOperator.INSERTION) + if (startingOp != CigarOperator.DELETION && endingOp != CigarOperator.DELETION) + return true; // we don't accept reads starting or ending in deletions (add any other constraint here) } return false; @@ -190,4 +191,18 @@ public static Cigar invertCigar (Cigar cigar) { return invertedCigar; } + /** + * Checks whether or not the read has any cigar element that is not H or S + * + * @param read + * @return true if it has any M, I or D, false otherwise + */ + public static boolean readHasNonClippedBases(GATKSAMRecord read) { + for (CigarElement cigarElement : read.getCigar().getCigarElements()) + if (cigarElement.getOperator() != CigarOperator.SOFT_CLIP && cigarElement.getOperator() != CigarOperator.HARD_CLIP) + return true; + return false; + } + + } diff --git a/public/java/test/org/broadinstitute/sting/utils/clipping/ReadClipperUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/clipping/ReadClipperUnitTest.java index 4dad68dc55..bc918c0a4f 100644 --- a/public/java/test/org/broadinstitute/sting/utils/clipping/ReadClipperUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/clipping/ReadClipperUnitTest.java @@ -30,12 +30,12 @@ import net.sf.samtools.CigarOperator; import org.broadinstitute.sting.BaseTest; import org.broadinstitute.sting.utils.Utils; -import org.broadinstitute.sting.utils.sam.ArtificialSAMUtils; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import java.util.HashMap; import java.util.List; /** @@ -59,10 +59,11 @@ public void testHardClipBothEndsByReferenceCoordinates() { int alnStart = read.getAlignmentStart(); int alnEnd = read.getAlignmentEnd(); int readLength = alnStart - alnEnd; - for (int i=0; i= alnStart + i, String.format("Clipped alignment start is less than original read (minus %d): %s -> %s", i, read.getCigarString(), clippedRead.getCigarString())); Assert.assertTrue(clippedRead.getAlignmentEnd() <= alnEnd + i, String.format("Clipped alignment end is greater than original read (minus %d): %s -> %s", i, read.getCigarString(), clippedRead.getCigarString())); + assertUnclippedLimits(read, clippedRead); } } } @@ -72,12 +73,14 @@ public void testHardClipByReadCoordinates() { for (Cigar cigar : cigarList) { GATKSAMRecord read = ReadClipperTestUtils.makeReadFromCigar(cigar); int readLength = read.getReadLength(); - for (int i=0; i %s", i, read.getCigarString(), clipLeft.getCigarString())); + Assert.assertTrue(clipLeft.getReadLength() <= readLength - i, String.format("Clipped read length is greater than original read length (minus %d): %s -> %s", i, read.getCigarString(), clipLeft.getCigarString())); + assertUnclippedLimits(read, clipLeft); - GATKSAMRecord clipRight = ReadClipper.hardClipByReadCoordinates(read, i, readLength-1); + GATKSAMRecord clipRight = ReadClipper.hardClipByReadCoordinates(read, i, readLength - 1); Assert.assertTrue(clipRight.getReadLength() <= i, String.format("Clipped read length is greater than original read length (minus %d): %s -> %s", i, read.getCigarString(), clipRight.getCigarString())); + assertUnclippedLimits(read, clipRight); } } } @@ -86,19 +89,27 @@ public void testHardClipByReadCoordinates() { public void testHardClipByReferenceCoordinates() { for (Cigar cigar : cigarList) { GATKSAMRecord read = ReadClipperTestUtils.makeReadFromCigar(cigar); - int alnStart = read.getAlignmentStart(); - int alnEnd = read.getAlignmentEnd(); - for (int i=alnStart; i<=alnEnd; i++) { - if (read.getSoftStart() == alnStart) { // we can't test left clipping if the read has hanging soft clips on the left side - GATKSAMRecord clipLeft = (new ReadClipper(read)).hardClipByReferenceCoordinates(alnStart, i); - if (!clipLeft.isEmpty()) - Assert.assertTrue(clipLeft.getAlignmentStart() >= i + 1, String.format("Clipped alignment start (%d) is less the expected (%d): %s -> %s", clipLeft.getAlignmentStart(), i + 1, read.getCigarString(), clipLeft.getCigarString())); + int start = read.getSoftStart(); + int stop = read.getSoftEnd(); + +// System.out.println(String.format("CIGAR: %s (%d, %d)", cigar.toString(), start, stop)); + +// if (ReadUtils.readIsEntirelyInsertion(read)) +// System.out.println("debug"); + + for (int i = start; i <= stop; i++) { + GATKSAMRecord clipLeft = (new ReadClipper(read)).hardClipByReferenceCoordinates(-1, i); + if (!clipLeft.isEmpty()) { +// System.out.println(String.format("\t left [%d] %s -> %s ", i-start+1, cigar.toString(), clipLeft.getCigarString())); + Assert.assertTrue(clipLeft.getAlignmentStart() >= Math.min(read.getAlignmentEnd(), i + 1), String.format("Clipped alignment start (%d) is less the expected (%d): %s -> %s", clipLeft.getAlignmentStart(), i + 1, read.getCigarString(), clipLeft.getCigarString())); + assertUnclippedLimits(read, clipLeft); } - if (read.getSoftEnd() == alnEnd) { // we can't test right clipping if the read has hanging soft clips on the right side - GATKSAMRecord clipRight = (new ReadClipper(read)).hardClipByReferenceCoordinates(i, alnEnd); - if (!clipRight.isEmpty() && clipRight.getAlignmentStart() <= clipRight.getAlignmentEnd()) // alnStart > alnEnd if the entire read is a soft clip now. We can't test those. - Assert.assertTrue(clipRight.getAlignmentEnd() <= i - 1, String.format("Clipped alignment end (%d) is greater than expected (%d): %s -> %s", clipRight.getAlignmentEnd(), i - 1, read.getCigarString(), clipRight.getCigarString())); + GATKSAMRecord clipRight = (new ReadClipper(read)).hardClipByReferenceCoordinates(i, -1); + if (!clipRight.isEmpty() && clipRight.getAlignmentStart() <= clipRight.getAlignmentEnd()) { // alnStart > alnEnd if the entire read is a soft clip now. We can't test those. +// System.out.println(String.format("\t right [%d] %s -> %s ", i-start+1, cigar.toString(), clipRight.getCigarString())); + Assert.assertTrue(clipRight.getAlignmentEnd() <= Math.max(read.getAlignmentStart(), i - 1), String.format("Clipped alignment end (%d) is greater than expected (%d): %s -> %s", clipRight.getAlignmentEnd(), i - 1, read.getCigarString(), clipRight.getCigarString())); + assertUnclippedLimits(read, clipRight); } } } @@ -111,10 +122,14 @@ public void testHardClipByReferenceCoordinatesLeftTail() { int alnStart = read.getAlignmentStart(); int alnEnd = read.getAlignmentEnd(); if (read.getSoftStart() == alnStart) { // we can't test left clipping if the read has hanging soft clips on the left side - for (int i=alnStart; i<=alnEnd; i++) { - GATKSAMRecord clipLeft = ReadClipper.hardClipByReferenceCoordinatesLeftTail(read, i); - if (!clipLeft.isEmpty()) - Assert.assertTrue(clipLeft.getAlignmentStart() >= i + 1, String.format("Clipped alignment start (%d) is less the expected (%d): %s -> %s", clipLeft.getAlignmentStart(), i + 1, read.getCigarString(), clipLeft.getCigarString())); + for (int i = alnStart; i <= alnEnd; i++) { + GATKSAMRecord clipLeft = ReadClipper.hardClipByReferenceCoordinatesLeftTail(read, i); + + if (!clipLeft.isEmpty()) { +// System.out.println(String.format("Left Tail [%d]: %s (%d,%d,%d : %d,%d,%d) -> %s (%d,%d,%d : %d,%d,%d)", i, cigar.toString(), read.getUnclippedStart(), read.getSoftStart(), read.getAlignmentStart(), read.getAlignmentEnd(), read.getSoftEnd(), read.getUnclippedEnd(), clipLeft.getCigarString(), clipLeft.getUnclippedStart(), clipLeft.getSoftStart(), clipLeft.getAlignmentStart(), clipLeft.getAlignmentEnd(), clipLeft.getSoftEnd(), clipLeft.getUnclippedEnd())); + Assert.assertTrue(clipLeft.getAlignmentStart() >= i + 1, String.format("Clipped alignment start (%d) is less the expected (%d): %s -> %s", clipLeft.getAlignmentStart(), i + 1, read.getCigarString(), clipLeft.getCigarString())); + assertUnclippedLimits(read, clipLeft); + } } } } @@ -127,10 +142,12 @@ public void testHardClipByReferenceCoordinatesRightTail() { int alnStart = read.getAlignmentStart(); int alnEnd = read.getAlignmentEnd(); if (read.getSoftEnd() == alnEnd) { // we can't test right clipping if the read has hanging soft clips on the right side - for (int i=alnStart; i<=alnEnd; i++) { + for (int i = alnStart; i <= alnEnd; i++) { GATKSAMRecord clipRight = ReadClipper.hardClipByReferenceCoordinatesRightTail(read, i); - if (!clipRight.isEmpty() && clipRight.getAlignmentStart() <= clipRight.getAlignmentEnd()) // alnStart > alnEnd if the entire read is a soft clip now. We can't test those. + if (!clipRight.isEmpty() && clipRight.getAlignmentStart() <= clipRight.getAlignmentEnd()) { // alnStart > alnEnd if the entire read is a soft clip now. We can't test those. Assert.assertTrue(clipRight.getAlignmentEnd() <= i - 1, String.format("Clipped alignment end (%d) is greater than expected (%d): %s -> %s", clipRight.getAlignmentEnd(), i - 1, read.getCigarString(), clipRight.getCigarString())); + assertUnclippedLimits(read, clipRight); + } } } } @@ -145,43 +162,36 @@ public void testHardClipLowQualEnds() { for (Cigar cigar : cigarList) { GATKSAMRecord read = ReadClipperTestUtils.makeReadFromCigar(cigar); int readLength = read.getReadLength(); - byte [] quals = new byte[readLength]; + byte[] quals = new byte[readLength]; for (int nLowQualBases = 0; nLowQualBases < readLength; nLowQualBases++) { - - // create a read with nLowQualBases in the left tail - Utils.fillArrayWithByte(quals, HIGH_QUAL); + Utils.fillArrayWithByte(quals, HIGH_QUAL); // create a read with nLowQualBases in the left tail for (int addLeft = 0; addLeft < nLowQualBases; addLeft++) quals[addLeft] = LOW_QUAL; read.setBaseQualities(quals); GATKSAMRecord clipLeft = ReadClipper.hardClipLowQualEnds(read, LOW_QUAL); - // Tests - - // Make sure the low qualities are gone - assertNoLowQualBases(clipLeft, LOW_QUAL); + assertUnclippedLimits(read, clipLeft); // Make sure limits haven't changed + assertNoLowQualBases(clipLeft, LOW_QUAL); // Make sure the low qualities are gone + Assert.assertEquals(clipLeft.getReadLength(), readLength - nLowQualBases, // Make sure only low quality bases were clipped + String.format("Clipped read size (%d) is different than the number high qual bases (%d) -- Cigars: %s -> %s", clipLeft.getReadLength(), readLength - nLowQualBases, read.getCigarString(), clipLeft.getCigarString())); - // Can't run this test with the current contract of no hanging insertions -// Assert.assertEquals(clipLeft.getReadLength(), readLength - nLowQualBases, String.format("Clipped read size (%d) is different than the number high qual bases (%d) -- Cigars: %s -> %s", clipLeft.getReadLength(), readLength - nLowQualBases, read.getCigarString(), clipLeft.getCigarString())); - // create a read with nLowQualBases in the right tail - Utils.fillArrayWithByte(quals, HIGH_QUAL); + Utils.fillArrayWithByte(quals, HIGH_QUAL); // create a read with nLowQualBases in the right tail for (int addRight = 0; addRight < nLowQualBases; addRight++) quals[readLength - addRight - 1] = LOW_QUAL; read.setBaseQualities(quals); GATKSAMRecord clipRight = ReadClipper.hardClipLowQualEnds(read, LOW_QUAL); - // Tests - - // Make sure the low qualities are gone - assertNoLowQualBases(clipRight, LOW_QUAL); +// System.out.println(String.format("Debug [%d]: %s -> %s / %s", nLowQualBases, cigar.toString(), clipLeft.getCigarString(), clipRight.getCigarString())); - // Make sure we haven't clipped any high quals -- Can't run this test with the current contract of no hanging insertions - //Assert.assertEquals(clipLeft.getReadLength(), readLength - nLowQualBases, String.format("Clipped read size (%d) is different than the number high qual bases (%d) -- Cigars: %s -> %s", clipRight.getReadLength(), readLength - nLowQualBases, read.getCigarString(), clipRight.getCigarString())); + assertUnclippedLimits(read, clipRight); // Make sure limits haven't changed + assertNoLowQualBases(clipRight, LOW_QUAL); // Make sure the low qualities are gone + Assert.assertEquals(clipLeft.getReadLength(), readLength - nLowQualBases, // Make sure only low quality bases were clipped + String.format("Clipped read size (%d) is different than the number high qual bases (%d) -- Cigars: %s -> %s", clipRight.getReadLength(), readLength - nLowQualBases, read.getCigarString(), clipRight.getCigarString())); - // create a read with nLowQualBases in the both tails - if (nLowQualBases <= readLength/2) { - Utils.fillArrayWithByte(quals, HIGH_QUAL); + if (nLowQualBases <= readLength / 2) { + Utils.fillArrayWithByte(quals, HIGH_QUAL); // create a read with nLowQualBases on both tails for (int addBoth = 0; addBoth < nLowQualBases; addBoth++) { quals[addBoth] = LOW_QUAL; quals[readLength - addBoth - 1] = LOW_QUAL; @@ -189,83 +199,25 @@ public void testHardClipLowQualEnds() { read.setBaseQualities(quals); GATKSAMRecord clipBoth = ReadClipper.hardClipLowQualEnds(read, LOW_QUAL); - // Tests - - // Make sure the low qualities are gone - assertNoLowQualBases(clipBoth, LOW_QUAL); - - // Can't run this test with the current contract of no hanging insertions - //Assert.assertEquals(clipLeft.getReadLength(), readLength - nLowQualBases, String.format("Clipped read size (%d) is different than the number high qual bases (%d) -- Cigars: %s -> %s", clipRight.getReadLength(), readLength - (2*nLowQualBases), read.getCigarString(), clipBoth.getCigarString())); + assertUnclippedLimits(read, clipBoth); // Make sure limits haven't changed + assertNoLowQualBases(clipBoth, LOW_QUAL); // Make sure the low qualities are gone + Assert.assertEquals(clipLeft.getReadLength(), readLength - nLowQualBases, // Make sure only low quality bases were clipped + String.format("Clipped read size (%d) is different than the number high qual bases (%d) -- Cigars: %s -> %s", clipRight.getReadLength(), readLength - (2 * nLowQualBases), read.getCigarString(), clipBoth.getCigarString())); } } -// logger.warn(String.format("Testing %s for all combinations of low/high qual... PASSED", read.getCigarString())); } - - // ONE OFF Testing clipping that ends inside an insertion ( Ryan's bug ) - final byte[] BASES = {'A','C','G','T','A','C','G','T'}; - final byte[] QUALS = {2, 2, 2, 2, 20, 20, 20, 2}; - final String CIGAR = "1S1M5I1S"; - - final byte[] CLIPPED_BASES = {}; - final byte[] CLIPPED_QUALS = {}; - final String CLIPPED_CIGAR = ""; - - - GATKSAMRecord read = ArtificialSAMUtils.createArtificialRead(BASES, QUALS, CIGAR); - GATKSAMRecord expected = ArtificialSAMUtils.createArtificialRead(CLIPPED_BASES, CLIPPED_QUALS, CLIPPED_CIGAR); - - ReadClipperTestUtils.assertEqualReads(ReadClipper.hardClipLowQualEnds(read, (byte) 2), expected); } @Test(enabled = true) public void testHardClipSoftClippedBases() { - - // Generate a list of cigars to test for (Cigar cigar : cigarList) { GATKSAMRecord read = ReadClipperTestUtils.makeReadFromCigar(cigar); GATKSAMRecord clippedRead = ReadClipper.hardClipSoftClippedBases(read); + CigarCounter original = new CigarCounter(read); + CigarCounter clipped = new CigarCounter(clippedRead); - int sumHardClips = 0; - int sumMatches = 0; - - boolean tail = true; - for (CigarElement element : read.getCigar().getCigarElements()) { - // Assuming cigars are well formed, if we see S or H, it means we're on the tail (left or right) - if (element.getOperator() == CigarOperator.HARD_CLIP || element.getOperator() == CigarOperator.SOFT_CLIP) - tail = true; - - // Adds all H, S and D's (next to hard/soft clips). - // All these should be hard clips after clipping. - if (tail && (element.getOperator() == CigarOperator.HARD_CLIP || element.getOperator() == CigarOperator.SOFT_CLIP || element.getOperator() == CigarOperator.DELETION)) - sumHardClips += element.getLength(); - - // this means we're no longer on the tail (insertions can still potentially be the tail because - // of the current contract of clipping out hanging insertions - else if (element.getOperator() != CigarOperator.INSERTION) - tail = false; - - // Adds all matches to verify that they remain the same after clipping - if (element.getOperator() == CigarOperator.MATCH_OR_MISMATCH) - sumMatches += element.getLength(); - } - - for (CigarElement element : clippedRead.getCigar().getCigarElements()) { - // Test if clipped read has Soft Clips (shouldn't have any!) - Assert.assertTrue( element.getOperator() != CigarOperator.SOFT_CLIP, String.format("Cigar %s -> %s -- FAILED (resulting cigar has soft clips)", read.getCigarString(), clippedRead.getCigarString())); - - // Keep track of the total number of Hard Clips after clipping to make sure everything was accounted for - if (element.getOperator() == CigarOperator.HARD_CLIP) - sumHardClips -= element.getLength(); - - // Make sure all matches are still there - if (element.getOperator() == CigarOperator.MATCH_OR_MISMATCH) - sumMatches -= element.getLength(); - } - Assert.assertTrue( sumHardClips == 0, String.format("Cigar %s -> %s -- FAILED (number of hard clips mismatched by %d)", read.getCigarString(), clippedRead.getCigarString(), sumHardClips)); - Assert.assertTrue( sumMatches == 0, String.format("Cigar %s -> %s -- FAILED (number of matches mismatched by %d)", read.getCigarString(), clippedRead.getCigarString(), sumMatches)); - - -// logger.warn(String.format("Cigar %s -> %s -- PASSED!", read.getCigarString(), clippedRead.getCigarString())); + assertUnclippedLimits(read, clippedRead); // Make sure limits haven't changed + original.assertHardClippingSoftClips(clipped); // Make sure we have only clipped SOFT_CLIPS } } @@ -276,38 +228,39 @@ public void testHardClipLeadingInsertions() { GATKSAMRecord read = ReadClipperTestUtils.makeReadFromCigar(cigar); GATKSAMRecord clippedRead = ReadClipper.hardClipLeadingInsertions(read); + assertUnclippedLimits(read, clippedRead); // Make sure limits haven't changed + int expectedLength = read.getReadLength() - leadingCigarElementLength(read.getCigar(), CigarOperator.INSERTION); if (cigarHasElementsDifferentThanInsertionsAndHardClips(read.getCigar())) expectedLength -= leadingCigarElementLength(ReadClipperTestUtils.invertCigar(read.getCigar()), CigarOperator.INSERTION); - if (! clippedRead.isEmpty()) { + if (!clippedRead.isEmpty()) { Assert.assertEquals(expectedLength, clippedRead.getReadLength(), String.format("%s -> %s", read.getCigarString(), clippedRead.getCigarString())); // check that everything else is still there Assert.assertFalse(startsWithInsertion(clippedRead.getCigar())); // check that the insertions are gone - } - else + } else Assert.assertTrue(expectedLength == 0, String.format("expected length: %d", expectedLength)); // check that the read was expected to be fully clipped } } } @Test(enabled = true) - public void testRevertSoftClippedBases() - { - for (Cigar cigar: cigarList) { + public void testRevertSoftClippedBases() { + for (Cigar cigar : cigarList) { final int leadingSoftClips = leadingCigarElementLength(cigar, CigarOperator.SOFT_CLIP); final int tailSoftClips = leadingCigarElementLength(ReadClipperTestUtils.invertCigar(cigar), CigarOperator.SOFT_CLIP); final GATKSAMRecord read = ReadClipperTestUtils.makeReadFromCigar(cigar); final GATKSAMRecord unclipped = ReadClipper.revertSoftClippedBases(read); - if ( leadingSoftClips > 0 || tailSoftClips > 0) { + assertUnclippedLimits(read, unclipped); // Make sure limits haven't changed + + if (leadingSoftClips > 0 || tailSoftClips > 0) { final int expectedStart = read.getAlignmentStart() - leadingSoftClips; final int expectedEnd = read.getAlignmentEnd() + tailSoftClips; Assert.assertEquals(unclipped.getAlignmentStart(), expectedStart); Assert.assertEquals(unclipped.getAlignmentEnd(), expectedEnd); - } - else + } else Assert.assertEquals(read.getCigarString(), unclipped.getCigarString()); } } @@ -315,12 +268,25 @@ public void testRevertSoftClippedBases() private void assertNoLowQualBases(GATKSAMRecord read, byte low_qual) { if (!read.isEmpty()) { - byte [] quals = read.getBaseQualities(); - for (int i=0; i 0; } @@ -335,10 +301,46 @@ private int leadingCigarElementLength(Cigar cigar, CigarOperator operator) { return 0; } - private boolean cigarHasElementsDifferentThanInsertionsAndHardClips (Cigar cigar) { + private boolean cigarHasElementsDifferentThanInsertionsAndHardClips(Cigar cigar) { for (CigarElement cigarElement : cigar.getCigarElements()) if (cigarElement.getOperator() != CigarOperator.INSERTION && cigarElement.getOperator() != CigarOperator.HARD_CLIP) return true; return false; } + + private class CigarCounter { + private HashMap counter; + + public Integer getCounterForOp(CigarOperator operator) { + return counter.get(operator); + } + + public CigarCounter(GATKSAMRecord read) { + CigarOperator[] operators = CigarOperator.values(); + counter = new HashMap(operators.length); + + for (CigarOperator op : operators) + counter.put(op, 0); + + for (CigarElement cigarElement : read.getCigar().getCigarElements()) + counter.put(cigarElement.getOperator(), counter.get(cigarElement.getOperator()) + cigarElement.getLength()); + } + + public boolean assertHardClippingSoftClips(CigarCounter clipped) { + for (CigarOperator op : counter.keySet()) { + if (op == CigarOperator.HARD_CLIP || op == CigarOperator.SOFT_CLIP) { + int counterTotal = counter.get(CigarOperator.HARD_CLIP) + counter.get(CigarOperator.SOFT_CLIP); + int clippedHard = clipped.getCounterForOp(CigarOperator.HARD_CLIP); + int clippedSoft = clipped.getCounterForOp(CigarOperator.SOFT_CLIP); + + Assert.assertEquals(counterTotal, clippedHard); + Assert.assertTrue(clippedSoft == 0); + } else + Assert.assertEquals(counter.get(op), clipped.getCounterForOp(op)); + } + return true; + } + + } + } \ No newline at end of file From cd68cc239b9dd200fb9a741e55dd60221c9d5c4a Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Thu, 29 Dec 2011 00:41:59 -0500 Subject: [PATCH 008/356] Added knuth-shuffle (KS) and randomSubset using KS to MathUtils * Knuth-shuffle is a simple, yet effective array permutator (hope this is good english). * added a simple randomSubset that returns a random subset without repeats of any given array with the same probability for every permutation. * added unit tests to both functions --- .../broadinstitute/sting/utils/MathUtils.java | 816 ++++++++++-------- .../sting/utils/MathUtilsUnitTest.java | 92 +- 2 files changed, 544 insertions(+), 364 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java index 759e1649df..4a3100a94d 100644 --- a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java @@ -25,6 +25,7 @@ package org.broadinstitute.sting.utils; +import com.google.java.contract.Ensures; import com.google.java.contract.Requires; import net.sf.samtools.SAMRecord; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; @@ -49,8 +50,11 @@ public class MathUtils { */ - /** Private constructor. No instantiating this class! */ - private MathUtils() {} + /** + * Private constructor. No instantiating this class! + */ + private MathUtils() { + } @Requires({"d > 0.0"}) public static int fastPositiveRound(double d) { @@ -58,21 +62,21 @@ public static int fastPositiveRound(double d) { } public static int fastRound(double d) { - if ( d > 0.0 ) { + if (d > 0.0) { return fastPositiveRound(d); } else { - return -1*fastPositiveRound(-1*d); + return -1 * fastPositiveRound(-1 * d); } } public static double sum(Collection numbers) { - return sum(numbers,false); + return sum(numbers, false); } - public static double sum( Collection numbers, boolean ignoreNan ) { + public static double sum(Collection numbers, boolean ignoreNan) { double sum = 0; - for ( Number n : numbers ) { - if ( ! ignoreNan || ! Double.isNaN(n.doubleValue())) { + for (Number n : numbers) { + if (!ignoreNan || !Double.isNaN(n.doubleValue())) { sum += n.doubleValue(); } } @@ -82,66 +86,72 @@ public static double sum( Collection numbers, boolean ignoreNan ) { public static int nonNanSize(Collection numbers) { int size = 0; - for ( Number n : numbers) { + for (Number n : numbers) { size += Double.isNaN(n.doubleValue()) ? 0 : 1; } return size; } - public static double average( Collection numbers, boolean ignoreNan) { - if ( ignoreNan ) { - return sum(numbers,true)/nonNanSize(numbers); + public static double average(Collection numbers, boolean ignoreNan) { + if (ignoreNan) { + return sum(numbers, true) / nonNanSize(numbers); } else { - return sum(numbers,false)/nonNanSize(numbers); + return sum(numbers, false) / nonNanSize(numbers); } } - public static double variance( Collection numbers, Number mean, boolean ignoreNan ) { + public static double variance(Collection numbers, Number mean, boolean ignoreNan) { double mn = mean.doubleValue(); double var = 0; - for ( Number n : numbers ) { var += ( ! ignoreNan || ! Double.isNaN(n.doubleValue())) ? (n.doubleValue()-mn)*(n.doubleValue()-mn) : 0; } - if ( ignoreNan ) { return var/(nonNanSize(numbers)-1); } - return var/(numbers.size()-1); + for (Number n : numbers) { + var += (!ignoreNan || !Double.isNaN(n.doubleValue())) ? (n.doubleValue() - mn) * (n.doubleValue() - mn) : 0; + } + if (ignoreNan) { + return var / (nonNanSize(numbers) - 1); + } + return var / (numbers.size() - 1); } public static double variance(Collection numbers, Number mean) { - return variance(numbers,mean,false); + return variance(numbers, mean, false); } public static double variance(Collection numbers, boolean ignoreNan) { - return variance(numbers,average(numbers,ignoreNan),ignoreNan); + return variance(numbers, average(numbers, ignoreNan), ignoreNan); } public static double variance(Collection numbers) { - return variance(numbers,average(numbers,false),false); + return variance(numbers, average(numbers, false), false); } public static double sum(double[] values) { double s = 0.0; - for ( double v : values) s += v; + for (double v : values) s += v; return s; } /** * Calculates the log10 cumulative sum of an array with log10 probabilities + * * @param log10p the array with log10 probabilites - * @param upTo index in the array to calculate the cumsum up to + * @param upTo index in the array to calculate the cumsum up to * @return the log10 of the cumulative sum */ - public static double log10CumulativeSumLog10(double [] log10p, int upTo) { + public static double log10CumulativeSumLog10(double[] log10p, int upTo) { return log10sumLog10(log10p, 0, upTo); } /** * Converts a real space array of probabilities into a log10 array + * * @param prRealSpace * @return */ public static double[] toLog10(double[] prRealSpace) { double[] log10s = new double[prRealSpace.length]; - for ( int i = 0; i < prRealSpace.length; i++ ) + for (int i = 0; i < prRealSpace.length; i++) log10s[i] = Math.log10(prRealSpace[i]); return log10s; } @@ -154,7 +164,7 @@ public static double log10sumLog10(double[] log10p, int start, int finish) { double sum = 0.0; double maxValue = Utils.findMaxEntry(log10p); - for ( int i = start; i < finish; i++ ) { + for (int i = start; i < finish; i++) { sum += Math.pow(10.0, log10p[i] - maxValue); } @@ -163,13 +173,13 @@ public static double log10sumLog10(double[] log10p, int start, int finish) { public static double sumDoubles(List values) { double s = 0.0; - for ( double v : values) s += v; + for (double v : values) s += v; return s; } public static int sumIntegers(List values) { int s = 0; - for ( int v : values) s += v; + for (int v : values) s += v; return s; } @@ -185,11 +195,11 @@ public static double log10sumLog10(double[] log10values) { } public static boolean wellFormedDouble(double val) { - return ! Double.isInfinite(val) && ! Double.isNaN(val); + return !Double.isInfinite(val) && !Double.isNaN(val); } public static double bound(double value, double minBoundary, double maxBoundary) { - return Math.max(Math.min(value, maxBoundary), minBoundary); + return Math.max(Math.min(value, maxBoundary), minBoundary); } public static boolean isBounded(double val, double lower, double upper) { @@ -197,7 +207,7 @@ public static boolean isBounded(double val, double lower, double upper) { } public static boolean isPositive(double val) { - return ! isNegativeOrZero(val); + return !isNegativeOrZero(val); } public static boolean isPositiveOrZero(double val) { @@ -209,17 +219,19 @@ public static boolean isNegativeOrZero(double val) { } public static boolean isNegative(double val) { - return ! isPositiveOrZero(val); + return !isPositiveOrZero(val); } /** * Compares double values for equality (within 1e-6), or inequality. * - * @param a the first double value - * @param b the second double value - * @return -1 if a is greater than b, 0 if a is equal to be within 1e-6, 1 if b is greater than a. + * @param a the first double value + * @param b the second double value + * @return -1 if a is greater than b, 0 if a is equal to be within 1e-6, 1 if b is greater than a. */ - public static byte compareDoubles(double a, double b) { return compareDoubles(a, b, 1e-6); } + public static byte compareDoubles(double a, double b) { + return compareDoubles(a, b, 1e-6); + } /** * Compares double values for equality (within epsilon), or inequality. @@ -227,23 +239,28 @@ public static boolean isNegative(double val) { * @param a the first double value * @param b the second double value * @param epsilon the precision within which two double values will be considered equal - * @return -1 if a is greater than b, 0 if a is equal to be within epsilon, 1 if b is greater than a. + * @return -1 if a is greater than b, 0 if a is equal to be within epsilon, 1 if b is greater than a. */ - public static byte compareDoubles(double a, double b, double epsilon) - { - if (Math.abs(a - b) < epsilon) { return 0; } - if (a > b) { return -1; } + public static byte compareDoubles(double a, double b, double epsilon) { + if (Math.abs(a - b) < epsilon) { + return 0; + } + if (a > b) { + return -1; + } return 1; } /** * Compares float values for equality (within 1e-6), or inequality. * - * @param a the first float value - * @param b the second float value - * @return -1 if a is greater than b, 0 if a is equal to be within 1e-6, 1 if b is greater than a. + * @param a the first float value + * @param b the second float value + * @return -1 if a is greater than b, 0 if a is equal to be within 1e-6, 1 if b is greater than a. */ - public static byte compareFloats(float a, float b) { return compareFloats(a, b, 1e-6f); } + public static byte compareFloats(float a, float b) { + return compareFloats(a, b, 1e-6f); + } /** * Compares float values for equality (within epsilon), or inequality. @@ -251,47 +268,50 @@ public static byte compareDoubles(double a, double b, double epsilon) * @param a the first float value * @param b the second float value * @param epsilon the precision within which two float values will be considered equal - * @return -1 if a is greater than b, 0 if a is equal to be within epsilon, 1 if b is greater than a. + * @return -1 if a is greater than b, 0 if a is equal to be within epsilon, 1 if b is greater than a. */ - public static byte compareFloats(float a, float b, float epsilon) - { - if (Math.abs(a - b) < epsilon) { return 0; } - if (a > b) { return -1; } + public static byte compareFloats(float a, float b, float epsilon) { + if (Math.abs(a - b) < epsilon) { + return 0; + } + if (a > b) { + return -1; + } return 1; } - public static double NormalDistribution(double mean, double sd, double x) - { - double a = 1.0 / (sd*Math.sqrt(2.0 * Math.PI)); - double b = Math.exp(-1.0 * (Math.pow(x - mean,2.0)/(2.0 * sd * sd))); + public static double NormalDistribution(double mean, double sd, double x) { + double a = 1.0 / (sd * Math.sqrt(2.0 * Math.PI)); + double b = Math.exp(-1.0 * (Math.pow(x - mean, 2.0) / (2.0 * sd * sd))); return a * b; } - public static double binomialCoefficient (int n, int k) { + public static double binomialCoefficient(int n, int k) { return Math.pow(10, log10BinomialCoefficient(n, k)); } + /** * Computes a binomial probability. This is computed using the formula - * - * B(k; n; p) = [ n! / ( k! (n - k)! ) ] (p^k)( (1-p)^k ) - * + *

+ * B(k; n; p) = [ n! / ( k! (n - k)! ) ] (p^k)( (1-p)^k ) + *

* where n is the number of trials, k is the number of successes, and p is the probability of success * - * @param n number of Bernoulli trials - * @param k number of successes - * @param p probability of success - * - * @return the binomial probability of the specified configuration. Computes values down to about 1e-237. + * @param n number of Bernoulli trials + * @param k number of successes + * @param p probability of success + * @return the binomial probability of the specified configuration. Computes values down to about 1e-237. */ - public static double binomialProbability (int n, int k, double p) { + public static double binomialProbability(int n, int k, double p) { return Math.pow(10, log10BinomialProbability(n, k, Math.log10(p))); } /** * Performs the cumulative sum of binomial probabilities, where the probability calculation is done in log space. - * @param start - start of the cumulant sum (over hits) - * @param end - end of the cumulant sum (over hits) - * @param total - number of attempts for the number of hits + * + * @param start - start of the cumulant sum (over hits) + * @param end - end of the cumulant sum (over hits) + * @param total - number of attempts for the number of hits * @param probHit - probability of a successful hit * @return - returns the cumulative probability */ @@ -300,11 +320,11 @@ public static double binomialCumulativeProbability(int start, int end, int total double prevProb; BigDecimal probCache = BigDecimal.ZERO; - for(int hits = start; hits < end; hits++) { + for (int hits = start; hits < end; hits++) { prevProb = cumProb; double probability = binomialProbability(total, hits, probHit); cumProb += probability; - if ( probability > 0 && cumProb - prevProb < probability/2 ) { // loss of precision + if (probability > 0 && cumProb - prevProb < probability / 2) { // loss of precision probCache = probCache.add(new BigDecimal(prevProb)); cumProb = 0.0; hits--; // repeat loop @@ -314,20 +334,20 @@ public static double binomialCumulativeProbability(int start, int end, int total return probCache.add(new BigDecimal(cumProb)).doubleValue(); } - + /** * Computes a multinomial coefficient efficiently avoiding overflow even for large numbers. * This is computed using the formula: - * - * M(x1,x2,...,xk; n) = [ n! / (x1! x2! ... xk!) ] - * + *

+ * M(x1,x2,...,xk; n) = [ n! / (x1! x2! ... xk!) ] + *

* where xi represents the number of times outcome i was observed, n is the number of total observations. * In this implementation, the value of n is inferred as the sum over i of xi. * - * @param k an int[] of counts, where each element represents the number of times a certain outcome was observed - * @return the multinomial of the specified configuration. + * @param k an int[] of counts, where each element represents the number of times a certain outcome was observed + * @return the multinomial of the specified configuration. */ - public static double multinomialCoefficient (int [] k) { + public static double multinomialCoefficient(int[] k) { int n = 0; for (int xi : k) { n += xi; @@ -339,37 +359,38 @@ public static double multinomialCoefficient (int [] k) { /** * Computes a multinomial probability efficiently avoiding overflow even for large numbers. * This is computed using the formula: - * - * M(x1,x2,...,xk; n; p1,p2,...,pk) = [ n! / (x1! x2! ... xk!) ] (p1^x1)(p2^x2)(...)(pk^xk) - * + *

+ * M(x1,x2,...,xk; n; p1,p2,...,pk) = [ n! / (x1! x2! ... xk!) ] (p1^x1)(p2^x2)(...)(pk^xk) + *

* where xi represents the number of times outcome i was observed, n is the number of total observations, and * pi represents the probability of the i-th outcome to occur. In this implementation, the value of n is * inferred as the sum over i of xi. * - * @param k an int[] of counts, where each element represents the number of times a certain outcome was observed - * @param p a double[] of probabilities, where each element represents the probability a given outcome can occur - * @return the multinomial probability of the specified configuration. + * @param k an int[] of counts, where each element represents the number of times a certain outcome was observed + * @param p a double[] of probabilities, where each element represents the probability a given outcome can occur + * @return the multinomial probability of the specified configuration. */ - public static double multinomialProbability (int[] k, double[] p) { + public static double multinomialProbability(int[] k, double[] p) { if (p.length != k.length) throw new UserException.BadArgumentValue("p and k", "Array of log10 probabilities must have the same size as the array of number of sucesses: " + p.length + ", " + k.length); int n = 0; - double [] log10P = new double[p.length]; - for (int i=0; i l) { double rms = 0.0; for (int i : l) - rms += i*i; + rms += i * i; rms /= l.size(); return Math.sqrt(rms); } - public static double distanceSquared( final double[] x, final double[] y ) { + public static double distanceSquared(final double[] x, final double[] y) { double dist = 0.0; - for(int iii = 0; iii < x.length; iii++) { + for (int iii = 0; iii < x.length; iii++) { dist += (x[iii] - y[iii]) * (x[iii] - y[iii]); } return dist; } public static double round(double num, int digits) { - double result = num * Math.pow(10.0, (double)digits); + double result = num * Math.pow(10.0, (double) digits); result = Math.round(result); - result = result / Math.pow(10.0, (double)digits); + result = result / Math.pow(10.0, (double) digits); return result; } @@ -442,11 +465,10 @@ public static double round(double num, int digits) { /** * normalizes the log10-based array. ASSUMES THAT ALL ARRAY ENTRIES ARE <= 0 (<= 1 IN REAL-SPACE). * - * @param array the array to be normalized + * @param array the array to be normalized * @param takeLog10OfOutput if true, the output will be transformed back into log10 units - * * @return a newly allocated array corresponding the normalized values in array, maybe log10 transformed - */ + */ public static double[] normalizeFromLog10(double[] array, boolean takeLog10OfOutput) { return normalizeFromLog10(array, takeLog10OfOutput, false); } @@ -476,7 +498,7 @@ public static double[] normalizeFromLog10(double[] array, boolean takeLog10OfOut sum += normalized[i]; for (int i = 0; i < array.length; i++) { double x = normalized[i] / sum; - if ( takeLog10OfOutput ) x = Math.log10(x); + if (takeLog10OfOutput) x = Math.log10(x); normalized[i] = x; } @@ -488,7 +510,7 @@ public static double[] normalizeFromLog10(List array, boolean takeLog10O // for precision purposes, we need to add (or really subtract, since they're // all negative) the largest value; also, we need to convert to normal-space. - double maxValue = MathUtils.arrayMaxDouble( array ); + double maxValue = MathUtils.arrayMaxDouble(array); for (int i = 0; i < array.size(); i++) normalized[i] = Math.pow(10, array.get(i) - maxValue); @@ -498,19 +520,19 @@ public static double[] normalizeFromLog10(List array, boolean takeLog10O sum += normalized[i]; for (int i = 0; i < array.size(); i++) { double x = normalized[i] / sum; - if ( takeLog10OfOutput ) x = Math.log10(x); + if (takeLog10OfOutput) x = Math.log10(x); normalized[i] = x; } return normalized; } + /** * normalizes the log10-based array. ASSUMES THAT ALL ARRAY ENTRIES ARE <= 0 (<= 1 IN REAL-SPACE). * - * @param array the array to be normalized - * + * @param array the array to be normalized * @return a newly allocated array corresponding the normalized values in array - */ + */ public static double[] normalizeFromLog10(double[] array) { return normalizeFromLog10(array, false); } @@ -520,11 +542,11 @@ public static double[] normalizeFromLog10(List array) { } public static int maxElementIndex(double[] array) { - if ( array == null ) throw new IllegalArgumentException("Array cannot be null!"); + if (array == null) throw new IllegalArgumentException("Array cannot be null!"); int maxI = -1; - for ( int i = 0; i < array.length; i++ ) { - if ( maxI == -1 || array[i] > array[maxI] ) + for (int i = 0; i < array.length; i++) { + if (maxI == -1 || array[i] > array[maxI]) maxI = i; } @@ -532,11 +554,11 @@ public static int maxElementIndex(double[] array) { } public static int maxElementIndex(int[] array) { - if ( array == null ) throw new IllegalArgumentException("Array cannot be null!"); + if (array == null) throw new IllegalArgumentException("Array cannot be null!"); int maxI = -1; - for ( int i = 0; i < array.length; i++ ) { - if ( maxI == -1 || array[i] > array[maxI] ) + for (int i = 0; i < array.length; i++) { + if (maxI == -1 || array[i] > array[maxI]) maxI = i; } @@ -556,11 +578,11 @@ public static byte arrayMin(byte[] array) { } public static int minElementIndex(double[] array) { - if ( array == null ) throw new IllegalArgumentException("Array cannot be null!"); + if (array == null) throw new IllegalArgumentException("Array cannot be null!"); int minI = -1; - for ( int i = 0; i < array.length; i++ ) { - if ( minI == -1 || array[i] < array[minI] ) + for (int i = 0; i < array.length; i++) { + if (minI == -1 || array[i] < array[minI]) minI = i; } @@ -568,32 +590,32 @@ public static int minElementIndex(double[] array) { } public static int minElementIndex(byte[] array) { - if ( array == null ) throw new IllegalArgumentException("Array cannot be null!"); + if (array == null) throw new IllegalArgumentException("Array cannot be null!"); int minI = -1; - for ( int i = 0; i < array.length; i++ ) { - if ( minI == -1 || array[i] < array[minI] ) + for (int i = 0; i < array.length; i++) { + if (minI == -1 || array[i] < array[minI]) minI = i; } return minI; - } + } public static int arrayMaxInt(List array) { - if ( array == null ) throw new IllegalArgumentException("Array cannot be null!"); - if ( array.size() == 0 ) throw new IllegalArgumentException("Array size cannot be 0!"); + if (array == null) throw new IllegalArgumentException("Array cannot be null!"); + if (array.size() == 0) throw new IllegalArgumentException("Array size cannot be 0!"); int m = array.get(0); - for ( int e : array ) m = Math.max(m, e); + for (int e : array) m = Math.max(m, e); return m; } public static double arrayMaxDouble(List array) { - if ( array == null ) throw new IllegalArgumentException("Array cannot be null!"); - if ( array.size() == 0 ) throw new IllegalArgumentException("Array size cannot be 0!"); + if (array == null) throw new IllegalArgumentException("Array cannot be null!"); + if (array.size() == 0) throw new IllegalArgumentException("Array size cannot be 0!"); double m = array.get(0); - for ( double e : array ) m = Math.max(m, e); + for (double e : array) m = Math.max(m, e); return m; } @@ -636,7 +658,7 @@ public static byte average(byte[] vals) { for (byte v : vals) { sum += v; } - return (byte) Math.floor(sum/vals.length); + return (byte) Math.floor(sum / vals.length); } public static double averageDouble(List vals) { @@ -749,7 +771,9 @@ public static List permuteList(List list, Integer[] permutation) { } - /** Draw N random elements from list. */ + /** + * Draw N random elements from list. + */ public static List randomSubset(List list, int N) { if (list.size() <= N) { return list; @@ -770,6 +794,25 @@ public static List randomSubset(List list, int N) { return ans; } + /** + * Draw N random elements from an array. + * + * @param array your objects + * @param n number of elements to select at random from the list + * @return a new list with the N randomly chosen elements from list + */ + @Requires({"array != null", "n>=0"}) + @Ensures({"result != null", "result.length == Math.min(n, array.length)"}) + public static Object[] randomSubset(final Object[] array, final int n) { + if (array.length <= n) + return array.clone(); + + Object[] shuffledArray = arrayShuffle(array); + Object[] result = new Object[n]; + System.arraycopy(shuffledArray, 0, result, 0, n); + return result; + } + public static double percentage(double x, double base) { return (base > 0 ? (x / base) * 100.0 : 0); } @@ -799,7 +842,7 @@ public static int countOccurrences(T x, List l) { return count; } - public static int countOccurrences(byte element, byte [] array) { + public static int countOccurrences(byte element, byte[] array) { int count = 0; for (byte y : array) { if (element == y) @@ -814,13 +857,13 @@ public static int countOccurrences(byte element, byte [] array) { * Better than sorting if N (number of elements to return) is small * * @param array the array - * @param n number of top elements to return + * @param n number of top elements to return * @return the n larger elements of the array */ - public static Collection getNMaxElements(double [] array, int n) { + public static Collection getNMaxElements(double[] array, int n) { ArrayList maxN = new ArrayList(n); double lastMax = Double.MAX_VALUE; - for (int i=0; i getNMaxElements(double [] array, int n) { */ static public ArrayList sampleIndicesWithReplacement(int n, int k) { - ArrayList chosen_balls = new ArrayList (k); - for (int i=0; i< k; i++) { + ArrayList chosen_balls = new ArrayList(k); + for (int i = 0; i < k; i++) { //Integer chosen_ball = balls[rand.nextInt(k)]; chosen_balls.add(GenomeAnalysisEngine.getRandomGenerator().nextInt(n)); //balls.remove(chosen_ball); @@ -872,11 +915,11 @@ static public ArrayList sampleIndicesWithoutReplacement(int n, int k) { /** * Given a list of indices into a list, return those elements of the list with the possibility of drawing list elements multiple times - - * @param indices the list of indices for elements to extract - * @param list the list from which the elements should be extracted - * @param the template type of the ArrayList - * @return a new ArrayList consisting of the elements at the specified indices + * + * @param indices the list of indices for elements to extract + * @param list the list from which the elements should be extracted + * @param the template type of the ArrayList + * @return a new ArrayList consisting of the elements at the specified indices */ static public ArrayList sliceListByIndices(List indices, List list) { ArrayList subset = new ArrayList(); @@ -898,18 +941,18 @@ public static Comparable orderStatisticSearch(int orderStat, List li ArrayList equalToX = new ArrayList(); ArrayList greaterThanX = new ArrayList(); - for(Comparable y : list) { - if(x.compareTo(y) > 0) { + for (Comparable y : list) { + if (x.compareTo(y) > 0) { lessThanX.add(y); - } else if(x.compareTo(y) < 0) { + } else if (x.compareTo(y) < 0) { greaterThanX.add(y); } else equalToX.add(y); } - if(lessThanX.size() > orderStat) + if (lessThanX.size() > orderStat) return orderStatisticSearch(orderStat, lessThanX); - else if(lessThanX.size() + equalToX.size() >= orderStat) + else if (lessThanX.size() + equalToX.size() >= orderStat) return orderStat; else return orderStatisticSearch(orderStat - lessThanX.size() - equalToX.size(), greaterThanX); @@ -918,7 +961,7 @@ else if(lessThanX.size() + equalToX.size() >= orderStat) public static Object getMedian(List list) { - return orderStatisticSearch((int) Math.ceil(list.size()/2), list); + return orderStatisticSearch((int) Math.ceil(list.size() / 2), list); } public static byte getQScoreOrderStatistic(List reads, List offsets, int k) { @@ -926,7 +969,7 @@ public static byte getQScoreOrderStatistic(List reads, List // list index maps to a q-score only through the offset index // returns the kth-largest q-score. - if( reads.size() == 0) { + if (reads.size() == 0) { return 0; } @@ -938,15 +981,15 @@ public static byte getQScoreOrderStatistic(List reads, List final byte qk = reads.get(k).getBaseQualities()[offsets.get(k)]; - for(int iter = 0; iter < reads.size(); iter ++) { + for (int iter = 0; iter < reads.size(); iter++) { SAMRecord read = reads.get(iter); int offset = offsets.get(iter); byte quality = read.getBaseQualities()[offset]; - if(quality < qk) { + if (quality < qk) { lessThanQReads.add(read); lessThanQOffsets.add(offset); - } else if(quality > qk) { + } else if (quality > qk) { greaterThanQReads.add(read); greaterThanQOffsets.add(offset); } else { @@ -954,9 +997,9 @@ public static byte getQScoreOrderStatistic(List reads, List } } - if(lessThanQReads.size() > k) + if (lessThanQReads.size() > k) return getQScoreOrderStatistic(lessThanQReads, lessThanQOffsets, k); - else if(equalToQReads.size() + lessThanQReads.size() >= k) + else if (equalToQReads.size() + lessThanQReads.size() >= k) return qk; else return getQScoreOrderStatistic(greaterThanQReads, greaterThanQOffsets, k - lessThanQReads.size() - equalToQReads.size()); @@ -964,10 +1007,11 @@ else if(equalToQReads.size() + lessThanQReads.size() >= k) } public static byte getQScoreMedian(List reads, List offsets) { - return getQScoreOrderStatistic(reads, offsets, (int)Math.floor(reads.size()/2.)); + return getQScoreOrderStatistic(reads, offsets, (int) Math.floor(reads.size() / 2.)); } - /** A utility class that computes on the fly average and standard deviation for a stream of numbers. + /** + * A utility class that computes on the fly average and standard deviation for a stream of numbers. * The number of observations does not have to be known in advance, and can be also very big (so that * it could overflow any naive summation-based scheme or cause loss of precision). * Instead, adding a new number observed @@ -983,20 +1027,31 @@ public static class RunningAverage { public void add(double obs) { obs_count++; double oldMean = mean; - mean += ( obs - mean ) / obs_count; // update mean - s += ( obs - oldMean ) * ( obs - mean ); + mean += (obs - mean) / obs_count; // update mean + s += (obs - oldMean) * (obs - mean); } public void addAll(Collection col) { - for ( Number o : col ) { + for (Number o : col) { add(o.doubleValue()); } } - public double mean() { return mean; } - public double stddev() { return Math.sqrt(s/(obs_count - 1)); } - public double var() { return s/(obs_count - 1); } - public long observationCount() { return obs_count; } + public double mean() { + return mean; + } + + public double stddev() { + return Math.sqrt(s / (obs_count - 1)); + } + + public double var() { + return s / (obs_count - 1); + } + + public long observationCount() { + return obs_count; + } public RunningAverage clone() { RunningAverage ra = new RunningAverage(); @@ -1007,71 +1062,86 @@ public RunningAverage clone() { } public void merge(RunningAverage other) { - if ( this.obs_count > 0 || other.obs_count > 0 ) { // if we have any observations at all - this.mean = ( this.mean * this.obs_count + other.mean * other.obs_count ) / ( this.obs_count + other.obs_count ); + if (this.obs_count > 0 || other.obs_count > 0) { // if we have any observations at all + this.mean = (this.mean * this.obs_count + other.mean * other.obs_count) / (this.obs_count + other.obs_count); this.s += other.s; } this.obs_count += other.obs_count; } } - + // // useful common utility routines // - public static double rate(long n, long d) { return n / (1.0 * Math.max(d, 1)); } - public static double rate(int n, int d) { return n / (1.0 * Math.max(d, 1)); } + public static double rate(long n, long d) { + return n / (1.0 * Math.max(d, 1)); + } + + public static double rate(int n, int d) { + return n / (1.0 * Math.max(d, 1)); + } + + public static long inverseRate(long n, long d) { + return n == 0 ? 0 : d / Math.max(n, 1); + } - public static long inverseRate(long n, long d) { return n == 0 ? 0 : d / Math.max(n, 1); } - public static long inverseRate(int n, int d) { return n == 0 ? 0 : d / Math.max(n, 1); } + public static long inverseRate(int n, int d) { + return n == 0 ? 0 : d / Math.max(n, 1); + } - public static double ratio(int num, int denom) { return ((double)num) / (Math.max(denom, 1)); } - public static double ratio(long num, long denom) { return ((double)num) / (Math.max(denom, 1)); } + public static double ratio(int num, int denom) { + return ((double) num) / (Math.max(denom, 1)); + } + + public static double ratio(long num, long denom) { + return ((double) num) / (Math.max(denom, 1)); + } public static final double[] log10Cache; public static final double[] jacobianLogTable; public static final int JACOBIAN_LOG_TABLE_SIZE = 101; public static final double JACOBIAN_LOG_TABLE_STEP = 0.1; - public static final double INV_JACOBIAN_LOG_TABLE_STEP = 1.0/JACOBIAN_LOG_TABLE_STEP; + public static final double INV_JACOBIAN_LOG_TABLE_STEP = 1.0 / JACOBIAN_LOG_TABLE_STEP; public static final double MAX_JACOBIAN_TOLERANCE = 10.0; private static final int MAXN = 11000; private static final int LOG10_CACHE_SIZE = 4 * MAXN; // we need to be able to go up to 2*(2N) when calculating some of the coefficients static { log10Cache = new double[LOG10_CACHE_SIZE]; - jacobianLogTable = new double[JACOBIAN_LOG_TABLE_SIZE]; + jacobianLogTable = new double[JACOBIAN_LOG_TABLE_SIZE]; log10Cache[0] = Double.NEGATIVE_INFINITY; - for (int k=1; k < LOG10_CACHE_SIZE; k++) + for (int k = 1; k < LOG10_CACHE_SIZE; k++) log10Cache[k] = Math.log10(k); - for (int k=0; k < JACOBIAN_LOG_TABLE_SIZE; k++) { - jacobianLogTable[k] = Math.log10(1.0+Math.pow(10.0,-((double)k) - * JACOBIAN_LOG_TABLE_STEP)); + for (int k = 0; k < JACOBIAN_LOG_TABLE_SIZE; k++) { + jacobianLogTable[k] = Math.log10(1.0 + Math.pow(10.0, -((double) k) + * JACOBIAN_LOG_TABLE_STEP)); - } + } } static public double softMax(final double[] vec) { double acc = vec[0]; - for (int k=1; k < vec.length; k++) - acc = softMax(acc,vec[k]); + for (int k = 1; k < vec.length; k++) + acc = softMax(acc, vec[k]); return acc; } static public double max(double x0, double x1, double x2) { - double a = Math.max(x0,x1); - return Math.max(a,x2); + double a = Math.max(x0, x1); + return Math.max(a, x2); } - + static public double softMax(final double x0, final double x1, final double x2) { - // compute naively log10(10^x[0] + 10^x[1]+...) - // return Math.log10(MathUtils.sumLog10(vec)); + // compute naively log10(10^x[0] + 10^x[1]+...) + // return Math.log10(MathUtils.sumLog10(vec)); - // better approximation: do Jacobian logarithm function on data pairs - double a = softMax(x0,x1); - return softMax(a,x2); + // better approximation: do Jacobian logarithm function on data pairs + double a = softMax(x0, x1); + return softMax(a, x2); } static public double softMax(final double x, final double y) { @@ -1084,49 +1154,50 @@ static public double softMax(final double x, final double y) { // slow exact version: // return Math.log10(Math.pow(10.0,x) + Math.pow(10.0,y)); - double diff = x-y; + double diff = x - y; if (diff > MAX_JACOBIAN_TOLERANCE) return x; else if (diff < -MAX_JACOBIAN_TOLERANCE) return y; else if (diff >= 0) { - int ind = (int)(diff*INV_JACOBIAN_LOG_TABLE_STEP+0.5); + int ind = (int) (diff * INV_JACOBIAN_LOG_TABLE_STEP + 0.5); return x + jacobianLogTable[ind]; - } - else { - int ind = (int)(-diff*INV_JACOBIAN_LOG_TABLE_STEP+0.5); + } else { + int ind = (int) (-diff * INV_JACOBIAN_LOG_TABLE_STEP + 0.5); return y + jacobianLogTable[ind]; } } - public static double phredScaleToProbability (byte q) { - return Math.pow(10,(-q)/10.0); + public static double phredScaleToProbability(byte q) { + return Math.pow(10, (-q) / 10.0); } - public static double phredScaleToLog10Probability (byte q) { - return ((-q)/10.0); + public static double phredScaleToLog10Probability(byte q) { + return ((-q) / 10.0); } /** * Returns the phred scaled value of probability p + * * @param p probability (between 0 and 1). * @return phred scaled probability of p */ - public static byte probabilityToPhredScale (double p) { + public static byte probabilityToPhredScale(double p) { return (byte) ((-10) * Math.log10(p)); } - public static double log10ProbabilityToPhredScale (double log10p) { + public static double log10ProbabilityToPhredScale(double log10p) { return (-10) * log10p; } /** * Converts LN to LOG10 + * * @param ln log(x) * @return log10(x) */ - public static double lnToLog10 (double ln) { + public static double lnToLog10(double ln) { return ln * Math.log10(Math.exp(1)); } @@ -1134,169 +1205,190 @@ public static double lnToLog10 (double ln) { * Constants to simplify the log gamma function calculation. */ private static final double - zero = 0.0, - one = 1.0, - half = .5, - a0 = 7.72156649015328655494e-02, - a1 = 3.22467033424113591611e-01, - a2 = 6.73523010531292681824e-02, - a3 = 2.05808084325167332806e-02, - a4 = 7.38555086081402883957e-03, - a5 = 2.89051383673415629091e-03, - a6 = 1.19270763183362067845e-03, - a7 = 5.10069792153511336608e-04, - a8 = 2.20862790713908385557e-04, - a9 = 1.08011567247583939954e-04, - a10 = 2.52144565451257326939e-05, - a11 = 4.48640949618915160150e-05, - tc = 1.46163214496836224576e+00, - tf = -1.21486290535849611461e-01, - tt = -3.63867699703950536541e-18, - t0 = 4.83836122723810047042e-01, - t1 = -1.47587722994593911752e-01, - t2 = 6.46249402391333854778e-02, - t3 = -3.27885410759859649565e-02, - t4 = 1.79706750811820387126e-02, - t5 = -1.03142241298341437450e-02, - t6 = 6.10053870246291332635e-03, - t7 = -3.68452016781138256760e-03, - t8 = 2.25964780900612472250e-03, - t9 = -1.40346469989232843813e-03, - t10 = 8.81081882437654011382e-04, - t11 = -5.38595305356740546715e-04, - t12 = 3.15632070903625950361e-04, - t13 = -3.12754168375120860518e-04, - t14 = 3.35529192635519073543e-04, - u0 = -7.72156649015328655494e-02, - u1 = 6.32827064025093366517e-01, - u2 = 1.45492250137234768737e+00, - u3 = 9.77717527963372745603e-01, - u4 = 2.28963728064692451092e-01, - u5 = 1.33810918536787660377e-02, - v1 = 2.45597793713041134822e+00, - v2 = 2.12848976379893395361e+00, - v3 = 7.69285150456672783825e-01, - v4 = 1.04222645593369134254e-01, - v5 = 3.21709242282423911810e-03, - s0 = -7.72156649015328655494e-02, - s1 = 2.14982415960608852501e-01, - s2 = 3.25778796408930981787e-01, - s3 = 1.46350472652464452805e-01, - s4 = 2.66422703033638609560e-02, - s5 = 1.84028451407337715652e-03, - s6 = 3.19475326584100867617e-05, - r1 = 1.39200533467621045958e+00, - r2 = 7.21935547567138069525e-01, - r3 = 1.71933865632803078993e-01, - r4 = 1.86459191715652901344e-02, - r5 = 7.77942496381893596434e-04, - r6 = 7.32668430744625636189e-06, - w0 = 4.18938533204672725052e-01, - w1 = 8.33333333333329678849e-02, - w2 = -2.77777777728775536470e-03, - w3 = 7.93650558643019558500e-04, - w4 = -5.95187557450339963135e-04, - w5 = 8.36339918996282139126e-04, - w6 = -1.63092934096575273989e-03; + zero = 0.0, + one = 1.0, + half = .5, + a0 = 7.72156649015328655494e-02, + a1 = 3.22467033424113591611e-01, + a2 = 6.73523010531292681824e-02, + a3 = 2.05808084325167332806e-02, + a4 = 7.38555086081402883957e-03, + a5 = 2.89051383673415629091e-03, + a6 = 1.19270763183362067845e-03, + a7 = 5.10069792153511336608e-04, + a8 = 2.20862790713908385557e-04, + a9 = 1.08011567247583939954e-04, + a10 = 2.52144565451257326939e-05, + a11 = 4.48640949618915160150e-05, + tc = 1.46163214496836224576e+00, + tf = -1.21486290535849611461e-01, + tt = -3.63867699703950536541e-18, + t0 = 4.83836122723810047042e-01, + t1 = -1.47587722994593911752e-01, + t2 = 6.46249402391333854778e-02, + t3 = -3.27885410759859649565e-02, + t4 = 1.79706750811820387126e-02, + t5 = -1.03142241298341437450e-02, + t6 = 6.10053870246291332635e-03, + t7 = -3.68452016781138256760e-03, + t8 = 2.25964780900612472250e-03, + t9 = -1.40346469989232843813e-03, + t10 = 8.81081882437654011382e-04, + t11 = -5.38595305356740546715e-04, + t12 = 3.15632070903625950361e-04, + t13 = -3.12754168375120860518e-04, + t14 = 3.35529192635519073543e-04, + u0 = -7.72156649015328655494e-02, + u1 = 6.32827064025093366517e-01, + u2 = 1.45492250137234768737e+00, + u3 = 9.77717527963372745603e-01, + u4 = 2.28963728064692451092e-01, + u5 = 1.33810918536787660377e-02, + v1 = 2.45597793713041134822e+00, + v2 = 2.12848976379893395361e+00, + v3 = 7.69285150456672783825e-01, + v4 = 1.04222645593369134254e-01, + v5 = 3.21709242282423911810e-03, + s0 = -7.72156649015328655494e-02, + s1 = 2.14982415960608852501e-01, + s2 = 3.25778796408930981787e-01, + s3 = 1.46350472652464452805e-01, + s4 = 2.66422703033638609560e-02, + s5 = 1.84028451407337715652e-03, + s6 = 3.19475326584100867617e-05, + r1 = 1.39200533467621045958e+00, + r2 = 7.21935547567138069525e-01, + r3 = 1.71933865632803078993e-01, + r4 = 1.86459191715652901344e-02, + r5 = 7.77942496381893596434e-04, + r6 = 7.32668430744625636189e-06, + w0 = 4.18938533204672725052e-01, + w1 = 8.33333333333329678849e-02, + w2 = -2.77777777728775536470e-03, + w3 = 7.93650558643019558500e-04, + w4 = -5.95187557450339963135e-04, + w5 = 8.36339918996282139126e-04, + w6 = -1.63092934096575273989e-03; /** * Efficient rounding functions to simplify the log gamma function calculation - * double to long with 32 bit shift + * double to long with 32 bit shift */ - private static final int HI (double x) { - return (int)(Double.doubleToLongBits(x) >> 32); + private static final int HI(double x) { + return (int) (Double.doubleToLongBits(x) >> 32); } /** * Efficient rounding functions to simplify the log gamma function calculation - * double to long without shift + * double to long without shift */ - private static final int LO (double x) { - return (int)Double.doubleToLongBits(x); + private static final int LO(double x) { + return (int) Double.doubleToLongBits(x); } /** * Most efficent implementation of the lnGamma (FDLIBM) * Use via the log10Gamma wrapper method. */ - private static double lnGamma (double x) { - double t,y,z,p,p1,p2,p3,q,r,w; + private static double lnGamma(double x) { + double t, y, z, p, p1, p2, p3, q, r, w; int i; int hx = HI(x); int lx = LO(x); /* purge off +-inf, NaN, +-0, and negative arguments */ - int ix = hx&0x7fffffff; + int ix = hx & 0x7fffffff; if (ix >= 0x7ff00000) return Double.POSITIVE_INFINITY; - if ((ix|lx)==0 || hx < 0) return Double.NaN; - if (ix<0x3b900000) { /* |x|<2**-70, return -log(|x|) */ + if ((ix | lx) == 0 || hx < 0) return Double.NaN; + if (ix < 0x3b900000) { /* |x|<2**-70, return -log(|x|) */ return -Math.log(x); } /* purge off 1 and 2 */ - if((((ix-0x3ff00000)|lx)==0)||(((ix-0x40000000)|lx)==0)) r = 0; - /* for x < 2.0 */ - else if(ix<0x40000000) { - if(ix<=0x3feccccc) { /* lgamma(x) = lgamma(x+1)-log(x) */ + if ((((ix - 0x3ff00000) | lx) == 0) || (((ix - 0x40000000) | lx) == 0)) r = 0; + /* for x < 2.0 */ + else if (ix < 0x40000000) { + if (ix <= 0x3feccccc) { /* lgamma(x) = lgamma(x+1)-log(x) */ r = -Math.log(x); - if(ix>=0x3FE76944) {y = one-x; i= 0;} - else if(ix>=0x3FCDA661) {y= x-(tc-one); i=1;} - else {y = x; i=2;} + if (ix >= 0x3FE76944) { + y = one - x; + i = 0; + } else if (ix >= 0x3FCDA661) { + y = x - (tc - one); + i = 1; + } else { + y = x; + i = 2; + } } else { r = zero; - if(ix>=0x3FFBB4C3) {y=2.0-x;i=0;} /* [1.7316,2] */ - else if(ix>=0x3FF3B4C4) {y=x-tc;i=1;} /* [1.23,1.73] */ - else {y=x-one;i=2;} + if (ix >= 0x3FFBB4C3) { + y = 2.0 - x; + i = 0; + } /* [1.7316,2] */ else if (ix >= 0x3FF3B4C4) { + y = x - tc; + i = 1; + } /* [1.23,1.73] */ else { + y = x - one; + i = 2; + } } - switch(i) { - case 0: - z = y*y; - p1 = a0+z*(a2+z*(a4+z*(a6+z*(a8+z*a10)))); - p2 = z*(a1+z*(a3+z*(a5+z*(a7+z*(a9+z*a11))))); - p = y*p1+p2; - r += (p-0.5*y); break; - case 1: - z = y*y; - w = z*y; - p1 = t0+w*(t3+w*(t6+w*(t9 +w*t12))); /* parallel comp */ - p2 = t1+w*(t4+w*(t7+w*(t10+w*t13))); - p3 = t2+w*(t5+w*(t8+w*(t11+w*t14))); - p = z*p1-(tt-w*(p2+y*p3)); - r += (tf + p); break; - case 2: - p1 = y*(u0+y*(u1+y*(u2+y*(u3+y*(u4+y*u5))))); - p2 = one+y*(v1+y*(v2+y*(v3+y*(v4+y*v5)))); - r += (-0.5*y + p1/p2); + switch (i) { + case 0: + z = y * y; + p1 = a0 + z * (a2 + z * (a4 + z * (a6 + z * (a8 + z * a10)))); + p2 = z * (a1 + z * (a3 + z * (a5 + z * (a7 + z * (a9 + z * a11))))); + p = y * p1 + p2; + r += (p - 0.5 * y); + break; + case 1: + z = y * y; + w = z * y; + p1 = t0 + w * (t3 + w * (t6 + w * (t9 + w * t12))); /* parallel comp */ + p2 = t1 + w * (t4 + w * (t7 + w * (t10 + w * t13))); + p3 = t2 + w * (t5 + w * (t8 + w * (t11 + w * t14))); + p = z * p1 - (tt - w * (p2 + y * p3)); + r += (tf + p); + break; + case 2: + p1 = y * (u0 + y * (u1 + y * (u2 + y * (u3 + y * (u4 + y * u5))))); + p2 = one + y * (v1 + y * (v2 + y * (v3 + y * (v4 + y * v5)))); + r += (-0.5 * y + p1 / p2); } - } - else if(ix<0x40200000) { /* x < 8.0 */ - i = (int)x; + } else if (ix < 0x40200000) { /* x < 8.0 */ + i = (int) x; t = zero; - y = x-(double)i; - p = y*(s0+y*(s1+y*(s2+y*(s3+y*(s4+y*(s5+y*s6)))))); - q = one+y*(r1+y*(r2+y*(r3+y*(r4+y*(r5+y*r6))))); - r = half*y+p/q; - z = one; /* lgamma(1+s) = log(s) + lgamma(s) */ - switch(i) { - case 7: z *= (y+6.0); /* FALLTHRU */ - case 6: z *= (y+5.0); /* FALLTHRU */ - case 5: z *= (y+4.0); /* FALLTHRU */ - case 4: z *= (y+3.0); /* FALLTHRU */ - case 3: z *= (y+2.0); /* FALLTHRU */ - r += Math.log(z); break; + y = x - (double) i; + p = y * (s0 + y * (s1 + y * (s2 + y * (s3 + y * (s4 + y * (s5 + y * s6)))))); + q = one + y * (r1 + y * (r2 + y * (r3 + y * (r4 + y * (r5 + y * r6))))); + r = half * y + p / q; + z = one; /* lgamma(1+s) = log(s) + lgamma(s) */ + switch (i) { + case 7: + z *= (y + 6.0); /* FALLTHRU */ + case 6: + z *= (y + 5.0); /* FALLTHRU */ + case 5: + z *= (y + 4.0); /* FALLTHRU */ + case 4: + z *= (y + 3.0); /* FALLTHRU */ + case 3: + z *= (y + 2.0); /* FALLTHRU */ + r += Math.log(z); + break; } /* 8.0 <= x < 2**58 */ } else if (ix < 0x43900000) { t = Math.log(x); - z = one/x; - y = z*z; - w = w0+z*(w1+y*(w2+y*(w3+y*(w4+y*(w5+y*w6))))); - r = (x-half)*(t-one)+w; + z = one / x; + y = z * z; + w = w0 + z * (w1 + y * (w2 + y * (w3 + y * (w4 + y * (w5 + y * w6))))); + r = (x - half) * (t - one) + w; } else /* 2**58 <= x <= inf */ - r = x*(Math.log(x)-one); + r = x * (Math.log(x) - one); return r; } @@ -1308,7 +1400,7 @@ else if(ix<0x40200000) { /* x < 8.0 */ * @param x the x parameter * @return the log10 of the gamma function at x. */ - public static double log10Gamma (double x) { + public static double log10Gamma(double x) { return lnToLog10(lnGamma(x)); } @@ -1320,13 +1412,13 @@ public static double log10Gamma (double x) { * @param k number of successes * @return the log10 of the binomial coefficient */ - public static double log10BinomialCoefficient (int n, int k) { - return log10Gamma(n+1) - log10Gamma(k+1) - log10Gamma(n-k+1); + public static double log10BinomialCoefficient(int n, int k) { + return log10Gamma(n + 1) - log10Gamma(k + 1) - log10Gamma(n - k + 1); } - public static double log10BinomialProbability (int n, int k, double log10p) { - double log10OneMinusP = Math.log10(1-Math.pow(10,log10p)); - return log10BinomialCoefficient(n, k) + log10p*k + log10OneMinusP*(n-k); + public static double log10BinomialProbability(int n, int k, double log10p) { + double log10OneMinusP = Math.log10(1 - Math.pow(10, log10p)); + return log10BinomialCoefficient(n, k) + log10p * k + log10OneMinusP * (n - k); } @@ -1338,38 +1430,74 @@ public static double log10BinomialProbability (int n, int k, double log10p) { * @param k array of any size with the number of successes for each grouping (k1, k2, k3, ..., km) * @return */ - public static double log10MultinomialCoefficient (int n, int [] k) { + public static double log10MultinomialCoefficient(int n, int[] k) { double denominator = 0.0; for (int x : k) { - denominator += log10Gamma(x+1); + denominator += log10Gamma(x + 1); } - return log10Gamma(n+1) - denominator; + return log10Gamma(n + 1) - denominator; } /** * Computes the log10 of the multinomial distribution probability given a vector * of log10 probabilities. Designed to prevent overflows even with very large numbers. * - * @param n number of trials - * @param k array of number of successes for each possibility + * @param n number of trials + * @param k array of number of successes for each possibility * @param log10p array of log10 probabilities * @return */ - public static double log10MultinomialProbability (int n, int [] k, double [] log10p) { + public static double log10MultinomialProbability(int n, int[] k, double[] log10p) { if (log10p.length != k.length) throw new UserException.BadArgumentValue("p and k", "Array of log10 probabilities must have the same size as the array of number of sucesses: " + log10p.length + ", " + k.length); double log10Prod = 0.0; - for (int i=0; i set = new HashSet(); + set.addAll(Arrays.asList(expected)); + set.removeAll(Arrays.asList(actual)); + return set.isEmpty(); + } + + + private void p (Object []x) { + for (Object v: x) + System.out.print((Integer) v + " "); + System.out.println(); + } } From 21ae3ef5f96e4b24bca21ea86d79dc7ed4b7f108 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Sat, 31 Dec 2011 13:56:41 -0500 Subject: [PATCH 009/356] Added downsampling support to ReduceReads * Downsampling is now a parameter to the walker with default value of 0 (no downsampling) * Downsampling selects reads at random at the variant region window and strives to achieve uniform coverage if possible around the desired downsampling value. * Added integration test --- .../sting/utils/sam/GATKSAMRecord.java | 6 + .../sting/utils/sam/ReadUtils.java | 113 ++++++++++++++++++ 2 files changed, 119 insertions(+) diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java index 5e0802fa60..913548ecc3 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java @@ -183,6 +183,12 @@ public boolean isReducedRead() { return getReducedReadCounts() != null; } + /** + * The number of bases corresponding the i'th base of the reduced read. + * + * @param i the read based coordinate inside the read + * @return the number of bases corresponding to the i'th base of the reduced read + */ public final byte getReducedCount(final int i) { byte firstCount = getReducedReadCounts()[0]; byte offsetCount = getReducedReadCounts()[i]; diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java b/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java index d52814ef7c..7fa2f6230b 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java @@ -29,6 +29,7 @@ import com.google.java.contract.Requires; import net.sf.samtools.*; import org.broadinstitute.sting.utils.GenomeLoc; +import org.broadinstitute.sting.utils.MathUtils; import org.broadinstitute.sting.utils.collections.Pair; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; @@ -531,4 +532,116 @@ else if (cigarElement.getOperator() != CigarOperator.HARD_CLIP && cigarElement.g return new Pair(false, null); } + /** + * Returns the coverage distribution of a list of reads within the desired region. + * + * See getCoverageDistributionOfRead for information on how the coverage is calculated. + * + * @param list the list of reads covering the region + * @param startLocation the first reference coordinate of the region (inclusive) + * @param stopLocation the last reference coordinate of the region (inclusive) + * @return an array with the coverage of each position from startLocation to stopLocation + */ + public static int [] getCoverageDistributionOfReads(List list, int startLocation, int stopLocation) { + int [] totalCoverage = new int[stopLocation - startLocation + 1]; + + for (GATKSAMRecord read : list) { + int [] readCoverage = getCoverageDistributionOfRead(read, startLocation, stopLocation); + totalCoverage = MathUtils.addArrays(totalCoverage, readCoverage); + } + + return totalCoverage; + } + + /** + * Returns the coverage distribution of a single read within the desired region. + * + * Note: This function counts DELETIONS as coverage (since the main purpose is to downsample + * reads for variant regions, and deletions count as variants) + * + * @param read the read to get the coverage distribution of + * @param startLocation the first reference coordinate of the region (inclusive) + * @param stopLocation the last reference coordinate of the region (inclusive) + * @return an array with the coverage of each position from startLocation to stopLocation + */ + public static int [] getCoverageDistributionOfRead(GATKSAMRecord read, int startLocation, int stopLocation) { + int [] coverage = new int[stopLocation - startLocation + 1]; + int refLocation = read.getSoftStart(); + for (CigarElement cigarElement : read.getCigar().getCigarElements()) { + switch (cigarElement.getOperator()) { + case S: + case M: + case EQ: + case N: + case X: + case D: + for (int i = 0; i < cigarElement.getLength(); i++) { + if (refLocation >= startLocation && refLocation <= stopLocation) { + int baseCount = read.isReducedRead() ? read.getReducedCount(refLocation - read.getSoftStart()) : 1; + coverage[refLocation - startLocation] += baseCount; // this may be a reduced read, so add the proper number of bases + } + refLocation++; + } + break; + + case P: + case I: + case H: + break; + } + + if (refLocation > stopLocation) + break; + } + return coverage; + } + + /** + * Makes association maps for the reads and loci coverage as described below : + * + * - First: locusToReadMap -- a HashMap that describes for each locus, which reads contribute to its coverage. + * Note: Locus is in reference coordinates. + * Example: Locus => {read1, read2, ..., readN} + * + * - Second: readToLocusMap -- a HashMap that describes for each read what loci it contributes to the coverage. + * Note: Locus is a boolean array, indexed from 0 (= startLocation) to N (= stopLocation), with true meaning it contributes to the coverage. + * Example: Read => {true, true, false, ... false} + * + * @param readList the list of reads to generate the association mappings + * @param startLocation the first reference coordinate of the region (inclusive) + * @param stopLocation the last reference coordinate of the region (inclusive) + * @return the two hashmaps described above + */ + public static Pair> , HashMap> getBothReadToLociMappings (List readList, int startLocation, int stopLocation) { + int arraySize = stopLocation - startLocation + 1; + + HashMap> locusToReadMap = new HashMap>(2*(stopLocation - startLocation + 1), 0.5f); + HashMap readToLocusMap = new HashMap(2*readList.size(), 0.5f); + + + for (int i = startLocation; i <= stopLocation; i++) + locusToReadMap.put(i, new HashSet()); // Initialize the locusToRead map with empty lists + + for (GATKSAMRecord read : readList) { + readToLocusMap.put(read, new Boolean[arraySize]); // Initialize the readToLocus map with empty arrays + + int [] readCoverage = getCoverageDistributionOfRead(read, startLocation, stopLocation); + + for (int i=0; i 0) { + // Update the hash for this locus + HashSet readSet = locusToReadMap.get(refLocation); + readSet.add(read); + + // Add this locus to the read hash + readToLocusMap.get(read)[refLocation - startLocation] = true; + } + else + // Update the boolean array with a 'no coverage' from this read to this locus + readToLocusMap.get(read)[refLocation-startLocation] = false; + } + } + return new Pair>, HashMap>(locusToReadMap, readToLocusMap); + } } From cce8511d290d4805be9657b0904a259a898ad657 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Sun, 1 Jan 2012 21:55:04 -0500 Subject: [PATCH 010/356] Some WGS performance upgrades for ReduceReads * Do not try to hard clip to the interval when doing WGS * Do not even add reads that have been completely clipped out in WGS From 4a208c7c06a55df1790d32b5de1c18608b12ddab Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Mon, 2 Jan 2012 18:26:31 -0500 Subject: [PATCH 011/356] Refactor of the downsampling machinery to accept different strategies * Implemented Adaptive downsampler * Added integration test * Added option to RRead scala script to choose downsampling strategy --- .../broadinstitute/sting/utils/MathUtils.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java index 4a3100a94d..737f4bb5f3 100644 --- a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java @@ -573,6 +573,10 @@ public static double arrayMin(double[] array) { return array[minElementIndex(array)]; } + public static int arrayMin(int[] array) { + return array[minElementIndex(array)]; + } + public static byte arrayMin(byte[] array) { return array[minElementIndex(array)]; } @@ -601,6 +605,18 @@ public static int minElementIndex(byte[] array) { return minI; } + public static int minElementIndex(int[] array) { + if (array == null) throw new IllegalArgumentException("Array cannot be null!"); + + int minI = -1; + for (int i = 0; i < array.length; i++) { + if (minI == -1 || array[i] < array[minI]) + minI = i; + } + + return minI; + } + public static int arrayMaxInt(List array) { if (array == null) throw new IllegalArgumentException("Array cannot be null!"); if (array.size() == 0) throw new IllegalArgumentException("Array size cannot be 0!"); From 0bdeda6f3fafcd8283ae745880b711460e2a51f2 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Mon, 2 Jan 2012 23:35:46 -0500 Subject: [PATCH 012/356] Added single sample option for the ReduceReads calling script From 18f06ad9131b93cee369377ec92ad26e463d65b7 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Mon, 2 Jan 2012 02:45:08 -0500 Subject: [PATCH 013/356] Script to calculate gc content of intervals independently * necessary for baits because we don't want the overlapping intervals to be merged by the GATK engine From caa5da2fd2ec51aaf9aa43d1651eefdabd662e00 Mon Sep 17 00:00:00 2001 From: Roger Zurawicki Date: Sat, 31 Dec 2011 02:14:29 -0500 Subject: [PATCH 014/356] Added parameter to combine RGs in CoverageByRG * -g takes a string of read groups separated by space " " * multiple -g creates multiple sum columns in the table Signed-off-by: Mauricio Carneiro From 3d4bf273de8e350d27b92ff8c5411a8e4485173e Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Mon, 2 Jan 2012 16:17:57 -0500 Subject: [PATCH 015/356] Added getPileupForReadGroups to ReadBackPileup * returns a pileup for all the read groups provided. * saves us from multiple calls to getPileup (which is very inefficient) --- .../pileup/AbstractReadBackedPileup.java | 36 +++++++++++++++++++ .../sting/utils/pileup/ReadBackedPileup.java | 8 +++++ 2 files changed, 44 insertions(+) diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java b/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java index 18051ce92d..586b86490b 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java @@ -526,6 +526,42 @@ public RBP getPileupForReadGroup(String targetReadGroupId) { } } + /** + * Gets the pileup for a set of read groups. Horrendously inefficient at this point. + * @param rgSet List of identifiers for the read groups. + * @return A read-backed pileup containing only the reads in the given read groups. + */ + @Override + public RBP getPileupForReadGroups(final HashSet rgSet) { + if(pileupElementTracker instanceof PerSamplePileupElementTracker) { + PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker)pileupElementTracker; + PerSamplePileupElementTracker filteredTracker = new PerSamplePileupElementTracker(); + + for(final String sample: tracker.getSamples()) { + PileupElementTracker perSampleElements = tracker.getElements(sample); + AbstractReadBackedPileup pileup = createNewPileup(loc,perSampleElements).getPileupForReadGroups(rgSet); + if(pileup != null) + filteredTracker.addElements(sample,pileup.pileupElementTracker); + } + return filteredTracker.size()>0 ? (RBP)createNewPileup(loc,filteredTracker) : null; + } + else { + UnifiedPileupElementTracker filteredTracker = new UnifiedPileupElementTracker(); + for(PE p: pileupElementTracker) { + GATKSAMRecord read = p.getRead(); + if(rgSet != null && !rgSet.isEmpty()) { + if(read.getReadGroup() != null && rgSet.contains(read.getReadGroup().getReadGroupId())) + filteredTracker.add(p); + } + else { + if(read.getReadGroup() == null || read.getReadGroup().getReadGroupId() == null) + filteredTracker.add(p); + } + } + return filteredTracker.size()>0 ? (RBP)createNewPileup(loc,filteredTracker) : null; + } + } + @Override public RBP getPileupForLane(String laneID) { if(pileupElementTracker instanceof PerSamplePileupElementTracker) { diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileup.java b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileup.java index 02767df7cd..ccd9d509fb 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileup.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileup.java @@ -30,6 +30,7 @@ import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import java.util.Collection; +import java.util.HashSet; import java.util.List; /** @@ -129,6 +130,13 @@ public interface ReadBackedPileup extends Iterable, HasGenomeLoca */ public ReadBackedPileup getPileupForReadGroup(String readGroupId); + /** + * Gets all the reads associated with a given read groups. + * @param rgSet Set of identifiers for the read group. + * @return A pileup containing only the reads in the given read groups. + */ + public ReadBackedPileup getPileupForReadGroups(final HashSet rgSet); + /** * Gets all reads in a given lane id. (Lane ID is the read group * id stripped of the last .XX sample identifier added by the GATK). From ca669ae74404df05f515b74960dfa4319d9b46d5 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Mon, 2 Jan 2012 16:19:31 -0500 Subject: [PATCH 016/356] Optimizations to the CoverageByRG walker * outputs only the groups of read groups necessary, avoiding multiple pileup creations every call to map * now also counts the number of variants associated with a given ROD (dbSNP) exist in the interval * new column: interval size From 055364d786a2fc1bb54b940b3186daead52590a9 Mon Sep 17 00:00:00 2001 From: David Roazen Date: Tue, 3 Jan 2012 10:18:45 -0500 Subject: [PATCH 017/356] Always use full, three-part version numbers. Previously, the initial release of a new GATK version had a version number with only one part (eg., "1.4"). This could potentially mislead people into thinking it's the most recent revision of a release, instead of the least recent. Now, initial releases will have full, three-part version numbers (eg., "1.4-0-g472fc94") like everything else. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index 2086d0c9ae..7c81c1f206 100644 --- a/build.xml +++ b/build.xml @@ -215,7 +215,7 @@ - + From 93e1417b6eaeb2be84efbf4bdc21e9dc90e10d79 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Tue, 3 Jan 2012 13:39:31 -0500 Subject: [PATCH 018/356] Update to the VSS GATK documentation. --- .../ValidationSiteSelectorWalker.java | 46 +++++++++++++++---- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/validationsiteselector/ValidationSiteSelectorWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/validationsiteselector/ValidationSiteSelectorWalker.java index ae11d8102e..cd4c571365 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/validationsiteselector/ValidationSiteSelectorWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/validationsiteselector/ValidationSiteSelectorWalker.java @@ -106,37 +106,70 @@ public enum SAMPLE_SELECTION_MODE { POLY_BASED_ON_GL } + /** + * The input VCF file + */ @Input(fullName="variant", shortName = "V", doc="Input VCF file, can be specified multiple times", required=true) public List> variants; + /** + * The output VCF file + */ @Output(doc="File to which variants should be written",required=true) protected VCFWriter vcfWriter = null; + /** + * Sample name(s) to subset the input VCF to, prior to selecting variants. -sn A -sn B subsets to samples A and B. + */ @Argument(fullName="sample_name", shortName="sn", doc="Include genotypes from this sample. Can be specified multiple times", required=false) public Set sampleNames = new HashSet(0); + /** + * Sample regexps to subset the input VCF to, prior to selecting variants. -sn NA12* subsets to all samples with prefix NA12 + */ @Argument(fullName="sample_expressions", shortName="se", doc="Regular expression to select many samples from the ROD tracks provided. Can be specified multiple times", required=false) public Set sampleExpressions ; + /** + * File containing a list of sample names to subset the input vcf to. Equivalent to specifying the contents of the file separately with -sn + */ @Input(fullName="sample_file", shortName="sf", doc="File containing a list of samples (one per line) to include. Can be specified multiple times", required=false) public Set sampleFiles; + /** + * A mode for selecting sites based on sample-level data. See the wiki documentation for more information. + */ @Argument(fullName="sampleMode", shortName="sampleMode", doc="Sample selection mode", required=false) private SAMPLE_SELECTION_MODE sampleMode = SAMPLE_SELECTION_MODE.NONE; + /** + * An P[nonref] threshold for SAMPLE_SELECTION_MODE=POLY_BASED_ON_GL. See the wiki documentation for more information. + */ @Argument(shortName="samplePNonref",fullName="samplePNonref", doc="GL-based selection mode only: the probability" + " that a site is non-reference in the samples for which to include the site",required=false) private double samplePNonref = 0.99; + /** + * The number of sites in your validation set + */ @Argument(fullName="numValidationSites", shortName="numSites", doc="Number of output validation sites", required=true) private int numValidationSites; + /** + * Do not exclude filtered sites (e.g. not PASS or .) from consideration for validation + */ @Argument(fullName="includeFilteredSites", shortName="ifs", doc="If true, will include filtered sites in set to choose variants from", required=false) private boolean INCLUDE_FILTERED_SITES = false; + /** + * Argument for the frequency selection mode. (AC/AF/AN) are taken from VCF info field, not recalculated. Typically specified for sites-only VCFs that still have AC/AF/AN information. + */ @Argument(fullName="ignoreGenotypes", shortName="ignoreGenotypes", doc="If true, will ignore genotypes in VCF, will take AC,AF from annotations and will make no sample selection", required=false) private boolean IGNORE_GENOTYPES = false; + /** + * Argument for the frequency selection mode. Allows reference (non-polymorphic) sites to be included in the validation set. + */ @Argument(fullName="ignorePolymorphicStatus", shortName="ignorePolymorphicStatus", doc="If true, will ignore polymorphic status in VCF, and will take VCF record directly without pre-selection", required=false) private boolean IGNORE_POLYMORPHIC = false; @@ -145,19 +178,14 @@ public enum SAMPLE_SELECTION_MODE { private int numFrequencyBins = 20; /** - * This argument selects allele frequency selection mode: - * KEEP_AF_SPECTRUM will choose variants so that the resulting allele frequency spectrum matches as closely as possible the input set - * UNIFORM will choose variants uniformly without regard to their allele frequency. - * - */ + * This argument selects allele frequency selection mode. See the wiki for more information. + */ @Argument(fullName="frequencySelectionMode", shortName="freqMode", doc="Allele Frequency selection mode", required=false) private AF_COMPUTATION_MODE freqMode = AF_COMPUTATION_MODE.KEEP_AF_SPECTRUM; /** - * This argument selects particular kinds of variants out of a list. If left empty, there is no type selection and all variant types are considered for other selection criteria. - * When specified one or more times, a particular type of variant is selected. - * - */ + * This argument selects particular kinds of variants (i.e. SNP, INDEL) out of a list. If left unspecified, all types are considered. + */ @Argument(fullName="selectTypeToInclude", shortName="selectType", doc="Select only a certain type of variants from the input file. Valid types are INDEL, SNP, MIXED, MNP, SYMBOLIC, NO_VARIATION. Can be specified multiple times", required=false) private List TYPES_TO_INCLUDE = new ArrayList(); From 2d093828a4473b5187fa3540c4f33bd0c0204e9c Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Tue, 3 Jan 2012 15:33:04 -0500 Subject: [PATCH 019/356] Final changes to Junky (been frozen for a while, but uncommitted) and the qscript for it. A first cursory implementation of the trellis-based Exact AC-constrained genotyping algorithm in UGE. Nothing calls into it, so this should be entirely safe (and, no surprise, it passes UG integration tests). --- .../genotyper/UnifiedGenotyperEngine.java | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java index aa33d39e3b..2159da6e6f 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java @@ -401,6 +401,11 @@ else if ( UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_M // *** note that calculating strand bias involves overwriting data structures, so we do that last final HashMap attributes = new HashMap(); + List mlecounts = new ArrayList(AFresult.log10AlleleFrequencyPosteriors.length); + for ( int i = 0; i < AFresult.log10AlleleFrequencyLikelihoods.length ; i++) { + mlecounts.add(MathUtils.maxElementIndex(AFresult.log10AlleleFrequencyLikelihoods[i])); + } + attributes.put("MLEAC",Utils.join(",",mlecounts)); // if the site was downsampled, record that fact if ( !limitedContext && rawContext.hasPileupBeenDownsampled() ) @@ -858,4 +863,171 @@ public static GenotypesContext assignGenotypes(final VariantContext vc, return calls; } + + /** + * @param vc variant context with genotype likelihoods + * @param allelesToUse bit vector describing which alternate alleles from the vc are okay to use + * @param exactAC integer array describing the AC from the exact model for the corresponding alleles + * @return genotypes + */ + public static GenotypesContext constrainedAssignGenotypes(VariantContext vc, boolean[] allelesToUse, int[] exactAC ) { + + final GenotypesContext GLs = vc.getGenotypes(); + + // samples + final List sampleIndices = GLs.getSampleNamesOrderedByName(); + + // we need to determine which of the alternate alleles (and hence the likelihoods) to use and carry forward + final int numOriginalAltAlleles = allelesToUse.length; + final List newAlleles = new ArrayList(numOriginalAltAlleles+1); + newAlleles.add(vc.getReference()); + final HashMap alleleIndexMap = new HashMap(); // need this for skipping dimensions + int[] alleleCount = new int[exactAC.length]; + for ( int i = 0; i < numOriginalAltAlleles; i++ ) { + if ( allelesToUse[i] ) { + newAlleles.add(vc.getAlternateAllele(i)); + alleleIndexMap.put(vc.getAlternateAllele(i),i); + alleleCount[i] = exactAC[i]; + } else { + alleleCount[i] = 0; + } + } + final List newAltAlleles = newAlleles.subList(1,newAlleles.size()); + final int numNewAltAlleles = newAltAlleles.size(); + ArrayList likelihoodIndexesToUse = null; + + // an optimization: if we are supposed to use all (or none in the case of a ref call) of the alleles, + // then we can keep the PLs as is; otherwise, we determine which ones to keep + final int[][] PLcache; + if ( numNewAltAlleles != numOriginalAltAlleles && numNewAltAlleles > 0 ) { + likelihoodIndexesToUse = new ArrayList(30); + PLcache = PLIndexToAlleleIndex[numOriginalAltAlleles]; + + for ( int PLindex = 0; PLindex < PLcache.length; PLindex++ ) { + int[] alleles = PLcache[PLindex]; + // consider this entry only if both of the alleles are good + if ( (alleles[0] == 0 || allelesToUse[alleles[0] - 1]) && (alleles[1] == 0 || allelesToUse[alleles[1] - 1]) ) + likelihoodIndexesToUse.add(PLindex); + } + } else { + PLcache = PLIndexToAlleleIndex[numOriginalAltAlleles]; + } + + // set up the trellis dimensions + // SAMPLE x alt 1 x alt 2 x alt 3 + // todo -- check that exactAC has alt counts at [1],[2],[3] (and not [0],[1],[2]) + double[][][][] transitionTrellis = new double[sampleIndices.size()+1][exactAC[1]][exactAC[2]][exactAC[3]]; + // N x AC1 x AC2 x AC3; worst performance in multi-allelic where all alleles are moderate frequency + // capped at the MLE ACs* + // todo -- there's an optimization: not all states in the rectangular matrix will be reached, in fact + // todo -- for tT[0] we only care about tT[0][0][0][0], and for tT[1], only combinations of 0,1,2. + int idx = 1; // index of which sample we're on + int prevMaxState = 0; // the maximum state (e.g. AC) reached by the previous sample. Symmetric. (AC capping handled by logic in loop) + // iterate over each sample + for ( String sample : sampleIndices ) { + // push the likelihoods into the next possible states, that is to say + // L[state] = L[prev state] + L[genotype getting into state] + // iterate over each previous state, by dimension + // and contribute the likelihoods for transitions to this state + double[][][] prevState = transitionTrellis[idx-1]; + double[][][] thisState = transitionTrellis[idx]; + Genotype genotype = GLs.get(sample); + if ( genotype.isNoCall() || genotype.isFiltered() ) { + thisState = prevState.clone(); + } else { + double[] likelihoods = genotype.getLikelihoods().getAsVector(); + int dim1min = Math.max(0, alleleCount[0]-2*(sampleIndices.size()-idx+1)); + int dim1max = Math.min(prevMaxState,alleleCount[0]); + int dim2min = Math.max(0,alleleCount[1]-2*(sampleIndices.size()-idx+1)); + int dim2max = Math.min(prevMaxState,alleleCount[1]); + int dim3min = Math.max(0,alleleCount[2]-2*(sampleIndices.size()-idx+1)); + int dim3max = Math.min(prevMaxState,alleleCount[2]); + // cue annoying nested for loop + for ( int a1 = dim1min ; a1 <= dim1max; a1++ ) { + for ( int a2 = dim2min; a2 <= dim2max; a2++ ) { + for ( int a3 = dim3min; a3 <= dim3max; a3++ ) { + double base = prevState[a1][a2][a3]; + for ( int likIdx : likelihoodIndexesToUse ) { + int[] offsets = calculateOffsets(PLcache[likIdx]); + thisState[a1+offsets[1]][a2+offsets[2]][a3+offsets[3]] = base + likelihoods[likIdx]; + } + } + } + } + prevMaxState += 2; + } + idx++; + } + + // after all that pain, we have a fully calculated trellis. Now just march backwards from the EAC state and + // assign genotypes along the greedy path + + GenotypesContext calls = GenotypesContext.create(sampleIndices.size()); + int[] state = alleleCount; + for ( String sample : Utils.reverse(sampleIndices) ) { + --idx; + // the next state will be the maximum achievable state + Genotype g = GLs.get(sample); + if ( g.isNoCall() || ! g.hasLikelihoods() ) { + calls.add(g); + continue; + } + + // subset to the new likelihoods. These are not used except for subsetting in the context iself. + // i.e. they are not a part of the calculation. + final double[] originalLikelihoods = GLs.get(sample).getLikelihoods().getAsVector(); + double[] newLikelihoods; + if ( likelihoodIndexesToUse == null ) { + newLikelihoods = originalLikelihoods; + } else { + newLikelihoods = new double[likelihoodIndexesToUse.size()]; + int newIndex = 0; + for ( int oldIndex : likelihoodIndexesToUse ) + newLikelihoods[newIndex++] = originalLikelihoods[oldIndex]; + + // might need to re-normalize + newLikelihoods = MathUtils.normalizeFromLog10(newLikelihoods, false, true); + } + + // todo -- alter this. For ease of programming, likelihood indeces are + // todo -- used to iterate over achievable states. + double max = Double.NEGATIVE_INFINITY; + int[] bestState = null; + int[] bestAlleles = null; + int bestLikIdx = -1; + for ( int likIdx : likelihoodIndexesToUse ) { + int[] offsets = calculateOffsets(PLcache[likIdx]); + double val = transitionTrellis[idx-1][state[0]-offsets[0]][state[1]-offsets[1]][state[2]-offsets[2]]; + if ( val > max ) { + max = val; + bestState = new int[] { state[0]-offsets[0],state[1]-offsets[1],state[2]-offsets[2]}; + bestAlleles = PLcache[likIdx]; + bestLikIdx = likIdx; + } + } + state = bestState; + List gtAlleles = new ArrayList(2); + gtAlleles.add(newAlleles.get(bestAlleles[0])); + gtAlleles.add(newAlleles.get(bestAlleles[1])); + + final double qual = numNewAltAlleles == 0 ? Genotype.NO_LOG10_PERROR : GenotypeLikelihoods.getQualFromLikelihoods(bestLikIdx, newLikelihoods); + Map attrs = new HashMap(g.getAttributes()); + if ( numNewAltAlleles == 0 ) + attrs.remove(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY); + else + attrs.put(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY, GenotypeLikelihoods.fromLog10Likelihoods(newLikelihoods)); + calls.add(new Genotype(sample, gtAlleles, qual, null, attrs, false)); + + } + return calls; + } + + private static int[] calculateOffsets(int[] alleleIndeces) { + int[] offsets = new int[4]; + for ( int i = 0; i < alleleIndeces.length; i++ ) { + offsets[alleleIndeces[i]]++; + } + + return offsets; + } } From 9093de1132be1c5a7cc657c552299bceb1f67c33 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Tue, 3 Jan 2012 15:58:51 -0500 Subject: [PATCH 020/356] Cleanup: remove code to calculate the MLE AC in the UGE. --- .../sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java index 2159da6e6f..34be88dbb7 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java @@ -401,11 +401,6 @@ else if ( UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_M // *** note that calculating strand bias involves overwriting data structures, so we do that last final HashMap attributes = new HashMap(); - List mlecounts = new ArrayList(AFresult.log10AlleleFrequencyPosteriors.length); - for ( int i = 0; i < AFresult.log10AlleleFrequencyLikelihoods.length ; i++) { - mlecounts.add(MathUtils.maxElementIndex(AFresult.log10AlleleFrequencyLikelihoods[i])); - } - attributes.put("MLEAC",Utils.join(",",mlecounts)); // if the site was downsampled, record that fact if ( !limitedContext && rawContext.hasPileupBeenDownsampled() ) From 31ecc38db871be69f4b8fd035c86e71017e1e629 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Wed, 4 Jan 2012 01:10:28 -0500 Subject: [PATCH 021/356] Initial implementation of a walker for redesigning low or high GC baits using a fairly textbook genetic algorithm. From 5cdde168af524a94972a08908fe7a78a6438fa5f Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Wed, 4 Jan 2012 14:25:43 -0500 Subject: [PATCH 022/356] Switch from using BWA to direct edit distance inspection. Seems to work quite well. From 58d45393048c6cdd429bfbb3bd58d96ca66d5caf Mon Sep 17 00:00:00 2001 From: Guillermo del Angel Date: Wed, 4 Jan 2012 15:28:26 -0500 Subject: [PATCH 023/356] Enabled banded indel computation by default. Reversed logic in input UG argument so that we can still disable it if required. Minor changes to integration tests due to minor differences in GL's and in annotations --- .../IndelGenotypeLikelihoodsCalculationModel.java | 2 +- .../gatk/walkers/genotyper/UnifiedArgumentCollection.java | 6 +++--- .../genotyper/UnifiedGenotyperIntegrationTest.java | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java index fe2086d474..8d279005b9 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java @@ -81,7 +81,7 @@ protected synchronized HashMap> initi protected IndelGenotypeLikelihoodsCalculationModel(UnifiedArgumentCollection UAC, Logger logger) { super(UAC, logger); pairModel = new PairHMMIndelErrorModel(UAC.INDEL_GAP_OPEN_PENALTY,UAC.INDEL_GAP_CONTINUATION_PENALTY, - UAC.OUTPUT_DEBUG_INDEL_INFO, UAC.BANDED_INDEL_COMPUTATION); + UAC.OUTPUT_DEBUG_INDEL_INFO, !UAC.DONT_DO_BANDED_INDEL_COMPUTATION); alleleList = new ArrayList(); getAlleleListFromVCF = UAC.GenotypingMode == GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES; minIndelCountForGenotyping = UAC.MIN_INDEL_COUNT_FOR_GENOTYPING; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java index 5713432b42..4639d67a7b 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java @@ -146,8 +146,8 @@ public class UnifiedArgumentCollection { public int INDEL_HAPLOTYPE_SIZE = 80; @Hidden - @Argument(fullName = "bandedIndel", shortName = "bandedIndel", doc = "Banded Indel likelihood computation", required = false) - public boolean BANDED_INDEL_COMPUTATION = false; + @Argument(fullName = "noBandedIndel", shortName = "noBandedIndel", doc = "Don't do Banded Indel likelihood computation", required = false) + public boolean DONT_DO_BANDED_INDEL_COMPUTATION = false; @Hidden @Argument(fullName = "indelDebug", shortName = "indelDebug", doc = "Output indel debug info", required = false) @@ -184,7 +184,7 @@ public UnifiedArgumentCollection clone() { // todo- arguments to remove uac.IGNORE_SNP_ALLELES = IGNORE_SNP_ALLELES; - uac.BANDED_INDEL_COMPUTATION = BANDED_INDEL_COMPUTATION; + uac.DONT_DO_BANDED_INDEL_COMPUTATION = DONT_DO_BANDED_INDEL_COMPUTATION; uac.MULTI_ALLELIC = MULTI_ALLELIC; return uac; } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java index 7c0dba5585..a91ea1c87b 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java @@ -265,7 +265,7 @@ public void testWithIndelAllelesPassedIn1() { WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec( baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "indelAllelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,100,000", 1, - Arrays.asList("fa4f3ee67d98b64102a8a3ec81a3bc81")); + Arrays.asList("c60a44ba94a80a0cb1fba8b6f90a13cd")); executeTest("test MultiSample Pilot2 indels with alleles passed in", spec1); } @@ -275,7 +275,7 @@ public void testWithIndelAllelesPassedIn2() { baseCommandIndels + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "indelAllelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,100,000", 1, - Arrays.asList("df90890e43d735573a3b3e4f289ca46b")); + Arrays.asList("36ce53ae4319718ad9c8ae391deebc8c")); executeTest("test MultiSample Pilot2 indels with alleles passed in and emitting all sites", spec2); } @@ -285,7 +285,7 @@ public void testWithIndelAllelesPassedIn3() { WalkerTest.WalkerTestSpec spec3 = new WalkerTest.WalkerTestSpec( baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "ALL.wgs.union_v2.20101123.indels.sites.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,080,000", 1, - Arrays.asList("cff6dd0f4eb1ef0b6fc476da6ffead19")); + Arrays.asList("d356cbaf240d7025d1aecdabaff3a3e0")); executeTest("test MultiSample Pilot2 indels with complicated records", spec3); } @@ -294,7 +294,7 @@ public void testWithIndelAllelesPassedIn4() { WalkerTest.WalkerTestSpec spec4 = new WalkerTest.WalkerTestSpec( baseCommandIndelsb37 + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "ALL.wgs.union_v2_chr20_100_110K.20101123.indels.sites.vcf -I " + validationDataLocation + "phase1_GBR_realigned.chr20.100K-110K.bam -o %s -L 20:100,000-110,000", 1, - Arrays.asList("1e2a4aab26e9ab0dae709d33a669e036")); + Arrays.asList("947c12ef2a8c29ae787cd11be8c565c8")); executeTest("test MultiSample Phase1 indels with complicated records", spec4); } From a6886a4cc0e21c368fcd06e36e144e41113bf0d0 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Wed, 4 Jan 2012 17:03:21 -0500 Subject: [PATCH 024/356] Initial commit of the Active Region Traversal. Not ready to be used by anyone yet. --- .../sting/gatk/GenomeAnalysisEngine.java | 2 +- .../executive/HierarchicalMicroScheduler.java | 1 - .../gatk/executive/LinearMicroScheduler.java | 8 +- .../sting/gatk/executive/MicroScheduler.java | 2 + .../traversals/TraverseActiveRegions.java | 213 ++++++++++++++++++ .../gatk/walkers/ActiveRegionWalker.java | 29 +++ .../broadinstitute/sting/utils/GenomeLoc.java | 6 +- .../sting/utils/activeregion/ActiveRead.java | 19 ++ .../utils/activeregion/ActiveRegion.java | 55 +++++ 9 files changed, 331 insertions(+), 4 deletions(-) create mode 100644 public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java create mode 100644 public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java create mode 100644 public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRead.java create mode 100644 public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRegion.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java b/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java index f954d76501..ede8e93406 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java @@ -443,7 +443,7 @@ protected Iterable getShardStrategy(SAMDataSource readsDataSource, Refere if(!readsDataSource.hasIndex() && intervals != null && !argCollection.allowIntervalsWithUnindexedBAM) throw new UserException.CommandLineException("Cannot perform interval processing when reads are present but no index is available."); - if(walker instanceof LocusWalker) { + if(walker instanceof LocusWalker || walker instanceof ActiveRegionWalker) { if (readsDataSource.getSortOrder() != SAMFileHeader.SortOrder.coordinate) throw new UserException.MissortedBAM(SAMFileHeader.SortOrder.coordinate, "Locus walkers can only traverse coordinate-sorted data. Please resort your input BAM file(s) or set the Sort Order tag in the header appropriately."); if(intervals == null) diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java index 39e1bdc726..eec4408200 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java +++ b/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java @@ -11,7 +11,6 @@ import org.broadinstitute.sting.gatk.walkers.TreeReducible; import org.broadinstitute.sting.gatk.walkers.Walker; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import org.broadinstitute.sting.utils.exceptions.UserException; import org.broadinstitute.sting.utils.threading.ThreadPoolMonitor; import java.util.Collection; diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java index ff5e1064bd..774b532f34 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java +++ b/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java @@ -10,6 +10,7 @@ import org.broadinstitute.sting.gatk.datasources.rmd.ReferenceOrderedDataSource; import org.broadinstitute.sting.gatk.io.DirectOutputTracker; import org.broadinstitute.sting.gatk.io.OutputTracker; +import org.broadinstitute.sting.gatk.traversals.TraverseActiveRegions; import org.broadinstitute.sting.gatk.walkers.LocusWalker; import org.broadinstitute.sting.gatk.walkers.Walker; import org.broadinstitute.sting.utils.SampleUtils; @@ -55,7 +56,6 @@ public Object execute(Walker walker, Iterable shardStrategy) { traversalEngine.startTimersIfNecessary(); if(shard.getShardType() == Shard.ShardType.LOCUS) { - LocusWalker lWalker = (LocusWalker)walker; WindowMaker windowMaker = new WindowMaker(shard, engine.getGenomeLocParser(), getReadIterator(shard), shard.getGenomeLocs(), SampleUtils.getSAMFileSamples(engine)); for(WindowMaker.WindowMakerIterator iterator: windowMaker) { @@ -77,6 +77,12 @@ public Object execute(Walker walker, Iterable shardStrategy) { done = walker.isDone(); } + // Special function call to empty out the work queue. Ugly for now but will be cleaned up when we push this functionality more into the engine + if( traversalEngine instanceof TraverseActiveRegions ) { + final Object result = ((TraverseActiveRegions) traversalEngine).endTraversal(walker, accumulator.getReduceInit()); + accumulator.accumulate(null, result); // Assumes only used with StandardAccumulator + } + Object result = accumulator.finishTraversal(); printOnTraversalDone(result); diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java index d013db7e84..5080997084 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java +++ b/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java @@ -128,6 +128,8 @@ protected MicroScheduler(GenomeAnalysisEngine engine, Walker walker, SAMDataSour traversalEngine = new TraverseDuplicates(); } else if (walker instanceof ReadPairWalker) { traversalEngine = new TraverseReadPairs(); + } else if (walker instanceof ActiveRegionWalker) { + traversalEngine = new TraverseActiveRegions(); } else { throw new UnsupportedOperationException("Unable to determine traversal type, the walker is an unknown type."); } diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java new file mode 100644 index 0000000000..01bfe396ae --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java @@ -0,0 +1,213 @@ +package org.broadinstitute.sting.gatk.traversals; + +import net.sf.samtools.SAMRecord; +import org.apache.log4j.Logger; +import org.broadinstitute.sting.gatk.WalkerManager; +import org.broadinstitute.sting.gatk.contexts.AlignmentContext; +import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.gatk.datasources.providers.*; +import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; +import org.broadinstitute.sting.gatk.walkers.ActiveRegionWalker; +import org.broadinstitute.sting.gatk.walkers.DataSource; +import org.broadinstitute.sting.gatk.walkers.Walker; +import org.broadinstitute.sting.utils.GenomeLoc; +import org.broadinstitute.sting.utils.activeregion.ActiveRegion; +import org.broadinstitute.sting.utils.pileup.PileupElement; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.Queue; + +/** + * Created by IntelliJ IDEA. + * User: rpoplin + * Date: 12/9/11 + */ + +public class TraverseActiveRegions extends TraversalEngine,LocusShardDataProvider> { + /** + * our log, which we want to capture anything from this class + */ + protected static Logger logger = Logger.getLogger(TraversalEngine.class); + + private final Queue workQueue = new LinkedList(); + private final LinkedHashSet myReads = new LinkedHashSet(); + + @Override + protected String getTraversalType() { + return "active regions"; + } + + @Override + public T traverse( final ActiveRegionWalker walker, + final LocusShardDataProvider dataProvider, + T sum) { + logger.debug(String.format("TraverseActiveRegion.traverse: Shard is %s", dataProvider)); + + LocusView locusView = getLocusView( walker, dataProvider ); + + int minStart = Integer.MAX_VALUE; + final LocusReferenceView referenceView = new LocusReferenceView( walker, dataProvider ); + + if ( locusView.hasNext() ) { // trivial optimization to avoid unnecessary processing when there's nothing here at all + + final ArrayList isActiveList = new ArrayList(); + + //ReferenceOrderedView referenceOrderedDataView = new ReferenceOrderedView( dataProvider ); + ReferenceOrderedView referenceOrderedDataView = null; + if ( WalkerManager.getWalkerDataSource(walker) != DataSource.REFERENCE_ORDERED_DATA ) + referenceOrderedDataView = new ManagingReferenceOrderedView( dataProvider ); + else + referenceOrderedDataView = (RodLocusView)locusView; + + // We keep processing while the next reference location is within the interval + while( locusView.hasNext() ) { + final AlignmentContext locus = locusView.next(); + GenomeLoc location = locus.getLocation(); + + dataProvider.getShard().getReadMetrics().incrementNumIterations(); + + if ( locus.hasExtendedEventPileup() ) { + // if the alignment context we received holds an "extended" pileup (i.e. pileup of insertions/deletions + // associated with the current site), we need to update the location. The updated location still starts + // at the current genomic position, but it has to span the length of the longest deletion (if any). + location = engine.getGenomeLocParser().setStop(location,location.getStop()+locus.getExtendedEventPileup().getMaxDeletionLength()); + + // it is possible that the new expanded location spans the current shard boundary; the next method ensures + // that when it is the case, the reference sequence held by the ReferenceView will be reloaded so that + // the view has all the bases we are gonna need. If the location fits within the current view bounds, + // the next call will not do anything to the view: + referenceView.expandBoundsToAccomodateLoc(location); + } + + // create reference context. Note that if we have a pileup of "extended events", the context will + // hold the (longest) stretch of deleted reference bases (if deletions are present in the pileup). + final ReferenceContext refContext = referenceView.getReferenceContext(location); + + // Iterate forward to get all reference ordered data covering this location + final RefMetaDataTracker tracker = referenceOrderedDataView.getReferenceOrderedDataAtLocus(locus.getLocation(), refContext); + + // Call the walkers isActive function for this locus and add them to the list to be integrated later + final boolean isActive = walker.isActive( tracker, refContext, locus ); + isActiveList.add( new ActiveRegion(location, isActive, engine.getGenomeLocParser()) ); + + // Grab all the previously unseen reads from this pileup and add them to the massive read list + for( final PileupElement p : locus.getBasePileup() ) { + final SAMRecord read = p.getRead(); + if( !myReads.contains(read) ) { + myReads.add(read); + } + } + + // If this is the last pileup for this shard then need to calculate the minimum alignment start so that + // we know which active regions in the work queue are now safe to process + if( !locusView.hasNext() ) { + for( final PileupElement p : locus.getBasePileup() ) { + final SAMRecord read = p.getRead(); + if( read.getAlignmentStart() < minStart ) { minStart = read.getAlignmentStart(); } + } + } + printProgress(dataProvider.getShard(),locus.getLocation()); + } + + // Take the individual isActive calls and integrate them into contiguous active regions and + // add these blocks of work to the work queue + final ArrayList activeRegions = integrateActiveList( isActiveList ); + logger.debug("Integrated " + isActiveList.size() + " isActive calls into " + activeRegions.size() + " regions." ); + workQueue.addAll( activeRegions ); + } + + while( workQueue.peek().getLocation().getStop() < minStart ) { + final ActiveRegion activeRegion = workQueue.remove(); + sum = processActiveRegion( activeRegion, myReads, workQueue, sum, walker ); + } + + return sum; + } + + // Special function called in LinearMicroScheduler to empty out the work queue. Ugly for now but will be cleaned up when we push this functionality more into the engine + public T endTraversal( final Walker walker, T sum) { + while( workQueue.peek() != null ) { + final ActiveRegion activeRegion = workQueue.remove(); + sum = processActiveRegion( activeRegion, myReads, workQueue, sum, (ActiveRegionWalker) walker ); + } + + return sum; + } + + private T processActiveRegion( final ActiveRegion activeRegion, final LinkedHashSet reads, final Queue workQueue, final T sum, final ActiveRegionWalker walker ) { + final ArrayList placedReads = new ArrayList(); + for( final SAMRecord read : reads ) { + final GenomeLoc readLoc = this.engine.getGenomeLocParser().createGenomeLoc( read ); + if( activeRegion.getLocation().overlapsP( readLoc ) ) { + // The region which the highest amount of overlap is chosen as the primary region for the read (tie breaking is done as right most region) + long maxOverlap = activeRegion.getLocation().sizeOfOverlap( readLoc ); + ActiveRegion bestRegion = activeRegion; + for( final ActiveRegion otherRegionToTest : workQueue ) { + if( otherRegionToTest.getLocation().sizeOfOverlap(readLoc) >= maxOverlap ) { + maxOverlap = otherRegionToTest.getLocation().sizeOfOverlap(readLoc); + bestRegion = otherRegionToTest; + } + } + bestRegion.add( (GATKSAMRecord) read, true ); + + // The read is also added to all other region in which it overlaps but marked as non-primary + if( !bestRegion.equals(activeRegion) ) { + activeRegion.add( (GATKSAMRecord) read, false ); + } + for( final ActiveRegion otherRegionToTest : workQueue ) { + if( !bestRegion.equals(otherRegionToTest) && otherRegionToTest.getLocation().overlapsP( readLoc ) ) { + activeRegion.add( (GATKSAMRecord) read, false ); + } + } + placedReads.add( read ); + } + } + reads.removeAll( placedReads ); // remove all the reads which have been placed into their active region + + logger.debug(">> Map call with " + activeRegion.getReads().size() + " " + (activeRegion.isActive ? "active" : "inactive") + " reads @ " + activeRegion.getLocation() + " with full extent: " + activeRegion.getReferenceLocation()); + final M x = walker.map( activeRegion, null ); // BUGBUG: tracker needs to be filled in and passed to the walker + return walker.reduce( x, sum ); + } + + /** + * Gets the best view of loci for this walker given the available data. + * @param walker walker to interrogate. + * @param dataProvider Data which which to drive the locus view. + * @return A view of the locus data, where one iteration of the locus view maps to one iteration of the traversal. + */ + private LocusView getLocusView( Walker walker, LocusShardDataProvider dataProvider ) { + DataSource dataSource = WalkerManager.getWalkerDataSource(walker); + if( dataSource == DataSource.READS ) + return new CoveredLocusView(dataProvider); + else if( dataSource == DataSource.REFERENCE ) //|| ! GenomeAnalysisEngine.instance.getArguments().enableRodWalkers ) + return new AllLocusView(dataProvider); + else if( dataSource == DataSource.REFERENCE_ORDERED_DATA ) + return new RodLocusView(dataProvider); + else + throw new UnsupportedOperationException("Unsupported traversal type: " + dataSource); + } + + // integrate active regions into contiguous chunks based on active status + private ArrayList integrateActiveList( final ArrayList activeList ) { + final ArrayList returnList = new ArrayList(); + ActiveRegion prevLocus = activeList.remove(0); + ActiveRegion startLocus = prevLocus; + for( final ActiveRegion thisLocus : activeList ) { + if( prevLocus.isActive != thisLocus.isActive ) { + returnList.add( new ActiveRegion( engine.getGenomeLocParser().createGenomeLoc(startLocus.getLocation().getContig(), startLocus.getLocation().getStart(), prevLocus.getLocation().getStart()), + prevLocus.isActive, engine.getGenomeLocParser() ) ); + startLocus = thisLocus; + } + prevLocus = thisLocus; + } + // output the last region if necessary + if( startLocus != prevLocus ) { + returnList.add( new ActiveRegion( engine.getGenomeLocParser().createGenomeLoc(startLocus.getLocation().getContig(), startLocus.getLocation().getStart(), prevLocus.getLocation().getStart()), + prevLocus.isActive, engine.getGenomeLocParser() ) ); + } + return returnList; + } +} diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java new file mode 100644 index 0000000000..d2891c959a --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java @@ -0,0 +1,29 @@ +package org.broadinstitute.sting.gatk.walkers; + +import org.broadinstitute.sting.gatk.contexts.AlignmentContext; +import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.gatk.refdata.ReadMetaDataTracker; +import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; +import org.broadinstitute.sting.utils.activeregion.ActiveRegion; + +/** + * Created by IntelliJ IDEA. + * User: rpoplin + * Date: 12/7/11 + */ + +@By(DataSource.READS) +@Requires({DataSource.READS, DataSource.REFERENCE_BASES}) +@PartitionBy(PartitionType.READ) +public abstract class ActiveRegionWalker extends Walker { + // Do we actually want to operate on the context? + public boolean filter(final RefMetaDataTracker tracker, final ReferenceContext ref, final AlignmentContext context) { + return true; // We are keeping all the reads + } + + // Determine active status over the AlignmentContext + public abstract boolean isActive(final RefMetaDataTracker tracker, final ReferenceContext ref, final AlignmentContext context); + + // Map over the ActiveRegion + public abstract MapType map(final ActiveRegion activeRegion, final ReadMetaDataTracker metaDataTracker); +} diff --git a/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java b/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java index 345161416b..6941b888b5 100644 --- a/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java +++ b/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java @@ -145,7 +145,7 @@ public GenomeLoc merge( GenomeLoc that ) throws ReviewedStingException { } return new GenomeLoc(getContig(), this.contigIndex, - Math.min(getStart(), that.getStart()), + Math.min( getStart(), that.getStart() ), Math.max( getStop(), that.getStop()) ); } @@ -465,4 +465,8 @@ public final double reciprocialOverlapFraction(final GenomeLoc o) { private final static double overlapPercent(final GenomeLoc gl1, final GenomeLoc gl2) { return (1.0 * gl1.intersect(gl2).size()) / gl1.size(); } + + public long sizeOfOverlap( final GenomeLoc that ) { + return ( this.overlapsP(that) ? Math.min( getStop(), that.getStop() ) - Math.max( getStart(), that.getStart() ) : 0L ); + } } diff --git a/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRead.java b/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRead.java new file mode 100644 index 0000000000..8d08a29b6e --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRead.java @@ -0,0 +1,19 @@ +package org.broadinstitute.sting.utils.activeregion; + +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; + +/** + * Created by IntelliJ IDEA. + * User: rpoplin + * Date: 1/4/12 + */ + +public class ActiveRead { + final public GATKSAMRecord read; + final public boolean isPrimaryRegion; + + ActiveRead( final GATKSAMRecord read, final boolean isPrimaryRegion ) { + this.read = read; + this.isPrimaryRegion = isPrimaryRegion; + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRegion.java b/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRegion.java new file mode 100644 index 0000000000..e8908480c6 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRegion.java @@ -0,0 +1,55 @@ +package org.broadinstitute.sting.utils.activeregion; + +import net.sf.picard.reference.IndexedFastaSequenceFile; +import org.broadinstitute.sting.utils.GenomeLoc; +import org.broadinstitute.sting.utils.GenomeLocParser; +import org.broadinstitute.sting.utils.HasGenomeLocation; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; + +import java.util.ArrayList; + +/** + * Created by IntelliJ IDEA. + * User: rpoplin + * Date: 1/4/12 + */ + +public class ActiveRegion implements HasGenomeLocation { + + private final ArrayList reads = new ArrayList(); + private byte[] reference = null; + private final GenomeLoc loc; + private GenomeLoc referenceLoc = null; + private final GenomeLocParser genomeLocParser; + public final boolean isActive; + + public ActiveRegion( final GenomeLoc loc, final boolean isActive, final GenomeLocParser genomeLocParser ) { + this.loc = loc; + this.isActive = isActive; + this.genomeLocParser = genomeLocParser; + referenceLoc = loc; + } + + // add each read to the bin and extend the reference genome loc if needed + public void add( final GATKSAMRecord read, final boolean isPrimaryRegion ) { + referenceLoc = referenceLoc.union( genomeLocParser.createGenomeLoc( read ) ); + reads.add( new ActiveRead(read, isPrimaryRegion) ); + } + + public ArrayList getReads() { return reads; } + + public byte[] getReference( final IndexedFastaSequenceFile referenceReader ) { + // set up the reference if we haven't done so yet + if ( reference == null ) { + reference = referenceReader.getSubsequenceAt(referenceLoc.getContig(), referenceLoc.getStart(), referenceLoc.getStop()).getBases(); + } + + return reference; + } + + public GenomeLoc getLocation() { return loc; } + + public GenomeLoc getReferenceLocation() { return referenceLoc; } + + public int size() { return reads.size(); } +} \ No newline at end of file From 43224ef364a9834b2b0f9384fa31f29ac2350513 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Thu, 5 Jan 2012 23:46:31 -0500 Subject: [PATCH 025/356] Turning the Adaptive Downsampler on with 100 by default From 616ff8ea017888945e4dd5dcdde6dc196e9b6bd2 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Fri, 6 Jan 2012 10:36:11 -0500 Subject: [PATCH 026/356] fixed typo in help text --- .../sting/gatk/walkers/variantutils/CombineVariants.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/CombineVariants.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/CombineVariants.java index 096085330d..af05c0dc4e 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/CombineVariants.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/CombineVariants.java @@ -105,7 +105,7 @@ public class CombineVariants extends RodWalker { * and each named argument will be labeled as such in the output (i.e., set=name rather than * set=variants2). The order of arguments does not matter unless except for the naming, so * if you provide an rod priority list and no explicit names than variants, variants2, etc - * are techincally order dependent. It is strongly recommended to provide explicit names when + * are technically order dependent. It is strongly recommended to provide explicit names when * a rod priority list is provided. */ @Input(fullName="variant", shortName = "V", doc="Input VCF file", required=true) From d4e7655d14d9bb9c30a79d53fafe309dd9922fa9 Mon Sep 17 00:00:00 2001 From: Guillermo del Angel Date: Fri, 6 Jan 2012 11:24:38 -0500 Subject: [PATCH 027/356] Added ability to call multiallelic indels, if -multiallelic is included in UG arguments. Simple idea: we genotype all alleles with count >= minIndelCnt. To support this, refactored code that computes consensus alleles. To ease merging of mulitple alt alleles, we create a single vc for each alt alleles and then use VariantContextUtils.simpleMerge to carry out merging, which takes care of handling all corner conditions already. In order to use this, interface to GenotypeLikelihoodsCalculationModel changed to pass in a GenomeLocParser object (why are these objects to hard to handle??). More testing is required and feature turned off my default. --- .../GenotypeLikelihoodsCalculationModel.java | 42 ++--- ...elGenotypeLikelihoodsCalculationModel.java | 147 ++++++++++-------- ...NPGenotypeLikelihoodsCalculationModel.java | 8 +- .../genotyper/UnifiedArgumentCollection.java | 2 +- .../genotyper/UnifiedGenotyperEngine.java | 2 +- 5 files changed, 106 insertions(+), 95 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java index b30a254148..ace780dd0f 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java @@ -31,6 +31,7 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; import org.broadinstitute.sting.utils.BaseUtils; +import org.broadinstitute.sting.utils.GenomeLocParser; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.pileup.PileupElement; import org.broadinstitute.sting.utils.pileup.ReadBackedPileup; @@ -72,25 +73,28 @@ protected GenotypeLikelihoodsCalculationModel(UnifiedArgumentCollection UAC, Log this.logger = logger; } - /** - * Must be overridden by concrete subclasses - * - * @param tracker rod data - * @param ref reference context - * @param contexts stratified alignment contexts - * @param contextType stratified context type - * @param priors priors to use for GLs - * @param alternateAlleleToUse the alternate allele to use, null if not set - * @param useBAQedPileup should we use the BAQed pileup or the raw one? - * @return variant context where genotypes are no-called but with GLs - */ - public abstract VariantContext getLikelihoods(RefMetaDataTracker tracker, - ReferenceContext ref, - Map contexts, - AlignmentContextUtils.ReadOrientation contextType, - GenotypePriors priors, - Allele alternateAlleleToUse, - boolean useBAQedPileup); + /** + * Can be overridden by concrete subclasses + * + * @param tracker rod data + * @param ref reference context + * @param contexts stratified alignment contexts + * @param contextType stratified context type + * @param priors priors to use for GLs + * @param alternateAlleleToUse the alternate allele to use, null if not set + * @param useBAQedPileup should we use the BAQed pileup or the raw one? + * @param locParser Genome Loc Parser + * @return variant context where genotypes are no-called but with GLs + */ + public abstract VariantContext getLikelihoods(RefMetaDataTracker tracker, + ReferenceContext ref, + Map contexts, + AlignmentContextUtils.ReadOrientation contextType, + GenotypePriors priors, + Allele alternateAlleleToUse, + boolean useBAQedPileup, + GenomeLocParser locParser); + protected int getFilteredDepth(ReadBackedPileup pileup) { int count = 0; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java index 8d279005b9..0756caf03a 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java @@ -33,6 +33,7 @@ import org.broadinstitute.sting.gatk.walkers.indels.PairHMMIndelErrorModel; import org.broadinstitute.sting.utils.BaseUtils; import org.broadinstitute.sting.utils.GenomeLoc; +import org.broadinstitute.sting.utils.GenomeLocParser; import org.broadinstitute.sting.utils.Haplotype; import org.broadinstitute.sting.utils.clipping.ReadClipper; import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants; @@ -54,17 +55,17 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood private final boolean getAlleleListFromVCF; private boolean DEBUG = false; - + private final boolean doMultiAllelicCalls; private boolean ignoreSNPAllelesWhenGenotypingIndels = false; - + private final int maxAlternateAlleles; private PairHMMIndelErrorModel pairModel; private static ThreadLocal>> indelLikelihoodMap = new ThreadLocal>>() { - protected synchronized HashMap> initialValue() { - return new HashMap>(); - } - }; + protected synchronized HashMap> initialValue() { + return new HashMap>(); + } + }; private LinkedHashMap haplotypeMap; @@ -87,6 +88,8 @@ protected IndelGenotypeLikelihoodsCalculationModel(UnifiedArgumentCollection UAC minIndelCountForGenotyping = UAC.MIN_INDEL_COUNT_FOR_GENOTYPING; HAPLOTYPE_SIZE = UAC.INDEL_HAPLOTYPE_SIZE; DEBUG = UAC.OUTPUT_DEBUG_INDEL_INFO; + maxAlternateAlleles = UAC.MAX_ALTERNATE_ALLELES; + doMultiAllelicCalls = UAC.MULTI_ALLELIC; haplotypeMap = new LinkedHashMap(); ignoreSNPAllelesWhenGenotypingIndels = UAC.IGNORE_SNP_ALLELES; @@ -95,7 +98,7 @@ protected IndelGenotypeLikelihoodsCalculationModel(UnifiedArgumentCollection UAC private ArrayList computeConsensusAlleles(ReferenceContext ref, Map contexts, - AlignmentContextUtils.ReadOrientation contextType) { + AlignmentContextUtils.ReadOrientation contextType, GenomeLocParser locParser) { Allele refAllele=null, altAllele=null; GenomeLoc loc = ref.getLocus(); ArrayList aList = new ArrayList(); @@ -114,7 +117,7 @@ private ArrayList computeConsensusAlleles(ReferenceContext ref, if (insCount < minIndelCountForGenotyping && delCount < minIndelCountForGenotyping) return aList; - + for ( Map.Entry sample : contexts.entrySet() ) { // todo -- warning, can be duplicating expensive partition here AlignmentContext context = AlignmentContextUtils.stratify(sample.getValue(), contextType); @@ -126,9 +129,9 @@ private ArrayList computeConsensusAlleles(ReferenceContext ref, for ( ExtendedEventPileupElement p : indelPileup.toExtendedIterable() ) { //SAMRecord read = p.getRead(); - GATKSAMRecord read = ReadClipper.hardClipAdaptorSequence(p.getRead()); + GATKSAMRecord read = ReadClipper.hardClipAdaptorSequence(p.getRead()); if (read == null) - continue; + continue; if(ReadUtils.is454Read(read)) { continue; } @@ -208,63 +211,69 @@ else if (p.isDeletion()) { } } -/* if (DEBUG) { - int icount = indelPileup.getNumberOfInsertions(); - int dcount = indelPileup.getNumberOfDeletions(); - if (icount + dcount > 0) - { - List> eventStrings = indelPileup.getEventStringsWithCounts(ref.getBases()); - System.out.format("#ins: %d, #del:%d\n", insCount, delCount); - - for (int i=0 ; i < eventStrings.size() ; i++ ) { - System.out.format("%s:%d,",eventStrings.get(i).first,eventStrings.get(i).second); - // int k=0; - } - System.out.println(); - } - } */ } + Collection vcs = new ArrayList(); int maxAlleleCnt = 0; String bestAltAllele = ""; + for (String s : consensusIndelStrings.keySet()) { - int curCnt = consensusIndelStrings.get(s); - if (curCnt > maxAlleleCnt) { - maxAlleleCnt = curCnt; - bestAltAllele = s; + int curCnt = consensusIndelStrings.get(s), stop = 0; + // if observed count if above minimum threshold, we will genotype this allele + if (curCnt < minIndelCountForGenotyping) + continue; + + if (s.startsWith("D")) { + // get deletion length + int dLen = Integer.valueOf(s.substring(1)); + // get ref bases of accurate deletion + int startIdxInReference = 1+loc.getStart()-ref.getWindow().getStart(); + stop = loc.getStart() + dLen; + byte[] refBases = Arrays.copyOfRange(ref.getBases(),startIdxInReference,startIdxInReference+dLen); + + if (Allele.acceptableAlleleBases(refBases)) { + refAllele = Allele.create(refBases,true); + altAllele = Allele.create(Allele.NULL_ALLELE_STRING, false); + } + } + else { + // insertion case + if (Allele.acceptableAlleleBases(s)) { + refAllele = Allele.create(Allele.NULL_ALLELE_STRING, true); + altAllele = Allele.create(s, false); + stop = loc.getStart(); + } } -// if (DEBUG) -// System.out.format("Key:%s, number: %d\n",s,consensusIndelStrings.get(s) ); - } //gdebug- - if (maxAlleleCnt < minIndelCountForGenotyping) - return aList; - if (bestAltAllele.startsWith("D")) { - // get deletion length - int dLen = Integer.valueOf(bestAltAllele.substring(1)); - // get ref bases of accurate deletion - int startIdxInReference = 1+loc.getStart()-ref.getWindow().getStart(); + ArrayList vcAlleles = new ArrayList(); + vcAlleles.add(refAllele); + vcAlleles.add(altAllele); - //System.out.println(new String(ref.getBases())); - byte[] refBases = Arrays.copyOfRange(ref.getBases(),startIdxInReference,startIdxInReference+dLen); + final VariantContextBuilder builder = new VariantContextBuilder().source(""); + builder.loc(loc.getContig(), loc.getStart(), stop); + builder.alleles(vcAlleles); + builder.referenceBaseForIndel(ref.getBase()); + builder.noGenotypes(); + if (doMultiAllelicCalls) + vcs.add(builder.make()); + else { + if (curCnt > maxAlleleCnt) { + maxAlleleCnt = curCnt; + vcs.clear(); + vcs.add(builder.make()); + } - if (Allele.acceptableAlleleBases(refBases)) { - refAllele = Allele.create(refBases,true); - altAllele = Allele.create(Allele.NULL_ALLELE_STRING, false); } } - else { - // insertion case - if (Allele.acceptableAlleleBases(bestAltAllele)) { - refAllele = Allele.create(Allele.NULL_ALLELE_STRING, true); - altAllele = Allele.create(bestAltAllele, false); - } - } - if (refAllele != null && altAllele != null) { - aList.add(0,refAllele); - aList.add(1,altAllele); - } + + if (vcs.isEmpty()) + return aList; // nothing else to do, no alleles passed minimum count criterion + + VariantContext mergedVC = VariantContextUtils.simpleMerge(locParser, vcs, null, VariantContextUtils.FilteredRecordMergeType.KEEP_IF_ANY_UNFILTERED, VariantContextUtils.GenotypeMergeType.UNSORTED, false, false, null, false, false); + + aList = new ArrayList(mergedVC.getAlleles()); + return aList; } @@ -277,7 +286,7 @@ public VariantContext getLikelihoods(RefMetaDataTracker tracker, AlignmentContextUtils.ReadOrientation contextType, GenotypePriors priors, Allele alternateAlleleToUse, - boolean useBAQedPileup) { + boolean useBAQedPileup, GenomeLocParser locParser) { if ( tracker == null ) return null; @@ -294,17 +303,17 @@ public VariantContext getLikelihoods(RefMetaDataTracker tracker, haplotypeMap.clear(); if (getAlleleListFromVCF) { - for( final VariantContext vc_input : tracker.getValues(UAC.alleles, loc) ) { - if( vc_input != null && - allowableTypes.contains(vc_input.getType()) && - ref.getLocus().getStart() == vc_input.getStart()) { - vc = vc_input; - break; - } - } - // ignore places where we don't have a variant - if ( vc == null ) - return null; + for( final VariantContext vc_input : tracker.getValues(UAC.alleles, loc) ) { + if( vc_input != null && + allowableTypes.contains(vc_input.getType()) && + ref.getLocus().getStart() == vc_input.getStart()) { + vc = vc_input; + break; + } + } + // ignore places where we don't have a variant + if ( vc == null ) + return null; alleleList.clear(); if (ignoreSNPAllelesWhenGenotypingIndels) { @@ -323,7 +332,7 @@ public VariantContext getLikelihoods(RefMetaDataTracker tracker, } else { - alleleList = computeConsensusAlleles(ref,contexts, contextType); + alleleList = computeConsensusAlleles(ref,contexts, contextType, locParser); if (alleleList.isEmpty()) return null; } @@ -340,7 +349,7 @@ public VariantContext getLikelihoods(RefMetaDataTracker tracker, if (alleleList.isEmpty()) return null; - + refAllele = alleleList.get(0); altAllele = alleleList.get(1); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java index eee89674ac..81c766e4de 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java @@ -30,10 +30,7 @@ import org.broadinstitute.sting.gatk.contexts.AlignmentContextUtils; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; -import org.broadinstitute.sting.utils.BaseUtils; -import org.broadinstitute.sting.utils.GenomeLoc; -import org.broadinstitute.sting.utils.MathUtils; -import org.broadinstitute.sting.utils.Utils; +import org.broadinstitute.sting.utils.*; import org.broadinstitute.sting.utils.baq.BAQ; import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants; import org.broadinstitute.sting.utils.exceptions.StingException; @@ -66,7 +63,8 @@ public VariantContext getLikelihoods(final RefMetaDataTracker tracker, final AlignmentContextUtils.ReadOrientation contextType, final GenotypePriors priors, final Allele alternateAlleleToUse, - final boolean useBAQedPileup) { + final boolean useBAQedPileup, + final GenomeLocParser locParser) { if ( !(priors instanceof DiploidSNPGenotypePriors) ) throw new StingException("Only diploid-based SNP priors are supported in the SNP GL model"); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java index 4639d67a7b..16159393f0 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java @@ -109,7 +109,7 @@ public class UnifiedArgumentCollection { * For advanced users only. */ @Advanced - @Argument(fullName = "multiallelic", shortName = "multiallelic", doc = "Allow the discovery of multiple alleles (SNPs only)", required = false) + @Argument(fullName = "multiallelic", shortName = "multiallelic", doc = "Allow the discovery of multiple alleles", required = false) public boolean MULTI_ALLELIC = false; /** diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java index 34be88dbb7..5d73e8d289 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java @@ -243,7 +243,7 @@ private VariantContext calculateLikelihoods(RefMetaDataTracker tracker, Referenc glcm.set(getGenotypeLikelihoodsCalculationObject(logger, UAC)); } - return glcm.get().get(model).getLikelihoods(tracker, refContext, stratifiedContexts, type, getGenotypePriors(model), alternateAlleleToUse, useBAQedPileup && BAQEnabledOnCMDLine); + return glcm.get().get(model).getLikelihoods(tracker, refContext, stratifiedContexts, type, getGenotypePriors(model), alternateAlleleToUse, useBAQedPileup && BAQEnabledOnCMDLine, genomeLocParser); } private VariantCallContext generateEmptyContext(RefMetaDataTracker tracker, ReferenceContext ref, Map stratifiedContexts, AlignmentContext rawContext) { From d9da37f9b4f2f57db62b2d726a0680854eeba38a Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 6 Jan 2012 16:35:53 -0500 Subject: [PATCH 028/356] Added SQL table creation and log loading to analyzeRunReports -- You can create (and drop the old) GATK_LOG table with the setupDB command -- You can load data into the database with the loadToDB command Currently I'm pushing up all of the GATK logs into the new MySQL server setup for the gsa group. Details of the server are in the code, for those interested. All of this is part of my experimentation with Tableau for visualizing GATK run logs. From f6a18aea63040e350817ac38f07fc62cbd899c2e Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Wed, 4 Jan 2012 12:05:08 -0500 Subject: [PATCH 029/356] Updated MDCP with INDEL best practices * chose 90.0 indel cut target for most datasets (this is arbitrary). --- .../MethodsDevelopmentCallingPipeline.scala | 166 +++++++++++------- 1 file changed, 103 insertions(+), 63 deletions(-) diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala index c06601a2d2..67cafe99f5 100755 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala @@ -46,20 +46,24 @@ class MethodsDevelopmentCallingPipeline extends QScript { val bamList: File, val goldStandard_VCF: File, val intervals: String, - val titvTarget: Double, - val trancheTarget: Double, + val indelTranchTarget: Double, + val snpTrancheTarget: Double, val isLowpass: Boolean, val isExome: Boolean, val nSamples: Int) { val name = qscript.outputDir + baseName val clusterFile = new File(name + ".clusters") - val rawVCF = new File(name + ".raw.vcf") + val rawSnpVCF = new File(name + ".raw.vcf") val rawIndelVCF = new File(name + ".raw.indel.vcf") val filteredIndelVCF = new File(name + ".filtered.indel.vcf") - val recalibratedVCF = new File(name + ".recalibrated.vcf") - val tranchesFile = new File(name + ".tranches") - val vqsrRscript = name + ".vqsr.r" - val recalFile = new File(name + ".tranches.recal") + val recalibratedSnpVCF = new File(name + ".snp.recalibrated.vcf") + val recalibratedIndelVCF = new File(name + ".indel.recalibrated.vcf") + val tranchesSnpFile = new File(name + ".snp.tranches") + val tranchesIndelFile = new File(name + ".indel.tranches") + val vqsrSnpRscript = name + ".snp.vqsr.r" + val vqsrIndelRscript = name + ".indel.vqsr.r" + val recalSnpFile = new File(name + ".snp.tranches.recal") + val recalIndelFile = new File(name + ".indel.tranches.recal") val goldStandardRecalibratedVCF = new File(name + "goldStandard.recalibrated.vcf") val goldStandardTranchesFile = new File(name + "goldStandard.tranches") val goldStandardRecalFile = new File(name + "goldStandard.tranches.recal") @@ -88,6 +92,7 @@ class MethodsDevelopmentCallingPipeline extends QScript { val training_1000G = "/humgen/1kg/processing/official_release/phase1/projectConsensus/phase1.wgs.projectConsensus.v2b.recal.highQuality.vcf" val badSites_1000G = "/humgen/1kg/processing/official_release/phase1/projectConsensus/phase1.wgs.projectConsensus.v2b.recal.terrible.vcf" val projectConsensus_1000G = "/humgen/1kg/processing/official_release/phase1/projectConsensus/ALL.wgs.projectConsensus_v2b.20101123.snps.sites.vcf" + val millsDevine_b37 = "/humgen/gsa-hpprojects/GATK/bundle/current/b37/Mills_Devine_2hit.indels.b37.sites.vcf" val lowPass: Boolean = true val exome: Boolean = true @@ -101,69 +106,69 @@ class MethodsDevelopmentCallingPipeline extends QScript { "NA12878_gold" -> new Target("NA12878.goldStandard", hg19, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/dev/carneiro/NA12878/data/goldStandard.list"), new File("/humgen/gsa-hpprojects/dev/carneiro/NA12878/analysis/snps/NA12878.HiSeq19.filtered.vcf"), // ** There is no gold standard for the gold standard ** - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.noChrY.hg19.intervals", 2.14, 99.0, lowPass, !exome, 391), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.noChrY.hg19.intervals", 90.0, 99.0, lowPass, !exome, 391), "NA12878_wgs_b37" -> new Target("NA12878.HiSeq.WGS.b37", hg19, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/NA12878.HiSeq.WGS.bwa.cleaned.recal.hg19.bam"), new File("/humgen/gsa-hpprojects/dev/carneiro/NA12878/analysis/snps/NA12878.HiSeq19.filtered.vcf"), - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.noChrY.hg19.intervals", 2.14, 99.0, !lowPass, !exome, 1), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.noChrY.hg19.intervals", 90.0, 99.0, !lowPass, !exome, 1), "NA12878_wgs_decoy" -> new Target("NA12878.HiSeq.WGS.b37_decoy", b37_decoy, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WGS.b37_decoy.NA12878.clean.dedup.recal.bam"), new File("/humgen/gsa-hpprojects/dev/carneiro/NA12878/analysis/snps/NA12878.HiSeq19.filtered.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.noChrY.hg19.intervals", 2.14, 99.0, !lowPass, !exome, 1), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.noChrY.hg19.intervals", 90.0, 99.0, !lowPass, !exome, 1), "NA12878_wgs_hg18" -> new Target("NA12878.HiSeq.WGS.hg18", hg18, dbSNP_hg18_129, hapmap_hg18, "/humgen/gsa-hpprojects/dev/depristo/oneOffProjects/1000GenomesProcessingPaper/wgs.v13/HiSeq.WGS.cleaned.indels.10.mask", new File("/humgen/gsa-hpprojects/NA12878Collection/bams/NA12878.HiSeq.WGS.bwa.cleaned.recal.bam"), new File("/home/radon01/depristo/work/oneOffProjects/1000GenomesProcessingPaper/wgs.v13/HiSeq.WGS.cleaned.ug.snpfiltered.indelfiltered.vcf"), - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg18.intervals", 2.14, 99.0, !lowPass, !exome, 1), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg18.intervals", 90.0, 99.0, !lowPass, !exome, 1), "NA12878_wex_b37" -> new Target("NA12878.HiSeq.WEx.b37", hg19, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/seq/picard_aggregation/C339/NA12878/v3/NA12878.bam"), new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 3.3, 98.0, !lowPass, exome, 1), + "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 90.0, 98.0, !lowPass, exome, 1), "NA12878_wex_hg18" -> new Target("NA12878.HiSeq.WEx.hg18", hg18, dbSNP_hg18_129, hapmap_hg18, "/humgen/gsa-hpprojects/dev/depristo/oneOffProjects/1000GenomesProcessingPaper/wgs.v13/GA2.WEx.cleaned.indels.10.mask", new File("/humgen/gsa-hpprojects/NA12878Collection/bams/NA12878.WEx.cleaned.recal.bam"), new File("/home/radon01/depristo/work/oneOffProjects/1000GenomesProcessingPaper/wgs.v13/GA2.WEx.cleaned.ug.snpfiltered.indelfiltered.vcf"), - "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.targets.interval_list", 3.3, 98.0, !lowPass, exome, 1), + "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.targets.interval_list", 90.0, 98.0, !lowPass, exome, 1), "NA12878_wex_decoy" -> new Target("NA12878.HiSeq.WEx.b37_decoy", b37_decoy, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WEx.b37_decoy.NA12878.clean.dedup.recal.bam"), new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 3.3, 98.0, !lowPass, exome, 1), + "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 90.0, 98.0, !lowPass, exome, 1), "CEUTrio_wex_b37" -> new Target("CEUTrio.HiSeq.WEx.b37", hg19, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WEx.bwa.cleaned.recal.bam"), new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"), - "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 3.3, 98.0, !lowPass, exome, 3), + "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 90.0, 98.0, !lowPass, exome, 3), "CEUTrio_wgs_b37" -> new Target("CEUTrio.HiSeq.WGS.b37", hg19, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WGS.bwa.cleaned.recal.bam"), new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 2.3, 99.0, !lowPass, !exome, 3), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 90.0, 99.0, !lowPass, !exome, 3), "CEUTrio_wex_decoy" -> new Target("CEUTrio.HiSeq.WEx.b37_decoy", b37_decoy, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WEx.b37_decoy.list"), new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 3.3, 98.0, !lowPass, exome, 3), + "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 90.0, 98.0, !lowPass, exome, 3), "CEUTrio_wgs_decoy" -> new Target("CEUTrio.HiSeq.WGS.b37_decoy", b37_decoy, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WGS.b37_decoy.list"), new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 2.3, 99.0, !lowPass, !exome, 3), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 90.0, 99.0, !lowPass, !exome, 3), "GA2hg19" -> new Target("NA12878.GA2.hg19", hg19, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/gsa-hpprojects/NA12878Collection/bams/NA12878.GA2.WGS.bwa.cleaned.hg19.bam"), new File("/humgen/gsa-hpprojects/dev/carneiro/NA12878/analysis/snps/NA12878.GA2.hg19.filtered.vcf"), - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 2.14, 99.0, !lowPass, !exome, 1), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 90.0, 99.0, !lowPass, !exome, 1), "FIN" -> new Target("FIN", b37, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/1kg/processing/pipeline_test_bams/FIN.79sample.Nov2010.chr20.bam"), new File("/humgen/gsa-hpprojects/dev/data/AugChr20Calls_v4_3state/ALL.august.v4.chr20.filtered.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.hg19.intervals", 2.3, 99.0, lowPass, !exome, 79), + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.hg19.intervals", 90.0, 99.0, lowPass, !exome, 79), "TGPWExGdA" -> new Target("1000G.WEx.GdA", b37, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/1kg/processing/pipeline_test_bams/Barcoded_1000G_WEx_Reduced_Plate_1.cleaned.list"), // BUGBUG: reduce from 60 to 20 people new File("/humgen/gsa-scr1/delangel/NewUG/calls/AugustRelease.filtered_Q50_QD5.0_SB0.0.allSamples.SNPs_hg19.WEx_UG_newUG_MQC.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 2.6, 99.0, !lowPass, exome, 96), + "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 90.0, 99.0, !lowPass, exome, 96), "LowPassN60" -> new Target("lowpass.N60", b36, dbSNP_b36, hapmap_b36, indelMask_b36, new File("/humgen/1kg/analysis/bamsForDataProcessingPapers/lowpass_b36/lowpass.chr20.cleaned.matefixed.bam"), // the bam list to call from new File("/home/radon01/depristo/work/oneOffProjects/VQSRCutByNRS/lowpass.N60.chr20.filtered.vcf"), // the gold standard VCF file to run through the VQSR - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.b36.intervals", 2.3, 99.0, lowPass, !exome, 60), // chunked interval list to use with Queue's scatter/gather functionality + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.b36.intervals", 90.0, 99.0, lowPass, !exome, 60), // chunked interval list to use with Queue's scatter/gather functionality "LowPassEUR363Nov" -> new Target("EUR.nov2010", b37, dbSNP_b37, hapmap_b37, indelMask_b37, new File("/humgen/1kg/processing/pipeline_test_bams/EUR.363sample.Nov2010.chr20.bam"), new File("/humgen/gsa-hpprojects/dev/data/AugChr20Calls_v4_3state/ALL.august.v4.chr20.filtered.vcf"), // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED ** - "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.hg19.intervals", 2.3, 99.0, lowPass, !exome, 363) + "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.chr20.hg19.intervals", 90.0, 99.0, lowPass, !exome, 363) ) @@ -181,15 +186,15 @@ class MethodsDevelopmentCallingPipeline extends QScript { val goldStandard = true for (target <- targets) { if( !skipCalling ) { - if (!noIndels) add(new indelCall(target), new indelFilter(target), new indelEvaluation(target)) + if (!noIndels) add(new indelCall(target), new indelRecal(target), new indelCut(target), new indelEvaluation(target)) add(new snpCall(target)) - add(new VQSR(target, !goldStandard)) - add(new applyVQSR(target, !goldStandard)) + add(new snpRecal(target, !goldStandard)) + add(new snpCut(target, !goldStandard)) add(new snpEvaluation(target)) } if ( runGoldStandard ) { - add(new VQSR(target, goldStandard)) - add(new applyVQSR(target, goldStandard)) + add(new snpRecal(target, goldStandard)) + add(new snpCut(target, goldStandard)) } } } @@ -222,7 +227,7 @@ class MethodsDevelopmentCallingPipeline extends QScript { this.min_base_quality_score = minimumBaseQuality if (qscript.deletions >= 0) this.max_deletion_fraction = qscript.deletions - this.out = t.rawVCF + this.out = t.rawSnpVCF this.glm = org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeLikelihoodsCalculationModel.Model.SNP this.baq = if (noBAQ || t.isExome) {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF} else {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY} this.analysisName = t.name + "_UGs" @@ -257,79 +262,114 @@ class MethodsDevelopmentCallingPipeline extends QScript { this.jobName = queueLogDir + t.name + ".indelfilter" } - // 3.) Variant Quality Score Recalibration - Generate Recalibration table - class VQSR(t: Target, goldStandard: Boolean) extends VariantRecalibrator with UNIVERSAL_GATK_ARGS { + class VQSRBase(t: Target) extends VariantRecalibrator with UNIVERSAL_GATK_ARGS { this.nt = 2 this.reference_sequence = t.reference this.intervalsString ++= List(t.intervals) - this.input :+= ( if ( goldStandard ) { t.goldStandard_VCF } else { t.rawVCF } ) - this.resource :+= new TaggedFile( t.hapmapFile, "training=true,truth=true,prior=15.0" ) - this.resource :+= new TaggedFile( omni_b37, "training=true,truth=true,prior=12.0" ) - this.resource :+= new TaggedFile( training_1000G, "training=true,prior=10.0" ) + this.allPoly = true + this.tranche ++= List("100.0", "99.9", "99.5", "99.3", "99.0", "98.9", "98.8", "98.5", "98.4", "98.3", "98.2", "98.1", "98.0", "97.9", "97.8", "97.5", "97.0", "95.0", "90.0") + } + + class snpRecal(t: Target, goldStandard: Boolean) extends VQSRBase(t) with UNIVERSAL_GATK_ARGS { + this.input :+= ( if ( goldStandard ) { t.goldStandard_VCF } else { t.rawSnpVCF } ) + this.resource :+= new TaggedFile( t.hapmapFile, "known=false,training=true,truth=true,prior=15.0" ) + this.resource :+= new TaggedFile( omni_b37, "known=false,training=true,truth=true,prior=12.0" ) + this.resource :+= new TaggedFile( training_1000G, "known=false,training=true,prior=10.0" ) this.resource :+= new TaggedFile( t.dbsnpFile, "known=true,prior=2.0" ) this.resource :+= new TaggedFile( projectConsensus_1000G, "prior=8.0" ) this.use_annotation ++= List("QD", "HaplotypeScore", "MQRankSum", "ReadPosRankSum", "MQ", "FS") - if(t.nSamples >= 10) { // InbreedingCoeff is a population-wide statistic that requires at least 10 samples to calculate - this.use_annotation ++= List("InbreedingCoeff") - } - if(!t.isExome) { + if(t.nSamples >= 10) + this.use_annotation ++= List("InbreedingCoeff") // InbreedingCoeff is a population-wide statistic that requires at least 10 samples to calculate + if(!t.isExome) this.use_annotation ++= List("DP") - } else { // exome specific parameters + else { // exome specific parameters this.resource :+= new TaggedFile( badSites_1000G, "bad=true,prior=2.0" ) this.mG = 6 - if(t.nSamples <= 3) { // very few exome samples means very few variants + if(t.nSamples <= 3) { // very few exome samples means very few variants this.mG = 4 this.percentBad = 0.04 } } - this.tranches_file = if ( goldStandard ) { t.goldStandardTranchesFile } else { t.tranchesFile } - this.recal_file = if ( goldStandard ) { t.goldStandardRecalFile } else { t.recalFile } - this.allPoly = true - this.tranche ++= List("100.0", "99.9", "99.5", "99.3", "99.0", "98.9", "98.8", "98.5", "98.4", "98.3", "98.2", "98.1", "98.0", "97.9", "97.8", "97.5", "97.0", "95.0", "90.0") - this.rscript_file = t.vqsrRscript - this.analysisName = t.name + "_VQSR" - this.jobName = queueLogDir + t.name + ".VQSR" + this.tranches_file = if ( goldStandard ) { t.goldStandardTranchesFile } else { t.tranchesSnpFile } + this.recal_file = if ( goldStandard ) { t.goldStandardRecalFile } else { t.recalSnpFile } + this.rscript_file = t.vqsrSnpRscript + this.mode = org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibratorArgumentCollection.Mode.SNP + this.analysisName = t.name + "_VQSRs" + this.jobName = queueLogDir + t.name + ".snprecal" } + class indelRecal(t: Target) extends VQSRBase(t) with UNIVERSAL_GATK_ARGS { + this.input :+= t.rawIndelVCF + this.resource :+= new TaggedFile( millsDevine_b37, "known=true,training=true,truth=true,prior=12.0" ) + this.use_annotation ++= List("QD", "HaplotypeScore", "ReadPosRankSum", "FS") + if(t.nSamples >= 10) + this.use_annotation ++= List("InbreedingCoeff") // InbreedingCoeff is a population-wide statistic that requires at least 10 samples to calculate + this.tranches_file = t.tranchesIndelFile + this.recal_file = t.recalIndelFile + this.rscript_file = t.vqsrIndelRscript + this.mode = org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibratorArgumentCollection.Mode.INDEL + this.analysisName = t.name + "_VQSRi" + this.jobName = queueLogDir + t.name + ".indelrecal" + } + + // 4.) Apply the recalibration table to the appropriate tranches - class applyVQSR (t: Target, goldStandard: Boolean) extends ApplyRecalibration with UNIVERSAL_GATK_ARGS { + class applyVQSRBase (t: Target) extends ApplyRecalibration with UNIVERSAL_GATK_ARGS { this.memoryLimit = 6 this.reference_sequence = t.reference this.intervalsString ++= List(t.intervals) - this.input :+= ( if ( goldStandard ) { t.goldStandard_VCF } else { t.rawVCF } ) - this.tranches_file = if ( goldStandard ) { t.goldStandardTranchesFile } else { t.tranchesFile} - this.recal_file = if ( goldStandard ) { t.goldStandardRecalFile } else { t.recalFile } - this.ts_filter_level = t.trancheTarget - this.out = t.recalibratedVCF - this.analysisName = t.name + "_AVQSR" - this.jobName = queueLogDir + t.name + ".applyVQSR" } + class snpCut (t: Target, goldStandard: Boolean) extends applyVQSRBase(t) { + this.input :+= ( if ( goldStandard ) { t.goldStandard_VCF } else { t.rawSnpVCF } ) + this.tranches_file = if ( goldStandard ) { t.goldStandardTranchesFile } else { t.tranchesSnpFile} + this.recal_file = if ( goldStandard ) { t.goldStandardRecalFile } else { t.recalSnpFile } + this.ts_filter_level = t.snpTrancheTarget + this.mode = org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibratorArgumentCollection.Mode.SNP + this.out = t.recalibratedSnpVCF + this.analysisName = t.name + "_AVQSRs" + this.jobName = queueLogDir + t.name + ".snpcut" + } + + class indelCut (t: Target) extends applyVQSRBase(t) { + this.input :+= t.rawIndelVCF + this.tranches_file = t.tranchesIndelFile + this.recal_file = t.recalIndelFile + this.ts_filter_level = t.indelTranchTarget + this.mode = org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibratorArgumentCollection.Mode.INDEL + this.out = t.recalibratedIndelVCF + this.analysisName = t.name + "_AVQSRi" + this.jobName = queueLogDir + t.name + ".indelcut" + } + + // 5.) Variant Evaluation Base(OPTIONAL) class EvalBase(t: Target) extends VariantEval with UNIVERSAL_GATK_ARGS { this.memoryLimit = 3 - this.reference_sequence = t.reference this.comp :+= new TaggedFile(t.hapmapFile, "hapmap" ) - this.intervalsString ++= List(t.intervals) this.D = new File(t.dbsnpFile) + this.reference_sequence = t.reference + this.intervalsString ++= List(t.intervals) this.sample = samples } // 5a.) SNP Evaluation (OPTIONAL) based on the cut vcf class snpEvaluation(t: Target) extends EvalBase(t) { if (t.reference == b37 || t.reference == hg19) this.comp :+= new TaggedFile( omni_b37, "omni" ) - this.eval :+= t.recalibratedVCF + this.eval :+= t.recalibratedSnpVCF this.out = t.evalFile this.analysisName = t.name + "_VEs" - this.jobName = queueLogDir + t.name + ".snp.eval" + this.jobName = queueLogDir + t.name + ".snpeval" } // 5b.) Indel Evaluation (OPTIONAL) class indelEvaluation(t: Target) extends EvalBase(t) { - this.eval :+= t.filteredIndelVCF - this.evalModule :+= "IndelStatistics" + this.eval :+= t.recalibratedIndelVCF + this.comp :+= new TaggedFile(millsDevine_b37, "mills" ) + this.noEV = true + this.evalModule = List("CompOverlap", "CountVariants", "TiTvVariantEvaluator", "ValidationReport", "IndelStatistics") this.out = t.evalIndelFile this.analysisName = t.name + "_VEi" - this.jobName = queueLogDir + queueLogDir + t.name + ".indel.eval" + this.jobName = queueLogDir + queueLogDir + t.name + ".indeleval" } } From 1f88a1bfe25a668122f826b86334a3e8515b5900 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Fri, 6 Jan 2012 17:25:04 -0500 Subject: [PATCH 030/356] Small fix to RRead script * fixing the downsample strategy variable From 5793625592d3b0c97e5af17d3e35c607fa5cfae0 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Sun, 8 Jan 2012 12:11:55 -0500 Subject: [PATCH 031/356] No more "Q-@". Generated log file names now use the first output + ".out" (ex. my.vcf.out) or the name of the first QScript plus the order the function was added (ex. MyScript-1.out). The same function added twice with the same outputs will now have the same default logs, meaning the 2nd instance of the function won't be added to the graph twice. QScript accessor to QSettings to specify a default runName and other default function settings. Because log files are no longer pseudo-random their presense can be used to tell if a job without other file outputs is "done". For now still using the log's .done file in addition to original outputs. Gathered log files concatenate all log files together into the stdout. InProcessFunctions now have PrintStreams for stdout and stderr. Updated ivy to use commons-io 2.1 for copying logs to the stdout PrintStream. Removed snakeyaml. During graph tracking of outputs the Index files, and now BAM MD5s, are tracked with the gathering of the original file. In Queue generated wrappers for the GATK the Index and MD5s used for tracking are switched to private scope. Added more detailed output when running with -l DEBUG. Simplified graphviz visualization for additional debugging. Switched usage of the scala class 'List' to the trait 'Seq' (think java.util.ArrayList vs. using the interface java.util.List) Minor cleanup to build including sending ant gsalib to R's default libloc. --- build.xml | 11 +- ivy.xml | 26 ++- .../sting/commandline/Gather.java | 5 +- .../gatk/ArgumentDefinitionField.java | 97 ++++---- .../queue/extensions/gatk/ArgumentField.java | 9 +- .../gatk/GATKExtensionsGenerator.java | 32 +-- .../qscripts/DataProcessingPipeline.scala | 54 ++--- .../qscripts/PacbioProcessingPipeline.scala | 4 +- .../sting/queue/QCommandLine.scala | 66 ++++-- .../broadinstitute/sting/queue/QScript.scala | 20 +- .../sting/queue/QScriptManager.scala | 4 +- .../sting/queue/QSettings.scala | 26 +-- .../sting/queue/engine/FunctionEdge.scala | 39 +++- .../sting/queue/engine/InProcessRunner.scala | 23 +- .../sting/queue/engine/MappingEdge.scala | 26 ++- .../sting/queue/engine/QEdge.scala | 28 ++- .../sting/queue/engine/QGraph.scala | 212 ++++++++++-------- .../sting/queue/engine/QGraphSettings.scala | 17 +- .../sting/queue/engine/QNode.scala | 28 ++- .../gridengine/GridEngineJobRunner.scala | 4 +- .../queue/engine/lsf/Lsf706JobRunner.scala | 13 +- .../extensions/gatk/BamGatherFunction.scala | 5 +- .../queue/extensions/gatk/GATKIntervals.scala | 15 +- .../extensions/gatk/GATKScatterFunction.scala | 21 +- .../sting/queue/extensions/gatk/RodBind.scala | 41 ---- .../picard/AddOrReplaceReadGroups.scala | 26 ++- .../extensions/picard/MarkDuplicates.scala | 26 ++- .../extensions/picard/MergeSamFiles.scala | 27 ++- .../extensions/picard/PicardBamFunction.scala | 24 +- .../queue/extensions/picard/ReorderSam.scala | 26 ++- .../queue/extensions/picard/RevertSam.scala | 28 ++- .../queue/extensions/picard/SamToFastq.scala | 26 ++- .../queue/extensions/picard/SortSam.scala | 27 ++- .../extensions/picard/ValidateSamFile.scala | 28 ++- .../samtools/SamtoolsIndexFunction.scala | 2 - .../samtools/SamtoolsMergeFunction.scala | 10 +- .../queue/function/CommandLineFunction.scala | 32 ++- .../queue/function/InProcessFunction.scala | 18 +- .../function/JavaCommandLineFunction.scala | 6 +- .../queue/function/ListWriterFunction.scala | 30 ++- .../sting/queue/function/QFunction.scala | 189 ++++++++-------- .../scattergather/CloneFunction.scala | 32 +-- .../ConcatenateLogsFunction.scala} | 33 ++- .../scattergather/GatherFunction.scala | 41 +++- .../scattergather/GathererFunction.scala | 7 +- .../scattergather/ScatterFunction.scala | 5 +- .../ScatterGatherableFunction.scala | 148 ++++++++---- .../sting/queue/util/EmailMessage.scala | 14 +- .../sting/queue/util/QJobReport.scala | 22 +- .../sting/queue/util/QScriptUtils.scala | 41 +++- .../sting/queue/util/ReflectionUtils.scala | 32 ++- .../ScalaCompoundArgumentTypeDescriptor.scala | 46 +++- .../queue/util/StringFileConversions.scala | 34 ++- .../sting/queue/util/SystemUtils.scala | 5 +- .../gatk/GATKIntervalsUnitTest.scala | 23 +- .../CommandLineFunctionUnitTest.scala | 58 +++-- .../sting/queue/pipeline/PipelineTest.scala | 6 +- .../queue/pipeline/PipelineTestEvalSpec.scala | 28 ++- .../queue/pipeline/PipelineTestSpec.scala | 26 ++- .../examples/HelloWorldPipelineTest.scala | 20 +- .../util/StringFileConversionsUnitTest.scala | 82 +++---- 61 files changed, 1356 insertions(+), 698 deletions(-) delete mode 100644 public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/RodBind.scala rename public/scala/src/org/broadinstitute/sting/queue/{extensions/gatk/AutoIndexGatherFunction.scala => function/scattergather/ConcatenateLogsFunction.scala} (53%) diff --git a/build.xml b/build.xml index 7c81c1f206..dbdafa3d96 100644 --- a/build.xml +++ b/build.xml @@ -1,5 +1,5 @@ - + + + + + + + + diff --git a/ivy.xml b/ivy.xml index 4f41904ba4..f5ff15c30c 100644 --- a/ivy.xml +++ b/ivy.xml @@ -1,3 +1,26 @@ + @@ -21,7 +44,6 @@ - @@ -40,7 +62,7 @@ - + diff --git a/public/java/src/org/broadinstitute/sting/commandline/Gather.java b/public/java/src/org/broadinstitute/sting/commandline/Gather.java index 59c3f50cbf..d452f708e0 100644 --- a/public/java/src/org/broadinstitute/sting/commandline/Gather.java +++ b/public/java/src/org/broadinstitute/sting/commandline/Gather.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -34,5 +34,6 @@ @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.FIELD}) public @interface Gather { - Class value(); + Class value() default Gather.class; + boolean enabled() default true; } diff --git a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java index cdfc329e81..71640c66a6 100644 --- a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java +++ b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -70,17 +70,18 @@ else if(getShortFieldGetter().equals(getFieldName())) " * Short name of %1$s%n" + " * @return Short name of %1$s%n" + " */%n" + - "def %3$s = this.%1$s%n" + + "%5$sdef %3$s = this.%1$s%n" + "%n" + "/**%n" + " * Short name of %1$s%n" + " * @param value Short name of %1$s%n" + " */%n" + - "def %4$s(value: %2$s) { this.%1$s = value }%n", + "%5$sdef %4$s(value: %2$s) { this.%1$s = value }%n", getFieldName(), getFieldType(), getShortFieldGetter(), - getShortFieldSetter()); + getShortFieldSetter(), + getPrivacy()); } protected static final String REQUIRED_TEMPLATE = " + required(\"%1$s\", %3$s, spaceSeparated=true, escape=true, format=%2$s)"; @@ -135,11 +136,8 @@ private static List getArgumentFields(ArgumentDefinitio new IntervalFileArgumentField(argumentDefinition), new IntervalStringArgumentField(argumentDefinition)); - // ROD Bindings are set by the RodBindField - } else if (RodBindArgumentField.ROD_BIND_FIELD.equals(argumentDefinition.fullName) && argumentDefinition.ioType == ArgumentIOType.INPUT) { - // TODO: Once everyone is using @Allows and @Requires correctly, we can stop blindly allowing Triplets - return Arrays.asList(new RodBindArgumentField(argumentDefinition), new InputIndexesArgumentField(argumentDefinition, Tribble.STANDARD_INDEX_EXTENSION)); - //return Collections.emptyList(); + } else if (NumThreadsArgumentField.NUM_THREADS_FIELD.equals(argumentDefinition.fullName)) { + return Arrays.asList(new NumThreadsArgumentField(argumentDefinition)); } else if ("input_file".equals(argumentDefinition.fullName) && argumentDefinition.ioType == ArgumentIOType.INPUT) { return Arrays.asList(new InputTaggedFileDefinitionField(argumentDefinition), new InputIndexesArgumentField(argumentDefinition, BAMIndex.BAMIndexSuffix, ".bam")); @@ -166,10 +164,13 @@ else if (VCFWriter.class.isAssignableFrom(argumentDefinition.argumentType)) fields.add(new OutputArgumentField(argumentDefinition, gatherClass)); - if (SAMFileWriter.class.isAssignableFrom(argumentDefinition.argumentType)) + if (SAMFileWriter.class.isAssignableFrom(argumentDefinition.argumentType)) { fields.add(new SAMFileWriterIndexArgumentField(argumentDefinition)); - else if (VCFWriter.class.isAssignableFrom(argumentDefinition.argumentType)) + fields.add(new SAMFileWriterMD5ArgumentField(argumentDefinition)); + } + else if (VCFWriter.class.isAssignableFrom(argumentDefinition.argumentType)) { fields.add(new VCFWriterIndexArgumentField(argumentDefinition)); + } return fields; @@ -228,7 +229,7 @@ public IntervalStringArgumentField(ArgumentDefinition argumentDefinition) { @Override protected String getRawFieldName() { return super.getRawFieldName() + "String"; } @Override protected String getFullName() { return super.getFullName() + "String"; } @Override protected String getRawShortFieldName() { return super.getRawShortFieldName() + "String"; } - @Override protected String getFieldType() { return "List[String]"; } + @Override protected String getFieldType() { return "Seq[String]"; } @Override protected String getDefaultValue() { return "Nil"; } @Override public String getCommandLineTemplate() { return REPEAT_TEMPLATE; } @@ -250,7 +251,7 @@ public InputArgumentField(ArgumentDefinition argumentDefinition) { } @Override protected Class getInnerType() { return File.class; } - @Override protected String getFieldType() { return isMultiValued() ? "List[File]" : "File"; } + @Override protected String getFieldType() { return isMultiValued() ? "Seq[File]" : "File"; } @Override protected String getDefaultValue() { return isMultiValued() ? "Nil" : "_"; } } @@ -294,7 +295,7 @@ public MultiValuedArgumentField(ArgumentDefinition argumentDefinition) { } @Override protected Class getInnerType() { return mapType(argumentDefinition.componentType); } - @Override protected String getFieldType() { return String.format("List[%s]", getType(getInnerType())); } + @Override protected String getFieldType() { return String.format("Seq[%s]", getType(getInnerType())); } @Override protected String getDefaultValue() { return "Nil"; } @Override protected String getCommandLineTemplate() { return REPEAT_TEMPLATE; } } @@ -336,17 +337,16 @@ public DefaultArgumentField(ArgumentDefinition argumentDefinition, boolean useFo } // Allows the user to specify the track name, track type, and the file. - public static class RodBindArgumentField extends ArgumentDefinitionField { - public static final String ROD_BIND_FIELD = "rodBind"; + public static class NumThreadsArgumentField extends OptionedArgumentField { + public static final String NUM_THREADS_FIELD = "num_threads"; - public RodBindArgumentField(ArgumentDefinition argumentDefinition) { - super(argumentDefinition); + public NumThreadsArgumentField(ArgumentDefinition argumentDefinition) { + super(argumentDefinition, false); } - @Override protected Class getInnerType() { return null; } // RodBind does not need to be imported. - @Override protected String getFieldType() { return "List[RodBind]"; } - @Override protected String getDefaultValue() { return "Nil"; } - @Override protected String getCommandLineTemplate() { - return " + repeat(\"%1$s\", %3$s, formatPrefix=RodBind.formatCommandLineParameter, spaceSeparated=true, escape=true, format=%2$s)"; + + @Override + protected String getFreezeFields() { + return String.format("if (num_threads.isDefined) nCoresRequest = num_threads%n"); } } @@ -356,7 +356,7 @@ public InputTaggedFileDefinitionField(ArgumentDefinition argumentDefinition) { super(argumentDefinition); } @Override protected Class getInnerType() { return null; } // TaggedFile does not need to be imported. - @Override protected String getFieldType() { return argumentDefinition.isMultiValued ? "List[File]" : "File"; } + @Override protected String getFieldType() { return argumentDefinition.isMultiValued ? "Seq[File]" : "File"; } @Override protected String getDefaultValue() { return argumentDefinition.isMultiValued ? "Nil" : "_"; } @Override protected String getCommandLineTemplate() { if (argumentDefinition.isMultiValued) { @@ -395,10 +395,11 @@ public InputIndexesArgumentField(ArgumentDefinition originalArgumentDefinition, } @Override protected String getFullName() { return this.indexFieldName; } @Override protected boolean isRequired() { return false; } - @Override protected String getFieldType() { return "List[File]"; } + @Override protected String getFieldType() { return "Seq[File]"; } @Override protected String getDefaultValue() { return "Nil"; } @Override protected Class getInnerType() { return File.class; } @Override protected String getRawFieldName() { return this.indexFieldName; } + @Override protected String getPrivacy() { return "private "; } @Override protected String getFreezeFields() { if (originalIsMultiValued) { if (originalSuffix == null) { @@ -434,53 +435,69 @@ public InputIndexesArgumentField(ArgumentDefinition originalArgumentDefinition, } } - // Tracks an automatically generated index - private static abstract class OutputIndexArgumentField extends ArgumentField { - protected final String indexFieldName; + // Tracks an automatically generated index, md5, etc. + private static abstract class AuxilliaryOutputArgumentField extends ArgumentField { protected final String originalFieldName; - public OutputIndexArgumentField(ArgumentDefinition originalArgumentDefinition) { - this.indexFieldName = originalArgumentDefinition.fullName + "Index"; + protected final String auxFieldName; + protected final String auxFieldLabel; + public AuxilliaryOutputArgumentField(ArgumentDefinition originalArgumentDefinition, String auxFieldLabel) { this.originalFieldName = originalArgumentDefinition.fullName; + this.auxFieldName = originalArgumentDefinition.fullName + auxFieldLabel; + this.auxFieldLabel = auxFieldLabel; } @Override protected Class getAnnotationIOClass() { return Output.class; } @Override public String getCommandLineAddition() { return ""; } - @Override protected String getDoc() { return "Automatically generated index for " + this.originalFieldName; } - @Override protected String getFullName() { return this.indexFieldName; } + @Override protected String getDoc() { return String.format("Automatically generated %s for %s", auxFieldLabel.toLowerCase(), this.originalFieldName); } + @Override protected String getFullName() { return this.auxFieldName; } @Override protected boolean isRequired() { return false; } @Override protected String getFieldType() { return "File"; } @Override protected String getDefaultValue() { return "_"; } @Override protected Class getInnerType() { return File.class; } - @Override protected String getRawFieldName() { return this.indexFieldName; } + @Override protected String getRawFieldName() { return this.auxFieldName; } + @Override protected String getPrivacy() { return "private "; } @Override public boolean isGather() { return true; } @Override protected String getGatherAnnotation() { - return String.format("@Gather(classOf[AutoIndexGatherFunction])%n"); + return String.format("@Gather(enabled=false)%n"); } } - private static class VCFWriterIndexArgumentField extends OutputIndexArgumentField { + private static class VCFWriterIndexArgumentField extends AuxilliaryOutputArgumentField { public VCFWriterIndexArgumentField(ArgumentDefinition originalArgumentDefinition) { - super(originalArgumentDefinition); + super(originalArgumentDefinition, "Index"); } @Override protected String getFreezeFields() { return String.format( ("if (%2$s != null)%n" + " if (!org.broadinstitute.sting.gatk.io.stubs.VCFWriterArgumentTypeDescriptor.isCompressed(%2$s.getPath))%n" + " %1$s = new File(%2$s.getPath + \"%3$s\")%n"), - indexFieldName, originalFieldName, Tribble.STANDARD_INDEX_EXTENSION); + auxFieldName, originalFieldName, Tribble.STANDARD_INDEX_EXTENSION); } } - private static class SAMFileWriterIndexArgumentField extends OutputIndexArgumentField { + private static class SAMFileWriterIndexArgumentField extends AuxilliaryOutputArgumentField { public SAMFileWriterIndexArgumentField(ArgumentDefinition originalArgumentDefinition) { - super(originalArgumentDefinition); + super(originalArgumentDefinition, "Index"); } @Override protected String getFreezeFields() { return String.format( ("if (%2$s != null)%n" + " if (!%3$s)%n" + " %1$s = new File(%2$s.getPath.stripSuffix(\".bam\") + \"%4$s\")%n"), - indexFieldName, originalFieldName, SAMFileWriterArgumentTypeDescriptor.DISABLE_INDEXING_FULLNAME, BAMIndex.BAMIndexSuffix); + auxFieldName, originalFieldName, SAMFileWriterArgumentTypeDescriptor.DISABLE_INDEXING_FULLNAME, BAMIndex.BAMIndexSuffix); + } + } + + private static class SAMFileWriterMD5ArgumentField extends AuxilliaryOutputArgumentField { + public SAMFileWriterMD5ArgumentField(ArgumentDefinition originalArgumentDefinition) { + super(originalArgumentDefinition, "MD5"); + } + @Override protected String getFreezeFields() { + return String.format( + ("if (%2$s != null)%n" + + " if (%3$s)%n" + + " %1$s = new File(%2$s.getPath + \"%4$s\")%n"), + auxFieldName, originalFieldName, SAMFileWriterArgumentTypeDescriptor.ENABLE_MD5_FULLNAME, ".md5"); } } diff --git a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java index e90933504a..2428a13a80 100644 --- a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java +++ b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentField.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -56,7 +56,7 @@ public final String getArgumentAddition() { return String.format("%n" + "/** %s */%n" + "@%s(fullName=\"%s\", shortName=\"%s\", doc=\"%s\", required=%s, exclusiveOf=\"%s\", validation=\"%s\")%n" + - "%svar %s: %s = %s%n" + + "%s%svar %s: %s = %s%n" + "%s", getDoc(), getAnnotationIOClass().getSimpleName(), @@ -66,7 +66,7 @@ public final String getArgumentAddition() { isRequired(), getExclusiveOf(), getValidation(), - getGatherAnnotation(), getFieldName(), getFieldType(), getDefaultValue(), + getGatherAnnotation(), getPrivacy(), getFieldName(), getFieldType(), getDefaultValue(), getDefineAddition()); } @@ -143,6 +143,9 @@ protected Collection> getDependentClasses() { /** @return True if this field uses @Gather. */ public boolean isGather() { return false; } + /** @return Privacy for the field. */ + protected String getPrivacy() { return ""; } + /** @return The raw field name, which will be checked against scala build in types. */ protected abstract String getRawFieldName(); /** @return The field name checked against reserved words. */ diff --git a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java index 9c40fb976a..a3f80af1c2 100644 --- a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java +++ b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/GATKExtensionsGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -34,13 +34,11 @@ import org.broadinstitute.sting.gatk.CommandLineGATK; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.gatk.WalkerManager; -import org.broadinstitute.sting.gatk.arguments.ValidationExclusion; import org.broadinstitute.sting.gatk.filters.FilterManager; import org.broadinstitute.sting.gatk.filters.ReadFilter; import org.broadinstitute.sting.gatk.io.stubs.OutputStreamArgumentTypeDescriptor; import org.broadinstitute.sting.gatk.io.stubs.SAMFileWriterArgumentTypeDescriptor; import org.broadinstitute.sting.gatk.io.stubs.VCFWriterArgumentTypeDescriptor; -import org.broadinstitute.sting.gatk.refdata.tracks.RMDTrackBuilder; import org.broadinstitute.sting.gatk.walkers.PartitionBy; import org.broadinstitute.sting.gatk.walkers.PartitionType; import org.broadinstitute.sting.gatk.walkers.Walker; @@ -85,7 +83,7 @@ public class GATKExtensionsGenerator extends CommandLineProgram { "%n" + "/** A dynamicly generated list of classes that the GATK Extensions depend on, but are not be detected by default by BCEL. */%n" + "class %s {%n" + - "val types = List(%n%s)%n" + + "val types = Seq(%n%s)%n" + "}%n"; @Output(fullName="output_directory", shortName="outDir", doc="Directory to output the generated scala", required=true) @@ -95,10 +93,6 @@ public class GATKExtensionsGenerator extends CommandLineProgram { GenomeAnalysisEngine GATKEngine = new GenomeAnalysisEngine(); WalkerManager walkerManager = new WalkerManager(); FilterManager filterManager = new FilterManager(); - // HACK: We're currently relying on the fact that RMDTrackBuilder is used only from RMD type lookups, not - // RMD track location. Therefore, no sequence dictionary is required. In the future, we should separate - // RMD track lookups from track creation. - RMDTrackBuilder trackBuilder = new RMDTrackBuilder(null,null,ValidationExclusion.TYPE.ALL); /** * Required main method implementation. @@ -147,7 +141,7 @@ protected int execute() { String clpConstructor = String.format("analysisName = \"%s\"%njavaMainClass = \"%s\"%n", clpClassName, clp.getName()); writeClass("org.broadinstitute.sting.queue.function.JavaCommandLineFunction", clpClassName, - false, clpConstructor, ArgumentDefinitionField.getArgumentFields(parser,clp), dependents, false); + false, clpConstructor, ArgumentDefinitionField.getArgumentFields(parser,clp), dependents); if (clp == CommandLineGATK.class) { for (Entry>> walkersByPackage: walkerManager.getWalkerNamesByPackage(false).entrySet()) { @@ -169,7 +163,7 @@ protected int execute() { } writeClass(GATK_EXTENSIONS_PACKAGE_NAME + "." + clpClassName, walkerName, - isScatter, constructor, argumentFields, dependents, true); + isScatter, constructor, argumentFields, dependents); } catch (Exception e) { throw new ReviewedStingException("Error generating wrappers for walker " + walkerType, e); } @@ -242,8 +236,8 @@ private String getScatterClass(Class walkerType) { */ private void writeClass(String baseClass, String className, boolean isScatter, String constructor, List argumentFields, - Set> dependents, boolean isGATKWalker) throws IOException { - String content = getContent(CLASS_TEMPLATE, baseClass, className, constructor, isScatter, "", argumentFields, dependents, isGATKWalker); + Set> dependents) throws IOException { + String content = getContent(CLASS_TEMPLATE, baseClass, className, constructor, isScatter, "", argumentFields, dependents); writeFile(GATK_EXTENSIONS_PACKAGE_NAME + "." + className, content); } @@ -257,7 +251,7 @@ private void writeClass(String baseClass, String className, boolean isScatter, */ private void writeFilter(String className, List argumentFields, Set> dependents) throws IOException { String content = getContent(TRAIT_TEMPLATE, "org.broadinstitute.sting.queue.function.CommandLineFunction", - className, "", false, String.format(" + \" -read_filter %s\"", className), argumentFields, dependents, false); + className, "", false, String.format(" + \" -read_filter %s\"", className), argumentFields, dependents); writeFile(GATK_EXTENSIONS_PACKAGE_NAME + "." + className, content); } @@ -351,8 +345,7 @@ private void writeFile(String fullClassName, String content) throws IOException */ private static String getContent(String scalaTemplate, String baseClass, String className, String constructor, boolean isScatter, String commandLinePrefix, - List argumentFields, Set> dependents, - boolean isGATKWalker) { + List argumentFields, Set> dependents) { StringBuilder arguments = new StringBuilder(); StringBuilder commandLine = new StringBuilder(commandLinePrefix); @@ -376,9 +369,6 @@ private static String getContent(String scalaTemplate, String baseClass, String if (isGather) importSet.add("import org.broadinstitute.sting.commandline.Gather"); - // Needed for ShellUtils.escapeShellArgument() - importSet.add("import org.broadinstitute.sting.queue.util.ShellUtils"); - // Sort the imports so that the are always in the same order. List sortedImports = new ArrayList(importSet); Collections.sort(sortedImports); @@ -386,10 +376,8 @@ private static String getContent(String scalaTemplate, String baseClass, String StringBuffer freezeFieldOverride = new StringBuffer(); for (String freezeField: freezeFields) freezeFieldOverride.append(freezeField); - if (freezeFieldOverride.length() > 0 || isGATKWalker) { - freezeFieldOverride.insert(0, String.format("override def freezeFieldValues = {%nsuper.freezeFieldValues%n")); - if ( isGATKWalker ) - freezeFieldOverride.append(String.format("if ( num_threads.isDefined ) nCoresRequest = num_threads%n")); + if (freezeFieldOverride.length() > 0) { + freezeFieldOverride.insert(0, String.format("override def freezeFieldValues() {%nsuper.freezeFieldValues()%n")); freezeFieldOverride.append(String.format("}%n%n")); } diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/DataProcessingPipeline.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/DataProcessingPipeline.scala index 621afe8170..e26541e987 100755 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/DataProcessingPipeline.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/DataProcessingPipeline.scala @@ -29,14 +29,14 @@ class DataProcessingPipeline extends QScript { var reference: File = _ @Input(doc="dbsnp ROD to use (must be in VCF format)", fullName="dbsnp", shortName="D", required=true) - var dbSNP: List[File] = List() + var dbSNP: Seq[File] = Seq() /**************************************************************************** * Optional Parameters ****************************************************************************/ @Input(doc="extra VCF files to use as reference indels for Indel Realignment", fullName="extra_indels", shortName="indels", required=false) - var indels: List[File] = List() + var indels: Seq[File] = Seq() @Input(doc="The path to the binary of bwa (usually BAM files have already been mapped - but if you want to remap this is the option)", fullName="path_to_bwa", shortName="bwa", required=false) var bwaPath: File = _ @@ -118,13 +118,13 @@ class DataProcessingPipeline extends QScript { // Because the realignment only happens after these scripts are executed, in case you are using // bwa realignment, this function will operate over the original bam files and output over the // (to be realigned) bam files. - def createSampleFiles(bamFiles: List[File], realignedBamFiles: List[File]): Map[String, List[File]] = { + def createSampleFiles(bamFiles: Seq[File], realignedBamFiles: Seq[File]): Map[String, Seq[File]] = { // Creating a table with SAMPLE information from each input BAM file - val sampleTable = scala.collection.mutable.Map.empty[String, List[File]] + val sampleTable = scala.collection.mutable.Map.empty[String, Seq[File]] val realignedIterator = realignedBamFiles.iterator for (bam <- bamFiles) { - val rBam = realignedIterator.next // advance to next element in the realignedBam list so they're in sync. + val rBam = realignedIterator.next() // advance to next element in the realignedBam list so they're in sync. val samReader = new SAMFileReader(bam) val header = samReader.getFileHeader @@ -138,12 +138,12 @@ class DataProcessingPipeline extends QScript { for (rg <- readGroups) { val sample = rg.getSample if (!sampleTable.contains(sample)) - sampleTable(sample) = List(rBam) + sampleTable(sample) = Seq(rBam) else if ( !sampleTable(sample).contains(rBam)) sampleTable(sample) :+= rBam } } - return sampleTable.toMap + sampleTable.toMap } // Rebuilds the Read Group string to give BWA @@ -161,8 +161,8 @@ class DataProcessingPipeline extends QScript { // Takes a list of processed BAM files and realign them using the BWA option requested (bwase or bwape). // Returns a list of realigned BAM files. - def performAlignment(bams: List[File]): List[File] = { - var realignedBams: List[File] = List() + def performAlignment(bams: Seq[File]): Seq[File] = { + var realignedBams: Seq[File] = Seq() var index = 1 for (bam <- bams) { // first revert the BAM file to the original qualities @@ -194,10 +194,10 @@ class DataProcessingPipeline extends QScript { realignedBams :+= rgRealignedBamFile index = index + 1 } - return realignedBams + realignedBams } - def getIndelCleaningModel(): ConsensusDeterminationModel = { + def getIndelCleaningModel: ConsensusDeterminationModel = { if (cleaningModel == "KNOWNS_ONLY") ConsensusDeterminationModel.KNOWNS_ONLY else if (cleaningModel == "USE_SW") @@ -206,17 +206,17 @@ class DataProcessingPipeline extends QScript { ConsensusDeterminationModel.USE_READS } - def revertBams(bams: List[File], removeAlignmentInformation: Boolean): List[File] = { - var revertedBAMList: List[File] = List() + def revertBams(bams: Seq[File], removeAlignmentInformation: Boolean): Seq[File] = { + var revertedBAMList: Seq[File] = Seq() for (bam <- bams) revertedBAMList :+= revertBAM(bam, removeAlignmentInformation) - return revertedBAMList + revertedBAMList } def revertBAM(bam: File, removeAlignmentInformation: Boolean): File = { val revertedBAM = swapExt(bam, ".bam", ".reverted.bam") add(revert(bam, revertedBAM, removeAlignmentInformation)) - return revertedBAM + revertedBAM } /**************************************************************************** @@ -224,22 +224,22 @@ class DataProcessingPipeline extends QScript { ****************************************************************************/ - def script = { + def script() { // final output list of processed bam files - var cohortList: List[File] = List() + var cohortList: Seq[File] = Seq() // sets the model for the Indel Realigner - cleanModelEnum = getIndelCleaningModel() + cleanModelEnum = getIndelCleaningModel // keep a record of the number of contigs in the first bam file in the list - val bams = QScriptUtils.createListFromFile(input) + val bams = QScriptUtils.createSeqFromFile(input) if (nContigs < 0) nContigs = QScriptUtils.getNumberOfContigs(bams(0)) val realignedBAMs = if (useBWApe || useBWAse || useBWAsw) {performAlignment(bams)} else {revertBams(bams, false)} // generate a BAM file per sample joining all per lane files if necessary - val sampleBAMFiles: Map[String, List[File]] = createSampleFiles(bams, realignedBAMs) + val sampleBAMFiles: Map[String, Seq[File]] = createSampleFiles(bams, realignedBAMs) // if this is a 'knowns only' indel realignment run, do it only once for all samples. val globalIntervals = new File(outputDir + projectName + ".intervals") @@ -317,7 +317,7 @@ class DataProcessingPipeline extends QScript { this.maxRecordsInRam = 100000 } - case class target (inBams: List[File], outIntervals: File) extends RealignerTargetCreator with CommandLineGATKArgs { + case class target (inBams: Seq[File], outIntervals: File) extends RealignerTargetCreator with CommandLineGATKArgs { if (cleanModelEnum != ConsensusDeterminationModel.KNOWNS_ONLY) this.input_file = inBams this.out = outIntervals @@ -330,7 +330,7 @@ class DataProcessingPipeline extends QScript { this.jobName = queueLogDir + outIntervals + ".target" } - case class clean (inBams: List[File], tIntervals: File, outBam: File) extends IndelRealigner with CommandLineGATKArgs { + case class clean (inBams: Seq[File], tIntervals: File, outBam: File) extends IndelRealigner with CommandLineGATKArgs { this.input_file = inBams this.targetIntervals = tIntervals this.out = outBam @@ -347,11 +347,11 @@ class DataProcessingPipeline extends QScript { case class cov (inBam: File, outRecalFile: File) extends CountCovariates with CommandLineGATKArgs { this.knownSites ++= qscript.dbSNP - this.covariate ++= List("ReadGroupCovariate", "QualityScoreCovariate", "CycleCovariate", "DinucCovariate") + this.covariate ++= Seq("ReadGroupCovariate", "QualityScoreCovariate", "CycleCovariate", "DinucCovariate") this.input_file :+= inBam this.recal_file = outRecalFile if (!defaultPlatform.isEmpty) this.default_platform = defaultPlatform - if (!qscript.intervalString.isEmpty()) this.intervalsString ++= List(qscript.intervalString) + if (!qscript.intervalString.isEmpty) this.intervalsString ++= Seq(qscript.intervalString) else if (qscript.intervals != null) this.intervals :+= qscript.intervals this.scatterCount = nContigs this.analysisName = queueLogDir + outRecalFile + ".covariates" @@ -363,7 +363,7 @@ class DataProcessingPipeline extends QScript { this.recal_file = inRecalFile this.baq = CalculationMode.CALCULATE_AS_NECESSARY this.out = outBam - if (!qscript.intervalString.isEmpty()) this.intervalsString ++= List(qscript.intervalString) + if (!qscript.intervalString.isEmpty) this.intervalsString ++= Seq(qscript.intervalString) else if (qscript.intervals != null) this.intervals :+= qscript.intervals this.no_pg_tag = qscript.testMode this.scatterCount = nContigs @@ -395,7 +395,7 @@ class DataProcessingPipeline extends QScript { this.jobName = queueLogDir + outBam + ".dedup" } - case class joinBams (inBams: List[File], outBam: File) extends MergeSamFiles with ExternalCommonArgs { + case class joinBams (inBams: Seq[File], outBam: File) extends MergeSamFiles with ExternalCommonArgs { this.input = inBams this.output = outBam this.analysisName = queueLogDir + outBam + ".joinBams" @@ -495,7 +495,7 @@ class DataProcessingPipeline extends QScript { this.jobName = queueLogDir + outBam + ".bwasw" } - case class writeList(inBams: List[File], outBamList: File) extends ListWriterFunction { + case class writeList(inBams: Seq[File], outBamList: File) extends ListWriterFunction { this.inputFiles = inBams this.listFile = outBamList this.analysisName = queueLogDir + outBamList + ".bamList" diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala index 4896eaed3c..2f954713e9 100755 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala @@ -53,9 +53,9 @@ class PacbioProcessingPipeline extends QScript { val queueLogDir: String = ".qlog/" - def script = { + def script() { - val fileList: List[File] = QScriptUtils.createListFromFile(input) + val fileList: Seq[File] = QScriptUtils.createSeqFromFile(input) for (file: File <- fileList) { diff --git a/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala b/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala index 32913deb47..7a22e700b8 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -36,6 +36,7 @@ import org.broadinstitute.sting.utils.io.IOUtils import org.broadinstitute.sting.utils.help.ApplicationDetails import java.util.{ResourceBundle, Arrays} import org.broadinstitute.sting.utils.text.TextFormattingUtils +import org.apache.commons.io.FilenameUtils /** * Entry point of Queue. Compiles and runs QScripts passed in to the command line. @@ -61,6 +62,7 @@ object QCommandLine extends Logging { CommandLineProgram.start(qCommandLine, argv) try { Runtime.getRuntime.removeShutdownHook(shutdownHook) + qCommandLine.shutdown() } catch { case _ => /* ignore, example 'java.lang.IllegalStateException: Shutdown in progress' */ } @@ -78,10 +80,10 @@ object QCommandLine extends Logging { class QCommandLine extends CommandLineProgram with Logging { @Input(fullName="script", shortName="S", doc="QScript scala file", required=true) @ClassType(classOf[File]) - private var scripts = List.empty[File] + var scripts = Seq.empty[File] @ArgumentCollection - private val settings = new QGraphSettings + val settings = new QGraphSettings private val qScriptManager = new QScriptManager private val qGraph = new QGraph @@ -91,7 +93,7 @@ class QCommandLine extends CommandLineProgram with Logging { private lazy val pluginManager = { qScriptClasses = IOUtils.tempDir("Q-Classes-", "", settings.qSettings.tempDirectory) qScriptManager.loadScripts(scripts, qScriptClasses) - new PluginManager[QScript](classOf[QScript], List(qScriptClasses.toURI.toURL)) + new PluginManager[QScript](classOf[QScript], Seq(qScriptClasses.toURI.toURL)) } QFunction.parsingEngine = new ParsingEngine(this) @@ -101,12 +103,16 @@ class QCommandLine extends CommandLineProgram with Logging { * functions, and then builds and runs a QGraph based on the dependencies. */ def execute = { + if (settings.qSettings.runName == null) + settings.qSettings.runName = FilenameUtils.removeExtension(scripts.head.getName) + qGraph.settings = settings val allQScripts = pluginManager.createAllTypes(); for (script <- allQScripts) { logger.info("Scripting " + pluginManager.getName(script.getClass.asSubclass(classOf[QScript]))) loadArgumentsIntoObject(script) + script.qSettings = settings.qSettings try { script.script() } catch { @@ -120,22 +126,34 @@ class QCommandLine extends CommandLineProgram with Logging { // Execute the job graph qGraph.run() + val functionsAndStatus = qGraph.getFunctionsAndStatus + val success = qGraph.success + // walk over each script, calling onExecutionDone for (script <- allQScripts) { - script.onExecutionDone(qGraph.getFunctionsAndStatus(script.functions), qGraph.success) - if ( ! settings.disableJobReport ) { - val jobStringName = (QScriptUtils.?(settings.jobReportFile)).getOrElse(settings.qSettings.jobNamePrefix + ".jobreport.txt") - - if (!shuttingDown) { - val reportFile = new File(jobStringName) - logger.info("Writing JobLogging GATKReport to file " + reportFile) - QJobReport.printReport(qGraph.getFunctionsAndStatus(script.functions), reportFile) - - if ( settings.run ) { - val pdfFile = new File(jobStringName + ".pdf") - logger.info("Plotting JobLogging GATKReport to file " + pdfFile) - QJobReport.plotReport(reportFile, pdfFile) - } + val scriptFunctions = functionsAndStatus.filterKeys(f => script.functions.contains(f)) + script.onExecutionDone(scriptFunctions, success) + } + + logger.info("Script %s with %d total jobs".format(if (success) "completed successfully" else "failed", functionsAndStatus.size)) + + if (!settings.disableJobReport) { + val jobStringName = { + if (settings.jobReportFile != null) + settings.jobReportFile + else + settings.qSettings.runName + ".jobreport.txt" + } + + if (!shuttingDown) { + val reportFile = IOUtils.absolute(settings.qSettings.runDirectory, jobStringName) + logger.info("Writing JobLogging GATKReport to file " + reportFile) + QJobReport.printReport(functionsAndStatus, reportFile) + + if (settings.run) { + val pdfFile = IOUtils.absolute(settings.qSettings.runDirectory, FilenameUtils.removeExtension(jobStringName) + ".pdf") + logger.info("Plotting JobLogging GATKReport to file " + pdfFile) + QJobReport.plotReport(reportFile, pdfFile) } } } @@ -179,20 +197,20 @@ class QCommandLine extends CommandLineProgram with Logging { override def getApplicationDetails : ApplicationDetails = { new ApplicationDetails(createQueueHeader(), - List.empty[String], + Seq.empty[String], ApplicationDetails.createDefaultRunningInstructions(getClass.asInstanceOf[Class[CommandLineProgram]]), "") } - private def createQueueHeader() : List[String] = { - List(String.format("Queue v%s, Compiled %s", getQueueVersion, getBuildTimestamp), - "Copyright (c) 2011 The Broad Institute", + private def createQueueHeader() : Seq[String] = { + Seq(String.format("Queue v%s, Compiled %s", getQueueVersion, getBuildTimestamp), + "Copyright (c) 2012 The Broad Institute", "Please view our documentation at http://www.broadinstitute.org/gsa/wiki", "For support, please view our support site at http://getsatisfaction.com/gsa") } private def getQueueVersion : String = { - var stingResources : ResourceBundle = TextFormattingUtils.loadResourceBundle("StingText") + val stingResources : ResourceBundle = TextFormattingUtils.loadResourceBundle("StingText") if ( stingResources.containsKey("org.broadinstitute.sting.queue.QueueVersion.version") ) { stingResources.getString("org.broadinstitute.sting.queue.QueueVersion.version") @@ -203,7 +221,7 @@ class QCommandLine extends CommandLineProgram with Logging { } private def getBuildTimestamp : String = { - var stingResources : ResourceBundle = TextFormattingUtils.loadResourceBundle("StingText") + val stingResources : ResourceBundle = TextFormattingUtils.loadResourceBundle("StingText") if ( stingResources.containsKey("build.timestamp") ) { stingResources.getString("build.timestamp") diff --git a/public/scala/src/org/broadinstitute/sting/queue/QScript.scala b/public/scala/src/org/broadinstitute/sting/queue/QScript.scala index fce65c9970..6f887ea002 100755 --- a/public/scala/src/org/broadinstitute/sting/queue/QScript.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/QScript.scala @@ -27,7 +27,6 @@ package org.broadinstitute.sting.queue import engine.JobRunInfo import org.broadinstitute.sting.queue.function.QFunction import annotation.target.field -import io.Source import util.{StringFileConversions, PrimitiveOptionConversions, Logging} /** @@ -53,6 +52,11 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon type ArgumentCollection = org.broadinstitute.sting.commandline.ArgumentCollection @field type Gather = org.broadinstitute.sting.commandline.Gather @field + /** + * Default settings for QFunctions + */ + var qSettings: QSettings = _ + /** * Builds the CommandLineFunctions that will be used to run this script and adds them to this.functions directly or using the add() utility method. */ @@ -60,18 +64,14 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon /** * A default handler for the onExecutionDone() function. By default this doesn't do anything - * except print out a fine status message. */ def onExecutionDone(jobs: Map[QFunction, JobRunInfo], success: Boolean) { - logger.info("Script %s with %d total jobs".format(if (success) "completed successfully" else "failed", jobs.size)) - // this is too much output - // for ( (f, info) <- jobs ) logger.info(" %s %s".format(f.jobName, info)) } /** * The command line functions that will be executed for this QScript. */ - var functions = List.empty[QFunction] + var functions = Seq.empty[QFunction] /** * Exchanges the extension on a file. @@ -98,22 +98,20 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon * Adds one or more command line functions to be run. * @param functions Functions to add. */ - def add(functions: QFunction*) = { + def add(functions: QFunction*) { functions.foreach(function => function.addOrder = QScript.nextAddOrder) this.functions ++= functions } - def addAll(functions: List[QFunction]) { + def addAll(functions: Seq[QFunction]) { functions.foreach( f => add(f) ) } - - def extractFileEntries(in: File): List[File] = Source.fromFile(in).getLines().toList } object QScript { private var addOrder = 0 private def nextAddOrder = { addOrder += 1 - List(addOrder) + Seq(addOrder) } } diff --git a/public/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala b/public/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala index 512a9f8dd1..74487917fc 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/QScriptManager.scala @@ -20,7 +20,7 @@ class QScriptManager() extends Logging { * Compiles and loads the scripts in the files into the current classloader. * Heavily based on scala/src/compiler/scala/tools/ant/Scalac.scala */ - def loadScripts(scripts: List[File], tempDir: File) { + def loadScripts(scripts: Seq[File], tempDir: File) { if (scripts.size > 0) { val settings = new Settings((error: String) => logger.error(error)) settings.deprecation.value = true @@ -36,7 +36,7 @@ class QScriptManager() extends Logging { logger.info("Compiling %s QScript%s".format(scripts.size, plural(scripts.size))) logger.debug("Compilation directory: " + settings.outdir.value) - run.compileFiles(scripts.map(new PlainFile(_))) + run.compileFiles(scripts.toList.map(new PlainFile(_))) reporter.printSummary() if (reporter.hasErrors) { diff --git a/public/scala/src/org/broadinstitute/sting/queue/QSettings.scala b/public/scala/src/org/broadinstitute/sting/queue/QSettings.scala index e8ac26a574..d9fed4ce8b 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/QSettings.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/QSettings.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -25,15 +25,14 @@ package org.broadinstitute.sting.queue import java.io.File -import org.broadinstitute.sting.commandline.{ArgumentCollection, Argument} -import org.broadinstitute.sting.queue.util.{SystemUtils, EmailSettings} +import org.broadinstitute.sting.commandline.Argument /** * Default settings settable on the command line and passed to CommandLineFunctions. */ class QSettings { - @Argument(fullName="job_name_prefix", shortName="jobPrefix", doc="Default name prefix for compute farm jobs.", required=false) - var jobNamePrefix: String = QSettings.processNamePrefix + @Argument(fullName="run_name", shortName="runName", doc="A name for this run used for various status messages.", required=false) + var runName: String = _ @Argument(fullName="job_project", shortName="jobProject", doc="Default project for compute farm jobs.", required=false) var jobProject: String = _ @@ -45,13 +44,13 @@ class QSettings { var jobPriority: Option[Int] = None @Argument(fullName="job_native_arg", shortName="jobNative", doc="Native arguments to pass to the job runner.", required=false) - var jobNativeArgs: List[String] = Nil + var jobNativeArgs: Seq[String] = Nil @Argument(fullName="job_resource_request", shortName="jobResReq", doc="Resource requests to pass to the job runner.", required=false) - var jobResourceRequests: List[String] = Nil + var jobResourceRequests: Seq[String] = Nil @Argument(fullName="job_environment_name", shortName="jobEnv", doc="Environment names for the job runner.", required=false) - var jobEnvironmentNames: List[String] = Nil + var jobEnvironmentNames: Seq[String] = Nil @Argument(fullName="memory_limit", shortName="memLimit", doc="Default memory limit for jobs, in gigabytes.", required=false) var memoryLimit: Option[Double] = None @@ -77,15 +76,4 @@ class QSettings { @Argument(fullName="job_scatter_gather_directory", shortName="jobSGDir", doc="Default directory to place scatter gather output for compute farm jobs.", required=false) var jobScatterGatherDirectory: File = _ - - @ArgumentCollection - val emailSettings = new EmailSettings -} - -/** - * Default settings settable on the command line and passed to CommandLineFunctions. - */ -object QSettings { - /** A semi-unique job prefix using the host name and the process id. */ - private val processNamePrefix = "Q-" + SystemUtils.pidAtHost } diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/FunctionEdge.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/FunctionEdge.scala index 55ed942672..8225d28ab3 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/FunctionEdge.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/FunctionEdge.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.engine import org.broadinstitute.sting.queue.function.QFunction @@ -28,15 +52,18 @@ class FunctionEdge(val function: QFunction, val inputs: QNode, val outputs: QNod val myRunInfo: JobRunInfo = JobRunInfo.default // purely for dryRun testing + /** + * When using reset status this variable tracks the old status + */ + var resetFromStatus: RunnerStatus.Value = null + /** * Initializes with the current status of the function. */ private var currentStatus = { - val isDone = function.isDone - val isFail = function.isFail - if (isFail.isDefined && isFail.get) + if (function.isFail) RunnerStatus.FAILED - else if (isDone.isDefined && isDone.get) + else if (function.isDone) RunnerStatus.DONE else RunnerStatus.PENDING @@ -136,13 +163,15 @@ class FunctionEdge(val function: QFunction, val inputs: QNode, val outputs: QNod * Resets the edge to pending status. */ def resetToPending(cleanOutputs: Boolean) { + if (resetFromStatus == null) + resetFromStatus = currentStatus currentStatus = RunnerStatus.PENDING if (cleanOutputs) function.deleteOutputs() runner = null } - override def dotString = function.dotString + override def shortDescription = function.shortDescription /** * Returns the path to the file to use for logging errors. diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/InProcessRunner.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/InProcessRunner.scala index d006cde4b4..be5622360b 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/InProcessRunner.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/InProcessRunner.scala @@ -3,7 +3,8 @@ package org.broadinstitute.sting.queue.engine import org.broadinstitute.sting.queue.function.InProcessFunction import java.util.Date import org.broadinstitute.sting.utils.Utils -import org.apache.commons.io.FileUtils +import org.apache.commons.io.{IOUtils, FileUtils} +import java.io.PrintStream /** * Runs a function that executes in process and does not fork out an external process. @@ -16,12 +17,24 @@ class InProcessRunner(val function: InProcessFunction) extends JobRunner[InProce getRunInfo.exechosts = Utils.resolveHostname() runStatus = RunnerStatus.RUNNING - function.run() + function.jobOutputStream = new PrintStream(FileUtils.openOutputStream(function.jobOutputFile)) + function.jobErrorStream = { + if (function.jobErrorFile != null) + new PrintStream(FileUtils.openOutputStream(function.jobErrorFile)) + else + function.jobOutputStream + } + try { + function.run() + function.jobOutputStream.println("%s%nDone.".format(function.description)) + } finally { + IOUtils.closeQuietly(function.jobOutputStream) + if (function.jobErrorFile != null) + IOUtils.closeQuietly(function.jobErrorStream) + } - getRunInfo.doneTime = new Date() - val content = "%s%nDone.".format(function.description) - FileUtils.writeStringToFile(function.jobOutputFile, content) runStatus = RunnerStatus.DONE + getRunInfo.doneTime = new Date() } def status = runStatus diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/MappingEdge.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/MappingEdge.scala index 1d56009f33..17f0561faa 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/MappingEdge.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/MappingEdge.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.engine /** @@ -10,5 +34,5 @@ class MappingEdge(val inputs: QNode, val outputs: QNode) extends QEdge { * @return */ override def toString = "" - override def dotString = "" + override def shortDescription = "" } diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/QEdge.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/QEdge.scala index 1608e3c088..e40a868675 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/QEdge.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/QEdge.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.engine /** @@ -15,9 +39,9 @@ trait QEdge { def outputs: QNode /** - * The function description in .dot files + * The short description */ - def dotString = "" + def shortDescription = "" override def hashCode = inputs.hashCode + outputs.hashCode diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala index 42ddf91040..cee2c6e56a 100755 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -30,7 +30,6 @@ import scala.collection.JavaConversions._ import org.jgrapht.alg.CycleDetector import org.jgrapht.EdgeFactory import org.jgrapht.ext.DOTExporter -import java.io.File import org.jgrapht.event.{TraversalListenerAdapter, EdgeTraversalEvent} import org.broadinstitute.sting.queue.QException import org.broadinstitute.sting.queue.function.{InProcessFunction, CommandLineFunction, QFunction} @@ -40,7 +39,8 @@ import collection.immutable.{TreeSet, TreeMap} import org.broadinstitute.sting.queue.function.scattergather.{ScatterFunction, CloneFunction, GatherFunction, ScatterGatherableFunction} import java.util.Date import org.broadinstitute.sting.utils.Utils -import org.broadinstitute.sting.utils.io.IOUtils +import org.apache.commons.io.{FileUtils, IOUtils} +import java.io.{OutputStreamWriter, File} /** * The internal dependency tracker between sets of function input and output files. @@ -69,7 +69,7 @@ class QGraph extends Logging { private val commandLinePluginManager = new CommandLinePluginManager private var commandLineManager: CommandLineJobManager[CommandLineJobRunner] = _ private val inProcessManager = new InProcessJobManager - private def managers = List[Any](inProcessManager, commandLineManager) + private def managers = Seq[Any](inProcessManager, commandLineManager) private class StatusCounts { var pending = 0 @@ -88,9 +88,14 @@ class QGraph extends Logging { runningLock.synchronized { if (running) { command.qSettings = settings.qSettings - command.freeze - val inputs = getQNode(command.inputs.toList.sorted(fileOrdering)) - val outputs = getQNode(command.outputs.toList.sorted(fileOrdering)) + command.freeze() + val inputFiles = command.inputs + var outputFiles = command.outputs + outputFiles :+= command.jobOutputFile + if (command.jobErrorFile != null) + outputFiles :+= command.jobErrorFile + val inputs = getQNode(inputFiles.sorted(fileOrdering)) + val outputs = getQNode(outputFiles.sorted(fileOrdering)) addEdge(new FunctionEdge(command, inputs, outputs)) } } @@ -106,8 +111,8 @@ class QGraph extends Logging { def run() { runningLock.synchronized { if (running) { - IOUtils.checkTempDir(settings.qSettings.tempDirectory) - fillGraph + org.broadinstitute.sting.utils.io.IOUtils.checkTempDir(settings.qSettings.tempDirectory) + fillGraph() val isReady = numMissingValues == 0 if (this.jobGraph.edgeSet.isEmpty) { @@ -133,11 +138,11 @@ class QGraph extends Logging { } } - private def fillGraph { + private def fillGraph() { logger.info("Generating graph.") - fill - if (settings.dotFile != null) - renderToDot(settings.dotFile) + fill() + if (settings.graphvizFile != null) + renderGraph(settings.graphvizFile) validate() if (running && numMissingValues == 0) { @@ -145,7 +150,7 @@ class QGraph extends Logging { if (!scatterGathers.isEmpty) { logger.info("Generating scatter gather jobs.") - var addedFunctions = List.empty[QFunction] + var addedFunctions = Seq.empty[QFunction] for (scatterGather <- scatterGathers) { val functions = scatterGather.asInstanceOf[FunctionEdge] .function.asInstanceOf[ScatterGatherableFunction] @@ -161,10 +166,10 @@ class QGraph extends Logging { addedFunctions.foreach(function => if (running) this.add(function)) logger.info("Regenerating graph.") - fill - val scatterGatherDotFile = if (settings.expandedDotFile != null) settings.expandedDotFile else settings.dotFile + fill() + val scatterGatherDotFile = if (settings.graphvizScatterGatherFile != null) settings.graphvizScatterGatherFile else settings.graphvizFile if (scatterGatherDotFile != null) - renderToDot(scatterGatherDotFile) + renderGraph(scatterGatherDotFile) validate() } } @@ -187,8 +192,8 @@ class QGraph extends Logging { * @param edge Graph edge to examine for the previous functions. * @return A list of prior function edges. */ - private def previousFunctions(edge: QEdge): List[FunctionEdge] = { - var previous = List.empty[FunctionEdge] + private def previousFunctions(edge: QEdge): Seq[FunctionEdge] = { + var previous = Seq.empty[FunctionEdge] val source = this.jobGraph.getEdgeSource(edge) for (incomingEdge <- this.jobGraph.incomingEdgesOf(source)) { incomingEdge match { @@ -208,8 +213,8 @@ class QGraph extends Logging { * @param edge Graph edge to examine for the next functions. * @return A list of prior function edges. */ - private def nextFunctions(edge: QEdge): List[FunctionEdge] = { - var next = List.empty[FunctionEdge] + private def nextFunctions(edge: QEdge): Seq[FunctionEdge] = { + var next = Seq.empty[FunctionEdge] val target = this.jobGraph.getEdgeTarget(edge) for (outgoingEdge <- this.jobGraph.outgoingEdgesOf(target)) { outgoingEdge match { @@ -238,7 +243,7 @@ class QGraph extends Logging { */ private def fillIn() { // clone since edgeSet is backed by the graph - asScalaSet(jobGraph.edgeSet).clone.foreach(edge => { + asScalaSet(jobGraph.edgeSet).clone().foreach(edge => { if (running) edge match { case cmd: FunctionEdge => { addCollectionOutputs(cmd.outputs) @@ -249,7 +254,7 @@ class QGraph extends Logging { }) } - private def getReadyJobs(): Set[FunctionEdge] = { + private def getReadyJobs: Set[FunctionEdge] = { jobGraph.edgeSet.filter{ case f: FunctionEdge => this.previousFunctions(f).forall(_.status == RunnerStatus.DONE) && f.status == RunnerStatus.PENDING @@ -317,33 +322,39 @@ class QGraph extends Logging { updateGraphStatus(false) - var readyJobs = getReadyJobs() + var readyJobs = getReadyJobs while (running && readyJobs.size > 0) { logger.debug("+++++++") - foreachFunction(readyJobs.toList, edge => { + foreachFunction(readyJobs.toSeq, edge => { if (running) { edge.myRunInfo.startTime = new Date() edge.getRunInfo.exechosts = Utils.resolveHostname() logEdge(edge) edge.myRunInfo.doneTime = new Date() - edge.markAsDone + edge.markAsDone() } }) - readyJobs = getReadyJobs() + readyJobs = getReadyJobs } } private def logEdge(edge: FunctionEdge) { logger.info("-------") + logger.info("%-8s %s".format(StringUtils.capitalize(edge.status.toString) + ":", edge.function.description)) if (logger.isDebugEnabled) { - logger.debug("Inputs: " + edge.inputs) + logger.debug("Inputs: " + edge.inputs) + logger.debug("Outputs: " + edge.outputs) + logger.debug("Done+: " + edge.function.doneOutputs.filter(_.exists())) + logger.debug("Done-: " + edge.function.doneOutputs.filterNot(_.exists())) + logger.debug("CmdDir: " + edge.function.commandDirectory) + logger.debug("Temp?: " + edge.function.isIntermediate) + logger.debug("Prev: " + + (if (edge.resetFromStatus == null) "none" else StringUtils.capitalize(edge.resetFromStatus.toString)) + + " (reset = " + (edge.resetFromStatus != null && edge.resetFromStatus != edge.status) + ")" ) } - logger.info(StringUtils.capitalize(edge.status.toString) + ": " + edge.function.description) - if (logger.isDebugEnabled) - logger.debug(edge.function.commandDirectory + " > " + edge.function.description) - logger.info("Log: " + edge.function.jobOutputFile.getAbsolutePath) + logger.info("Log: " + edge.function.jobOutputFile.getAbsolutePath) if (edge.function.jobErrorFile != null) - logger.info("Error: " + edge.function.jobErrorFile.getAbsolutePath) + logger.info("Error: " + edge.function.jobErrorFile.getAbsolutePath) } /** @@ -380,7 +391,7 @@ class QGraph extends Logging { updateGraphStatus(true) var readyJobs = TreeSet.empty[FunctionEdge](functionOrdering) - readyJobs ++= getReadyJobs() + readyJobs ++= getReadyJobs runningJobs = Set.empty[FunctionEdge] var lastRunningCheck = System.currentTimeMillis var logNextStatusCounts = true @@ -407,7 +418,7 @@ class QGraph extends Logging { statusCounts.running += startedJobs.size if (logNextStatusCounts) - logStatusCounts + logStatusCounts() logNextStatusCounts = false deleteCleanup(lastRunningCheck) @@ -456,10 +467,10 @@ class QGraph extends Logging { checkRetryJobs(failedJobs) } - readyJobs ++= getReadyJobs() + readyJobs ++= getReadyJobs } - logStatusCounts + logStatusCounts() deleteCleanup(-1) } catch { case e => @@ -476,7 +487,7 @@ class QGraph extends Logging { private def nextRunningCheck(lastRunningCheck: Long) = ((30 * 1000L) - (System.currentTimeMillis - lastRunningCheck)) - private def logStatusCounts { + private def logStatusCounts() { logger.info("%d Pend, %d Run, %d Fail, %d Done".format( statusCounts.pending, statusCounts.running, statusCounts.failed, statusCounts.done)) } @@ -532,7 +543,8 @@ class QGraph extends Logging { } if (edge.status == RunnerStatus.DONE || edge.status == RunnerStatus.SKIPPED) { - logger.debug("Already done: " + edge.function.description) + if (logger.isDebugEnabled) + logEdge(edge) addCleanup(edge) } } @@ -546,12 +558,12 @@ class QGraph extends Logging { } /** - * Checks if the function should have their outptus removed after they finish running - * @param edges Function to check + * Checks if the function should have their outputs removed after they finish running + * @param edge Function to check */ private def addCleanup(edge: FunctionEdge) { if (!settings.keepIntermediates) - if (edge.function.isIntermediate && edge.function.deleteIntermediateOutputs) + if (edge.function.isIntermediate) cleanupJobs += edge } @@ -601,14 +613,16 @@ class QGraph extends Logging { * From the previous edges, resets any that are marked as skipped to pending. * If those that are reset have skipped edges, those skipped edges are recursively also set * to pending. + * Any edges after this edge are also reset to pending. * @param edge Dependent edge. * @param previous Previous edges that provide inputs to edge. - * @param cleanOutputs If true will clean up the output files when resetting skipped jobs to pending. + * @param cleanOutputs If true will clean up the output files when resetting jobs to pending. */ - private def resetPreviousSkipped(edge: FunctionEdge, previous: List[FunctionEdge], cleanOutputs: Boolean) { - for (previousEdge <- previous.filter(_.status == RunnerStatus.SKIPPED)) { - previousEdge.resetToPending(cleanOutputs) - resetPreviousSkipped(previousEdge, this.previousFunctions(previousEdge), cleanOutputs) + private def resetPreviousSkipped(edge: FunctionEdge, previous: Seq[FunctionEdge], cleanOutputs: Boolean) { + val edges = previous.filter(_.status == RunnerStatus.SKIPPED) ++ this.nextFunctions(edge).filter(_.status != RunnerStatus.PENDING) + for (resetEdge <- edges) { + resetEdge.resetToPending(cleanOutputs) + resetPreviousSkipped(resetEdge, this.previousFunctions(resetEdge), cleanOutputs) } } @@ -628,9 +642,9 @@ class QGraph extends Logging { val emailMessage = new EmailMessage emailMessage.from = settings.statusEmailFrom emailMessage.to = settings.statusEmailTo - emailMessage.subject = "Queue function: Started: " + settings.qSettings.jobNamePrefix - addStartedFunctions(emailMessage, started.toList) - emailMessage.trySend(settings.qSettings.emailSettings) + emailMessage.subject = "Queue function: Started: " + settings.qSettings.runName + addStartedFunctions(emailMessage, started.toSeq) + emailMessage.trySend(settings.emailSettings) } } @@ -639,9 +653,9 @@ class QGraph extends Logging { val emailMessage = new EmailMessage emailMessage.from = settings.statusEmailFrom emailMessage.to = settings.statusEmailTo - emailMessage.subject = "Queue function: Failure: " + settings.qSettings.jobNamePrefix - addFailedFunctions(emailMessage, failed.toList) - emailMessage.trySend(settings.qSettings.emailSettings) + emailMessage.subject = "Queue function: Failure: " + settings.qSettings.runName + addFailedFunctions(emailMessage, failed.toSeq) + emailMessage.trySend(settings.emailSettings) } } @@ -665,7 +679,7 @@ class QGraph extends Logging { private def emailStatus() { if (running && settings.statusEmailTo.size > 0) { - var failed = List.empty[FunctionEdge] + var failed = Seq.empty[FunctionEdge] foreachFunction(edge => { if (edge.status == RunnerStatus.FAILED) { failed :+= edge @@ -677,16 +691,16 @@ class QGraph extends Logging { emailMessage.to = settings.statusEmailTo emailMessage.body = getStatus + nl if (failed.size == 0) { - emailMessage.subject = "Queue run: Success: " + settings.qSettings.jobNamePrefix + emailMessage.subject = "Queue run: Success: " + settings.qSettings.runName } else { - emailMessage.subject = "Queue run: Failure: " + settings.qSettings.jobNamePrefix + emailMessage.subject = "Queue run: Failure: " + settings.qSettings.runName addFailedFunctions(emailMessage, failed) } - emailMessage.trySend(settings.qSettings.emailSettings) + emailMessage.trySend(settings.emailSettings) } } - private def addStartedFunctions(emailMessage: EmailMessage, started: List[FunctionEdge]) { + private def addStartedFunctions(emailMessage: EmailMessage, started: Seq[FunctionEdge]) { if (emailMessage.body == null) emailMessage.body = "" emailMessage.body += """ @@ -697,7 +711,7 @@ class QGraph extends Logging { started.map(edge => emailDescription(edge)).mkString(nl+nl)) } - private def addFailedFunctions(emailMessage: EmailMessage, failed: List[FunctionEdge]) { + private def addFailedFunctions(emailMessage: EmailMessage, failed: Seq[FunctionEdge]) { val logs = failed.flatMap(edge => logFiles(edge)) if (emailMessage.body == null) @@ -725,7 +739,7 @@ class QGraph extends Logging { } private def logFiles(edge: FunctionEdge) = { - var failedOutputs = List.empty[File] + var failedOutputs = Seq.empty[File] failedOutputs :+= edge.function.jobOutputFile if (edge.function.jobErrorFile != null) failedOutputs :+= edge.function.jobErrorFile @@ -762,14 +776,14 @@ class QGraph extends Logging { private def getStatus = { val buffer = new StringBuilder doStatus(status => buffer.append(status).append(nl)) - buffer.toString + buffer.toString() } /** * Gets job statuses by traversing the graph and looking for status-related files */ - private def doStatus(statusFunc: String => Unit) = { - var statuses = List.empty[AnalysisStatus] + private def doStatus(statusFunc: String => Unit) { + var statuses = Seq.empty[AnalysisStatus] var maxWidth = 0 foreachFunction(edge => { val name = edge.function.analysisName @@ -860,7 +874,7 @@ class QGraph extends Logging { private def newGraph = new SimpleDirectedGraph[QNode, QEdge](new EdgeFactory[QNode, QEdge] { def createEdge(input: QNode, output: QNode) = new MappingEdge(input, output)}) - private def getQNode(files: List[File]) = { + private def getQNode(files: Seq[File]) = { nodeMap.get(files) match { case Some(node) => node @@ -888,7 +902,7 @@ class QGraph extends Logging { if (inputs.files.size > 1) for (file <- inputs.files) { if (running) { - val input = getQNode(List(file)) + val input = getQNode(Seq(file)) if (!jobGraph.containsEdge(input, inputs)) addEdge(new MappingEdge(input, inputs)) } @@ -903,7 +917,7 @@ class QGraph extends Logging { if (outputs.files.size > 1) for (file <- outputs.files) { if (running) { - val output = getQNode(List(file)) + val output = getQNode(Seq(file)) if (!jobGraph.containsEdge(outputs, output)) addEdge(new MappingEdge(outputs, output)) } @@ -937,37 +951,36 @@ class QGraph extends Logging { /** * Utility function for running a method over all function edges. - * @param edgeFunction Function to run for each FunctionEdge. + * @param f Function to run for each FunctionEdge. */ private def foreachFunction(f: (FunctionEdge) => Unit) { - foreachFunction(jobGraph.edgeSet.toList.filter(_.isInstanceOf[FunctionEdge]).asInstanceOf[List[FunctionEdge]], f) + foreachFunction(jobGraph.edgeSet.toSeq.filter(_.isInstanceOf[FunctionEdge]).asInstanceOf[Seq[FunctionEdge]], f) } /** * Utility function for running a method over a list of function edges. - * @param edegs Edges to traverse. - * @param edgeFunction Function to run for each FunctionEdge. + * @param edges Edges to traverse. + * @param f Function to run for each FunctionEdge. */ - private def foreachFunction(edges: List[FunctionEdge], f: (FunctionEdge) => Unit) { + private def foreachFunction(edges: Seq[FunctionEdge], f: (FunctionEdge) => Unit) { edges.sorted(functionOrdering).foreach(edge => if (running) f(edge)) } /** - * Utility function for running a method over all function edges. - * @param edgeFunction Function to run for each FunctionEdge. + * Utility function returning all function edges. */ - private def getFunctionEdges: List[FunctionEdge] = { - jobGraph.edgeSet.toList.filter(_.isInstanceOf[FunctionEdge]).asInstanceOf[List[FunctionEdge]] + private def getFunctionEdges: Seq[FunctionEdge] = { + jobGraph.edgeSet.toSeq.filter(_.isInstanceOf[FunctionEdge]).asInstanceOf[Seq[FunctionEdge]] } /** * Utility function for running a method over all functions, but traversing the nodes in order of dependency. - * @param edgeFunction Function to run for each FunctionEdge. + * @param f Function to run for each FunctionEdge. */ private def traverseFunctions(f: (FunctionEdge) => Unit) { val iterator = new TopologicalOrderIterator(this.jobGraph) iterator.addTraversalListener(new TraversalListenerAdapter[QNode, QEdge] { - override def edgeTraversed(event: EdgeTraversalEvent[QNode, QEdge]) = { + override def edgeTraversed(event: EdgeTraversalEvent[QNode, QEdge]) { if (running) { event.getEdge match { case functionEdge: FunctionEdge => f(functionEdge) @@ -980,23 +993,44 @@ class QGraph extends Logging { } /** - * Outputs the graph to a .dot file. + * Outputs the graph to a .gv DOT file. + * http://www.graphviz.org/Documentation.php * http://en.wikipedia.org/wiki/DOT_language - * @param file Path to output the .dot file. + * @param file Path to output the .gv file. */ - private def renderToDot(file: java.io.File) { - val out = new java.io.FileWriter(file) + private def renderGraph(file: java.io.File) { + val vertexIDProvider = new org.jgrapht.ext.VertexNameProvider[QNode] { + def getVertexName(node: QNode) = node.id.toString + } + + val vertexLabelProvider = new org.jgrapht.ext.VertexNameProvider[QNode] { + // The QGraph fills in with single file nodes between nodes that contain more than one file. + // We only need to display the single element nodes. + def getVertexName(node: QNode) = { + if (!node.files.isEmpty && node.files.tail.isEmpty) + node.files.head.getName + else + "" + } + } - // todo -- we need a nice way to visualize the key pieces of information about commands. Perhaps a - // todo -- visualizeString() command, or something that shows inputs / outputs - val ve = new org.jgrapht.ext.EdgeNameProvider[QEdge] { - def getEdgeName(function: QEdge) = if (function.dotString == null) "" else function.dotString.replace("\"", "\\\"") + val edgeNameProvider = new org.jgrapht.ext.EdgeNameProvider[QEdge] { + def getEdgeName(edge: QEdge) = { + if (edge.shortDescription != null) + edge.shortDescription.replace("\"", "\\\"") + else + "" + } } - //val iterator = new TopologicalOrderIterator(qGraph.jobGraph) - (new DOTExporter(new org.jgrapht.ext.IntegerNameProvider[QNode](), null, ve)).export(out, jobGraph) + val exporter = new DOTExporter(vertexIDProvider, vertexLabelProvider, edgeNameProvider) - out.close + val out = new OutputStreamWriter(FileUtils.openOutputStream(file)) + try { + exporter.export(out, jobGraph) + } finally { + IOUtils.closeQuietly(out) + } } /** @@ -1054,7 +1088,7 @@ class QGraph extends Logging { */ def isShutdown = !running - def getFunctionsAndStatus(functions: List[QFunction]): Map[QFunction, JobRunInfo] = { + def getFunctionsAndStatus: Map[QFunction, JobRunInfo] = { getFunctionEdges.map(edge => (edge.function, edge.getRunInfo)).toMap } diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/QGraphSettings.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/QGraphSettings.scala index 56d6975a51..6d81d4bd71 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/QGraphSettings.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/QGraphSettings.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -26,7 +26,7 @@ package org.broadinstitute.sting.queue.engine import java.io.File import org.broadinstitute.sting.queue.QSettings -import org.broadinstitute.sting.queue.util.SystemUtils +import org.broadinstitute.sting.queue.util.{EmailSettings, SystemUtils} import org.broadinstitute.sting.commandline.{Advanced, ArgumentCollection, Argument} /** @@ -58,16 +58,16 @@ class QGraphSettings { var keepIntermediates = false @Argument(fullName="status_email_to", shortName="statusTo", doc="Email address to send emails to upon completion or on error.", required=false) - var statusEmailTo: List[String] = Nil + var statusEmailTo: Seq[String] = Nil @Argument(fullName="status_email_from", shortName="statusFrom", doc="Email address to send emails from upon completion or on error.", required=false) var statusEmailFrom: String = System.getProperty("user.name") + "@" + SystemUtils.mailName - @Argument(fullName="dot_graph", shortName="dot", doc="Outputs the queue graph to a .dot file. See: http://en.wikipedia.org/wiki/DOT_language", required=false) - var dotFile: File = _ + @Argument(fullName="graphviz", shortName="gv", doc="Outputs the queue graph to a Graphviz .gv file. See: http://www.graphviz.org/Documentation.php", required=false) + var graphvizFile: File = _ - @Argument(fullName="expanded_dot_graph", shortName="expandedDot", doc="Outputs the queue graph of scatter gather to a .dot file. Otherwise overwrites the dot_graph", required=false) - var expandedDotFile: File = _ + @Argument(fullName="graphviz_scatter_gather", shortName="gvsg", doc="Outputs the scatter/gather queue graph to a Graphviz .gv file. Otherwise overwrites the --graphviz file.", required=false) + var graphvizScatterGatherFile: File = _ @Argument(fullName="jobReport", shortName="jobReport", doc="File where we will write the Queue job report", required=false) var jobReportFile: String = _ @@ -76,6 +76,9 @@ class QGraphSettings { @Argument(fullName="disableJobReport", shortName="disabpleJobReport", doc="If provided, we will not create a job report", required=false) var disableJobReport: Boolean = false + @ArgumentCollection + val emailSettings = new EmailSettings + @ArgumentCollection val qSettings = new QSettings } diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/QNode.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/QNode.scala index a86c08aae5..a5c039a530 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/QNode.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/QNode.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.engine import java.io.File @@ -6,7 +30,7 @@ import java.io.File * Represents a state between QFunctions the directed acyclic QGraph * @param files The list of files that represent this node state ordered by file name. */ -class QNode (val id: Int, val files: List[File]) { +class QNode (val id: Int, val files: Seq[File]) { override def equals(obj: Any) = { obj match { case other: QNode => this.id == other.id @@ -16,5 +40,5 @@ class QNode (val id: Int, val files: List[File]) { override def hashCode = id - override def toString = files.toString + override def toString = files.toString() } diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/gridengine/GridEngineJobRunner.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/gridengine/GridEngineJobRunner.scala index fca92a7a17..239f834820 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/gridengine/GridEngineJobRunner.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/gridengine/GridEngineJobRunner.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -69,7 +69,7 @@ class GridEngineJobRunner(session: Session, function: CommandLineFunction) exten if ( function.nCoresRequest.getOrElse(1) > 1 ) { if ( function.qSettings.dontRequestMultipleCores ) logger.warn("Sending multicore job %s to farm without requesting appropriate number of cores (%d)".format( - function.jobName, function.nCoresRequest.get)) + function.shortDescription, function.nCoresRequest.get)) else nativeSpec += " -pe %s %d".format(function.qSettings.parallelEnvironmentName, function.nCoresRequest.get) } diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/lsf/Lsf706JobRunner.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/lsf/Lsf706JobRunner.scala index 5ef78500c8..de996d1870 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/lsf/Lsf706JobRunner.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/lsf/Lsf706JobRunner.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -35,8 +35,8 @@ import org.broadinstitute.sting.queue.engine.{RunnerStatus, CommandLineJobRunner import java.util.regex.Pattern import java.lang.StringBuffer import java.util.Date -import com.sun.jna.{Pointer, Structure, StringArray, NativeLong} -import com.sun.jna.ptr.{PointerByReference, IntByReference} +import com.sun.jna.{Structure, StringArray, NativeLong} +import com.sun.jna.ptr.IntByReference /** * Runs jobs on an LSF compute cluster. @@ -60,7 +60,6 @@ class Lsf706JobRunner(val function: CommandLineFunction) extends CommandLineJobR /** * Dispatches the function on the LSF cluster. - * @param function Command to run. */ def start() { Lsf706JobRunner.lsfLibLock.synchronized { @@ -110,7 +109,7 @@ class Lsf706JobRunner(val function: CommandLineFunction) extends CommandLineJobR if ( function.nCoresRequest.getOrElse(1) > 1 ) { if ( function.qSettings.dontRequestMultipleCores ) logger.warn("Sending multicore job %s to farm without requesting appropriate number of cores (%d)".format( - function.jobName, function.nCoresRequest.get)) + function.shortDescription, function.nCoresRequest.get)) else { request.numProcessors = function.nCoresRequest.get request.maxNumProcessors = request.numProcessors @@ -298,7 +297,7 @@ object Lsf706JobRunner extends Logging { runner.getRunInfo.doneTime = new Date(jobInfo.endTime.longValue * 1000) val exHostsRaw = jobInfo.exHosts.getStringArray(0) //logger.warn("exHostsRaw = " + exHostsRaw) - val exHostsList = exHostsRaw.toList + val exHostsList = exHostsRaw.toSeq //logger.warn("exHostsList = " + exHostsList) val exHosts = exHostsList.reduceLeft(_ + "," + _) //logger.warn("exHosts = " + exHosts) @@ -363,7 +362,7 @@ object Lsf706JobRunner extends Logging { /** * Returns the run limit in seconds for the queue. * If the queue name is null returns the length of the default queue. - * @param queue Name of the queue or null for the default queue. + * @param queueName Name of the queue or null for the default queue. * @return the run limit in seconds for the queue. */ private def getRlimitRun(queueName: String) = { diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala index 9751012a46..6cd4b06bc5 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/BamGatherFunction.scala @@ -53,6 +53,9 @@ class BamGatherFunction extends GatherFunction with PicardBamFunction { val disableIndex = QFunction.findField(originalFunction.getClass, SAMFileWriterArgumentTypeDescriptor.DISABLE_INDEXING_FULLNAME) this.createIndex = Some(!originalGATK.getFieldValue(disableIndex).asInstanceOf[Boolean]) - super.freezeFieldValues + val enableMD5 = QFunction.findField(originalFunction.getClass, SAMFileWriterArgumentTypeDescriptor.ENABLE_MD5_FULLNAME) + this.createMD5 = Some(originalGATK.getFieldValue(enableMD5).asInstanceOf[Boolean]) + + super.freezeFieldValues() } } diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala index 9e47f64a1c..42a027be1d 100755 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -32,9 +32,8 @@ import net.sf.samtools.SAMFileHeader import java.util.Collections import org.broadinstitute.sting.utils.{GenomeLoc, GenomeLocSortedSet, GenomeLocParser} -case class GATKIntervals(reference: File, intervals: List[String]) { +case class GATKIntervals(reference: File, intervals: Seq[String]) { private lazy val referenceDataSource = new ReferenceDataSource(reference) -// private var splitsBySize = Map.empty[Int, java.util.List[java.lang.Integer]] lazy val samFileHeader = { val header = new SAMFileHeader @@ -53,13 +52,5 @@ case class GATKIntervals(reference: File, intervals: List[String]) { Collections.unmodifiableList(parsedLocs) } - lazy val contigs = locs.map(_.getContig).distinct.toList - -// def getSplits(size: Int) = { -// splitsBySize.getOrElse(size, { -// val splits: java.util.List[java.lang.Integer] = IntervalUtils.splitFixedIntervals(locs, size) -// splitsBySize += size -> splits -// splits -// }) -// } + lazy val contigs = locs.map(_.getContig).distinct.toSeq } diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKScatterFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKScatterFunction.scala index c9adff0264..28c3f41e98 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKScatterFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKScatterFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -26,7 +26,6 @@ package org.broadinstitute.sting.queue.extensions.gatk import org.broadinstitute.sting.utils.interval.IntervalUtils import java.io.File -import collection.JavaConversions._ import org.broadinstitute.sting.utils.io.IOUtils import org.broadinstitute.sting.queue.function.scattergather.{CloneFunction, ScatterFunction} import org.broadinstitute.sting.commandline.Output @@ -39,7 +38,7 @@ trait GATKScatterFunction extends ScatterFunction { private final val intervalsStringField = "intervalsString" @Output(doc="Scatter function outputs") - var scatterOutputFiles: List[File] = Nil + var scatterOutputFiles: Seq[File] = Nil /** The original GATK function. */ protected var originalGATK: CommandLineGATK = _ @@ -48,7 +47,7 @@ trait GATKScatterFunction extends ScatterFunction { protected var referenceSequence: File = _ /** The list of interval files ("/path/to/interval.list") or interval strings ("chr1", "chr2") to parse into smaller parts. */ - protected var intervals: List[String] = Nil + protected var intervals: Seq[String] = Nil /** Whether the last scatter job should also include any unmapped reads. */ protected var includeUnmapped: Boolean = _ @@ -57,7 +56,7 @@ trait GATKScatterFunction extends ScatterFunction { this.originalGATK = this.originalFunction.asInstanceOf[CommandLineGATK] this.referenceSequence = this.originalGATK.reference_sequence if (this.originalGATK.intervals.isEmpty && (this.originalGATK.intervalsString == null || this.originalGATK.intervalsString.isEmpty)) { - this.intervals ++= GATKScatterFunction.getGATKIntervals(this.referenceSequence, List.empty[String]).contigs + this.intervals ++= GATKScatterFunction.getGATKIntervals(this.referenceSequence, Seq.empty[String]).contigs } else { this.intervals ++= this.originalGATK.intervals.map(_.toString) this.intervals ++= this.originalGATK.intervalsString.filterNot(interval => IntervalUtils.isUnmapped(interval)) @@ -70,16 +69,16 @@ trait GATKScatterFunction extends ScatterFunction { } override def initCloneInputs(cloneFunction: CloneFunction, index: Int) { - cloneFunction.setFieldValue(this.intervalsField, List(new File("scatter.intervals"))) + cloneFunction.setFieldValue(this.intervalsField, Seq(new File("scatter.intervals"))) if (index == this.scatterCount && this.includeUnmapped) - cloneFunction.setFieldValue(this.intervalsStringField, List("unmapped")) + cloneFunction.setFieldValue(this.intervalsStringField, Seq("unmapped")) else - cloneFunction.setFieldValue(this.intervalsStringField, List.empty[String]) + cloneFunction.setFieldValue(this.intervalsStringField, Seq.empty[String]) } override def bindCloneInputs(cloneFunction: CloneFunction, index: Int) { val scatterPart = cloneFunction.getFieldValue(this.intervalsField) - .asInstanceOf[List[File]] + .asInstanceOf[Seq[File]] .map(file => IOUtils.absolute(cloneFunction.commandDirectory, file)) cloneFunction.setFieldValue(this.intervalsField, scatterPart) this.scatterOutputFiles ++= scatterPart @@ -100,9 +99,9 @@ trait GATKScatterFunction extends ScatterFunction { } object GATKScatterFunction { - var gatkIntervals = List.empty[GATKIntervals] + var gatkIntervals = Seq.empty[GATKIntervals] - def getGATKIntervals(reference: File, intervals: List[String]) = { + def getGATKIntervals(reference: File, intervals: Seq[String]) = { gatkIntervals.find(gi => gi.reference == reference && gi.intervals == intervals) match { case Some(gi) => gi case None => diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/RodBind.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/RodBind.scala deleted file mode 100644 index deb83bf5a2..0000000000 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/RodBind.scala +++ /dev/null @@ -1,41 +0,0 @@ -package org.broadinstitute.sting.queue.extensions.gatk - -import java.io.File -import org.broadinstitute.sting.utils.io.FileExtension -import java.lang.String - -/** - * Used to provide -B rodBind arguments to the GATK. - */ -class RodBind(var trackName: String, var trackType: String, path: String, val tag: String) extends File(path) with FileExtension { - def this(trackName: String, trackType: String, path: String) = - this(trackName, trackType, path, null) - def this(trackName: String, trackType: String, file: File, tag: String) = - this(trackName, trackType, file.getPath, tag) - def this(trackName: String, trackType: String, file: File) = - this(trackName, trackType, file.getPath, null) - require(trackName != null, "RodBind trackName cannot be null") - require(trackType != null, "RodBind trackType cannot be null") - def withPath(newPath: String) = new RodBind(trackName, trackType, newPath, tag) -} - -/** - * Used to provide -B rodBind arguments to the GATK. - */ -object RodBind { - def apply(trackName: String, trackType: String, path: String, tag: String) = new RodBind(trackName, trackType, path, tag) - def apply(trackName: String, trackType: String, path: String) = new RodBind(trackName, trackType, path, null) - def apply(trackName: String, trackType: String, file: File, tag: String) = new RodBind(trackName, trackType, file, tag) - def apply(trackName: String, trackType: String, file: File) = new RodBind(trackName, trackType, file, null) - - def formatCommandLineParameter( cmdLineParam: String, value: Any ) = { - value match { - case rodBind: RodBind if (rodBind.tag != null) => - "%s:%s,%s,%s".format(cmdLineParam, rodBind.trackName, rodBind.trackType, rodBind.tag) - case rodBind: RodBind => - "%s:%s,%s".format(cmdLineParam, rodBind.trackName, rodBind.trackType) - case x => - "" - } - } -} diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/AddOrReplaceReadGroups.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/AddOrReplaceReadGroups.scala index 93735e4ac2..2faa659084 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/AddOrReplaceReadGroups.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/AddOrReplaceReadGroups.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.extensions.picard import org.broadinstitute.sting.commandline._ @@ -15,7 +39,7 @@ class AddOrReplaceReadGroups extends org.broadinstitute.sting.queue.function.Jav javaMainClass = "net.sf.picard.sam.AddOrReplaceReadGroups" @Input(doc="The input SAM or BAM files to analyze. Must be coordinate sorted.", shortName = "input", fullName = "input_bam_files", required = true) - var input: List[File] = Nil + var input: Seq[File] = Nil @Output(doc="The output BAM file with the modified/added read groups", shortName = "output", fullName = "output_bam_file", required = true) var output: File = _ diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MarkDuplicates.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MarkDuplicates.scala index d73c556af7..06c6e3fdc1 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MarkDuplicates.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MarkDuplicates.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.extensions.picard import org.broadinstitute.sting.commandline._ @@ -15,7 +39,7 @@ class MarkDuplicates extends org.broadinstitute.sting.queue.function.JavaCommand javaMainClass = "net.sf.picard.sam.MarkDuplicates" @Input(doc="The input SAM or BAM files to analyze. Must be coordinate sorted.", shortName = "input", fullName = "input_bam_files", required = true) - var input: List[File] = Nil + var input: Seq[File] = Nil @Output(doc="The output file to write marked records to", shortName = "output", fullName = "output_bam_file", required = true) var output: File = _ diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MergeSamFiles.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MergeSamFiles.scala index 036932cc68..8c23775775 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MergeSamFiles.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/MergeSamFiles.scala @@ -1,9 +1,32 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.extensions.picard import org.broadinstitute.sting.commandline._ import java.io.File -import org.broadinstitute.sting.queue.QScript._ /* * Created by IntelliJ IDEA. @@ -16,7 +39,7 @@ class MergeSamFiles extends org.broadinstitute.sting.queue.function.JavaCommandL javaMainClass = "net.sf.picard.sam.MergeSamFiles" @Input(doc="The input SAM or BAM files to analyze. Must be coordinate sorted.", shortName = "input", fullName = "input_bam_files", required = true) - var input: List[File] = Nil + var input: Seq[File] = Nil @Output(doc="The output merged BAM file", shortName = "output", fullName = "output_bam_file", required = true) var output: File = _ diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamFunction.scala index 76856dc366..defb43e4e8 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/PicardBamFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -41,20 +41,22 @@ trait PicardBamFunction extends JavaCommandLineFunction { var sortOrder = SortOrder.coordinate var compressionLevel: Option[Int] = None var createIndex: Option[Boolean] = None + var createMD5: Option[Boolean] = None var maxRecordsInRam: Option[Int] = None var assumeSorted: Option[Boolean] = None - protected def inputBams: List[File] + protected def inputBams: Seq[File] protected def outputBam: File abstract override def commandLine = super.commandLine + - repeat("INPUT=", inputBams, spaceSeparated=false) + - required("TMP_DIR=" + jobTempDir) + - optional("OUTPUT=", outputBam, spaceSeparated=false) + - optional("COMPRESSION_LEVEL=", compressionLevel, spaceSeparated=false) + - optional("VALIDATION_STRINGENCY=", validationStringency, spaceSeparated=false) + - optional("SO=", sortOrder, spaceSeparated=false) + - optional("MAX_RECORDS_IN_RAM=", maxRecordsInRam, spaceSeparated=false) + - optional("ASSUME_SORTED=", assumeSorted, spaceSeparated=false) + - optional("CREATE_INDEX=", createIndex, spaceSeparated=false) + repeat("INPUT=", inputBams, spaceSeparated=false) + + required("TMP_DIR=" + jobTempDir) + + optional("OUTPUT=", outputBam, spaceSeparated=false) + + optional("COMPRESSION_LEVEL=", compressionLevel, spaceSeparated=false) + + optional("VALIDATION_STRINGENCY=", validationStringency, spaceSeparated=false) + + optional("SO=", sortOrder, spaceSeparated=false) + + optional("MAX_RECORDS_IN_RAM=", maxRecordsInRam, spaceSeparated=false) + + optional("ASSUME_SORTED=", assumeSorted, spaceSeparated=false) + + optional("CREATE_INDEX=", createIndex, spaceSeparated=false) + + optional("CREATE_MD5_FILE=", createMD5, spaceSeparated=false) } diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ReorderSam.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ReorderSam.scala index b1968bee5a..46188586e7 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ReorderSam.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ReorderSam.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.extensions.picard import org.broadinstitute.sting.commandline._ @@ -14,7 +38,7 @@ class ReorderSam extends org.broadinstitute.sting.queue.function.JavaCommandLine javaMainClass = "net.sf.picard.sam.ReorderSam" @Input(doc="Input file (bam or sam) to extract reads from.", shortName = "input", fullName = "input_bam_files", required = true) - var input: List[File] = Nil + var input: Seq[File] = Nil @Output(doc="Output file (bam or sam) to write extracted reads to.", shortName = "output", fullName = "output_bam_file", required = true) var output: File = _ diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/RevertSam.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/RevertSam.scala index 60d8bfaf81..c2161b5518 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/RevertSam.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/RevertSam.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.extensions.picard import org.broadinstitute.sting.commandline._ @@ -15,7 +39,7 @@ class RevertSam extends org.broadinstitute.sting.queue.function.JavaCommandLineF javaMainClass = "net.sf.picard.sam.RevertSam" @Input(shortName = "input", fullName = "input_bam_files", required = true, doc = "The input SAM or BAM files to revert.") - var input: List[File] = Nil + var input: Seq[File] = Nil @Output(shortName = "output", fullName = "output_bam_file", required = true, doc = "The reverted BAM or SAM output file.") var output: File = _ @@ -33,7 +57,7 @@ class RevertSam extends org.broadinstitute.sting.queue.function.JavaCommandLineF var removeAlignmentInformation: Boolean = true @Argument(shortName = "atc", fullName = "attributes_to_clear", required = false, doc = "When removing alignment information, the set of optional tags to remove.") - var attributesToClear: List[String] = Nil + var attributesToClear: Seq[String] = Nil @Argument(shortName = "sa", fullName = "sample_alias", required = false, doc = "The sample alias to use in the reverted output file. This will override the existing sample alias in the file and is used only if all the read groups in the input file have the same sample alias.") var sampleAlias: String = null diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SamToFastq.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SamToFastq.scala index 3eb4e8e064..6c658b1055 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SamToFastq.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SamToFastq.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.extensions.picard import org.broadinstitute.sting.commandline._ @@ -15,7 +39,7 @@ class SamToFastq extends org.broadinstitute.sting.queue.function.JavaCommandLine javaMainClass = "net.sf.picard.sam.SamToFastq" @Input(shortName = "input", fullName = "input_bam_files", required = true, doc = "Input SAM/BAM file to extract reads from.") - var input: List[File] = Nil + var input: Seq[File] = Nil @Output(shortName = "fastq", fullName = "output_fastq_file", required = true, doc = "Output fastq file (single-end fastq or, if paired, first end of the pair fastq).") var fastq: File = _ diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SortSam.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SortSam.scala index a56093be8d..9257cc7c28 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SortSam.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/SortSam.scala @@ -1,9 +1,32 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.extensions.picard import org.broadinstitute.sting.commandline._ import java.io.File -import org.broadinstitute.sting.queue.QScript._ /* * Created by IntelliJ IDEA. @@ -16,7 +39,7 @@ class SortSam extends org.broadinstitute.sting.queue.function.JavaCommandLineFun javaMainClass = "net.sf.picard.sam.SortSam" @Input(doc="The input SAM or BAM files to sort.", shortName = "input", fullName = "input_bam_files", required = true) - var input: List[File] = Nil + var input: Seq[File] = Nil @Output(doc="The sorted BAM or SAM output file.", shortName = "output", fullName = "output_bam_file", required = true) var output: File = _ diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ValidateSamFile.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ValidateSamFile.scala index 030e4b07d3..43d4ab442a 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ValidateSamFile.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/picard/ValidateSamFile.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.extensions.picard import org.broadinstitute.sting.commandline._ @@ -17,7 +41,7 @@ class ValidateSamFile extends org.broadinstitute.sting.queue.function.JavaComman javaMainClass = "net.sf.picard.sam.ValidateSamFile" @Input(doc="The input SAM or BAM files to analyze. Must be coordinate sorted.", shortName = "input", fullName = "input_bam_files", required = true) - var input: List[File] = Nil + var input: Seq[File] = Nil @Output(doc="Send output to a file instead of stdout", shortName = "output", fullName = "output_file", required = false) var output: File = _ @@ -26,7 +50,7 @@ class ValidateSamFile extends org.broadinstitute.sting.queue.function.JavaComman var MODE: Mode = Mode.VERBOSE @Argument(doc="List of validation error types to ignore.", shortName = "ignore", fullName = "ignore_error_types", required = false) - var IGNORE: List[String] = Nil + var IGNORE: Seq[String] = Nil @Argument(doc = "The maximum number of lines output in verbose mode.", shortName = "max", fullName = "max_output", required = false) var MAX_OUTPUT: Int = 100 diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsIndexFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsIndexFunction.scala index 83a03b904e..1ad758b585 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsIndexFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsIndexFunction.scala @@ -52,6 +52,4 @@ class SamtoolsIndexFunction extends SamtoolsCommandLineFunction { required("index") + required(bamFile) + required(bamFileIndex) - - override def dotString = "Index: %s".format(bamFile.getName) } diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsMergeFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsMergeFunction.scala index aff9a25c0d..1949d9add8 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsMergeFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/samtools/SamtoolsMergeFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -34,7 +34,7 @@ class SamtoolsMergeFunction extends SamtoolsCommandLineFunction { analysisName = "samtools merge" @Input(doc="BAM file input") - var inputBams: List[File] = Nil + var inputBams: Seq[File] = Nil @Output(doc="BAM file output") var outputBam: File = _ @@ -43,10 +43,10 @@ class SamtoolsMergeFunction extends SamtoolsCommandLineFunction { var region: String = _ @Input(doc="BAM file input indexes") - var inputBamIndexes: List[File] = Nil + var inputBamIndexes: Seq[File] = Nil - override def freezeFieldValues = { - super.freezeFieldValues + override def freezeFieldValues() { + super.freezeFieldValues() inputBamIndexes ++= inputBams .filter(orig => orig != null && orig.getName.endsWith(".bam")) .flatMap(orig => Array( diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala index 167dcb593f..eff4a2ba91 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/CommandLineFunction.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.function import org.broadinstitute.sting.queue.util._ @@ -27,13 +51,13 @@ trait CommandLineFunction extends QFunction with Logging { var jobQueue: String = _ /** Native arguments to pass to the job runner */ - var jobNativeArgs: List[String] = Nil + var jobNativeArgs: Seq[String] = Nil /** Native arguments to pass to the job runner */ - var jobResourceRequests: List[String] = Nil + var jobResourceRequests: Seq[String] = Nil /** Environment names to pass to the job runner */ - var jobEnvironmentNames: List[String] = Nil + var jobEnvironmentNames: Seq[String] = Nil override def copySettingsTo(function: QFunction) { super.copySettingsTo(function) @@ -270,7 +294,7 @@ trait CommandLineFunction extends QFunction with Logging { } // Trim leading and trailing whitespace off our three tokens, and unwrap Some(x) to x for the param - val trimmedValues : List[String] = List((if ( prefix != null ) prefix.trim else ""), + val trimmedValues : Seq[String] = Seq((if ( prefix != null ) prefix.trim else ""), (param match { case Some(x) => paramFormat.format(x).trim case x => paramFormat.format(x).trim diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/InProcessFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/InProcessFunction.scala index 783eef1bfb..653b87b2f9 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/InProcessFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/InProcessFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -24,10 +24,24 @@ package org.broadinstitute.sting.queue.function +import java.io.PrintStream + + /** * Runs a function in process. */ trait InProcessFunction extends QFunction { + analysisName = this.getClass.getSimpleName + def run() - def description = this.getClass.getSimpleName + " " + this.commandOutputs.mkString(" ") + + /** + * During run() this stream will write to the stdout. + */ + var jobOutputStream: PrintStream = null + + /** + * Write errors to this stream run(). + */ + var jobErrorStream: PrintStream = null } diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/JavaCommandLineFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/JavaCommandLineFunction.scala index 5b19cf9b66..534d68069c 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/JavaCommandLineFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/JavaCommandLineFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -42,7 +42,7 @@ trait JavaCommandLineFunction extends CommandLineFunction { * Class path for the main class. * Defaults to the current classpath. */ - var javaClasspath: List[String] = Nil + var javaClasspath: Seq[String] = Nil /** * Memory limit for the java executable, or if None will use the default memoryLimit. @@ -82,5 +82,5 @@ trait JavaCommandLineFunction extends CommandLineFunction { object JavaCommandLineFunction { val currentClasspath = System.getProperty("java.class.path") - .split(File.pathSeparatorChar).map(path => IOUtils.absolute(new File(path)).getPath).toList + .split(File.pathSeparatorChar).map(path => IOUtils.absolute(new File(path)).getPath).toSeq } diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/ListWriterFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/ListWriterFunction.scala index f60302ef48..becc64f04c 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/ListWriterFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/ListWriterFunction.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.function import org.broadinstitute.sting.commandline.{Input, Output} @@ -9,10 +33,12 @@ import org.apache.commons.io.IOUtils * Custom formats can override addFile. */ class ListWriterFunction extends InProcessFunction { - @Input(doc="input files") var inputFiles: List[File] = Nil + analysisName = "WriteList" + + @Input(doc="input files") var inputFiles: Seq[File] = Nil @Output(doc="output file") var listFile: File = _ - def run { + def run() { val writer = new PrintWriter(listFile) try { for (inputFile <- inputFiles) diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala index 59f2ada446..dee1acfacc 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -29,7 +29,7 @@ import java.lang.annotation.Annotation import org.broadinstitute.sting.commandline._ import org.broadinstitute.sting.queue.{QException, QSettings} import collection.JavaConversions._ -import org.broadinstitute.sting.queue.function.scattergather.SimpleTextGatherFunction +import java.lang.IllegalStateException import org.broadinstitute.sting.queue.util._ import org.broadinstitute.sting.utils.io.IOUtils @@ -39,13 +39,18 @@ import org.broadinstitute.sting.utils.io.IOUtils * Inputs are matched to other outputs by using .equals() */ trait QFunction extends Logging with QJobReport { - /** A short description of this step in the graph */ + /** + * A short description of what this class of function does. + * By default does not include the output specific to this function. + * See shortDescription for a description of what this instance of the function outputs. + */ var analysisName: String = "" - /** Prefix for automatic job name creation */ - var jobNamePrefix: String = _ - - /** The name name of the job */ + /** + * The name name of the job, must be file system safe and unique to the graph. + * Defaults to "runName-". + * Use shortDescription for an alternative that is display friendly. + */ var jobName: String = _ /** Default settings */ @@ -58,7 +63,7 @@ trait QFunction extends Logging with QJobReport { var jobTempDir: File = null /** Order the function was added to the graph. */ - var addOrder: List[Int] = Nil + var addOrder: Seq[Int] = Nil /** Job priority */ var jobPriority: Option[Int] = None @@ -78,12 +83,6 @@ trait QFunction extends Logging with QJobReport { */ var isIntermediate = false - /** - * If true and isIntermediate is true, the files listed - * via outputs will deleted after the command completes. - */ - var deleteIntermediateOutputs = true - // ------------------------------------------------------- // // job run information @@ -95,8 +94,6 @@ trait QFunction extends Logging with QJobReport { * @param function QFunction to copy values to. */ override def copySettingsTo(function: QFunction) { - function.analysisName = this.analysisName - function.jobName = this.jobName function.qSettings = this.qSettings function.commandDirectory = this.commandDirectory function.jobTempDir = this.jobTempDir @@ -105,79 +102,80 @@ trait QFunction extends Logging with QJobReport { function.jobRestartable = this.jobRestartable function.updateJobRun = this.updateJobRun function.isIntermediate = this.isIntermediate - function.deleteIntermediateOutputs = this.deleteIntermediateOutputs function.reportGroup = this.reportGroup function.reportFeatures = this.reportFeatures } /** File to redirect any output. Defaults to .out */ - @Output(doc="File to redirect any output", required=false) - @Gather(classOf[SimpleTextGatherFunction]) var jobOutputFile: File = _ /** File to redirect any errors. Defaults to .out */ - @Output(doc="File to redirect any errors", required=false) - @Gather(classOf[SimpleTextGatherFunction]) var jobErrorFile: File = _ /** * Description of this command line function. */ - def description: String + def description: String = "%s: %s > %s".format(analysisName, inputs, outputs) /** - * The function description in .dot files + * A short description of the function. */ - def dotString = jobName + " => " + description + def shortDescription = { + firstOutput match { + case file: File => analysisName + ": " + file.getName + case _ => analysisName + } + } /** - * Returns true if the function is done, false if it's - * not done and None if the done status is unknown. + * Returns true if the function is done. */ - def isDone = { + def isDone: Boolean = { val files = doneOutputs if (files.size == 0) - None - else - Some(files.forall(_.exists)) + throw new IllegalStateException("Function should have at least one output: " + analysisName) + files.forall(_.exists) } /** - * Returns true if the function has failed, false if it - * has not failed and None if the fail status is unknown. + * Returns true if the function has failed. */ - def isFail = { + def isFail: Boolean = { val files = failOutputs if (files.size == 0) - None - else - Some(files.exists(_.exists)) + throw new IllegalStateException("Function should have at least one output: " + analysisName) + files.exists(_.exists) } /** - * Returns true if the file is a log file for this function. + * Returns files to track for hidden done/fail files. + * @return Seq[String] files. */ - protected def isLogFile(file: File) = - file == jobOutputFile || file == jobErrorFile + protected def statusPaths = { + var paths = outputs + paths :+= jobOutputFile + if (jobErrorFile != null) + paths :+= jobErrorFile + paths + } /** - * Returns the output files for this function. - * @return Set[File] outputs for this function. + * Returns prefixes for hidden done/fail files. + * @return prefixes. */ - private def statusPaths = - commandOutputs.map(file => file.getParentFile + "/." + file.getName) - + private def statusPrefixes = statusPaths.map(file => file.getParentFile + "/." + file.getName) + /** * Returns the output files for this function. - * @return Set[File] outputs for this function. + * @return outputs for this function. */ - def doneOutputs = statusPaths.map(path => new File(path + ".done")) + def doneOutputs: Seq[File] = statusPrefixes.map(path => new File(path + ".done")) /** * Returns the output files for this function. - * @return Set[File] outputs for this function. + * @return outputs for this function. */ - def failOutputs = statusPaths.map(path => new File(path + ".fail")) + def failOutputs: Seq[File] = statusPrefixes.map(path => new File(path + ".fail")) /** The complete list of fields on this CommandLineFunction. */ def functionFields = QFunction.classFields(this.functionFieldClass).functionFields @@ -195,21 +193,21 @@ trait QFunction extends Logging with QJobReport { /** * Returns the input files for this function. - * @return Set[File] inputs for this function. + * @return inputs for this function. */ - def inputs = getFieldFiles(inputFields) + def inputs: Seq[File] = getFieldFiles(inputFields) /** * Returns the output files for this function. - * @return Set[File] outputs for this function. + * @return outputs for this function. */ - def outputs = getFieldFiles(outputFields) + def outputs: Seq[File] = getFieldFiles(outputFields) /** - * Returns the non-log outputs for this function. - * @return the non-log outputs for this function. + * Returns the first output file. + * @return first output for this function. */ - def commandOutputs = outputs.filterNot(file => isLogFile(file)) + def firstOutput: File = outputs.headOption.getOrElse(null) /** * Returns the set of directories where files may be written. @@ -218,6 +216,9 @@ trait QFunction extends Logging with QJobReport { var dirs = Set.empty[File] dirs += commandDirectory dirs += jobTempDir + dirs += jobOutputFile.getParentFile + if (jobErrorFile != null) + dirs += jobErrorFile.getParentFile dirs ++= outputs.map(_.getParentFile) dirs } @@ -235,7 +236,7 @@ trait QFunction extends Logging with QJobReport { * Deletes the output files and all the status files for this function. */ def deleteOutputs() { - commandOutputs.foreach(file => IOUtils.tryDelete(file)) + outputs.foreach(file => IOUtils.tryDelete(file)) doneOutputs.foreach(file => IOUtils.tryDelete(file)) failOutputs.foreach(file => IOUtils.tryDelete(file)) } @@ -252,63 +253,63 @@ trait QFunction extends Logging with QJobReport { /** * Returns fields that do not have values which are required. - * @return List[String] names of fields missing values. + * @return Seq[String] names of fields missing values. */ - def missingFields: List[String] = { + def missingFields: Seq[String] = { val missingInputs = missingFields(inputFields, classOf[Input]) val missingOutputs = missingFields(outputFields, classOf[Output]) val missingArguments = missingFields(argumentFields, classOf[Argument]) - (missingInputs | missingOutputs | missingArguments).toList.sorted + (missingInputs ++ missingOutputs ++ missingArguments).distinct.sorted } /** * Returns fields that do not have values which are required. * @param sources Fields to check. * @param annotation Annotation. - * @return Set[String] names of fields missing values. + * @return names of fields missing values. */ - private def missingFields(sources: List[ArgumentSource], annotation: Class[_ <: Annotation]): Set[String] = { - var missing = Set.empty[String] + private def missingFields(sources: Seq[ArgumentSource], annotation: Class[_ <: Annotation]): Seq[String] = { + var missing: Seq[String] = Nil for (source <- sources) { if (isRequired(source, annotation)) if (!hasFieldValue(source)) if (!exclusiveOf(source, annotation).exists(otherSource => hasFieldValue(otherSource))) - missing += "@%s: %s - %s".format(annotation.getSimpleName, source.field.getName, doc(source, annotation)) + missing :+= "@%s: %s - %s".format(annotation.getSimpleName, source.field.getName, doc(source, annotation)) } missing } /** - * Gets the files from the fields. The fields must be a File, a FileExtension, or a List or Set of either. + * Gets the files from the fields. The fields must be a File, a FileExtension, or a Seq or Set of either. * @param fields Fields to get files. - * @return Set[File] for the fields. + * @return for the fields. */ - private def getFieldFiles(fields: List[ArgumentSource]): Set[File] = { - var files = Set.empty[File] + private def getFieldFiles(fields: Seq[ArgumentSource]): Seq[File] = { + var files: Seq[File] = Nil for (field <- fields) files ++= getFieldFiles(field) - files + files.distinct } /** - * Gets the files from the field. The field must be a File, a FileExtension, or a List or Set of either. - * @param fields Field to get files. - * @return Set[File] for the field. + * Gets the files from the field. The field must be a File, a FileExtension, or a Seq or Set of either. + * @param field Field to get files. + * @return for the field. */ - def getFieldFiles(field: ArgumentSource): Set[File] = { - var files = Set.empty[File] + def getFieldFiles(field: ArgumentSource): Seq[File] = { + var files: Seq[File] = Nil CollectionUtils.foreach(getFieldValue(field), (fieldValue) => { val file = fieldValueToFile(field, fieldValue) if (file != null) - files += file + files :+= file }) - files + files.distinct } /** - * Gets the file from the field. The field must be a File or a FileExtension and not a List or Set. + * Gets the file from the field. The field must be a File or a FileExtension and not a Seq or Set. * @param field Field to get the file. - * @return File for the field. + * @return for the field. */ def getFieldFile(field: ArgumentSource): File = fieldValueToFile(field, getFieldValue(field)) @@ -340,14 +341,15 @@ trait QFunction extends Logging with QJobReport { * Sets all field values. */ def freezeFieldValues() { - if (jobNamePrefix == null) - jobNamePrefix = qSettings.jobNamePrefix - if (jobName == null) - jobName = QFunction.nextJobName(jobNamePrefix) + jobName = qSettings.runName + "-" + this.addOrder.mkString("-") - if (jobOutputFile == null) - jobOutputFile = new File(jobName + ".out") + if (jobOutputFile == null) { + jobOutputFile = firstOutput match { + case file: File => new File(file.getParentFile, file.getName + ".out") + case _ => new File(jobName + ".out") + } + } if (jobTempDir == null) jobTempDir = qSettings.tempDirectory @@ -378,6 +380,10 @@ trait QFunction extends Logging with QJobReport { fieldValue = CollectionUtils.updated(fieldValue, canon).asInstanceOf[AnyRef] this.setFieldValue(field, fieldValue) } + + this.jobOutputFile = canon(this.jobOutputFile).asInstanceOf[File] + if (this.jobErrorFile != null) + this.jobErrorFile = canon(this.jobErrorFile).asInstanceOf[File] } /** @@ -443,7 +449,7 @@ trait QFunction extends Logging with QJobReport { /** * Returns false if the value is null or an empty collection. - * @param value Value to test for null, or a collection to test if it is empty. + * @param param Value to test for null, or a collection to test if it is empty. * @return false if the value is null, or false if the collection is empty, otherwise true. */ protected def hasValue(param: Any) = CollectionUtils.isNotNullOrNotEmpty(param) @@ -472,28 +478,15 @@ trait QFunction extends Logging with QJobReport { } object QFunction { - /** Job index counter for this run of Queue. */ - private var jobIndex = 0 - var parsingEngine: ParsingEngine = _ - /** - * Returns the next job name using the prefix. - * @param prefix Prefix of the job name. - * @return the next job name. - */ - private def nextJobName(prefix: String) = { - jobIndex += 1 - prefix + "-" + jobIndex - } - /** * The list of fields defined on a class * @param clazz The class to lookup fields. */ private class ClassFields(clazz: Class[_]) { /** The complete list of fields on this CommandLineFunction. */ - val functionFields: List[ArgumentSource] = parsingEngine.extractArgumentSources(clazz).toList + val functionFields: Seq[ArgumentSource] = parsingEngine.extractArgumentSources(clazz).toSeq /** The @Input fields on this CommandLineFunction. */ val inputFields = functionFields.filter(source => ReflectionUtils.hasAnnotation(source.field, classOf[Input])) /** The @Output fields on this CommandLineFunction. */ diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/CloneFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/CloneFunction.scala index b5cef3d5c2..5b4f2b7e6d 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/CloneFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/CloneFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -25,7 +25,6 @@ package org.broadinstitute.sting.queue.function.scattergather import org.broadinstitute.sting.commandline.ArgumentSource -import java.io.File import org.broadinstitute.sting.queue.function.{QFunction, CommandLineFunction} /** @@ -62,9 +61,8 @@ class CloneFunction extends CommandLineFunction { } } - override def commandOutputs = withScatterPart(() => originalFunction.commandOutputs) - override def dotString = withScatterPart(() => originalFunction.dotString) override def description = withScatterPart(() => originalFunction.description) + override def shortDescription = withScatterPart(() => originalFunction.shortDescription) override protected def functionFieldClass = originalFunction.getClass def commandLine = withScatterPart(() => originalFunction.commandLine) @@ -75,30 +73,22 @@ class CloneFunction extends CommandLineFunction { } override def getFieldValue(source: ArgumentSource): AnyRef = { - source.field.getName match { - case "jobOutputFile" => jobOutputFile - case "jobErrorFile" => jobErrorFile - case _ => overriddenFields.get(source) match { - case Some(value) => value.asInstanceOf[AnyRef] - case None => { - val value = originalFunction.getFieldValue(source) - overriddenFields += source -> value - value - } + overriddenFields.get(source) match { + case Some(value) => value.asInstanceOf[AnyRef] + case None => { + val value = originalFunction.getFieldValue(source) + overriddenFields += source -> value + value } } } - def setFieldValue(field: String, value: Any): Unit = { + def setFieldValue(field: String, value: Any) { val source = QFunction.findField(originalFunction.getClass, field) setFieldValue(source, value) } - override def setFieldValue(source: ArgumentSource, value: Any): Unit = { - source.field.getName match { - case "jobOutputFile" => jobOutputFile = value.asInstanceOf[File] - case "jobErrorFile" => jobErrorFile = value.asInstanceOf[File] - case _ => overriddenFields += source -> value - } + override def setFieldValue(source: ArgumentSource, value: Any) { + overriddenFields += source -> value } } diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/AutoIndexGatherFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ConcatenateLogsFunction.scala similarity index 53% rename from public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/AutoIndexGatherFunction.scala rename to public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ConcatenateLogsFunction.scala index 7fb96e0741..9261dd7674 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/AutoIndexGatherFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ConcatenateLogsFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -22,15 +22,34 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -package org.broadinstitute.sting.queue.extensions.gatk +package org.broadinstitute.sting.queue.function.scattergather -import org.broadinstitute.sting.queue.function.scattergather.GatherFunction import org.broadinstitute.sting.queue.function.InProcessFunction +import org.broadinstitute.sting.queue.QException +import org.broadinstitute.sting.commandline.Input +import org.apache.commons.io.FileUtils +import java.io.File +import collection.JavaConversions._ /** - * A no-op for index files that were automatically generated during the gather step. - * TODO: Allow graph to know that this isn't needed, and/or that one gather job can actually gather N-outputs, and/or look more into generic source->sinks. + * Concatenate log files to the jobOutputFile. */ -class AutoIndexGatherFunction extends InProcessFunction with GatherFunction { - def run() {} +class ConcatenateLogsFunction extends InProcessFunction { + analysisName = "Concat" + + @Input(doc="Parts to gather back into the original output") + var logs: Seq[File] = Nil + + override def description = "%s: %s > %s".format(analysisName, logs, jobOutputFile) + override def shortDescription = analysisName + ": " + jobOutputFile.getName + + def run() { + val missing = org.broadinstitute.sting.utils.io.IOUtils.waitFor(logs, 120) + if (!missing.isEmpty) + throw new QException("Unable to find log: " + missing.mkString(", ")) + logs.foreach(log => { + FileUtils.copyFile(log, this.jobOutputStream) + this.jobOutputStream.println() + }) + } } diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GatherFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GatherFunction.scala index 6b8b5d143f..c8b9d52fbe 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GatherFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GatherFunction.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.function.scattergather import java.io.File @@ -11,22 +35,31 @@ import collection.JavaConversions._ * Base class for Gather command line functions. */ trait GatherFunction extends QFunction { + analysisName = "Gather" + var originalFunction: ScatterGatherableFunction = _ @Output(doc="The original output of the scattered function") var originalOutput: File = _ @Input(doc="Parts to gather back into the original output") - var gatherParts: List[File] = Nil - - @Input(doc="Other log files that will be gathered before this output", required=false) - var originalLogFiles: List[File] = Nil + var gatherParts: Seq[File] = Nil /** * Called to initialize the gather function values after all other values have been setup for this function. */ def init() {} + /** + * Don't include this @Gather's log file when tracking .done. + * The done files for original log file being produced will do. + * + * The logs from the scatter/gather jobs are concatenated together into the original log. + * Without removing the logs a .done file would be created for the logs. If a SGF is switched + * from scatterCount=1 to >1 then this Gather would be "missing" its logs and re-run. + */ + override protected def statusPaths = outputs + /** * Waits for gather parts to propagate over NFS or throws an exception. */ diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GathererFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GathererFunction.scala index c1204fd1d3..536bbf5fc6 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GathererFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/GathererFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -32,11 +32,12 @@ import collection.JavaConversions._ * Runs a Gatherer in process. */ class GathererFunction(gathererClass: Class[_ <: Gatherer]) extends InProcessFunction with GatherFunction { + analysisName = this.gathererClass.getSimpleName + def run() { val gatherer = gathererClass.newInstance if (gatherer.waitForInputs) - waitForGatherParts + waitForGatherParts() gatherer.gather(this.gatherParts, this.originalOutput) } - override def description = this.gathererClass.getSimpleName + " " + this.commandOutputs.mkString(" ") } diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterFunction.scala index 632e2d39fe..a407476719 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterFunction.scala @@ -32,14 +32,17 @@ import org.broadinstitute.sting.queue.function.QFunction * Base class for Scatter functions. */ trait ScatterFunction extends QFunction { + analysisName = "Scatter" + var originalFunction: ScatterGatherableFunction = _ @Input(doc="Original inputs to scatter") var originalInputs: Set[File] = _ + override def shortDescription = analysisName + ": %s ...".format(firstOutput.getName) + /** * Called to initialize scatter function values after all other values have been setup for this function. - * @param originalFunction The original function to with inputs bind to this scatter function. */ def init() {} diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala index 402da4a7a2..921928bce0 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -30,6 +30,7 @@ import org.broadinstitute.sting.commandline.{Gatherer, Gather, ArgumentSource} import org.broadinstitute.sting.queue.function.{QFunction, CommandLineFunction} import org.broadinstitute.sting.queue.QException import org.broadinstitute.sting.utils.io.IOUtils +import collection.immutable.ListMap /** * A function that can be run faster by splitting it up into pieces and then joining together the results. @@ -47,28 +48,28 @@ trait ScatterGatherableFunction extends CommandLineFunction { /** * Function that returns the class to use for gathering a directory. If it returns null then @Gather annotation will be used. - * @param gatherField Field that is to be gathered. + * PartialFunction param gatherField Field that is to be gathered. * @return The class of the GatherFunction to be used or null. */ var gatherClass: PartialFunction[ArgumentSource, Class[_ <: GatherFunction]] = _ /** * Allows external modification of the ScatterFunction that will create the scatter pieces in the temporary directories. - * @param scatterFunction The function that will create the scatter pieces in the temporary directories. + * PartialFunction param scatterFunction The function that will create the scatter pieces in the temporary directories. */ var setupScatterFunction: PartialFunction[ScatterFunction, Unit] = _ /** * Allows external modification of the GatherFunction that will collect the gather pieces in the temporary directories. - * @param gatherFunction The function that will merge the gather pieces from the temporary directories. - * @param gatherField The output field being gathered. + * PartialFunction param gatherFunction The function that will merge the gather pieces from the temporary directories. + * PartialFunction param gatherField The output field being gathered. */ var setupGatherFunction: PartialFunction[(GatherFunction, ArgumentSource), Unit] = _ /** * Allows external modification of the cloned function. - * @param cloneFunction A clone wrapper of this ScatterGatherableFunction - * @param index The one based index (from 1..scatterCount inclusive) of the scatter piece. + * PartialFunction param cloneFunction A clone wrapper of this ScatterGatherableFunction + * PartialFunction param index The one based index (from 1..scatterCount inclusive) of the scatter piece. */ var setupCloneFunction: PartialFunction[(CloneFunction, Int), Unit] = _ @@ -108,8 +109,9 @@ trait ScatterGatherableFunction extends CommandLineFunction { scatterFunction.originalFunction = this scatterFunction.originalInputs = inputFiles scatterFunction.commandDirectory = this.scatterGatherTempDir("scatter") - scatterFunction.isIntermediate = true + scatterFunction.jobOutputFile = new File("scatter.out") scatterFunction.addOrder = this.addOrder :+ 1 + scatterFunction.isIntermediate = true initScatterFunction(scatterFunction) scatterFunction.absoluteCommandDirectory() @@ -121,69 +123,61 @@ trait ScatterGatherableFunction extends CommandLineFunction { * Returns a list of scatter / gather and clones of this function * that can be run in parallel to produce the same output as this * command line function. - * @return List[QFunction] to run instead of this function. + * @return Seq[QFunction] to run instead of this function. */ def generateFunctions() = { - var functions = List.empty[QFunction] - - // Only gather up fields that will have a value - val outputFieldsWithValues = this.outputFields.filter(hasFieldValue(_)) - - // Create the scatter function based on @Scatter - functions :+= scatterFunction - // Ask the scatter function how many clones to create. val numClones = scatterFunction.scatterCount - // List of the log files that are output by this function. - var logFiles = List(this.jobOutputFile) - if (this.jobErrorFile != null) - logFiles :+= this.jobErrorFile - // Create the gather functions for each output field - var gatherFunctions = Map.empty[ArgumentSource, GatherFunction] - var gatherOutputs = Map.empty[ArgumentSource, File] + var gatherFunctions = ListMap.empty[ArgumentSource, GatherFunction] + var gatherOutputs = ListMap.empty[ArgumentSource, File] var gatherAddOrder = numClones + 2 + + // Only track fields that will have a value + val outputFieldsWithValues = this.outputFields.filter(hasFieldValue(_)) + for (gatherField <- outputFieldsWithValues) { - val gatherOutput = getFieldFile(gatherField) + gatherOutputs += gatherField -> getFieldFile(gatherField) + } + + // Only gather fields that are @Gather(enabled=true) + val outputFieldsWithGathers = outputFieldsWithValues.filter(hasGatherFunction(_)) + + for (gatherField <- outputFieldsWithGathers) { + val gatherOutput = gatherOutputs(gatherField) val gatherFunction = this.newGatherFunction(gatherField) this.copySettingsTo(gatherFunction) gatherFunction.originalFunction = this gatherFunction.originalOutput = gatherOutput gatherFunction.commandDirectory = this.scatterGatherTempDir("gather-" + gatherField.field.getName) - // If this is a gather for a log file, make the gather intermediate just in case the log file name changes - // Otherwise have the regular output function wait on the log files to gather - if (isLogFile(gatherOutput)) { - gatherFunction.isIntermediate = true - // Only delete the log files if the original function is an intermediate - // and the intermediate files are supposed to be deleted - gatherFunction.deleteIntermediateOutputs = this.isIntermediate && this.deleteIntermediateOutputs - } else { - gatherFunction.originalLogFiles = logFiles - } + gatherFunction.jobOutputFile = new File("gather-" + gatherOutput.getName + ".out") gatherFunction.addOrder = this.addOrder :+ gatherAddOrder initGatherFunction(gatherFunction, gatherField) gatherFunction.absoluteCommandDirectory() gatherFunction.init() - functions :+= gatherFunction gatherFunctions += gatherField -> gatherFunction - gatherOutputs += gatherField -> gatherOutput gatherAddOrder += 1 } // Create the clone functions for running the parallel jobs - var cloneFunctions = List.empty[CloneFunction] + var cloneFunctions = Seq.empty[CloneFunction] + val dirFormat = "temp_%%0%dd_of_%d".format(numClones.toString.length(), numClones) for (i <- 1 to numClones) { val cloneFunction = this.newCloneFunction() this.copySettingsTo(cloneFunction) cloneFunction.originalFunction = this + cloneFunction.analysisName = this.analysisName cloneFunction.cloneIndex = i - cloneFunction.commandDirectory = this.scatterGatherTempDir("temp-"+i) + cloneFunction.commandDirectory = this.scatterGatherTempDir(dirFormat.format(i)) + cloneFunction.jobOutputFile = new File(this.jobOutputFile.getName) + if (this.jobErrorFile != null) + cloneFunction.jobErrorFile = new File(this.jobErrorFile.getName) cloneFunction.addOrder = this.addOrder :+ (i+1) cloneFunction.isIntermediate = true @@ -200,17 +194,39 @@ trait ScatterGatherableFunction extends CommandLineFunction { // If the command directory is relative, insert the run directory ahead of it. cloneFunction.absoluteCommandDirectory() - // Get absolute paths to the files and bind the sg functions to the clone function via the absolute paths. + // Allow the scatter function to set the specific input for this clone scatterFunction.bindCloneInputs(cloneFunction, i) + + // Set each of the clone outputs to be absolute paths. for (gatherField <- outputFieldsWithValues) { val gatherPart = IOUtils.absolute(cloneFunction.commandDirectory, cloneFunction.getFieldFile(gatherField)) cloneFunction.setFieldValue(gatherField, gatherPart) - gatherFunctions(gatherField).gatherParts :+= gatherPart + } + + // For the outputs that are being gathered add this clone's output to be gathered. + for (gatherField <- outputFieldsWithGathers) { + gatherFunctions(gatherField).gatherParts :+= cloneFunction.getFieldFile(gatherField) } cloneFunctions :+= cloneFunction } - functions ++= cloneFunctions + + // Track the functions starting with the scatter function. + var functions: Seq[QFunction] = Seq(scatterFunction) ++ cloneFunctions ++ gatherFunctions.values + + // Make all log file paths absolute. + for (function <- functions) { + function.jobOutputFile = IOUtils.absolute(function.commandDirectory, function.jobOutputFile) + if (function.jobErrorFile != null) + function.jobErrorFile = IOUtils.absolute(function.commandDirectory, function.jobErrorFile) + } + + val jobOutputGather = gatherLogFile(_.jobOutputFile, functions, gatherAddOrder) + if (this.jobErrorFile != null) { + val jobErrorGather = gatherLogFile(_.jobErrorFile, functions, gatherAddOrder + 1) + functions :+= jobErrorGather + } + functions :+= jobOutputGather // Return all the various created functions. functions @@ -237,6 +253,25 @@ trait ScatterGatherableFunction extends CommandLineFunction { this.setupScatterFunction(scatterFunction) } + /** + * Returns true if the field should be gathered. + * @param gatherField Field that defined @Gather. + * @return true if the field should be gathered. + */ + protected def hasGatherFunction(gatherField: ArgumentSource) : Boolean = { + // Check if there is a function that will return the gather class for this field. + if (this.gatherClass != null && this.gatherClass.isDefinedAt(gatherField)) + true + + // Check for an annotation defining the gather class. + else if (ReflectionUtils.hasAnnotation(gatherField.field, classOf[Gather])) + ReflectionUtils.getAnnotation(gatherField.field, classOf[Gather]).enabled + + // Nothing else to disable this field. + else + true + } + /** * Creates a new GatherFunction for the gatherField. * @param gatherField Field that defined @Gather. @@ -255,16 +290,18 @@ trait ScatterGatherableFunction extends CommandLineFunction { if (ReflectionUtils.hasAnnotation(gatherField.field, classOf[Gather])) { gatherClass = ReflectionUtils.getAnnotation(gatherField.field, classOf[Gather]).value } else { - throw new QException("Missing @Gather annotation: " + gatherField.field) + throw new QException("Missing @Gather annotation on %s".format(gatherField.field)) } } - if (classOf[GatherFunction].isAssignableFrom(gatherClass)) { + if (gatherClass == classOf[GatherFunction]) { + throw new QException("@Gather did not specify class type on %s".format(gatherField.field)) + } else if (classOf[GatherFunction].isAssignableFrom(gatherClass)) { gatherClass.newInstance.asInstanceOf[GatherFunction] } else if (classOf[Gatherer].isAssignableFrom(gatherClass)) { new GathererFunction(gatherClass.asSubclass(classOf[Gatherer])) } else { - throw new QException("Unsupported @Gather class type: " + gatherClass) + throw new QException("Unsupported @Gather class type on %s: %s".format(gatherField.field, gatherClass)) } } @@ -298,10 +335,27 @@ trait ScatterGatherableFunction extends CommandLineFunction { this.setupCloneFunction(cloneFunction, index) } + /** + * Gathers up the logs files from other functions. + * @param logFile Takes the QFunction and return the log file. + * @param functions The functions for which the logs will be concatenated. + * @param addOrder The order this function should be added in the graph. + */ + private def gatherLogFile(logFile: (QFunction) => File, functions: Seq[QFunction], addOrder: Int) = { + val gatherLogFunction = new ConcatenateLogsFunction + this.copySettingsTo(gatherLogFunction) + gatherLogFunction.logs = functions.map(logFile).filter(_ != null) + gatherLogFunction.jobOutputFile = logFile(this) + gatherLogFunction.commandDirectory = this.scatterGatherTempDir() + gatherLogFunction.addOrder = this.addOrder :+ addOrder + gatherLogFunction.isIntermediate = false + gatherLogFunction + } + /** * Returns a temporary directory under this scatter gather directory. - * @param Sub directory under the scatter gather directory. + * @param subDir directory under the scatter gather directory. * @return temporary directory under this scatter gather directory. */ - private def scatterGatherTempDir(subDir: String) = IOUtils.absolute(this.scatterGatherDirectory, this.jobName + "-sg/" + subDir) + private def scatterGatherTempDir(subDir: String = "") = IOUtils.absolute(this.scatterGatherDirectory, this.jobName + "-sg/" + subDir) } diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala b/public/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala index cda981d29c..2ef7aa06f0 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/util/EmailMessage.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -34,12 +34,12 @@ import scala.collection.JavaConversions._ */ class EmailMessage extends Logging { var from: String = _ - var to: List[String] = Nil - var cc: List[String] = Nil - var bcc: List[String] = Nil + var to: Seq[String] = Nil + var cc: Seq[String] = Nil + var bcc: Seq[String] = Nil var subject: String = _ var body: String = _ - var attachments: List[File] = Nil + var attachments: Seq[File] = Nil /** * Sends the email and throws an exception if the email can't be sent. @@ -111,10 +111,10 @@ class EmailMessage extends Logging { /** * Converts the email addresses to a collection of InternetAddress which can bypass client side validation, * specifically that the domain is specified. - * @param addresses List of email addresses. + * @param addresses Seq of email addresses. * @return java.util.List of InternetAddress'es */ - private def convert(addresses: List[String]): java.util.List[InternetAddress] = { + private def convert(addresses: Seq[String]): java.util.List[InternetAddress] = { addresses.map(address => new InternetAddress(address, false)) } diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/QJobReport.scala b/public/scala/src/org/broadinstitute/sting/queue/util/QJobReport.scala index 73d1c028a2..e548e5c5e1 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/util/QJobReport.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/util/QJobReport.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -28,9 +28,10 @@ import org.broadinstitute.sting.queue.function.QFunction import org.broadinstitute.sting.gatk.report.{GATKReportTable, GATKReport} import org.broadinstitute.sting.utils.exceptions.UserException import org.broadinstitute.sting.queue.engine.JobRunInfo -import java.io.{FileOutputStream, PrintStream, File} +import java.io.{PrintStream, File} import org.broadinstitute.sting.utils.R.{RScriptLibrary, RScriptExecutor} import org.broadinstitute.sting.utils.io.Resource +import org.apache.commons.io.{IOUtils, FileUtils} /** * A mixin to add Job info to the class @@ -67,7 +68,7 @@ trait QJobReport extends Logging { def getReportGroup = self.analysisName.replaceAll(GATKReportTable.INVALID_TABLE_NAME_REGEX, "_") def getReportFeatures = reportFeatures - def getReportFeatureNames: List[String] = getReportFeatures.keys.toList + def getReportFeatureNames: Seq[String] = getReportFeatures.keys.toSeq def getReportFeature(key: String): String = { getReportFeatures.get(key) match { case Some(x) => x @@ -102,9 +103,12 @@ object QJobReport { def printReport(jobsRaw: Map[QFunction, JobRunInfo], dest: File) { val jobs = jobsRaw.filter(_._2.isFilledIn).filter(_._1.includeInReport) jobs foreach {case (qf, info) => qf.setRunInfo(info)} - val stream = new PrintStream(new FileOutputStream(dest)) - printJobLogging(jobs.keys.toList, stream) - stream.close() + val stream = new PrintStream(FileUtils.openOutputStream(dest)) + try { + printJobLogging(jobs.keys.toSeq, stream) + } finally { + IOUtils.closeQuietly(stream) + } } def plotReport(reportFile: File, pdfFile: File) { @@ -129,7 +133,7 @@ object QJobReport { * Prints the JobLogging logs to a GATKReport. First splits up the * logs by group, and for each group generates a GATKReportTable */ - private def printJobLogging(logs: List[QFunction], stream: PrintStream) { + private def printJobLogging(logs: Seq[QFunction], stream: PrintStream) { // create the report val report: GATKReport = new GATKReport @@ -151,11 +155,11 @@ object QJobReport { report.print(stream) } - private def groupLogs(logs: List[QFunction]): Map[String, List[QFunction]] = { + private def groupLogs(logs: Seq[QFunction]): Map[String, Seq[QFunction]] = { logs.groupBy(_.getReportGroup) } - private def logKeys(logs: List[QFunction]): Set[String] = { + private def logKeys(logs: Seq[QFunction]): Set[String] = { // the keys should be the same for each log, but we will check that val keys = Set[String](logs(0).getReportFeatureNames : _*) diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala b/public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala index 3b1b2ece15..5d76f39ed7 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.util import java.io.File @@ -12,23 +36,22 @@ import collection.JavaConversions._ * User: carneiro * Date: 7/14/11 * Time: 4:57 PM - * To change this template use File | Settings | File Templates. */ object QScriptUtils { /** - * Takes a bam list file and produces a scala list with each file allowing the bam list + * Takes a bam list file and produces a scala sequence with each file allowing the bam list * to have empty lines and comment lines (lines starting with #). */ - def createListFromFile(in: File):List[File] = { + def createSeqFromFile(in: File):Seq[File] = { // If the file provided ends with .bam, .fasta or .fq, it is not a bam list, we treat it as a single file. // and return a list with only this file. if (in.toString.endsWith(".bam") || in.toString.endsWith(".fasta") || in.toString.endsWith(".fq")) - return List(in) + return Seq(in) - var list: List[File] = List() - for (file <- fromFile(in).getLines) + var list: Seq[File] = Seq() + for (file <- fromFile(in).getLines()) if (!file.startsWith("#") && !file.isEmpty ) list :+= new File(file.trim()) list.sortWith(_.compareTo(_) < 0) @@ -55,8 +78,4 @@ object QScriptUtils { } false } - - - def ?[A <: AnyRef](ref: A): Option[A] = - if (ref eq null) None else Some(ref) -} \ No newline at end of file +} diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/ReflectionUtils.scala b/public/scala/src/org/broadinstitute/sting/queue/util/ReflectionUtils.scala index f6a174dd66..980a22e8e4 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/util/ReflectionUtils.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/util/ReflectionUtils.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.util import org.broadinstitute.sting.queue.QException @@ -64,17 +88,17 @@ object ReflectionUtils { /** * Returns all the declared fields on a class in order of sub type to super type. * @param clazz Base class to start looking for fields. - * @return List[Field] found on the class and all super classes. + * @return Seq[Field] found on the class and all super classes. */ - def getAllFields(clazz: Class[_]) = getAllTypes(clazz).map(_.getDeclaredFields).flatMap(_.toList) + def getAllFields(clazz: Class[_]) = getAllTypes(clazz).map(_.getDeclaredFields).flatMap(_.toSeq) /** * Gets all the types on a class in order of sub type to super type. * @param clazz Base class. - * @return List[Class] including the class and all super classes. + * @return Seq[Class] including the class and all super classes. */ def getAllTypes(clazz: Class[_]) = { - var types = List.empty[Class[_]] + var types = Seq.empty[Class[_]] var c = clazz while (c != null) { types :+= c diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala b/public/scala/src/org/broadinstitute/sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala index 58341a0a5e..6b615e6d9d 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/util/ScalaCompoundArgumentTypeDescriptor.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.util import collection.JavaConversions._ @@ -14,32 +38,34 @@ class ScalaCompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor { /** * Checks if the class type is a scala collection. * @param classType Class type to check. - * @return true if the class is a List, Set, or an Option. + * @return true if the class is a Seq, Set, or an Option. */ def supports(classType: Class[_]) = isCompound(classType) /** * Checks if the class type is a scala collection. * @param source Argument source to check. - * @return true if the source is a List, Set, or an Option. + * @return true if the source is a Seq, Set, or an Option. */ override def isMultiValued(source: ArgumentSource) = isCompound(source.field.getType) /** * Checks if the class type is a scala collection. * @param classType Class type to check. - * @return true if the class is a List, Set, or an Option. + * @return true if the class is a Seq, Set, or an Option. */ private def isCompound(classType: Class[_]) = { - classOf[List[_]].isAssignableFrom(classType) || + classOf[Seq[_]].isAssignableFrom(classType) || + classOf[List[_]].isAssignableFrom(classType) || // see comment below re: List vs. Seq classOf[Set[_]].isAssignableFrom(classType) || classOf[Option[_]].isAssignableFrom(classType) } /** * Parses the argument matches based on the class type of the argument source's field. + * @param parsingEngine Parsing engine. * @param source Argument source that contains the field being populated. - * @param classType Class type being parsed. + * @param typeType Type of the argument source's field. * @param argumentMatches The argument match strings that were found for this argument source. * @return The parsed object. */ @@ -51,7 +77,15 @@ class ScalaCompoundArgumentTypeDescriptor extends ArgumentTypeDescriptor { val componentType = ReflectionUtils.getCollectionType(source.field) val componentArgumentParser = parsingEngine.selectBestTypeDescriptor(componentType) - if (classOf[List[_]].isAssignableFrom(classType)) { + if (classOf[Seq[_]].isAssignableFrom(classType)) { + var seq = Seq.empty[Any] + for (argumentMatch <- argumentMatches) + for (value <- argumentMatch) + seq :+= componentArgumentParser.parse(parsingEngine, source, componentType, new ArgumentMatches(value)) + seq + } else if (classOf[List[_]].isAssignableFrom(classType)) { + // QScripts should be using the interface Seq instead of the class List. + // Leaving this here for now for legacy support until the effects of switching have been tested for a while. -ks var list = List.empty[Any] for (argumentMatch <- argumentMatches) for (value <- argumentMatch) diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/StringFileConversions.scala b/public/scala/src/org/broadinstitute/sting/queue/util/StringFileConversions.scala index 62240b6045..2c6d62ae91 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/util/StringFileConversions.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/util/StringFileConversions.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -44,7 +44,7 @@ object StringFileConversions { // and mixins all correct so this doesn't have to be duplicated with concrete implementations? // http://programming-scala.labs.oreilly.com/ch12.html is your friend. - implicit def stringsAsFiles(x: List[Comparable[_ >: String with File <: Comparable[_ >: String with File <: Serializable] with Serializable] with Serializable]): List[File] = { + implicit def stringsAsFiles(x: Seq[Comparable[_ >: String with File <: Comparable[_ >: String with File <: Serializable] with Serializable] with Serializable]): Seq[File] = { x.map(_ match { case string: String => stringAsFile(string) case file: File => file @@ -52,7 +52,23 @@ object StringFileConversions { }) } - implicit def filesAsStrings(x: List[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable]): List[String] = { + implicit def filesAsStrings(x: Seq[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable]): Seq[String] = { + x.map(_ match { + case file: File => fileAsString(file) + case string: String => string + case null => null + }) + } + + implicit def stringsAsFilesList(x: List[Comparable[_ >: String with File <: Comparable[_ >: String with File <: Serializable] with Serializable] with Serializable]): List[File] = { + x.map(_ match { + case string: String => stringAsFile(string) + case file: File => file + case null => null + }) + } + + implicit def filesAsStringsList(x: List[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable]): List[String] = { x.map(_ match { case file: File => fileAsString(file) case string: String => string @@ -91,14 +107,22 @@ trait StringFileConversions { StringFileConversions.fileAsString(x) } - implicit def stringsAsFiles(x: List[Comparable[_ >: String with File <: Comparable[_ >: String with File <: Serializable] with Serializable] with Serializable]): List[File] = { + implicit def stringsAsFiles(x: Seq[Comparable[_ >: String with File <: Comparable[_ >: String with File <: Serializable] with Serializable] with Serializable]): Seq[File] = { StringFileConversions.stringsAsFiles(x) } - implicit def filesAsStrings(x: List[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable]): List[String] = { + implicit def filesAsStrings(x: Seq[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable]): Seq[String] = { StringFileConversions.filesAsStrings(x) } + implicit def stringsAsFilesList(x: List[Comparable[_ >: String with File <: Comparable[_ >: String with File <: Serializable] with Serializable] with Serializable]): List[File] = { + StringFileConversions.stringsAsFilesList(x) + } + + implicit def filesAsStringsList(x: List[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable]): List[String] = { + StringFileConversions.filesAsStringsList(x) + } + implicit def stringsAsFiles(x: Set[Comparable[_ >: File with String <: Comparable[_ >: File with String <: Comparable[_ >: File with String <: Serializable] with Serializable] with Serializable] with Serializable]): Set[File] = { StringFileConversions.stringsAsFiles(x) } diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/SystemUtils.scala b/public/scala/src/org/broadinstitute/sting/queue/util/SystemUtils.scala index 9002def78f..ed149f8a43 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/util/SystemUtils.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/util/SystemUtils.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -24,7 +24,6 @@ package org.broadinstitute.sting.queue.util -import java.lang.management.ManagementFactory import java.net.InetAddress import java.io.File import io.Source @@ -56,6 +55,4 @@ object SystemUtils extends Logging { else hostName.split('.').takeRight(2).mkString(".") } - - val pidAtHost = ManagementFactory.getRuntimeMXBean.getName.split('.').head } diff --git a/public/scala/test/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervalsUnitTest.scala b/public/scala/test/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervalsUnitTest.scala index 38abe24efb..db0d187c9d 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervalsUnitTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervalsUnitTest.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -40,9 +40,6 @@ class GATKIntervalsUnitTest { createSetFromSequenceDictionary(new ReferenceDataSource(hg18Reference).getReference.getSequenceDictionary).toList private final lazy val hg19Reference = new File(BaseTest.hg19Reference) - private final lazy val hg19GenomeLocParser = new GenomeLocParser(new CachingIndexedFastaSequenceFile(hg19Reference)) - private final lazy val hg19ReferenceLocs = GenomeLocSortedSet. - createSetFromSequenceDictionary(new ReferenceDataSource(hg19Reference).getReference.getSequenceDictionary).toList @Test def testWithIntervals() { @@ -50,16 +47,14 @@ class GATKIntervalsUnitTest { val chr2 = hg18GenomeLocParser.parseGenomeLoc("chr2:2-3") val chr3 = hg18GenomeLocParser.parseGenomeLoc("chr3:3-5") - val gi = new GATKIntervals(hg18Reference, List("chr1:1-1", "chr2:2-3", "chr3:3-5")) - Assert.assertEquals(gi.locs.toList, List(chr1, chr2, chr3)) - Assert.assertEquals(gi.contigs, List("chr1", "chr2", "chr3")) -// Assert.assertEquals(gi.getSplits(2).toList, List(2, 3)) -// Assert.assertEquals(gi.getSplits(3).toList, List(1, 2, 3)) + val gi = new GATKIntervals(hg18Reference, Seq("chr1:1-1", "chr2:2-3", "chr3:3-5")) + Assert.assertEquals(gi.locs.toSeq, Seq(chr1, chr2, chr3)) + Assert.assertEquals(gi.contigs, Seq("chr1", "chr2", "chr3")) } - @Test(timeOut = 30000) + @Test(timeOut = 30000L) def testIntervalFile() { - var gi = new GATKIntervals(hg19Reference, List(BaseTest.hg19Intervals)) + var gi = new GATKIntervals(hg19Reference, Seq(BaseTest.hg19Intervals)) Assert.assertEquals(gi.locs.size, 189894) // Timeout check is because of bad: // for(Item item: javaConvertedScalaList) @@ -74,14 +69,12 @@ class GATKIntervalsUnitTest { val gi = new GATKIntervals(hg18Reference, Nil) Assert.assertEquals(gi.locs, hg18ReferenceLocs) Assert.assertEquals(gi.contigs.size, hg18ReferenceLocs.size) -// Assert.assertEquals(gi.getSplits(2).toList, List(10, 45)) -// Assert.assertEquals(gi.getSplits(4).toList, List(5, 10, 16, 45)) } @Test def testContigCounts() { Assert.assertEquals(new GATKIntervals(hg18Reference, Nil).contigs, hg18ReferenceLocs.map(_.getContig)) - Assert.assertEquals(new GATKIntervals(hg18Reference, List("chr1", "chr2", "chr3")).contigs, List("chr1", "chr2", "chr3")) - Assert.assertEquals(new GATKIntervals(hg18Reference, List("chr1:1-2", "chr1:4-5", "chr2:1-1", "chr3:2-2")).contigs, List("chr1", "chr2", "chr3")) + Assert.assertEquals(new GATKIntervals(hg18Reference, Seq("chr1", "chr2", "chr3")).contigs, Seq("chr1", "chr2", "chr3")) + Assert.assertEquals(new GATKIntervals(hg18Reference, Seq("chr1:1-2", "chr1:4-5", "chr2:1-1", "chr3:2-2")).contigs, Seq("chr1", "chr2", "chr3")) } } diff --git a/public/scala/test/org/broadinstitute/sting/queue/function/CommandLineFunctionUnitTest.scala b/public/scala/test/org/broadinstitute/sting/queue/function/CommandLineFunctionUnitTest.scala index eb50c3a2ef..9c5b648d28 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/function/CommandLineFunctionUnitTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/function/CommandLineFunctionUnitTest.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.function import org.testng.Assert @@ -114,20 +138,20 @@ class CommandLineFunctionUnitTest extends CommandLineFunction { @DataProvider( name = "repeatTestData" ) def repeatDataProvider = { - Array(Array("", List("a", "bc", "d"), "", " ", true, true, " 'a' 'bc' 'd' "), - Array("", List("a", "bc", "d"), "", " ", true, false, " a bc d "), - Array("", List("a", "bc", "d"), "", "", true, true, " 'a''bc''d' "), - Array("", List("a", "bc", "d"), "", "", true, false, " abcd "), - Array("-f", List("file1", "file2", "file3"), "", " ", true, true, " '-f' 'file1' '-f' 'file2' '-f' 'file3' "), - Array("-f", List("file1", "file2", "file3"), "", " ", true, false, " -f file1 -f file2 -f file3 "), - Array("-f", List("file1", "file2", "file3"), "", " ", false, true, " '-ffile1' '-ffile2' '-ffile3' "), - Array("-f", List("file1", "file2", "file3"), "", " ", false, false, " -ffile1 -ffile2 -ffile3 "), - Array("-f", List("file1", "file2", "file3"), "", "", false, true, " '-ffile1''-ffile2''-ffile3' "), - Array("-f", List("file1", "file2", "file3"), "", "", false, false, " -ffile1-ffile2-ffile3 "), - Array("-f", List("file1", "file2", "file3"), "suffix", " ", true, true, " '-f' 'file1' 'suffix' '-f' 'file2' 'suffix' '-f' 'file3' 'suffix' "), - Array("-f", List("file1", "file2", "file3"), "suffix", " ", true, false, " -f file1 suffix -f file2 suffix -f file3 suffix "), - Array("-f", List("file1", "file2", "file3"), "suffix", " ", false, true, " '-ffile1suffix' '-ffile2suffix' '-ffile3suffix' "), - Array("-f", List("file1", "file2", "file3"), "suffix", " ", false, false, " -ffile1suffix -ffile2suffix -ffile3suffix "), + Array(Array("", Seq("a", "bc", "d"), "", " ", true, true, " 'a' 'bc' 'd' "), + Array("", Seq("a", "bc", "d"), "", " ", true, false, " a bc d "), + Array("", Seq("a", "bc", "d"), "", "", true, true, " 'a''bc''d' "), + Array("", Seq("a", "bc", "d"), "", "", true, false, " abcd "), + Array("-f", Seq("file1", "file2", "file3"), "", " ", true, true, " '-f' 'file1' '-f' 'file2' '-f' 'file3' "), + Array("-f", Seq("file1", "file2", "file3"), "", " ", true, false, " -f file1 -f file2 -f file3 "), + Array("-f", Seq("file1", "file2", "file3"), "", " ", false, true, " '-ffile1' '-ffile2' '-ffile3' "), + Array("-f", Seq("file1", "file2", "file3"), "", " ", false, false, " -ffile1 -ffile2 -ffile3 "), + Array("-f", Seq("file1", "file2", "file3"), "", "", false, true, " '-ffile1''-ffile2''-ffile3' "), + Array("-f", Seq("file1", "file2", "file3"), "", "", false, false, " -ffile1-ffile2-ffile3 "), + Array("-f", Seq("file1", "file2", "file3"), "suffix", " ", true, true, " '-f' 'file1' 'suffix' '-f' 'file2' 'suffix' '-f' 'file3' 'suffix' "), + Array("-f", Seq("file1", "file2", "file3"), "suffix", " ", true, false, " -f file1 suffix -f file2 suffix -f file3 suffix "), + Array("-f", Seq("file1", "file2", "file3"), "suffix", " ", false, true, " '-ffile1suffix' '-ffile2suffix' '-ffile3suffix' "), + Array("-f", Seq("file1", "file2", "file3"), "suffix", " ", false, false, " -ffile1suffix -ffile2suffix -ffile3suffix "), Array("-f", null, "", " ", true, true, ""), Array("-f", Nil, "", " ", true, true, "") ) @@ -148,11 +172,11 @@ class CommandLineFunctionUnitTest extends CommandLineFunction { @DataProvider( name = "repeatWithPrefixFormattingTestData" ) def repeatWithPrefixFormattingDataProvider = { - Array(Array("-f", List("file1", "file2", "file3"), "", " ", true, true, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value), + Array(Array("-f", Seq("file1", "file2", "file3"), "", " ", true, true, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value), " '-f:tagfile1' 'file1' '-f:tagfile2' 'file2' '-f:tagfile3' 'file3' "), - Array("-f", List("file1", "file2", "file3"), "", " ", true, false, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value), + Array("-f", Seq("file1", "file2", "file3"), "", " ", true, false, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value), " -f:tagfile1 file1 -f:tagfile2 file2 -f:tagfile3 file3 "), - Array("", List("file1", "file2", "file3"), "", " ", true, true, (prefix: String, value: Any) => "-%s".format(value), + Array("", Seq("file1", "file2", "file3"), "", " ", true, true, (prefix: String, value: Any) => "-%s".format(value), " '-file1' 'file1' '-file2' 'file2' '-file3' 'file3' "), Array("-f", null, "", " ", true, true, (prefix: String, value: Any) => "%s:tag%s".format(prefix, value), ""), diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala index aedbc1cd39..f0feb207b8 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTest.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -47,10 +47,10 @@ object PipelineTest extends BaseTest with Logging { final val allJobRunners = { val commandLinePluginManager = new CommandLinePluginManager - commandLinePluginManager.getPlugins.map(commandLinePluginManager.getName(_)).toList + commandLinePluginManager.getPlugins.map(commandLinePluginManager.getName(_)).toSeq } - final val defaultJobRunners = List("Lsf706", "GridEngine") + final val defaultJobRunners = Seq("Lsf706", "GridEngine") /** * Returns the top level output path to this test. diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTestEvalSpec.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTestEvalSpec.scala index 33b8c1c392..3996f2ca3a 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTestEvalSpec.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTestEvalSpec.scala @@ -1,14 +1,38 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.pipeline /** * Data validations to evaluate on a GATKReport. */ class PipelineTestEvalSpec { - /** List of eval modules to output. */ + /** Eval modules to output. */ var evalReport: String = _ /** Validations to assert. */ - var validations: List[PipelineValidation[_]] = Nil + var validations: Seq[PipelineValidation[_]] = Nil } /** A VariantEval JEXL and range of values to validate. */ diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTestSpec.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTestSpec.scala index a7b3f3a47f..0900246982 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTestSpec.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/PipelineTestSpec.scala @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.queue.pipeline class PipelineTestSpec(var name: String = null) { @@ -9,7 +33,7 @@ class PipelineTestSpec(var name: String = null) { var jobQueue: String = _ /** Job runners to run the test. Default is null which means use the default. */ - var jobRunners: List[String] = _ + var jobRunners: Seq[String] = _ /** Expected MD5 results for each file path. */ var fileMD5s = Map.empty[String, String] diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/HelloWorldPipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/HelloWorldPipelineTest.scala index f320cb3a6a..a43727ba60 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/HelloWorldPipelineTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/HelloWorldPipelineTest.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -38,11 +38,11 @@ class HelloWorldPipelineTest { } @Test - def testHelloWorldWithPrefix() { + def testHelloWorldWithRunName() { val spec = new PipelineTestSpec - spec.name = "HelloWorldWithPrefix" + spec.name = "HelloWorldWithRunName" spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" + - " -jobPrefix HelloWorld" + " -runName HelloWorld" spec.jobRunners = PipelineTest.allJobRunners PipelineTest.executeTest(spec) } @@ -73,7 +73,7 @@ class HelloWorldPipelineTest { spec.name = "HelloWorldWithLsfResource" spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" + " -jobResReq rusage[iodine_io=1] -jobResReq select[swp>0] -jobResReq order[swp]" - spec.jobRunners = List("Lsf706") + spec.jobRunners = Seq("Lsf706") PipelineTest.executeTest(spec) } @@ -83,7 +83,7 @@ class HelloWorldPipelineTest { spec.name = "HelloWorldWithLsfResourceAndMemoryLimit" spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" + " -memLimit 1.25 -jobResReq rusage[iodine_io=1] -jobResReq select[swp>0] -jobResReq order[swp]" - spec.jobRunners = List("Lsf706") + spec.jobRunners = Seq("Lsf706") PipelineTest.executeTest(spec) } @@ -93,7 +93,7 @@ class HelloWorldPipelineTest { spec.name = "HelloWorldWithLsfEnvironment" spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" + " -jobEnv tv" - spec.jobRunners = List("Lsf706") + spec.jobRunners = Seq("Lsf706") PipelineTest.executeTest(spec) } @@ -103,7 +103,7 @@ class HelloWorldPipelineTest { spec.name = "HelloWorldWithGridEngineResource" spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" + " -jobResReq s_core=1000M" - spec.jobRunners = List("GridEngine") + spec.jobRunners = Seq("GridEngine") PipelineTest.executeTest(spec) } @@ -113,7 +113,7 @@ class HelloWorldPipelineTest { spec.name = "HelloWorldWithGridEngineResourceAndMemoryLimit" spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" + " -memLimit 1.25 -jobResReq s_core=1000M" - spec.jobRunners = List("GridEngine") + spec.jobRunners = Seq("GridEngine") PipelineTest.executeTest(spec) } @@ -123,7 +123,7 @@ class HelloWorldPipelineTest { spec.name = "HelloWorldWithGridEngineEnvironment" spec.args = "-S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/HelloWorld.scala" + " -jobEnv \"make 1\"" - spec.jobRunners = List("GridEngine") + spec.jobRunners = Seq("GridEngine") PipelineTest.executeTest(spec) } } diff --git a/public/scala/test/org/broadinstitute/sting/queue/util/StringFileConversionsUnitTest.scala b/public/scala/test/org/broadinstitute/sting/queue/util/StringFileConversionsUnitTest.scala index a735edebee..4d364040a8 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/util/StringFileConversionsUnitTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/util/StringFileConversionsUnitTest.scala @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -50,40 +50,40 @@ class StringFileConversionsUnitTest { @Test def testStringToFileList() { - var files = List(new File("foo")) + var files = Seq(new File("foo")) files :+= "bar" - Assert.assertEquals(files, List(new File("foo"), new File("bar"))) + Assert.assertEquals(files, Seq(new File("foo"), new File("bar"))) - files = List(new File("foo")) + files = Seq(new File("foo")) files :+= null.asInstanceOf[String] - Assert.assertEquals(files, List(new File("foo"), null)) + Assert.assertEquals(files, Seq(new File("foo"), null)) - files = List[File](null) + files = Seq[File](null) files :+= "foo" - Assert.assertEquals(files, List(null, new File("foo"))) + Assert.assertEquals(files, Seq(null, new File("foo"))) - files = List[File](null) + files = Seq[File](null) files :+= null.asInstanceOf[String] - Assert.assertEquals(files, List(null, null)) + Assert.assertEquals(files, Seq(null, null)) } @Test def testFileToStringList() { - var strings = List("foo") + var strings = Seq("foo") strings :+= new File("bar") - Assert.assertEquals(strings, List("foo", "bar")) + Assert.assertEquals(strings, Seq("foo", "bar")) - strings = List("foo") + strings = Seq("foo") strings :+= null.asInstanceOf[File] - Assert.assertEquals(strings, List("foo", null)) + Assert.assertEquals(strings, Seq("foo", null)) - strings = List[String](null) + strings = Seq[String](null) strings :+= new File("foo") - Assert.assertEquals(strings, List(null, "foo")) + Assert.assertEquals(strings, Seq(null, "foo")) - strings = List[String](null) + strings = Seq[String](null) strings :+= null.asInstanceOf[File] - Assert.assertEquals(strings, List(null, null)) + Assert.assertEquals(strings, Seq(null, null)) } @Test @@ -126,40 +126,40 @@ class StringFileConversionsUnitTest { @Test def testStringListToFileList() { - var files = List(new File("foo")) - files ++= List("bar") - Assert.assertEquals(files, List(new File("foo"), new File("bar"))) + var files = Seq(new File("foo")) + files ++= Seq("bar") + Assert.assertEquals(files, Seq(new File("foo"), new File("bar"))) - files = List(new File("foo")) - files ++= List[String](null) - Assert.assertEquals(files, List(new File("foo"), null)) + files = Seq(new File("foo")) + files ++= Seq[String](null) + Assert.assertEquals(files, Seq(new File("foo"), null)) - files = List[File](null) - files ++= List("foo") - Assert.assertEquals(files, List(null, new File("foo"))) + files = Seq[File](null) + files ++= Seq("foo") + Assert.assertEquals(files, Seq(null, new File("foo"))) - files = List[File](null) - files ++= List[String](null) - Assert.assertEquals(files, List(null, null)) + files = Seq[File](null) + files ++= Seq[String](null) + Assert.assertEquals(files, Seq(null, null)) } @Test def testFileListToStringList() { - var strings = List("foo") - strings ++= List(new File("bar")) - Assert.assertEquals(strings, List("foo", "bar")) + var strings = Seq("foo") + strings ++= Seq(new File("bar")) + Assert.assertEquals(strings, Seq("foo", "bar")) - strings = List("foo") - strings ++= List[File](null) - Assert.assertEquals(strings, List("foo", null)) + strings = Seq("foo") + strings ++= Seq[File](null) + Assert.assertEquals(strings, Seq("foo", null)) - strings = List[String](null) - strings ++= List(new File("foo")) - Assert.assertEquals(strings, List(null, "foo")) + strings = Seq[String](null) + strings ++= Seq(new File("foo")) + Assert.assertEquals(strings, Seq(null, "foo")) - strings = List[String](null) - strings ++= List[File](null) - Assert.assertEquals(strings, List(null, null)) + strings = Seq[String](null) + strings ++= Seq[File](null) + Assert.assertEquals(strings, Seq(null, null)) } @Test From 1ddac59a49da129c5767833c62ab69043a15a749 Mon Sep 17 00:00:00 2001 From: Menachem Fromer Date: Sun, 8 Jan 2012 12:22:18 -0500 Subject: [PATCH 032/356] Added alpha version of Exome CNV calling pipeline script. To run it, you would need to checkout and compile our C++ code by 'git clone /psych/genetics_data/projects/seq/exome/CNV/git_master/xhmm', though this is not yet recommended since this process is all still preliminary From f741ec6c6a453c84a5888eae8d50c93b6f04e159 Mon Sep 17 00:00:00 2001 From: Menachem Fromer Date: Sun, 8 Jan 2012 12:51:50 -0500 Subject: [PATCH 033/356] Replaced dotFile with shortDescription, as per Khalid's latest update From 6f2abd76df30a2b90f6f90bf538cc45c715ef3dd Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Mon, 9 Jan 2012 15:31:18 -0500 Subject: [PATCH 034/356] Updating the MDCP with the new indel gold standard from Ryan. --- .../queue/qscripts/MethodsDevelopmentCallingPipeline.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala index 67cafe99f5..b50bf3d674 100755 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala @@ -92,7 +92,7 @@ class MethodsDevelopmentCallingPipeline extends QScript { val training_1000G = "/humgen/1kg/processing/official_release/phase1/projectConsensus/phase1.wgs.projectConsensus.v2b.recal.highQuality.vcf" val badSites_1000G = "/humgen/1kg/processing/official_release/phase1/projectConsensus/phase1.wgs.projectConsensus.v2b.recal.terrible.vcf" val projectConsensus_1000G = "/humgen/1kg/processing/official_release/phase1/projectConsensus/ALL.wgs.projectConsensus_v2b.20101123.snps.sites.vcf" - val millsDevine_b37 = "/humgen/gsa-hpprojects/GATK/bundle/current/b37/Mills_Devine_2hit.indels.b37.sites.vcf" + val indelGoldStandardCallset = "/humgen/gsa-hpprojects/GATK/data/Comparisons/Unvalidated/GoldStandardIndel/gold.standard.indel.MillsAnd1000G.b37.vcf" val lowPass: Boolean = true val exome: Boolean = true @@ -300,7 +300,7 @@ class MethodsDevelopmentCallingPipeline extends QScript { class indelRecal(t: Target) extends VQSRBase(t) with UNIVERSAL_GATK_ARGS { this.input :+= t.rawIndelVCF - this.resource :+= new TaggedFile( millsDevine_b37, "known=true,training=true,truth=true,prior=12.0" ) + this.resource :+= new TaggedFile( indelGoldStandardCallset, "known=true,training=true,truth=true,prior=12.0" ) this.use_annotation ++= List("QD", "HaplotypeScore", "ReadPosRankSum", "FS") if(t.nSamples >= 10) this.use_annotation ++= List("InbreedingCoeff") // InbreedingCoeff is a population-wide statistic that requires at least 10 samples to calculate @@ -365,7 +365,7 @@ class MethodsDevelopmentCallingPipeline extends QScript { // 5b.) Indel Evaluation (OPTIONAL) class indelEvaluation(t: Target) extends EvalBase(t) { this.eval :+= t.recalibratedIndelVCF - this.comp :+= new TaggedFile(millsDevine_b37, "mills" ) + this.comp :+= new TaggedFile(indelGoldStandardCallset, "indelGS" ) this.noEV = true this.evalModule = List("CompOverlap", "CountVariants", "TiTvVariantEvaluator", "ValidationReport", "IndelStatistics") this.out = t.evalIndelFile From 6b9dcaf9797fbeb92815bce12028758264745239 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Mon, 9 Jan 2012 16:53:48 -0500 Subject: [PATCH 035/356] added a -o option to AssessLikelihoodsAtTruth From 133739a76eba6a26a4ac00b557e64ce8a160bb3d Mon Sep 17 00:00:00 2001 From: Menachem Fromer Date: Mon, 9 Jan 2012 18:17:39 -0500 Subject: [PATCH 036/356] Add option to run the longer components on a different queue From f2cecce10f0bf9e0d1c015884708e79a6ec24be2 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Tue, 10 Jan 2012 11:34:23 -0500 Subject: [PATCH 037/356] Much better implementation of the approximate summing of an array of log10 values (including more efficient rounding). Now effectively takes 0% of UG runtime on T2D GENES (as opposed to 11% previously). --- .../genotyper/ExactAFCalculationModel.java | 44 ++++++++++++------- .../UnifiedGenotyperIntegrationTest.java | 4 +- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java index aa743f86fc..295d3f9f01 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java @@ -70,15 +70,25 @@ private static final ArrayList getGLs(GenotypesContext GLs) { return genotypeLikelihoods; } - final static double approximateLog10SumLog10(double[] vals) { - if ( vals.length < 2 ) - throw new ReviewedStingException("Passing array with fewer than 2 values when computing approximateLog10SumLog10"); - double approx = approximateLog10SumLog10(vals[0], vals[1]); - for ( int i = 2; i < vals.length; i++ ) - approx = approximateLog10SumLog10(approx, vals[i]); - return approx; + final int maxElementIndex = MathUtils.maxElementIndex(vals); + double approxSum = vals[maxElementIndex]; + if ( approxSum == Double.NEGATIVE_INFINITY ) + return approxSum; + + for ( int i = 0; i < vals.length; i++ ) { + if ( i == maxElementIndex || vals[i] == Double.NEGATIVE_INFINITY ) + continue; + + final double diff = approxSum - vals[i]; + if ( diff < MathUtils.MAX_JACOBIAN_TOLERANCE ) { + final int ind = fastRound(diff / MathUtils.JACOBIAN_LOG_TABLE_STEP); // hard rounding + approxSum += MathUtils.jacobianLogTable[ind]; + } + } + + return approxSum; } final static double approximateLog10SumLog10(double small, double big) { @@ -89,27 +99,29 @@ final static double approximateLog10SumLog10(double small, double big) { small = t; } - if (small == Double.NEGATIVE_INFINITY || big == Double.NEGATIVE_INFINITY ) + if ( small == Double.NEGATIVE_INFINITY || big == Double.NEGATIVE_INFINITY ) return big; - if (big >= small + MathUtils.MAX_JACOBIAN_TOLERANCE) + final double diff = big - small; + if ( diff >= MathUtils.MAX_JACOBIAN_TOLERANCE ) return big; // OK, so |y-x| < tol: we use the following identity then: // we need to compute log10(10^x + 10^y) // By Jacobian logarithm identity, this is equal to // max(x,y) + log10(1+10^-abs(x-y)) - // we compute the second term as a table lookup - // with integer quantization + // we compute the second term as a table lookup with integer quantization // we have pre-stored correction for 0,0.1,0.2,... 10.0 - //final int ind = (int)(((big-small)/JACOBIAN_LOG_TABLE_STEP)); // hard rounding - int ind = (int)(Math.round((big-small)/MathUtils.JACOBIAN_LOG_TABLE_STEP)); // hard rounding - - //double z =Math.log10(1+Math.pow(10.0,-diff)); - //System.out.format("x: %f, y:%f, app: %f, true: %f ind:%d\n",x,y,t2,z,ind); + final int ind = fastRound(diff / MathUtils.JACOBIAN_LOG_TABLE_STEP); // hard rounding return big + MathUtils.jacobianLogTable[ind]; } + // A fast implementation of the Math.round() method. This method does not perform + // under/overflow checking, so this shouldn't be used in the general case (but is fine + // here because we already make those checks before calling in to the rounding). + final static int fastRound(double d) { + return (d > 0) ? (int)(d + 0.5d) : (int)(d - 0.5d); + } // ------------------------------------------------------------------------------------- // diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java index 7c0dba5585..646ede836f 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java @@ -28,7 +28,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest { public void testMultiSamplePilot1() { WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec( baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000", 1, - Arrays.asList("66ed60c6c1190754abd8a0a9d1d8d61e")); + Arrays.asList("d61c7055bd09024abb8902bde6bd3960")); executeTest("test MultiSample Pilot1", spec); } @@ -294,7 +294,7 @@ public void testWithIndelAllelesPassedIn4() { WalkerTest.WalkerTestSpec spec4 = new WalkerTest.WalkerTestSpec( baseCommandIndelsb37 + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "ALL.wgs.union_v2_chr20_100_110K.20101123.indels.sites.vcf -I " + validationDataLocation + "phase1_GBR_realigned.chr20.100K-110K.bam -o %s -L 20:100,000-110,000", 1, - Arrays.asList("1e2a4aab26e9ab0dae709d33a669e036")); + Arrays.asList("69bfc9bec43a4fdd85dda3b947e6a98e")); executeTest("test MultiSample Phase1 indels with complicated records", spec4); } From e923a2e51268c49c7697a40855af8bce001e0fec Mon Sep 17 00:00:00 2001 From: Matt Hanna Date: Tue, 10 Jan 2012 12:12:33 -0500 Subject: [PATCH 038/356] Revving Picard to incorporate final version of ReadWalker performance improvements. --- .../picard/sam/MergingSamRecordIterator.java | 247 ------ .../sf/picard/sam/SamFileHeaderMerger.java | 744 ------------------ .../gatk/datasources/reads/SAMDataSource.java | 17 +- ...2164.jar => picard-private-parts-2181.jar} | Bin 40954 -> 40954 bytes ...2164.xml => picard-private-parts-2181.xml} | 2 +- .../repository/net.sf/picard-1.58.1057.xml | 3 - ...ard-1.58.1057.jar => picard-1.59.1066.jar} | Bin 1201269 -> 1206214 bytes .../repository/net.sf/picard-1.59.1066.xml | 3 + settings/repository/net.sf/sam-1.58.1057.xml | 3 - .../{sam-1.58.1057.jar => sam-1.59.1066.jar} | Bin 569648 -> 569649 bytes settings/repository/net.sf/sam-1.59.1066.xml | 3 + 11 files changed, 20 insertions(+), 1002 deletions(-) delete mode 100644 public/java/src/net/sf/picard/sam/MergingSamRecordIterator.java delete mode 100644 public/java/src/net/sf/picard/sam/SamFileHeaderMerger.java rename settings/repository/edu.mit.broad/{picard-private-parts-2164.jar => picard-private-parts-2181.jar} (88%) rename settings/repository/edu.mit.broad/{picard-private-parts-2164.xml => picard-private-parts-2181.xml} (58%) delete mode 100644 settings/repository/net.sf/picard-1.58.1057.xml rename settings/repository/net.sf/{picard-1.58.1057.jar => picard-1.59.1066.jar} (91%) create mode 100644 settings/repository/net.sf/picard-1.59.1066.xml delete mode 100644 settings/repository/net.sf/sam-1.58.1057.xml rename settings/repository/net.sf/{sam-1.58.1057.jar => sam-1.59.1066.jar} (96%) create mode 100644 settings/repository/net.sf/sam-1.59.1066.xml diff --git a/public/java/src/net/sf/picard/sam/MergingSamRecordIterator.java b/public/java/src/net/sf/picard/sam/MergingSamRecordIterator.java deleted file mode 100644 index 4b1c7a9994..0000000000 --- a/public/java/src/net/sf/picard/sam/MergingSamRecordIterator.java +++ /dev/null @@ -1,247 +0,0 @@ -/* - * Copyright (c) 2011, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ -package net.sf.picard.sam; - -import net.sf.picard.PicardException; - -import java.util.*; -import java.lang.reflect.Constructor; - -import net.sf.samtools.*; -import net.sf.samtools.util.CloseableIterator; - -/** - * Provides an iterator interface for merging multiple underlying iterators into a single - * iterable stream. The underlying iterators/files must all have the same sort order unless - * the requested output format is unsorted, in which case any combination is valid. - */ -public class MergingSamRecordIterator implements CloseableIterator { - private final PriorityQueue pq; - private final SamFileHeaderMerger samHeaderMerger; - private final Collection readers; - private final SAMFileHeader.SortOrder sortOrder; - private final SAMRecordComparator comparator; - - private boolean initialized = false; - private boolean iterationStarted = false; - - /** - * Constructs a new merging iterator with the same set of readers and sort order as - * provided by the header merger parameter. - * @param headerMerger The merged header and contents of readers. - * @param forcePresorted True to ensure that the iterator checks the headers of the readers for appropriate sort order. - * @deprecated replaced by (SamFileHeaderMerger, Collection, boolean) - */ - public MergingSamRecordIterator(final SamFileHeaderMerger headerMerger, final boolean forcePresorted) { - this(headerMerger, headerMerger.getReaders(), forcePresorted); - } - - /** - * Constructs a new merging iterator with the same set of readers and sort order as - * provided by the header merger parameter. - * @param headerMerger The merged header and contents of readers. - * @param assumeSorted false ensures that the iterator checks the headers of the readers for appropriate sort order. - */ - public MergingSamRecordIterator(final SamFileHeaderMerger headerMerger, Collection readers, final boolean assumeSorted) { - this.samHeaderMerger = headerMerger; - this.sortOrder = headerMerger.getMergedHeader().getSortOrder(); - this.comparator = getComparator(); - this.readers = readers; - - this.pq = new PriorityQueue(readers.size()); - - for (final SAMFileReader reader : readers) { - if (!assumeSorted && this.sortOrder != SAMFileHeader.SortOrder.unsorted && - reader.getFileHeader().getSortOrder() != this.sortOrder){ - throw new PicardException("Files are not compatible with sort order"); - } - } - } - - /** - * Add a given SAM file iterator to the merging iterator. Use this to restrict the merged iteration to a given genomic interval, - * rather than iterating over every read in the backing file or stream. - * @param reader Reader to add to the merging iterator. - * @param iterator Iterator traversing over reader contents. - */ - public void addIterator(final SAMFileReader reader, final CloseableIterator iterator) { - if(iterationStarted) - throw new PicardException("Cannot add another iterator; iteration has already begun"); - if(!samHeaderMerger.containsHeader(reader.getFileHeader())) - throw new PicardException("All iterators to be merged must be accounted for in the SAM header merger"); - final ComparableSamRecordIterator comparableIterator = new ComparableSamRecordIterator(reader,iterator,comparator); - addIfNotEmpty(comparableIterator); - initialized = true; - } - - private void startIterationIfRequired() { - if(initialized) - return; - for(SAMFileReader reader: readers) - addIterator(reader,reader.iterator()); - iterationStarted = true; - } - - /** - * Close down all open iterators. - */ - public void close() { - // Iterators not in the priority queue have already been closed; only close down the iterators that are still in the priority queue. - for(CloseableIterator iterator: pq) - iterator.close(); - } - - /** Returns true if any of the underlying iterators has more records, otherwise false. */ - public boolean hasNext() { - startIterationIfRequired(); - return !this.pq.isEmpty(); - } - - /** Returns the next record from the top most iterator during merging. */ - public SAMRecord next() { - startIterationIfRequired(); - - final ComparableSamRecordIterator iterator = this.pq.poll(); - final SAMRecord record = iterator.next(); - addIfNotEmpty(iterator); - record.setHeader(this.samHeaderMerger.getMergedHeader()); - - // Fix the read group if needs be - if (this.samHeaderMerger.hasReadGroupCollisions()) { - final String oldGroupId = (String) record.getAttribute(ReservedTagConstants.READ_GROUP_ID); - if (oldGroupId != null ) { - final String newGroupId = this.samHeaderMerger.getReadGroupId(iterator.getReader().getFileHeader(),oldGroupId); - record.setAttribute(ReservedTagConstants.READ_GROUP_ID, newGroupId); - } - } - - // Fix the program group if needs be - if (this.samHeaderMerger.hasProgramGroupCollisions()) { - final String oldGroupId = (String) record.getAttribute(ReservedTagConstants.PROGRAM_GROUP_ID); - if (oldGroupId != null ) { - final String newGroupId = this.samHeaderMerger.getProgramGroupId(iterator.getReader().getFileHeader(),oldGroupId); - record.setAttribute(ReservedTagConstants.PROGRAM_GROUP_ID, newGroupId); - } - } - - // Fix up the sequence indexes if needs be - if (this.samHeaderMerger.hasMergedSequenceDictionary()) { - if (record.getReferenceIndex() != SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX) { - record.setReferenceIndex(this.samHeaderMerger.getMergedSequenceIndex(iterator.getReader().getFileHeader(),record.getReferenceIndex())); - } - - if (record.getReadPairedFlag() && record.getMateReferenceIndex() != SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX) { - record.setMateReferenceIndex(this.samHeaderMerger.getMergedSequenceIndex(iterator.getReader().getFileHeader(),record.getMateReferenceIndex())); - } - } - - return record; - } - - /** - * Adds iterator to priority queue. If the iterator has more records it is added - * otherwise it is closed and not added. - */ - private void addIfNotEmpty(final ComparableSamRecordIterator iterator) { - if (iterator.hasNext()) { - pq.offer(iterator); - } - else { - iterator.close(); - } - } - - /** Unsupported operation. */ - public void remove() { - throw new UnsupportedOperationException("MergingSAMRecorderIterator.remove()"); - } - - /** - * Get the right comparator for a given sort order (coordinate, alphabetic). In the - * case of "unsorted" it will return a comparator that gives an arbitrary but reflexive - * ordering. - */ - private SAMRecordComparator getComparator() { - // For unsorted build a fake comparator that compares based on object ID - if (this.sortOrder == SAMFileHeader.SortOrder.unsorted) { - return new SAMRecordComparator() { - public int fileOrderCompare(final SAMRecord lhs, final SAMRecord rhs) { - return System.identityHashCode(lhs) - System.identityHashCode(rhs); - } - - public int compare(final SAMRecord lhs, final SAMRecord rhs) { - return fileOrderCompare(lhs, rhs); - } - }; - } - if (samHeaderMerger.hasMergedSequenceDictionary() && sortOrder.equals(SAMFileHeader.SortOrder.coordinate)) { - return new MergedSequenceDictionaryCoordinateOrderComparator(); - } - - // Otherwise try and figure out what kind of comparator to return and build it - return this.sortOrder.getComparatorInstance(); - } - - /** Returns the merged header that the merging iterator is working from. */ - public SAMFileHeader getMergedHeader() { - return this.samHeaderMerger.getMergedHeader(); - } - - /** - * Ugh. Basically does a regular coordinate compare, but looks up the sequence indices in the merged - * sequence dictionary. I hate the fact that this extends SAMRecordCoordinateComparator, but it avoids - * more copy & paste. - */ - private class MergedSequenceDictionaryCoordinateOrderComparator extends SAMRecordCoordinateComparator { - - public int fileOrderCompare(final SAMRecord samRecord1, final SAMRecord samRecord2) { - final int referenceIndex1 = getReferenceIndex(samRecord1); - final int referenceIndex2 = getReferenceIndex(samRecord2); - if (referenceIndex1 != referenceIndex2) { - if (referenceIndex1 == SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX) { - return 1; - } else if (referenceIndex2 == SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX) { - return -1; - } else { - return referenceIndex1 - referenceIndex2; - } - } - if (referenceIndex1 == SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX) { - // Both are unmapped. - return 0; - } - return samRecord1.getAlignmentStart() - samRecord2.getAlignmentStart(); - } - - private int getReferenceIndex(final SAMRecord samRecord) { - if (samRecord.getReferenceIndex() != SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX) { - return samHeaderMerger.getMergedSequenceIndex(samRecord.getHeader(), samRecord.getReferenceIndex()); - } - if (samRecord.getMateReferenceIndex() != SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX) { - return samHeaderMerger.getMergedSequenceIndex(samRecord.getHeader(), samRecord.getMateReferenceIndex()); - } - return SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX; - } - } -} diff --git a/public/java/src/net/sf/picard/sam/SamFileHeaderMerger.java b/public/java/src/net/sf/picard/sam/SamFileHeaderMerger.java deleted file mode 100644 index f78cd81dac..0000000000 --- a/public/java/src/net/sf/picard/sam/SamFileHeaderMerger.java +++ /dev/null @@ -1,744 +0,0 @@ -/* - * The MIT License - * - * Copyright (c) 2009 The Broad Institute - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -package net.sf.picard.sam; - -import java.util.*; - -import net.sf.picard.PicardException; -import net.sf.samtools.AbstractSAMHeaderRecord; -import net.sf.samtools.SAMFileHeader; -import net.sf.samtools.SAMFileReader; -import net.sf.samtools.SAMProgramRecord; -import net.sf.samtools.SAMReadGroupRecord; -import net.sf.samtools.SAMSequenceDictionary; -import net.sf.samtools.SAMSequenceRecord; -import net.sf.samtools.util.SequenceUtil; - -/** - * Merges SAMFileHeaders that have the same sequences into a single merged header - * object while providing read group translation for cases where read groups - * clash across input headers. - */ -public class SamFileHeaderMerger { - //Super Header to construct - private final SAMFileHeader mergedHeader; - private Collection readers; - private final Collection headers; - - //Translation of old group ids to new group ids - private final Map> samReadGroupIdTranslation = - new IdentityHashMap>(); - - //the read groups from different files use the same group ids - private boolean hasReadGroupCollisions = false; - - //the program records from different files use the same program record ids - private boolean hasProgramGroupCollisions = false; - - //Translation of old program group ids to new program group ids - private Map> samProgramGroupIdTranslation = - new IdentityHashMap>(); - - private boolean hasMergedSequenceDictionary = false; - - // Translation of old sequence dictionary ids to new dictionary ids - // This is an IdentityHashMap because it can be quite expensive to compute the hashCode for - // large SAMFileHeaders. It is possible that two input files will have identical headers so that - // the regular HashMap would fold them together, but the value stored in each of the two - // Map entries will be the same, so it should not hurt anything. - private final Map> samSeqDictionaryIdTranslationViaHeader = - new IdentityHashMap>(); - - //HeaderRecordFactory that creates SAMReadGroupRecord instances. - private static final HeaderRecordFactory READ_GROUP_RECORD_FACTORY = new HeaderRecordFactory() { - public SAMReadGroupRecord createRecord(String id, SAMReadGroupRecord srcReadGroupRecord) { - return new SAMReadGroupRecord(id, srcReadGroupRecord); - } - }; - - //HeaderRecordFactory that creates SAMProgramRecord instances. - private static final HeaderRecordFactory PROGRAM_RECORD_FACTORY = new HeaderRecordFactory() { - public SAMProgramRecord createRecord(String id, SAMProgramRecord srcProgramRecord) { - return new SAMProgramRecord(id, srcProgramRecord); - } - }; - - //comparator used to sort lists of program group and read group records - private static final Comparator RECORD_ID_COMPARATOR = new Comparator() { - public int compare(AbstractSAMHeaderRecord o1, AbstractSAMHeaderRecord o2) { - return o1.getId().compareTo(o2.getId()); - } - }; - - /** - * Create SAMFileHeader with additional information. Required that sequence dictionaries agree. - * - * @param readers sam file readers to combine - * @param sortOrder sort order new header should have - * @deprecated replaced by SamFileHeaderMerger(Collection, SAMFileHeader.SortOrder, boolean) - */ - public SamFileHeaderMerger(final Collection readers, final SAMFileHeader.SortOrder sortOrder) { - this(readers, sortOrder, false); - } - - /** - * Create SAMFileHeader with additional information. - * - * @param readers sam file readers to combine - * @param sortOrder sort order new header should have - * @param mergeDictionaries If true, merge sequence dictionaries in new header. If false, require that - * all input sequence dictionaries be identical. - * @deprecated replaced by SamFileHeaderMerger(Collection, SAMFileHeader.SortOrder, boolean) - */ - public SamFileHeaderMerger(final Collection readers, final SAMFileHeader.SortOrder sortOrder, final boolean mergeDictionaries) { - this(sortOrder, getHeadersFromReaders(readers), mergeDictionaries); - this.readers = readers; - } - - /** - * Create SAMFileHeader with additional information.. This is the preferred constructor. - * - * @param sortOrder sort order new header should have - * @param headers sam file headers to combine - * @param mergeDictionaries If true, merge sequence dictionaries in new header. If false, require that - * all input sequence dictionaries be identical. - */ - public SamFileHeaderMerger(final SAMFileHeader.SortOrder sortOrder, final Collection headers, final boolean mergeDictionaries) { - this.headers = headers; - this.mergedHeader = new SAMFileHeader(); - - SAMSequenceDictionary sequenceDictionary; - try { - sequenceDictionary = getSequenceDictionary(headers); - this.hasMergedSequenceDictionary = false; - } - catch (SequenceUtil.SequenceListsDifferException pe) { - if (mergeDictionaries) { - sequenceDictionary = mergeSequenceDictionaries(headers); - this.hasMergedSequenceDictionary = true; - } - else { - throw pe; - } - } - - this.mergedHeader.setSequenceDictionary(sequenceDictionary); - - // Set program that creates input alignments - for (final SAMProgramRecord program : mergeProgramGroups(headers)) { - this.mergedHeader.addProgramRecord(program); - } - - // Set read groups for merged header - final List readGroups = mergeReadGroups(headers); - this.mergedHeader.setReadGroups(readGroups); - this.mergedHeader.setGroupOrder(SAMFileHeader.GroupOrder.none); - - this.mergedHeader.setSortOrder(sortOrder); - - for (final SAMFileHeader header : headers) { - for (final String comment : header.getComments()) { - this.mergedHeader.addComment(comment); - } - } - } - - // Utilility method to make use with old constructor - private static List getHeadersFromReaders(Collection readers) { - List headers = new ArrayList(readers.size()); - for (SAMFileReader reader : readers) { - headers.add(reader.getFileHeader()); - } - return headers; - } - - - /** - * Checks to see if there are clashes where different readers are using the same read - * group IDs. If yes, then those IDs that collided are remapped. - * - * @param headers headers to combine - * @return new list of read groups constructed from all the readers - */ - private List mergeReadGroups(final Collection headers) { - //prepare args for mergeHeaderRecords(..) call - final HashSet idsThatAreAlreadyTaken = new HashSet(); - - final List> readGroupsToProcess = new LinkedList>(); - for (final SAMFileHeader header : headers) { - for (final SAMReadGroupRecord readGroup : header.getReadGroups()) { - //verify that there are no existing id collisions in this input file - if(!idsThatAreAlreadyTaken.add(readGroup.getId())) - throw new PicardException("Input file: " + header + " contains more than one RG with the same id (" + readGroup.getId() + ")"); - - readGroupsToProcess.add(new HeaderRecordAndFileHeader(readGroup, header)); - } - idsThatAreAlreadyTaken.clear(); - } - - final List result = new LinkedList(); - - hasReadGroupCollisions = mergeHeaderRecords(readGroupsToProcess, READ_GROUP_RECORD_FACTORY, idsThatAreAlreadyTaken, samReadGroupIdTranslation, result); - - //sort the result list by record id - Collections.sort(result, RECORD_ID_COMPARATOR); - - return result; - } - - - /** - * Checks to see if there are clashes where different readers are using the same program - * group IDs. If yes, then those IDs that collided are remapped. - * - * @param headers headers to combine - * @return new list of program groups constructed from all the readers - */ - private List mergeProgramGroups(final Collection headers) { - - final List overallResult = new LinkedList(); - - //this Set will accumulate all SAMProgramRecord ids that have been encountered so far. - final HashSet idsThatAreAlreadyTaken = new HashSet(); - - //need to process all program groups - List> programGroupsLeftToProcess = new LinkedList>(); - for (final SAMFileHeader header : headers) { - for (final SAMProgramRecord programGroup : header.getProgramRecords()) { - //verify that there are no existing id collisions in this input file - if(!idsThatAreAlreadyTaken.add(programGroup.getId())) - throw new PicardException("Input file: " + header + " contains more than one PG with the same id (" + programGroup.getId() + ")"); - - programGroupsLeftToProcess.add(new HeaderRecordAndFileHeader(programGroup, header)); - } - idsThatAreAlreadyTaken.clear(); - } - - //A program group header (lets say ID=2 PN=B PP=1) may have a PP (previous program) attribute which chains it to - //another program group header (lets say ID=1 PN=A) to indicate that the given file was - //processed by program A followed by program B. These PP attributes potentially - //connect headers into one or more tree structures. Merging is done by - //first merging all headers that don't have PP attributes (eg. tree roots), - //then updating and merging all headers whose PPs point to the tree-root headers, - //and so on until all program group headers are processed. - - //currentProgramGroups is the list of records to merge next. Start by merging the programGroups that don't have a PP attribute (eg. the tree roots). - List< HeaderRecordAndFileHeader > currentProgramGroups = new LinkedList>(); - for(final Iterator> programGroupsLeftToProcessIterator = programGroupsLeftToProcess.iterator(); programGroupsLeftToProcessIterator.hasNext(); ) { - final HeaderRecordAndFileHeader pair = programGroupsLeftToProcessIterator.next(); - if(pair.getHeaderRecord().getAttribute(SAMProgramRecord.PREVIOUS_PROGRAM_GROUP_ID_TAG) == null) { - programGroupsLeftToProcessIterator.remove(); - currentProgramGroups.add(pair); - } - } - - //merge currentProgramGroups - while(!currentProgramGroups.isEmpty()) - { - final List currentResult = new LinkedList(); - - hasProgramGroupCollisions |= mergeHeaderRecords(currentProgramGroups, PROGRAM_RECORD_FACTORY, idsThatAreAlreadyTaken, samProgramGroupIdTranslation, currentResult); - - //add currentResults to overallResults - overallResult.addAll(currentResult); - - //apply the newly-computed id translations to currentProgramGroups and programGroupsLeftToProcess - currentProgramGroups = translateIds(currentProgramGroups, samProgramGroupIdTranslation, false); - programGroupsLeftToProcess = translateIds(programGroupsLeftToProcess, samProgramGroupIdTranslation, true); - - //find all records in programGroupsLeftToProcess whose ppId points to a record that was just processed (eg. a record that's in currentProgramGroups), - //and move them to the list of programGroupsToProcessNext. - LinkedList> programGroupsToProcessNext = new LinkedList>(); - for(final Iterator> programGroupsLeftToProcessIterator = programGroupsLeftToProcess.iterator(); programGroupsLeftToProcessIterator.hasNext(); ) { - final HeaderRecordAndFileHeader pairLeftToProcess = programGroupsLeftToProcessIterator.next(); - final Object ppIdOfRecordLeftToProcess = pairLeftToProcess.getHeaderRecord().getAttribute(SAMProgramRecord.PREVIOUS_PROGRAM_GROUP_ID_TAG); - //find what currentProgramGroups this ppId points to (NOTE: they have to come from the same file) - for(final HeaderRecordAndFileHeader justProcessedPair : currentProgramGroups) { - String idJustProcessed = justProcessedPair.getHeaderRecord().getId(); - if(pairLeftToProcess.getFileHeader() == justProcessedPair.getFileHeader() && ppIdOfRecordLeftToProcess.equals(idJustProcessed)) { - programGroupsLeftToProcessIterator.remove(); - programGroupsToProcessNext.add(pairLeftToProcess); - break; - } - } - } - - currentProgramGroups = programGroupsToProcessNext; - } - - //verify that all records were processed - if(!programGroupsLeftToProcess.isEmpty()) { - StringBuffer errorMsg = new StringBuffer(programGroupsLeftToProcess.size() + " program groups weren't processed. Do their PP ids point to existing PGs? \n"); - for( final HeaderRecordAndFileHeader pair : programGroupsLeftToProcess ) { - SAMProgramRecord record = pair.getHeaderRecord(); - errorMsg.append("@PG ID:"+record.getProgramGroupId()+" PN:"+record.getProgramName()+" PP:"+record.getPreviousProgramGroupId() +"\n"); - } - throw new PicardException(errorMsg.toString()); - } - - //sort the result list by record id - Collections.sort(overallResult, RECORD_ID_COMPARATOR); - - return overallResult; - } - - - /** - * Utility method that takes a list of program groups and remaps all their - * ids (including ppIds if requested) using the given idTranslationTable. - * - * NOTE: when remapping, this method creates new SAMProgramRecords and - * doesn't mutate any records in the programGroups list. - * - * @param programGroups The program groups to translate. - * @param idTranslationTable The translation table. - * @param translatePpIds Whether ppIds should be translated as well. - * - * @return The list of translated records. - */ - private List> translateIds( - List> programGroups, - Map> idTranslationTable, - boolean translatePpIds) { - - //go through programGroups and translate any IDs and PPs based on the idTranslationTable. - List> result = new LinkedList>(); - for(final HeaderRecordAndFileHeader pair : programGroups ) { - final SAMProgramRecord record = pair.getHeaderRecord(); - final String id = record.getProgramGroupId(); - final String ppId = (String) record.getAttribute(SAMProgramRecord.PREVIOUS_PROGRAM_GROUP_ID_TAG); - - final SAMFileHeader header = pair.getFileHeader(); - final Map translations = idTranslationTable.get(header); - - //see if one or both ids need to be translated - SAMProgramRecord translatedRecord = null; - if(translations != null) - { - String translatedId = translations.get( id ); - String translatedPpId = translatePpIds ? translations.get( ppId ) : null; - - boolean needToTranslateId = translatedId != null && !translatedId.equals(id); - boolean needToTranslatePpId = translatedPpId != null && !translatedPpId.equals(ppId); - - if(needToTranslateId && needToTranslatePpId) { - translatedRecord = new SAMProgramRecord(translatedId, record); - translatedRecord.setAttribute(SAMProgramRecord.PREVIOUS_PROGRAM_GROUP_ID_TAG, translatedPpId); - } else if(needToTranslateId) { - translatedRecord = new SAMProgramRecord(translatedId, record); - } else if(needToTranslatePpId) { - translatedRecord = new SAMProgramRecord(id, record); - translatedRecord.setAttribute(SAMProgramRecord.PREVIOUS_PROGRAM_GROUP_ID_TAG, translatedPpId); - } - } - - if(translatedRecord != null) { - result.add(new HeaderRecordAndFileHeader(translatedRecord, header)); - } else { - result.add(pair); //keep the original record - } - } - - return result; - } - - - /** - * Utility method for merging a List of AbstractSAMHeaderRecords. If it finds - * records that have identical ids and attributes, it will collapse them - * into one record. If it finds records that have identical ids but - * non-identical attributes, this is treated as a collision. When collision happens, - * the records' ids are remapped, and an old-id to new-id mapping is added to the idTranslationTable. - * - * NOTE: Non-collided records also get recorded in the idTranslationTable as - * old-id to old-id. This way, an idTranslationTable lookup should never return null. - * - * @param headerRecords The header records to merge. - * @param headerRecordFactory Constructs a specific subclass of AbstractSAMHeaderRecord. - * @param idsThatAreAlreadyTaken If the id of a headerRecord matches an id in this set, it will be treated as a collision, and the headRecord's id will be remapped. - * @param idTranslationTable When records collide, their ids are remapped, and an old-id to new-id - * mapping is added to the idTranslationTable. Non-collided records also get recorded in the idTranslationTable as - * old-id to old-id. This way, an idTranslationTable lookup should never return null. - * - * @param result The list of merged header records. - * - * @return True if there were collisions. - */ - private boolean mergeHeaderRecords(final List> headerRecords, HeaderRecordFactory headerRecordFactory, - final HashSet idsThatAreAlreadyTaken, Map> idTranslationTable, List result) { - - //The outer Map bins the header records by their ids. The nested Map further collapses - //header records which, in addition to having the same id, also have identical attributes. - //In other words, each key in the nested map represents one or more - //header records which have both identical ids and identical attributes. The List of - //SAMFileHeaders keeps track of which readers these header record(s) came from. - final Map>> idToRecord = - new HashMap>>(); - - //Populate the idToRecord and seenIds data structures - for (final HeaderRecordAndFileHeader pair : headerRecords) { - final RecordType record = pair.getHeaderRecord(); - final SAMFileHeader header = pair.getFileHeader(); - final String recordId = record.getId(); - Map> recordsWithSameId = idToRecord.get(recordId); - if(recordsWithSameId == null) { - recordsWithSameId = new LinkedHashMap>(); - idToRecord.put(recordId, recordsWithSameId); - } - - List fileHeaders = recordsWithSameId.get(record); - if(fileHeaders == null) { - fileHeaders = new LinkedList(); - recordsWithSameId.put(record, fileHeaders); - } - - fileHeaders.add(header); - } - - //Resolve any collisions between header records by remapping their ids. - boolean hasCollisions = false; - for (final Map.Entry>> entry : idToRecord.entrySet() ) - { - final String recordId = entry.getKey(); - final Map> recordsWithSameId = entry.getValue(); - - - for( Map.Entry> recordWithUniqueAttr : recordsWithSameId.entrySet()) { - final RecordType record = recordWithUniqueAttr.getKey(); - final List fileHeaders = recordWithUniqueAttr.getValue(); - - String newId; - if(!idsThatAreAlreadyTaken.contains(recordId)) { - //don't remap 1st record. If there are more records - //with this id, they will be remapped in the 'else'. - newId = recordId; - idsThatAreAlreadyTaken.add(recordId); - } else { - //there is more than one record with this id. - hasCollisions = true; - - //find a unique newId for this record - int idx=1; - while(idsThatAreAlreadyTaken.contains(newId = recordId + "." + Integer.toString(idx++))) - ; - - idsThatAreAlreadyTaken.add( newId ); - } - - for(SAMFileHeader fileHeader : fileHeaders) { - Map readerTranslationTable = idTranslationTable.get(fileHeader); - if(readerTranslationTable == null) { - readerTranslationTable = new HashMap(); - idTranslationTable.put(fileHeader, readerTranslationTable); - } - readerTranslationTable.put(recordId, newId); - } - - result.add( headerRecordFactory.createRecord(newId, record) ); - } - } - - return hasCollisions; - } - - - /** - * Get the sequences off the SAMFileHeader. Throws runtime exception if the sequence - * are different from one another. - * - * @param headers headers to pull sequences from - * @return sequences from files. Each file should have the same sequence - */ - private SAMSequenceDictionary getSequenceDictionary(final Collection headers) { - SAMSequenceDictionary sequences = null; - for (final SAMFileHeader header : headers) { - - if (sequences == null) { - sequences = header.getSequenceDictionary(); - } - else { - final SAMSequenceDictionary currentSequences = header.getSequenceDictionary(); - SequenceUtil.assertSequenceDictionariesEqual(sequences, currentSequences); - } - } - - return sequences; - } - - /** - * Get the sequences from the SAMFileHeader, and merge the resulting sequence dictionaries. - * - * @param headers headers to pull sequences from - * @return sequences from files. Each file should have the same sequence - */ - private SAMSequenceDictionary mergeSequenceDictionaries(final Collection headers) { - SAMSequenceDictionary sequences = new SAMSequenceDictionary(); - for (final SAMFileHeader header : headers) { - final SAMSequenceDictionary currentSequences = header.getSequenceDictionary(); - sequences = mergeSequences(sequences, currentSequences); - } - // second pass, make a map of the original seqeunce id -> new sequence id - createSequenceMapping(headers, sequences); - return sequences; - } - - /** - * They've asked to merge the sequence headers. What we support right now is finding the sequence name superset. - * - * @param mergeIntoDict the result of merging so far. All SAMSequenceRecords in here have been cloned from the originals. - * @param mergeFromDict A new sequence dictionary to merge into mergeIntoDict. - * @return A new sequence dictionary that resulting from merging the two inputs. - */ - private SAMSequenceDictionary mergeSequences(SAMSequenceDictionary mergeIntoDict, SAMSequenceDictionary mergeFromDict) { - - // a place to hold the sequences that we haven't found a home for, in the order the appear in mergeFromDict. - LinkedList holder = new LinkedList(); - - // Return value will be created from this. - LinkedList resultingDict = new LinkedList(); - for (final SAMSequenceRecord sequenceRecord : mergeIntoDict.getSequences()) { - resultingDict.add(sequenceRecord); - } - - // Index into resultingDict of previous SAMSequenceRecord from mergeFromDict that already existed in mergeIntoDict. - int prevloc = -1; - // Previous SAMSequenceRecord from mergeFromDict that already existed in mergeIntoDict. - SAMSequenceRecord previouslyMerged = null; - - for (SAMSequenceRecord sequenceRecord : mergeFromDict.getSequences()) { - // Does it already exist in resultingDict? - int loc = getIndexOfSequenceName(resultingDict, sequenceRecord.getSequenceName()); - if (loc == -1) { - // If doesn't already exist in resultingDict, save it an decide where to insert it later. - holder.add(sequenceRecord.clone()); - } else if (prevloc > loc) { - // If sequenceRecord already exists in resultingDict, but prior to the previous one - // from mergeIntoDict that already existed, cannot merge. - throw new PicardException("Cannot merge sequence dictionaries because sequence " + - sequenceRecord.getSequenceName() + " and " + previouslyMerged.getSequenceName() + - " are in different orders in two input sequence dictionaries."); - } else { - // Since sequenceRecord already exists in resultingDict, don't need to add it. - // Add in all the sequences prior to it that have been held in holder. - resultingDict.addAll(loc, holder); - // Remember the index of sequenceRecord so can check for merge imcompatibility. - prevloc = loc + holder.size(); - previouslyMerged = sequenceRecord; - holder.clear(); - } - } - // Append anything left in holder. - if (holder.size() != 0) { - resultingDict.addAll(holder); - } - return new SAMSequenceDictionary(resultingDict); - } - - /** - * Find sequence in list. - * @param list List to search for the sequence name. - * @param sequenceName Name to search for. - * @return Index of SAMSequenceRecord with the given name in list, or -1 if not found. - */ - private static int getIndexOfSequenceName(final List list, final String sequenceName) { - for (int i = 0; i < list.size(); ++i) { - if (list.get(i).getSequenceName().equals(sequenceName)) { - return i; - } - } - return -1; - } - - /** - * create the sequence mapping. This map is used to convert the unmerged header sequence ID's to the merged - * list of sequence id's. - * @param headers the collections of headers. - * @param masterDictionary the superset dictionary we've created. - */ - private void createSequenceMapping(final Collection headers, SAMSequenceDictionary masterDictionary) { - LinkedList resultingDictStr = new LinkedList(); - for (SAMSequenceRecord r : masterDictionary.getSequences()) { - resultingDictStr.add(r.getSequenceName()); - } - for (final SAMFileHeader header : headers) { - Map seqMap = new HashMap(); - SAMSequenceDictionary dict = header.getSequenceDictionary(); - for (SAMSequenceRecord rec : dict.getSequences()) { - seqMap.put(rec.getSequenceIndex(), resultingDictStr.indexOf(rec.getSequenceName())); - } - this.samSeqDictionaryIdTranslationViaHeader.put(header, seqMap); - } - } - - - - /** - * Returns the read group id that should be used for the input read and RG id. - * - * @deprecated replaced by getReadGroupId(SAMFileHeader, String) - * */ - public String getReadGroupId(final SAMFileReader reader, final String originalReadGroupId) { - return getReadGroupId(reader.getFileHeader(), originalReadGroupId); - } - - /** Returns the read group id that should be used for the input read and RG id. */ - public String getReadGroupId(final SAMFileHeader header, final String originalReadGroupId) { - return this.samReadGroupIdTranslation.get(header).get(originalReadGroupId); - } - - /** - * @param reader one of the input files - * @param originalProgramGroupId a program group ID from the above input file - * @return new ID from the merged list of program groups in the output file - * @deprecated replaced by getProgramGroupId(SAMFileHeader, String) - */ - public String getProgramGroupId(final SAMFileReader reader, final String originalProgramGroupId) { - return getProgramGroupId(reader.getFileHeader(), originalProgramGroupId); - } - - /** - * @param header one of the input headers - * @param originalProgramGroupId a program group ID from the above input file - * @return new ID from the merged list of program groups in the output file - */ - public String getProgramGroupId(final SAMFileHeader header, final String originalProgramGroupId) { - return this.samProgramGroupIdTranslation.get(header).get(originalProgramGroupId); - } - - /** Returns true if there are read group duplicates within the merged headers. */ - public boolean hasReadGroupCollisions() { - return this.hasReadGroupCollisions; - } - - /** Returns true if there are program group duplicates within the merged headers. */ - public boolean hasProgramGroupCollisions() { - return hasProgramGroupCollisions; - } - - /** @return if we've merged the sequence dictionaries, return true */ - public boolean hasMergedSequenceDictionary() { - return hasMergedSequenceDictionary; - } - - /** Returns the merged header that should be written to any output merged file. */ - public SAMFileHeader getMergedHeader() { - return this.mergedHeader; - } - - /** Returns the collection of readers that this header merger is working with. May return null. - * @deprecated replaced by getHeaders() - */ - public Collection getReaders() { - return this.readers; - } - - /** Returns the collection of readers that this header merger is working with. - */ - public Collection getHeaders() { - return this.headers; - } - - /** - * Tells whether this header merger contains a given SAM file header. Note that header presence - * is confirmed / blocked by == equality, rather than actually testing SAMFileHeader.equals(), for - * reasons of performance. - * @param header header to check for. - * @return True if the header exists in this HeaderMerger. False otherwise. - */ - boolean containsHeader(SAMFileHeader header) { - for(SAMFileHeader headerMergerHeader: headers) { - if(headerMergerHeader == header) - return true; - } - return false; - } - - /** - * returns the new mapping for a specified reader, given it's old sequence index - * @param reader the reader - * @param oldReferenceSequenceIndex the old sequence (also called reference) index - * @return the new index value - * @deprecated replaced by getMergedSequenceIndex(SAMFileHeader, Integer) - */ - public Integer getMergedSequenceIndex(SAMFileReader reader, Integer oldReferenceSequenceIndex) { - return this.getMergedSequenceIndex(reader.getFileHeader(), oldReferenceSequenceIndex); - } - - /** - * Another mechanism for getting the new sequence index, for situations in which the reader is not available. - * Note that if the SAMRecord has already had its header replaced with the merged header, this won't work. - * @param header The original header for the input record in question. - * @param oldReferenceSequenceIndex The original sequence index. - * @return the new index value that is compatible with the merged sequence index. - */ - public Integer getMergedSequenceIndex(final SAMFileHeader header, Integer oldReferenceSequenceIndex) { - final Map mapping = this.samSeqDictionaryIdTranslationViaHeader.get(header); - if (mapping == null) { - throw new PicardException("No sequence dictionary mapping available for header: " + header); - } - - final Integer newIndex = mapping.get(oldReferenceSequenceIndex); - if (newIndex == null) { - throw new PicardException("No mapping for reference index " + oldReferenceSequenceIndex + " from header: " + header); - } - - return newIndex; - } - - - /** - * Implementations of this interface are used by mergeHeaderRecords(..) to instantiate - * specific subclasses of AbstractSAMHeaderRecord. - */ - private static interface HeaderRecordFactory { - - /** - * Constructs a new instance of RecordType. - * @param id The id of the new record. - * @param srcRecord Except for the id, the new record will be a copy of this source record. - */ - public RecordType createRecord(final String id, RecordType srcRecord); - } - - /** - * Struct that groups together a subclass of AbstractSAMHeaderRecord with the - * SAMFileHeader that it came from. - */ - private static class HeaderRecordAndFileHeader { - private RecordType headerRecord; - private SAMFileHeader samFileHeader; - - public HeaderRecordAndFileHeader(RecordType headerRecord, SAMFileHeader samFileHeader) { - this.headerRecord = headerRecord; - this.samFileHeader = samFileHeader; - } - - public RecordType getHeaderRecord() { - return headerRecord; - } - public SAMFileHeader getFileHeader() { - return samFileHeader; - } - } -} diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java index 2e243b8473..c0537334d3 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java @@ -556,7 +556,7 @@ private void initializeReaderPositions(SAMReaders readers) { */ private StingSAMIterator getIterator(SAMReaders readers, Shard shard, boolean enableVerification) { // Set up merging to dynamically merge together multiple BAMs. - MergingSamRecordIterator mergingIterator = readers.createMergingIterator(); + Map> iteratorMap = new HashMap>(); for(SAMReaderID id: getReaderIDs()) { CloseableIterator iterator = null; @@ -573,9 +573,13 @@ private StingSAMIterator getIterator(SAMReaders readers, Shard shard, boolean en iterator = readers.getReader(id).iterator(shard.getFileSpans().get(id)); if(shard.getGenomeLocs().size() > 0) iterator = new IntervalOverlapFilteringIterator(iterator,shard.getGenomeLocs()); - mergingIterator.addIterator(readers.getReader(id),iterator); + iteratorMap.put(readers.getReader(id), iterator); } + MergingSamRecordIterator mergingIterator = readers.createMergingIterator(iteratorMap); + + + return applyDecoratingIterators(shard.getReadMetrics(), enableVerification, readProperties.useOriginalBaseQualities(), @@ -847,8 +851,13 @@ public String getReadGroupId(final SAMReaderID readerID, final String originalRe return headerMerger.getReadGroupId(header,originalReadGroupID); } - public MergingSamRecordIterator createMergingIterator() { - return new MergingSamRecordIterator(headerMerger,readers.values(),true); + /** + * Creates a new merging iterator from the given map, with the given header. + * @param iteratorMap A map of readers to iterators. + * @return An iterator which will merge those individual iterators. + */ + public MergingSamRecordIterator createMergingIterator(final Map> iteratorMap) { + return new MergingSamRecordIterator(headerMerger,iteratorMap,true); } /** diff --git a/settings/repository/edu.mit.broad/picard-private-parts-2164.jar b/settings/repository/edu.mit.broad/picard-private-parts-2181.jar similarity index 88% rename from settings/repository/edu.mit.broad/picard-private-parts-2164.jar rename to settings/repository/edu.mit.broad/picard-private-parts-2181.jar index 4465f91f578165fb7f49ea5c891b40da3ac8e70a..ef33718276988af18e549760e2fe06dd91544302 100644 GIT binary patch delta 820 zcmeyhpXt|rCcXe~W)?061`Y;>U|%hViG1oTK_J?4Hp>rhAbmp)LjO_%(~}h#HNXlc z`!MQ581;;nEWdzCH}7GTV}}UcJXGcqu+FflMVPTm+SG5LZR z=jI8VIgDU+lekSFMp$s`vO!pxf`3^cW@L%kvVhFkyjskM6(adt`WpvC?5oO3PB3e- z6sO+g3wmM@J+JiKA@ZjBe;6TpObtbtAuL}LZAOS}jp;ic2rJKNJ2Ql(={A!Q?A6W3 z+#8r6I=sB@vq8jk1B;n>K>=g7Sa`c9*f;0h%_cvX&A$0na4=N4V;E2)$jOV3MsPq( zIGG?0m3)#^4w2j(pLz`{Ynhh;wS8&fb*Q?Ak~l^|kWJq&zQ1xzh=E~`1_Of$FzgIB zE0kRmf*AX&Baedx6mOGrCYnRMzI>t`B&J?Y)PpdjCmBH)fs>%o(mTl>qWHojPq0Cg zWhZArM7k$?Kp2M8r6zxzYz6kmWc?}G5Yc&4Tp${rO-Y0>yr*VBq_$7>g(&8j29?U4 z77Yo0$e-%$QjOVerjLfcT^c z$N;6e$=7CuLPYduL!~AI8DOdJvy&mZ;^xFcME1`~hA`CTLZy1=WPHxZ3GRKV(dS#L~T`!X=80|W56nkBG{uM z6?Z3XlBP)yMMONw9&#z<)Pf=yQKKjIPy~BWJZK?SaCg2ry&vEAy*K-2wpx;^CCPcR z$02VfL?Ohuln?37y|QlCU`~EFY}HRqsMTG(vS!-C1FV;N0rzN7u2@lXi8j?j`mD4A zJZh)wgXP`HoUIR0LS%`M4)!9lgO%N?`B+U*o-^wF7|~wWS_8V``X^(?xci7~&oH06 zdpE)S+xSO;_Ivj;l@s%*8Y^pV^sH%N_yO+*MNh!D%>nB3@27C@27c6olE;b;(7xkY z%2zd4!c!7Dy1T#BKx^&4E^VCfvrp=4#EgNJPXt&RC81Coe7!Dwyyx zwq2O3!?$3X7e^|>4JOCL=})eG6uGGzQR=c!T7UcV{YnENOD%->t-JG?%{MCz7+bxc zRAl?zSVBL9)t~A|@lxM)4bW&D1oRu?S*DE;!eyh24`MsnIFyI%1i+W}uwU#j--CHm z6Hp7OPBeT=4FP&)#$Xj^`VdxTg>`lIJe1d#V&4bzrB5OJAUyzOHGLa%hB707SD6?r zEqfV|$xZ^Cb1`f(X@ULEvC7}-$qQ>bKLKSUKZ@{Z T;T)9ZLL8G1EJUF^T1fl{#xPWO diff --git a/settings/repository/edu.mit.broad/picard-private-parts-2164.xml b/settings/repository/edu.mit.broad/picard-private-parts-2181.xml similarity index 58% rename from settings/repository/edu.mit.broad/picard-private-parts-2164.xml rename to settings/repository/edu.mit.broad/picard-private-parts-2181.xml index 6a22ea2c36..d11423b598 100644 --- a/settings/repository/edu.mit.broad/picard-private-parts-2164.xml +++ b/settings/repository/edu.mit.broad/picard-private-parts-2181.xml @@ -1,3 +1,3 @@ - + diff --git a/settings/repository/net.sf/picard-1.58.1057.xml b/settings/repository/net.sf/picard-1.58.1057.xml deleted file mode 100644 index 15c5b5620c..0000000000 --- a/settings/repository/net.sf/picard-1.58.1057.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/settings/repository/net.sf/picard-1.58.1057.jar b/settings/repository/net.sf/picard-1.59.1066.jar similarity index 91% rename from settings/repository/net.sf/picard-1.58.1057.jar rename to settings/repository/net.sf/picard-1.59.1066.jar index 4a82a3058e5c8939773c8c4f3e78dae6105547a3..1bbfd5a19efbc0395c83e8d35c3e212834891633 100644 GIT binary patch delta 58768 zcmb@v2YeMp_cwlK_TJsSn@zjP4Y`m&LMMeP0Ya}z?;;37fKUVyN>H(IK~Y3R!5|~3 z2#CajfAea(^SqG=qbEFAsA?M)?7Q>h9@$0e(6Cw=ki`$H-?b;~ z#y%)(+Q7kndB$6*h9EM8S{!dBGgb+u$B!qfhdxf!ODq2jsO9VIl+wFDZLYFKAi4C= z@yya!U%uOgJmHn4emD00i8s91;dkHjVh?`&oEIxRb)V_x6F)>fOrQC)m)8#8`3`Lz ztfJl9h*$k_Oo=HC>Q{c5L-b{i$}AS?>QJgkR&~Ej)YXssMIbu1gZL`Pff(&GMYNM{ zbn!G%Pdde49Q)PTCg{K>KK1jxn$xO4Y4PgB=tozJc5b!NTE(J|+aJ+_+2UT?Md7a- zMX$J6rr@?y`@cF27>7%L3e;V*Lez>bS}sobT===2E7F2QX~Al$1=LetB-T|i_xc0T zhgXX9pp&sn#gX4X(TG0LRJU5Xi-@X>6|qiZqUSe>Z#1{!@3x5RQ=F)pwqNACE!whQ zT<-R9#b@FP!%f`wwTSqg&bxeO!^rY3M9aIV*@b3ubjWFOsEZ3DIY+i}v-i)DO1gD? z)m;AWMNhVppC&pe&OMXc~1y5dNHU;@VrsNslGfE7oY57j+IFu{ z_d42(J{MG6hNfvwdEbn2_czA;ZYzmM3 zH~E~g811%7d)RC3*Y|2`y_PJwPkUK9`Jeo)!|Wj#7(+~TctdQmLViR2xIQ5! zf;*w3ryth7^`aka)uPfF6n73hHFFOz+@prBvqNsdmV<%di`%pp9oSpOv}{$3QB|zf zfc?-aU!rRM8z%YKRazkW;dX7d%fh0AcWL=m-L*$+;ifVhZRJ3Fw(AF}m!gX@ zU%D#(JuxfVXTMfkibbm6pq7`&M0w!G7Xz0X$c4WiN^JJmR*?_1M(XQ>S}-O=T}|p1 zpk(T^2#29 z^iI=@rk8XPG(bHaiEXE5pVIBpg9}YHm@*rq7!_vtWUs7%yRX;T+XXOKfJK&XESSQ$IheT{6BI zbpju^DV1v36y~5d)v+m!gLDq+a!`+h3=T4>zD*4{Xvje$4pOL=AbDjEo0@Q&F1KkE zU188@L7mi`PqlW8onO6l1XNqyL{Cy9{|fY1>z~tFN0KRo4v#0nFc1@rhw}s~&J)-; zPaxzxfs^wDTFw)gIZq(xJb|C{1d7fRSUOK2>O31e&t%4wP? z`PI_^p%K<0x}4i5dy!ApCQH_#1es1DS&wSVOsXdvP(#^>n#m^ATsEUNvIVu5t*EnX zPd#KGx-=p$r2+CX8Y=tJm9igAk^N~to;S#WlndZpfa_K0q~>TkI2GMUEvPk~9BTve zpe6Y^v< zo}$-{+Y$J!FVSy(OV0$raUSsnREq;ztw{4pGYjU;d+Ge%11R5GzHZTeafcz6TO`FD z7Dnxz99vrq5YM7{5pl$#w&FR4 z@;nDeExJn_v&0KxF9$C&30|_q%iMUG%;jTi`K|y{E}tLY*Xei#9Tu*x99%k88Wc5SdJV9I=cb~I%tes+$LT`-1Z zSBsvO-H;+)F=cmC_Aq5nQ}!}tZ%*57%08yN)RLFUzJ~0_D^GpZ@VGWrP3-TxEShja zi}8j1v*wo+UMDEFNB3bp2lwhdzUSzk19}U(N9DY&wJONS7=cAAV^Tp;QE^Ge#KMgE z^9mGYd5hI87&X`{07MoNsuTYJ9TPqZs)RDKf?*ET!t7=J~z5_0JAB<$_m8wbYRLOq2lf5?+Ubr7Rf}}t z%;~eHmlVzg;d&Mq%_}J=D!II1=KMlIwW_Kg-h1fC-h+CAqJsyF95`srhgv`N$uV<` z+OWkRcKDn+@Szqo#B?me6DO%59pdUrFfA@#4E5^~ zk)r#j7fmj_PF<3V9(?X&%`cMrgO`9e8ghWc4AvWRp!)O^ZBkqR#7P4Ri<}i+kgs6w z+=hZ&VZ}GIum~?nRT9ShiSr8Q%r7jOR5(wK{8SsCh(hxw8lod69vGM*2dTZEYMFY^ zrjT{UsKaGGrvXh=%g;35nAoEEvj$F|H>;pz(zL>Pf)aRS46gxGP~ybGc_lqu0cT)g z$=vCa1cge9OA2OsiNjE&2ZjTRTL3qxxF>pIp1R>Ptw%${g*+L|u?EhCbLNdG9u81N zQw4QUtrm&o(%Eg2REA$mo@9D0133R=?khp@lZxjTmDI#Dz^%b=v%Z4tiCp=J;-1qA zC(RHfu6B|ffvm@Z5=JZjAFWdHC$%QnJ)hj(LCv4+k5ANRc?~nX%GJpXY_Zpz)aLu{ zGvr{ES!Om1r%o-bA_O{TYRR-dbBky7nLe{nee}6D#?t2&&YM581oK`E_*`q5t4{-A zc$mq#lLZAPPI6n!Wb$emG#`yaSL1_v`4}_}_kFHis@k2>>V+Uff|eeI=Yd@a%4w>) zzY)`HO@S{>^?y;T>l21B)ziwX6OcpXP+Ja@!=s;_(uPTM1YYSdJ*!UqX{FZskUp~M zOZv*DZ#ejlgCFShns2n)&RCvm$dR_ZT#mBo4E=1=Z=B(GI%?A&^rsD}^f}7P0*<}HP=29bZ8=d+QYThe zq14F?uF#fKB42 z$hkJ1rE@m@i~eoPc@l&lz<}o4B2R$(USrE^L7$UdjP~l)Z?%R4hRN%=*aBN#&wy{> z;6{;d%bPe@$U%gIMIzpoi#c|)yahATmP_QVw!97P5m$-vwp=Q2S0~2>Qq=J8w0fpQsjb(0DmW|U#eJLL+@K3m=;?>6L0TdtD#*g~h%>hHJBI%r<0Ty4mE zZMg;n5SNL*wirgIL0ri20>u>25SNIyhP=;~Yvnqm25eEtWWS$Le?UHH%k><4NN!;C zHrjHNFb(;zEjP z%0%v}^vM}j%-&fw)rs%5dOmTDxVH4?Uy_qB=WTh2@i4tPn4t#zp!G;NEJ48| z@;O63Z_A^it@_kVyd<4LPq={d_$roc%VY8d_0tbp$m9Y4q9I>W8~>Hi-fzp7da|vwCeG1EhpOdM{TcA z@%Q>dcFg3$d6VW&pIuTscb+=2EvCC_nV8VXSF&Jsq1xC3%hK^1^%y1h`RZ}ZOci8! zEyD)brM#fDraem&kEe-$Y=ooF{-RBg>YWkLVJw{)$WUL-G3%>Z$9c%~IM|zW(Zt{aUQL^*3!{>G2O!)sI6HEOn~XXs9}kO$e%)ziV-7)eikK zb^Y(!6{lqoa=or!&{vD1AGU=25KK8m!wae+(`~h2i;G-+@B!{)5-(qWr%RC zC%qgW;u~18m!n3DPjN+GARUW>_caF3WB5byukpY_@t+^QMG_W||NQVhlCYp4jBwMc zYCO9qSK2dHEqB zcyBBzgK7)uzF+PnY(;`rBg0d8_tkih;@w~4eRo1CGiH>H1yHQ22m=@|wzvJ>q@I4? z%28kZt&NWCrn==+FDsZ)PMOd|)H5omK0mbzYT#_lvF#!1Lx~8NV#nW!xkO3ema~U)1r5sGL(K(X_D7_JphMI$ z-xv1nqEpzwAn<7VSmD)3c6&}5*_-L@3YgEND=l|P~7mc z${ktzsO@MWvPNYGGj>y6c9hzcQ~L-i?NCAagK{gV<5~*h(J7yGI8o|cR(Tp<(~d&B zgGU8*$r9z1Rz{8Tefy}}Xb**8_X_eOL!AohfvUJ<&x3exzl(b1`P@kFJjN68bf9e7^eR{y@+~bcr#^8V zj0?~M;eZFd4aYp-Rv7;Pys8Ml^hu2vYCSn0`gAbLDql=B;*ijr!Sf$yF;Ky!c=2$* z0KMLSLici)Rel`uUxaDzr6E6Kdf7iPjr(FqILK4FZ6?!M5ncM`bld{KWi=M7iKP^J@m|1 zQ>sV+AJ)<4#v)M!q1qfy`65XqgMoFWp&}$wzzX+~E^3KXq4Ys)RS8+Yssx(im5`CD5@>r> zi4Q4G)J4UXe4d)P@FHk&Yf+B@qOFEgT|k}#u3R0?N3>Qh6Q!1#5E&v9LBfS=_uXD=n2B8FfE^sQ7U z{XAtFv;|8^?s>{GL~}#55YX=aN!DK^&r-S}T84r>4ACmH5*=U~qP3g7CDt?)VpVY7 zIckI*d~TjoqCR4OQltN#R*&=jNwpfEqXa{=!OLHODtY(73xyit^VN^I95<^@In~a} zsuQKbD&s8Uj{~SlG9phK|;%c=0gsVld+2%=0T~L>V<0Sd&$cyogojFsFVR z1Px$~8;wDuIqP5^R#a|`Ns+(t={#hEO6bWTp<%_G%Ag}E)J$mLkiMhk<+VDVib zEN9`%z?d6D^_-E$>h$Zq-b0M{sAw9KJ3zib@J5i5Ta29_LM)9KM>;Q!f>#P&7b930 zFCVCNN%!7yk}$-L=6EUT&G*nK$*Ut4z&u}0dUo}P3c8|#Mi1XXW7bh}PI~JDA>Tt3 zvyZOCx;ggX5)CWl}P? z7q0_(rj{271w!T~suMOsf!6s}SPxsfsSp$%wIl%GUjUP|;1m;U`)Dn!bB~OM{%`nb z9E0qlsl!J*tM;_rG=2DJgOjf2=x&-(K{L5%14~b83JYZ-XV(#%ymVlBgf% zdI8?2Vx|;}T6C=l)9skyD=~}KVs0P7tU8Z5;}@Au-z-7LwnOi==XnBNI@;kN8ttGM zaF7l-Id}lmGGBBAuiZ%XMW+ZdnzR|+8A6PuE4`{a5o5N3O(gv#I-`h$T%<%7NMssa zOLvQ|q8lo?ofeAjklOrU(z8SlAg((Dpc5JAD9#W)#osjSFY=$m;4(z7?uO`H3GQz3 z*FX7aNM*O+}Hlq~oY+ ziOt=)#6FRJGcNfj8ppsxlaDD|{Q%+OMAR?miPM+X=B!tYhG&grN#3J8> za_W_~mSXv>H^21`Mf7!4m*21D_iORBIlo=UZ`YMkW=QvKpgNqifRh$f6~CU}t`7%L z(F7NZnkcXc-X3MDt4#0`+*-qd?Ny^bLKS@61*R=UHW+#imaOJrK`p46Xh|aw9|LYV zLFCaC4Bw?9pGq+j*I~6@FFMmE(G~2x8*Rl1+$nlsCGUk5ybqlc{pklWn0^yO=$sfT zd}5fWEkL_2Y%=qMs%MQ^B&`ib$-Urm5sra%;mNn(bWEaoG&Kui(0 zKy|qcDG!Ju@t7zUPeNg`SClyO?JdYEJu$cXLC%OpLwIiSENjhk3ll1a+Imxq&x_kQ zb1;C^!Ok4iDItO>&KV6#qD!4QXi_OP1pZ$E%u*a<@iLAR9qL=PTw?G$dk#=-zZd{& zV8$TE^HPF}`@}$$&_K)GVi2JCK+$s00?Q@nI1UtHkz$G%C59pvG3XS1z$z}Vfghkd z;z=EFCh(1vR0*sC&P3%&I-D0lXM$FuF|q%Vj`ME(@&rz;L}mZ2;2-2aPbtn~dX8EG zWJCnq)wfYMagN$ED~L_2m!6sqB%n}qr#-f}5)^_>m&cN40YmqX%3+2KM$PBYAK^<}^_auCz7CSz1i~zgfYn5BXtoF<%0kqCv(v4Y zp|-k|495E`vRc@t<%>+vG!Cqld&`G?B%~XMroalLDihdoY!vTbLCUrcCzK8$b4ja2 z0h}yv@z@*&EvR&s!Z;WNH^hdv`Wv7~Ru6hD{|&Wr%SUNl!0|XyQKb8yN0j;h_g$mGsMUB4uXyItElBQL{ zP{fsX7GYx=Ts~n|UtH(kF{R_;*{Z6}5+*+zJ z?^{ElX~Xsbok?QYz>P`h%2BMxQofw5Pu-m-FU3TBh9~`BCAGgJLn$BEjy@mc$8L`QwJ75 zp)k0&oSKzUClp@8g>Q?JLdt)E)eLa{0p>migEGgcxmOv_((I~kTkG}=kcjjMu5)|E zYY7@1lZEy4{xVv$j~)P_9vlb|pi51aNnw@BBs<@i6ZY+)^%CMoW_1M*0cO~TAh_WI zbSL9RNuyjfN!SO$I>M5h4-IbzK@bXnuzwG2LYA`1`}o!C4N&@F#t)R=ybdFEdp5@) z_Epd$hiD6rCTAxL3AiqLxD1@-af%mDP@>ohX8t5N|90vnc3=|iq}gH@7O!%8OjOX; zh=|fYv75dW`{;M^3?!}tqOLe78jHiCIZo#Wh$Grsl?4QQAZxwfr5)CmE z8=bS%3B&&n$_<)e7mb~Ix=e;h^-zmIC6Ys<1QfgjyXk6lpF!PEfQyp3(-J5GaK zHCsH^U1qV84=X%aN5C3kjYs|V(c?TomqzJ{0olQ=74#&}_^o9W%nm-~y*&kqN({_7 zMt*H8dTrYt+Rhrcrz2Pyc3?hrK_b{8z`5M8E^}S+3F!AJc>d>D0Zw5p{t}(|6?JQx`Eu%tiCEu(h2BT1)B$S-8aEb1s|nou%4_m>=5>#5s!vLG+HGNjTj= z2-#cP00xnOdLOFMq=7jXH7V!fCJo9xMs2EwJrjXB;VvHdpo2eH&H-rP4^|)m@8ePY zbWSh|vS2S2+zt9ana3mFQBvmV-jvHXK$kuF)%qP_R|~yQLmbsH^z$uWUW{AN)o140 zVcXLNjo_H4(u@tqx+={_2vebRgZ=(N2iHO3EU>fF`PdG((L2`>rVM=u$dw8Jx0n87%4wt zq@1B~;upF~{0manzo|(4M%RhoX|ec&9u$AlW@sGBuqS#M+N1YHZe4Js`0r zLlP~A2@z{oY;5g*71v|#SkR4~5jb6iv>L{i?M9Ta#ncE!^-YMzit8yI5)iaGajGeVGnTNF5buKSYCPBtfGXArVXrmATu@foJ0*Q+t0Vn4K zMBE0BxU}k60+K$9I5-7KqFNR7%sOuyx0A+P;BJnvK^>fSLBxse)&Zh}xE-aWSVjl& ztPQTl=7|R!p}!V*lnKS<;}B5Ag}V-piSl;|zP~-2-fvTUcym(UZQ^TP&yB`j>RZ~~ zjox>=FI*>f7O2R^PPOm*QOnQ2Tr%z3TyI4*FWhI#BFAe?K}rk!W}#8-O=b+s`E?LHy~t?!v+*Vg{CbG|O_Wr3l_C9R*diJr98$u>5ykp6dUH-GJTf55Z5 zZj$I<=vlNsR@=YIi*{+~&+}}wuWRZLx~b}}R{j$XYRjH`^zzMEL^k`>VHuyS&V21p zRfpR8?C2w{{S{up@$LNM8oBiy8tYH?X#QS-|0*vvf2qIBy-@Jf6GLuWf|X;5;o8a{ z@(0tR^;YtEJ(?m|q+^lhs^S5??*$Q0Z-;o+qQ*U?QiyT0y=uD?V->fIBW)?Kfi>fqmN z@Zf##qQZBHhetz|-p?PS4vyBpb{qQm82w#$G;Zy-C6|?ywFDDqO`)E+#tw#}FOJnS zz4rV#PCuhNtyp;I)}<%Wu_x7>_?S>dp&svc(AZYj)|!qHJ>4DA0D=n%GxV6~kgN3t z9@$=1r8T91?(gZ0XxYe0UJp}q*;&}oNVc) zsLRRqOp(FQOb+UsqJb$Inxc^@U?*85nwX-gDVmug%M{tB$T3AObSS21!5 zwl+l@Q-F82wV<}o(^$^nSUXd+x4^3NP0^9xA-8olMHj}Qt0}rMjP9oBVPWIn%c6Tk zZ&UO!#ibm*%oKeM(a*xsKz}Yez!U=wG01`-g!g*{jQj72h#?#uYKUPL^%TQRF~Sfd z4RN_emx@sVe7eHI7IQRb8pFYrrWk8sn|Kvx81HPb3OIeDg#(XCxGo?j8=}y{267Yw zoMMQn7G1%hN&!kt6YVUR|6Og08T>gf!ewW2QYZe{$rQ6pQDi}!oXub67-Fu4rD{I6 z^BNAW<>c!)wt!>TbL<8~+{n+HI9SNZ4%kKfy4Vyqo8lHzEMWxE?jhn;=81&n=t;inAi(M<3CfnXAPHTWo|i>Z)5rCzZEF zuOEC?JZy{20;W(~I4Hx%tGDQ_v_}zHzeR6sJ|-Tw#S`Lj^~x6gCT*+O6dh2e-yjm7 z6i>k@hB1WM5l-Z`srMe$yO^u#0b6VrPpi6*=>t-B*kY&H1r92`^E}I@jFvX+uphyuzWK!Kp_5*3LvyyrhQ-QW*J8@~GG&fU z&(L1=>tlKcv}!L`xldj4xZW%FARV?Ldm(s<4y&Pk<3I%QjC$m8{mK@r#eQ2HpvP=l zEe_f^k$m39GJgp6g(FAw$!%|lLpFALFvVJpjUJ__?hCaPRrFn48my=G&|Xe|j)Ujb z>?iaF`cZ5FshLX5PE1z`*CyIAAiE)6u(4@-QN8*E$p0F>u1-Ip*N%C~W@~nv-lxyh z$@_HNm)xoc`>qzR+Su#8&cSgTHYuOm*g0;p>3jOY7H^16Q1{ogDzn9#=p-QU7F!{f z+4K$qY|s9sCT-OlsO4Mr5unC9ws==OqQ2g$XE%FKye~+A>>r2^ZSfI(Y>SV@Cmeig zi_f@gcZ!q9(L&wzxJ*}9JgK*=^|@QeFEv1PZ*Bx_AN^ac8NsL7STYg=;Sk zpohK6`$zR*YF~xk8NmLm_y-1?t3l{gg~1~i3NqFoM*kOOwqQ%ZWXavF|AjJ5_vphT zjPzKvXd3pekK!2Jz}=b!I7)v44Y`zN(^kYZ8i>Q`Cvn{F!-4cc2hUjQb|};R1N>LHAtnqkS(Y9VF#^7`}JFZ)?@qi z23?p$j8;bv8SdFJ9x@zz7S?cX&w^ONTK>Ln3&HIiP}TvxX>oPMun(vrIb^wND78E& z0Wis%zdO<5e2K70?$Q4F7dNZ&o97 zyBe8Ot22YUJDGbO)cXyt?t2_LCE~~_h!e{yG6idpsa16YTh$;FMgeX;@_@p+;Q;;k zpne6&op(sD*PG$6a>>H!(kt{TdM%>{#IDs4yIu@&ybBR4fCF*eA-y*ses)N2GKL{u zf=Q}_BOS+dg)R>_$WCCoYJeS6JzaBPD)T1t67I3R1v^Q%QxnMtQxeQis4tRwpq3aQ zj)lOfrr@wX5!F0*SkE3(RZRfZybC}QCWEoCu6!HSG^oK9`l1@ca55%K^Z1lno230K@{or^p`QHPI>d?X&t=G_21NZ(i~x!wfX&6sq)t?PS4L^GiU% zQ4mxbOvz+eEPag_YbE=`Sd*J5(Zp}7r$53`)oqGy|S-h#6s04Usb^f$nehOOL+YC&dJo`|{hVIU;Sr2WBlO{E)S&}|`VKA{#!FoX7az=B>o~ahQNFE* ziysr}N7#M3y&B>JjZ_C0pNGpIP(K8Co%bsK3r*qEr;#xHid#!Vpd_6Y8HJbW19J|z z2U6*qD4&m*^8Nd$Xf!euvpHgG9Ejs&9x)V)4g16TZkkg@mK&SvoIV|8wTR|0B=_uT zH-y_g&?W7q2GEAL70=*2Z9jGahiDMeMu_K}j*IZ&Tw4mma+TZOfU+Gb_(6r9UXLVG zH?K!(fsa=8$ViV0Xb$c^R1LZ?`OZ@@n! zNcL`;HxN3&88|tI?G=n94XCosU`Yj8NN!L;c9dcdB0B3IIdJ}TQI74f+|fq|uYr|L z%Y&$hN6i3SVs(j9AN@<>1U6!CKt1;+HHFyKLcGH##_!Pp@jeX{A2<|YR-scLnC`B} za2r@-I$weFD^^tjZ`>T=`N6Ain3oRRPTqv2gHNvL5ByfG9)E%(N@vt{0|^+XlzlV= zjT|~UYc~yxQsmkl)Hh1k{gWGi=HTM0>r*@y^%sh>5l3^-@^7fzTw3O#fehHjKw1#I zK6pa~-S|&vU)F$TLn7c-u%S{VMO+|@0falGA%L`o+W)2sTG%;lJ%!S;cM!DN3pe9@ zOv54AqV3rX+@)y>+C<_zun|{bTSuJ6N&Ok8GG5>S&lydGeD#1MlMFY)F;5=@RhW2z z2#Ir49~8qJ6%q)!_$>19Y48txk}u#B$Obx#x8o@2Vsw53I=^Gw|A8~tKVkducMWtJ zRHIYZLuarJ1xO8i7@bTH9jzLj^WJzZ20Awn2;MR<`zUq9>}<`aqD$6Mi>gt3t6FwU zuN%2}Kw0Inob3bl(QVZO8fOU>iZ=+}jtI&w!vljGhjVwhqqR2%UDarHrm&Rcml~N8 z{73q6k6Xt*Zi9Ns0QHgB(90O7X*^*4)ZJ+sAC`9!5g$m+5@yqC5vhB z>r!u-fi`D4?d6p>2?hp^2|GA+6A=)+pk+YNjX}dTx~dOmZI$pj6c+Bh2y@HRnLT9~ z0@fCBXncxU$rH-pLoi3gzNiO1 z-QBA%=^wkEaiUKh*C#b|)OZcA3fFp2CwfqK-Lq`>8Hwtq1S26j;xGM4-N{q;zGw5> zVaBx6)b{;Q^*MQ-iZPO+mkQ%j&B?R5L+`$G(un4yv8RpXDDvEs=Agh1O6sZ&NkLQ1 z%rrdh@Aa96@HBam`o?!&>a1MjvCb~uwPzTX=YOT?EaOQJ@Au~#hUr56;(DXCm-PnY zi#P{~y%XF_)7Ke$65VLytww!M^E}{?vDAeZefI@pYPthu zNqOJyJK~6T#5oQ(q7%L{KJdJdbosZj)l(J!_@}YNL-3ljMz%|ZZ+7Bs0fNtdnfc^xWGluVud6%jC{{`p5@qYUs}i zwbkr!&{i2&nYE+q$C{}20`TORZtSOFZTNvR4}=XKQW zyX{2BS(3`A7sym$5qQVRTv&eWZ5+wtSe~8qxS69ZWnj10${&1zC*>{34G*HFsC5!T!K@y_g~Ey ze~Y2>ZGz+RMWLu)Z72yOs^b|TTl2YrU)_wqym)K7$1pk`i(>~W$tqz(AU!(amcWZ~ zE;hOM1X_4hov=1=mpg-w_*3ny=bype7{z!PPPSRn+f$b)ej&B#{zyB zDNtjBnv@=XI#w_E->Roxe*k`ghJ0$dv{l<)fgh+zPr_jk1DCBjs_x6IhGFbg;%o%XY{>}u_yo|@;JeE za^M(lF??qiH|*qwtAiZJQLa&9yI7@&;}ovh#i179#$r=~%i_o}7=Ta(8Cls7nFZI( z$X6*eV10tPYn5SC*JfZN1O*8fqcnFPiC8*yt_x@q`)US~flV1V0P+j=RY|BZ6;Dd| z$2$6A{^Y}_hr!j?Fck?~scJ)`U~;6&sfvapir>V02K<1q?lXX8`6E+&%+F5@@hPkB z;U+|i&p0^A!RH|7ks%yA#laVP*W62GyR5GnrP*q6By$dJJfLb@}$-Pucsl&Kb-WLG99W!PWo{cz6O273$X2ag^y-E|d$ zy$%{YP-bvSCRb8FMm9hLWrGM>C>vTdf?bD5b{#T8HsQQ*9x{TRhm4R}{F==H9EgmN zxg2ZG0glK<$d)m(m2A!XWTN}lEP?|OQ(nSIwKZiPe`{y5&k!j)aOL?ivZMIfkey;^ z7QBXFKP>L|Y+;7;>m7ae|g9hnsSQMYqb4oc(e`j*4*P3Pa)uZL!1= z+H!fNA;(&@QjWvbe;iiH@%%CYzGP&9#TWifIhmh@rkujhsivI9&*^%%!HXO!nDS~< z&M@RmL(a0Gk$4Z6|1B7<@R*aumYgl;aBMCI^CE^UVGk+;n3nS`d5y#onv~aZwgmxs zy}W@x;QXwGyvdX}JsTk-F>;Yy9H5QzW{%y$u_YYb%E4_MEal*K&b*AF!B5RbiQ}`4 zlKm9P6)wf&dvr5po^7vMjPb=rhJeG-g=9=%7-kPD>rbkk^5kiDIex?oAquP4wW!d9x>#W ziA?`8Lq5tw<}p(~jtaSNrF?>et^D~(Mgqg>Dc~Wuaq@OkKFtMoaIlk8c5wjDMp9OA zEXtE>w;}f!a<7_m63(2oVk;E7XdUW{tNH~v$5T(&F&n7JPZm74eQk_TpZ#Pli0L;N zcle9vvJ)81DKi9C(bRHyFj&i zJkVB!7NLg8i>w2oMAyv**5xWJ1stt){}eIRP3)1J+8XNiqDg9|Mb2gpTT^Y+cZ^feWx5z_@;UJilnVjioNw^;A0Sxc-I)?sHt z!A9wQwPlI5NOic?>L%($Z@SgOo!y<$cW<+NQoOD{y4|`%YbVmjJejwtGL&`314VLcj9i7$&3vxUF`XKT?$6--P> zQ4n5npdX3OU#e1jabMP+lT6h9+^CbQeCa*qPfUR_0cRX%5R*TP0$cvV zv0pj(7r*|SgWu%uw)}%*e{%4be1&~n$iL-THEx9!?siVJhmV~eg;Us>Q}O%~jJ=FW zGmGcJ*UEYR6zo0))A%1>DmAh-p-Ds2)P{^et$^l(!wJnVUsc_A!3XEL6;}Orc>TcE zbkvHAhV3!RZOzmIwgz(oTZ_?bLyNVwI4$1R60}5H3vy?*lP{{oZ6Y%cvE#OuBtN#b zWb7PpJYPRK#+exzQ*gqbVQVSe9ktcs2NUX`oLFvawY1cWhr<;6xbvTECknDkdMR$ z@+%Ckjv0<;qYKFmIXSjt#;k(tE+jURU#W-gvUa9ujWL?FCbrg8%eDn>h1w!pG*gGq zTJX|wx0MjlaJSEkw8NbPduxN<zrmk;*F8e6g?< zP~t(ej!IZ!Hdnu0Y6R8fGqH8m&J-iHRJ*pR_@8%XzE&NvcWPeGUa!fi@Ghpq$BMk) znkFWwvj+nYmp;3of9bEoLh9rLR*7o3%hysJ`Pi%#o%x_OONavX!Fp>&H1d!&S*SPf z$8c=8(OMAs4WdL4|6E5JiH?R_@@9p%B78iAEtsO7> z-q!~FwIR~6uz6o=)Y$|eFvs!1B~2FIG2JhE$8zA14T~b4L0tQ8>3okcEnDFa>!b6* zy_@X<%ENwAfb+FS*a2Uk!G@8Wbrm+iA}fjHZqnErYS3SfBX&=rUQU55q*@%rWCiVV zip@%_*R7o5vXUttZbJD(5V@0ZXp|f&r%;x!oKld$7a~&2DeP=(1@uCckG(g%bz;xk z2C;nXQP~=xAI>NSQZqO%X+a}kN--K|Tob4VO+@M>cmYY&oR=nsYx+C3(>uo(>@=u=z!K_E8I9+%nh-rx_?2te`f*OAxQ1woc({U{x9O+>GH0 zYIk=9wGVcY48kwBy^oy(*|TPC?yRpEAZ2hvTX%h!4ltr~q%_IH!U* zzW=}#aPd)VamhFy_j|9f3d-qNxaStB8hBM>WirM_3eJMs2MaHZlCtxA-CCz) z1*c*xa{o^glog!L1G9pz&X2`Fhm*rSG!xw!$MkeY%B>u8x^q@OXt@XWp6G-(61EgcB?1T8s=lt}Cm&7MLx7PHqF$4KtS4GnO}Sz<8m49KKf2O<^9Mjc{zL zZFQzM*bj_ah9X`9b~7J`?AL%1T}SO<4R#r>xecHjz-n%USBaZo`Lz( zn_+--3v769rI+Y7*m*3)>Fw<}rdk1a26u@ZT8aC~tHcNx(H2l?L`;EgVKLn+u7OM6 z#dII8hpZDz>3&>5eLyH&n0Xj?<({C2aA$6VIE-EVQRpLHr-#Km&_|r4N5!|eNB28D zfm6nC2JMthX_stHQQ3ud%K@}cj-rFIfDXx7bVSalWAX-iNiL>WWGO|C%Lm}U z1s-DLoABELpJ4K1_-(21REyCU@0!KPi#8k9y8ipFqNj;pW17{bhtD*UI z5txDX#w|LNi#jD|p=1$C9-y?d>{6omFA7(}Mo>4zY{RkcHpD#Vx0wB%)H_Gb3{m3i z0edFfhwN~F(ie_gaJY=O`RCz;UR+~{YYlN-C_dIokA>p~Tt}|;7t}%LD8~>BP^J>Y z9yf&k0l&oOV-Ri@T-b7<7`Sl-vd0k&b^lv|h*N^C4kT}I-4#{*3@tR9-TcBb1?DAI zRcOnjw8%Ndu9?7BW8F2Ey@@$IX9glXwJDQ*cr6~y-nwq))vJPT$>-%LW*xQWHERi0 zxX2^aWFOs%BY$^Iy$wu{@6F=qzcnBG^P(E1We2mv`X0IiSLkw>_i?1G@*Z##pass- zU>_|XovrPmJ9(8_ky900j5BrIZ@qi_K3X{%wPkBKN*}b3?iroS-d$o~Oj627E+3s+ zL94?)CokIVbEF^FABqj@x>(p~#8Wpgk^Z=nKMagyGH$uu4u2@CAd0TTeYOW7d6vN+ z$}v%o-Vzyb0FmiPEo{o#p9bUVYy>nNMwdB*R)>Ai7-u5F?jQ$luzAvHutcjX7J&(A zF!74V2))@kZ1}~S1&PpA-vaSiP<=WsmLMjbLF*cO^0!;7zuks!JS?;M^d7@zsqPic zy3lk%XTL(+g-I`2h2tXVZ;f;?3Aq>8RnP->0Kvo=$@F0HU7h{|SH7Ov8a|8K@NMjB zIW37c?u*bSNHP!aqs@3cQU@is?4~mK@Pc@~ao_*P$Hy)RYu-IsTS5Pj(LI)?y@?POOT&iaqU%Z5F;aHVh6vETqpWv<)*+RU6h{64*-wH*oE0P z{x*88XwJ_u>qKLZ_+fJgZLgrGac4BR6HVgJxOB@+gD?jVcWwN(aj6nKBv{1{Q$m6m#Hk$%vD9&?(Izce^0> zxF8QQNCt8-To+_G1_IzAe2>!M38m$T?xIIoWZjnHXEDp4OwT%E7W;F$cmAMu>9sH+8v<&_n?gXR2 z#V5!neIa}F!zJ?m^bA(!gIHycVTFAI?&Cg$+mi2bxiuaa_v^#aOcU^&)^LH*Ta3qb z{t3|OOu}W^LUA2dzDPt&6}RDH(j#IfG(@xT9~l(kB1o}#2^ZZ@AlrLluK1U@23LWv zm2u)GSywESGsI$9ByN$L#S-}h*3}*2cDUPJ=1gKZfP#qdtXPSSF4RZxEwBo)Sg_AZ zaSx`mCF;`;xUIVy2drCsf(6zFsx9s02yLij~ zGTBNfaUWRZowB)Di-p~fqo$_L!frT>kxc>Jl$nrlzz&5@v6XP`XJ9Ra3DaC~L0tb` z$IB+)#|59f<~-eMi2I$j^Zs*i%L1#|hJ2Z~D!-TAS*5*f&@@Q*8dN2^^8Yt4RHb}n zN&u5fDHiqr&!DTa|F2-9B@S%iucR>F9R5$(EQO3^P70+XIB5iW1Ea|wPPFQ+}#U)=eQc^>v>l+(em-|_Bw7?YB-Vq=!d zPk2d7aEqL9AXkL_k*sW%58f!pevq^5Vlm;ZwKOU0hnnK;urK%y*c@+JbSyZ1;TTOn zN`1n{QR>dqi8rr~j{ryV0GISckAC@Zp;+&5=PU$&&Uaz}|=@9Qr5Fy?V(z z?F??=r16ku70794V)!sgzQFCM)ez2EROP|#0(saS+}J05%NI0tSu!4;^4>xxobqz*w zq=HqtVYZ8xxGw35f!b))g+)WW?}>rXaY73G(8hc}T)x0( z^1$32Z`(p0v#^H&&-;2U#phP1t)S`*;fNppY&STcfB|a! zj$I``S_7N!85`FtAyOe+e8s*AJ)BB8=HX=I9Hkz>!oYS2ShUM!R?2&-kZ&EByiq|v ztgZ3^2rtQQT7-V6@+gt#FQe|&=*F`L6Dun>gf^Y7!rB8EHsE&`;NS+4A2k3}PWMW< zs5*^#tVUyg^hn#Qf_`#DB$rcfp=RjpAF(I<3H~Y1K$q|foSpm%ZJ6uiM8Sytei*T@ z$Njy>Fl}~Wk{<#WeMt&>9aiCQOFw-gb$G%J(6_J%KLd;Ke?zeG%XneR1Q8>HB16d37klhuH|GB|_8s6+72V%+Z?^Bvw$x1_g@lq25?X*jAXJeWic+Ns zsPrlz!V(Lp6hV*?kfKIVq=`Z(O0%LM*j`0IKn3Z7jVk&7&dj~L*}T5r^X-$_+?_dd zX6DS9GiT zK`JDC0b(PH6c>UiCZh-h#2;x2crgdLK$-%a!K7&Hm;EmL8f%@eSF;*;+>XcXt64H0 zci?fyYMgw8g#aGkGu8rhcHB~yk-U_pB550vwn?a+c-&bLDkS{HN>R;$8Gl{YG@#aF zz14<-uu|&x3yP3_%3hF(1Y194pCA}L?FVfl@aCc>bW5Ta2|JCM^NPS18A=i9ZS*El zmw9B{M+@F4Fk$PIZR{#dR6aUo(kIrglK+IWr`iOjH%*yOr9(-jk1g^AYZC=$=s~EB zs^D!VzCkz~dKdvg1^Z{btL=<^3sPJ{0;5Jnf#Ek}S(AiW9VaVk;zyz1+<9%F{G0#E z2bC!}sG+mvypESSlw(gByCxoX;bCq0S9G4)@1cc8U66sPZDHhd53Y#2m!+%iVI0)~ z7j1Q9P1ItLxho7|yRnDW?l28klqLbxz8gesI_BX6ps^W9awS8v-tbV7ajjS# zJ&w-hmG6D%=`-*yiTEk-35STsWbmkR5LhCD%8Z11AH6~Sl+LVoF_#Ip1#hOqzF;|y z3kE~pA@zL>PBJ#$M8Z9h-UaJ~b^tZz1J*T*j_Lhl8yv;|+~uef@CUbtUIE)~XGfD~ zZ)1OjXd)C1bpQ;52Vrw>FnI1kxKa#(QSdO)$<%Z>ykT}rz})5r18u=JunC(eDPndr z!;>@#O}C3~?iy@*SK0IcM8Q4E5o95|);3sL*~8jSU`E@qNsGgf#VSRN*{?u`uW0g% z!Q_{uzhtCV=k;?1QrUi7iJQ$W%6sJ{C@(EI{MYjZz(W=z5Xj>dU!^^>Gc4dttbz1=GyWf zLUb1=L91iHhhuRK<2aU&4f00nqu|5I;KM2GA$2P393KN8&cI?Wz?P~r+3V_4qEE&` zw@ATMK&)rNa}B|I5kYpez#&hvqd9%uzOll0GjbKVU*xUax5M$_>okzHNnnX zqw_7rk<67FZqmh!(u^LOz-ITrjpD1T9MBX3m z>>&)~dmkFSYa+^}FqjPF{Y&DVF=gkYo$p$d|L~QG&M`HQVr={XbLwL(iXCV1c-@cy zU)fZ462tv8s}ASMT5y}J&Cc=sBA92cbx29w$*_Fx{YGsWsGUfi8oa{>1&sjy+;_{hdnggO=)|qY_E;&gP|nb zZ)Xp%4{Yp1XMk>#apLO}C%n72TiF3C`;7SXbNV`HWrrMS@)uP2VZwwP9yKbZD*KY+ zzM`LB)7LiyheE$~upIUuD)~D*T$zv3&mXMp7$qO4Xx#Id!+xZvlK=>?Q}lS6zJ8*V zpB*6V8Txsae!`qPhn=IZ^YjHU0`9{-L2my5nNo#8_wJvKLaEDT;D)$r48azm)2{ zhrc$|8N~~qaa+p^Ww)*D4#=v2WyZatB`&G2X?$gh7RxR9U9<8khirCE&JasB*kgEcdo7O4C3)`enk|%OgY_i)M`5eO%j6x}Yf1kC9?`5x zAO2~EGe_EvCS*9<@ps5=L(6n_;jdMJkH)|~HVYq<%-?) zcP6FZ76z2arkFJxiP1yDW+95Wqp`Vcp2p_0Rl3og71m`?A#ubE>~)0E9iLaj+1^1} zoiM6o19)r=XSN3vfPtmr(y)t&Gt@m~1Q^R3ne1_V?>3tsmUUjXN>hA<77!Lq73&a)ALE)Y-(#=ix-Q0O~6-$%0oyC1*7 z8F=>fBG!ocVHcG@(38Q5R$`F)5TJ1`_5HXQm5$-bB z3ezMMhmAMuRxD+aVkc{)T4as1i)p>F4u-jI8G~7e8_yoR7?YZ?mkf4#l1A$zWhF|S zhpTL3(X=HPvxCK|I3OpI%#?ibWz4@Pc^Rt)6%TJh5?*IX0tjVN%VGd31>;PoabUKO-+l(CF5mI+^MvI;YYC@1!rz=Rqq zo(uN74^u!Nkbqc+2$Ks2o(>xfGxOtaD}yk7gM{fDBurm66RT1pOkY8mfTRq)#*|2& z01`@^2{R5!BR`!K;SZ7MDM_MRP}}5Cc^ZHaHlWWiWL<;$5aYK*gUm~r>23UX5~~Tr zFhP+F=&X{>It2tlYT-2|u~U!$M}q{25Ue8>2pDZvh-Dl4GZW*WDaKk06lsi~Oe_b8 zJLuSvEI|YsKP`14P;cX;b=w`B9LlEN!cK-lVvSV+w>Kbd* zHMJAE7cP8j%YO}Zj68m56idXY%s}ZzT@pcrXW4V8iw%Y!)FD)tOua4!F9Iq6Fj6zA zdQMVPanK5C2TMwtJoL(S!hgyijqzMlRxr!h%@GUs3PNQ9#>7NX56SJ(721!@J6{o@ zg0#gq+N}h(0G@jMTnKmeUY6PsQl!>imeYw=ubK3|9>M@(g%0$_Q|?k$tJW5lw~f`= z%j(h&EnVTFAX7bv3+&a_$I0ACrxi~FnI&IlX&9a*P$5qvk*8sWJdKyKZeWc9TwSKF zVo9PJdKoghDN?qvLi}j9lvT&S=Kl_gXbDFBK) zP=-rb9hfpT#TY2U6>wc)&eR9JIRsrb2K_PtJ@XWZI1i_cakdz*0$zms{!2KkXDL+Z z6@vP-A=?#VZUI{e{7vEa{{o&Okk5!4LvaBSN$$x~^t;(naKs|?P8<&Cp`*(rZ@B0X zo<3H##L9TxoGip*wwHZ~A%Tq|bYVP5(*paNptl>3EqktIR{U>8>?((m*wevPSvUt5 z%I}*fBk-1}tvnAWUatgQmnbViYzb3`C|LeL{u@KQKF|?AI(D(vy?t%USld1Lr3#{2 zM9QWL9O%%os`}w-G~8waD`99FKxJtXS;Mql?4I6W!giY_zo73EWa7QsSbJDd?qVH! zXTeCMBLU#8aHmjDO8K}0^XX2s>q89xk6=!8ph6{@Ru~4YvB>#|&9K-pnTk*NlC1B#~v!zI1-9d;HtJah^{Sb-Z+E)Slar+P5R< zREBsy_PW!@Yx!}i#Kzj1y{zDx^N!xyg1?+?ydn?JIOtzf9n(~G>D0d<&xg}`uQ}ai zWfpB)by=@*wY9tKECxq+?#* zMr#rn=6Y&Dcs=zfdOL>>@ZnY|KOw6IW!#6P( zN0j9>JcoD(C6{1*dXRS#4D= zIay$wa@h&%vMW~QH)`os8~KGUR(^NFjuk6;ukxouSq?Wr_4&sz&k4?X(9d-_Xdfv(!ys&yN9Lt$h1Ah#uKLt(Jy=9BZS zaeQo+&Aw>n?x?cT9@^Ito>DLt7z5>^*Ra+0|IWw<_R^xt{_LgIQmj9Nc@^b;KBzp> zUslvd+hVpxgXZXjvYP|6vk}UzvQrOgb~C(1$6a=(@&R=;zn>0x{anM@Y!~S13k@Q5 z4W5))yy+uam87@WdJPBVZPM6g`g)swZlR}l*m@pN)VN5jB>}I-YnVpY2!}V1#vq?H zrM=R@ikDWb8CL?4Y5d?LT1rY61(&KPsEHc(6EA3Ll9~*9X?UA8NmEnQRDLC2%c!BI zsp%Rv+yGFOyjL?dwYpkEQ){YOnwqWF0=YFcN3AVA9&^<^as-XDlm9wZrJmSju&ecz z$(q_g&8IuF`N83uKcXS>kQ1ayZKO7qXYqC(p}B1ZY7ZW@)gc*rq92O16e+c9&Hx($p4eOHFMB^uSOz8&2+c%}`vd zscqGJG_@V_sQ03YFv;DfsqNJc(qUEYsOIy_uzbgj+XZlvoi$SH(`tg+81G+Gi#r!+ zYG?hq7|&hkS#b~~@OD+ZVcjph*d|SUWMnS3qNsx1HMNI&ANRU&5>dlZT2j*eYEOl^ zLU0`)nJ`gPdy#W+A1+)f)xK(~ruGBv>+G{d@uEB}4wuV6pys2r+L~IZ_Se(_>Ogd- z3cpVH%`Z7^tAWekiFjE_wZZ67}^V^a39dY793BcN@dr=D{j~$koLJB-*VSu7_`ra za3msQa?=Cbw$NGVW1~R`T=euY!wWcEt&Ja=9`!noq*b8Q(MQtYGyKLdrh<9lV_I5( zN|8&l2`FtMl=?}cq%72lJmQR!Ji(GYD2WQ9)Po=gZV1P-ERvuq7kNk&`2}*It?DM= z#g4y{2)z|cdw$|EEjMu&bM=Pz61=@USloJH)8*TW7n_KR=Bd-P=s+y2Zm_=B;0xq8 zliySnEqUSyB7O&qPjqji1lp2Gq`iVf+8{_0=G#$ZRF%B8{4_yV5xX=|wEL1mXtuD_ zZLsmd6V(26ELW=nm#{(*xQfUMAyH`yWL~=DavRH##R(%Wy*L^Rf7S z0Sl}bp>Hn%%PxaRU5-tXm(epT(Rr&OgkQny^jGnU=XKFr^U;Uh;4N_&hn1RuEfy=G zudzs7fu$22OKQPFB%Q%qgP8-!DE8&?{g)nT1H#&P)lmvY`x=Tc0a~b&jPA!1bvhUg zPc(+m{dkhyKOQl%;LX@Hp-bP*f+fTXNx-iIyi7a60x)O*zjR~vax~p!Whbrdloht( zbU@or|6oS~H|&)o12&k9BrDzsV1p^z%Ff(kY2=f@&f-~|br-@iG#LE;u~s9~L9qcZ;R!ZMuTiV3w0@QVxPdHozX*#d0uKYQx+|cq7n@dg5R4 zqFyv>Hf^Mdc6p^oLmha~(QpZ|cz?;_;PAS*#E4iN`w{h*u{w0C&n}ja0vh^Y72lY~ zOo0!32aR>0#y)Hiz&Eh4j5XVnvx_zFjhlnaWvs{`-b(B)v{+j{Nv|Hpa8}NiRMwI< zCGc7nSM9?2lN<+IZ)a`%CRzm2C@BbN4Fg12az-e|rsh=~U20_o*u0XAiF>#NvzoE! zo-g94V-6Y2iYF=NZIF_`!(E8ZJbE7|lO9L+Y{C}8UwASJ7L`(t;Malv68>+34~Puc zgBeaB19eYTx{om)uaGKn;2qY*%5L6d(Kj)0Bp(t-LUoaSX6_8^F37+6Ix)~^qVM3{ zA^J|d&Zf>2ud@@trAyf}K}qJN4ucb)6h6XUU-a()vV_uQoh^q9zwh}Uorl~7zIN1k z5-6z?xqd0PW~WhsM2J z0c@b4DWE4DKq1h(`89m4VJUtGIZAe43NJ&Tr{;;f+k0T=2)A{;ySDsEzqP=;wmgw4 zBwD&`7rW0GNEC^;fbgsE^)kjzZ}csqsXC#K4fG23Y;&?0meoN!EXI)RLw$*w^~HZ* zKm7-8Zf@-Bum2c8KL+9pLv$M(gdQywH+6v(2GfwmZX*pmvAriHoVZ#RgTu!7g(tL) zVi5k0LDn1ZX6|4bvElW@Ffrh)tRZ_;3_=U!-SZ;OjJLgMa9+HN!MY#AwVY-VWh1N? z7`Esll~=H)RFJB{X{U?G;Z!EDhZGfYb~q39QpjN+cbTx6BCjo=xM6c|MujX&5BNh` zFhf|?#;hsc2!6}#YA$OlinFPHyj~*NfMpr(=%d3zF@52PJY9RilP(d|S(yHg(+G!s z2FZp9=2s`-Jt9ej9m=-~9cwM8ucugmj^aj(6qnH=RLCORfjB6nRC_$xWu1{~lU0W2 zl~H9cDkUlkI0$V~6(MY>Du6SpLKl~cs^BH1UX=hIW~5_jt=oVKW1y-c!QyZ*uuRmvmg>Ccs@kIt6TtP(#3>u8IJrI zcB+vyGKLx(`}uIbrdSt^i=n)DmKGaGqMPZgWo#I^cl$cf4S5f(W%Xj*OIg$|Rv~pQ zs}kc`%KRb{6%Ehg*FJ^|u@sZymOEpTgJ}^Q+I9JM_fl{c1NnwS&F5!kX%CBWv>ucj z0l(cu8VgVX14T2m538>J`XAB7!(?-j?qM85T~3gadKQj}ntX|_y3{!kCs2>X zz2>CMfd{KAk}m}lV&kF){egXz>fl{GGggFe()E-YWcU~j!$&fIj4lT?Bmcj!;7+)5 zkVy^}Dl(v#NxmeymZ*^TAlBvhy$>eklQ86>(XMZIv4^4XJOb~H$`ZNczXUS47~BUg zCUzT+xx_+qh|bO6woh36E;a(_0wckENTgE*M$wS~a9Yy=N6Qlm04N6(&L{B0J70&3 zq~RfF&S??(@FfvW@H^n!p}Ucg5}>5fa1m42IGX5ozzGWjDwFE91AfZnw2sk5b%O`J z%q(u66dJ@&R4F|(jyljYPox1RGqceJF#Cpw|DCd;X(cR|2|+)HVG{4U3j=lVU#ouA|e9QpP6w=F%1dU)ZqZ}Q{* zTAh;c*B6wO9fO6R-{+6g2^59t7|#AgR2<~sH`nre{5U~iN(9;|>6Wne?lIi`z8*|T zy5-^Lw6|zDg7cGEZEQ^9{x9%K12#w>QcdCLnuB5H#7p&#N^Oh}4Xd#XSZo}}u&s{O z84WHQ2JAI+vlkHX9N+ZD8aob@H z0f+eoR(krRhsmt5KG45a3ZQe)K>rq#UVvsjJc`i|%^8t2=5c;MObVNz))qJmHHI_a zCQN&uf^e@wBuQEoqA)=hrCE?Jmkqio%`yFnmP1}d>5VMJe!OhtWr#S@w185CETSDi zW3;0MP*bHED=md!L7kOW!8+6ID66~y{V1xev^J^?KSR32f?+mwiyL*TV;(EV5;G_- z+LFc&5hEl}0I5R!+#>`JBv5r}WzNph)#6q|$b@C0dDa6t0=!NB2VP3GD(y(1QBh|Y z7zHj#h(JMSaj7s<%2_t^p?oO|!wRUJJD3spAZTwfi+IzGMK5kZB6)d(xyUCP@8X5_Rf8?d0&~ljRRtr{{Ap?anhI+h+Am{Q^%3k@t+^bVg1^-OqTmuzQGT2f63D6w( zq(H;&i!sO8OIiJjI|ENa$P-8j3IC_*2g(#bW=Zi+D-FLX#*W40?4IJ^GXwMCH(p>t z%(H9Dr{do;aLDnS@bDb28lNUkV3=d<2J~I1^%R}(7{+E@bV4_!J9sw<>M)5$Gt5_p zDm@@|?JzbNCsJ)Nf_hG*T8VPOj=8Epn<3$Xi+%4GEN5fq;B-tU2YA^@r5Bz|tO!oZ zy@Ah+#m)w}BcrrHa6e0C=o`$?7a534!5w#55~Q0{Di7c zWhhj-+j1!s?=paMyDxoe9w zWiWnGLmYJ1iK?XBLi;5{KFHL7@KP4468xAXLzWo~8RDwYzEcvdXoVP=Szv}wm|zLw z#8Xn-RM?^#7X-x?RZ#t+EK}vO0W8~%bcN8$Uii)G*s+oz^=saCHa~|Rz+vK94|K&*@uGMO@n z1tKG`QAnkJ=}2ojD9CaTG@Xp|X)dax%m33PMns~V9HdmC8K6E6vC+3#bs%7qJsY4~ z=5ev7&Wr6(6v=1@*+XWdTNg;PjQxnS(6CYa1X+t1#b}{Jzj+C9$~dr=s=sj{++@WC zIfi9~ciPsIjI3~K=1IQ>floJIrwcm2m#DCytpkAg5M1N%m0l3qs0BnF45(J)uVzO$tL zWPwo*ShG$M1)@r1=DFpz*y^E5@Gm}DE~{R?n~w^Pbv;KymmD!1+{lEYKE+(rL_N^ zV6Pd*)h|>=wNjvxuh#yZs{_hRpp!Kvwtqo0P7lnF9LXS|2Qk#olGd6$O1 zMVJaPreKsO_sR{nUrciF6FO1I0B-1e5wIA01wNb-Q!!I;XMuZ{>S~E}D;yde9z%n} zV<>K+rz*QxPi(BM2LT$x4gE>P$$f8PBasG~BjpQ}kvp_CRQ(RkR!?D`eOj5NH?uh~ z(I;gW<4~$zdWW^dIe<&?xa=L)WfxnHrhf@S3!#r7CkU7@1(!h|v#X>@V> zYW!~xp-PwXtwQ=M`2Q*_E_P!E-c#gFQwDYbtHtX8FP+l{bdvzL z3nHiK7l}W1Ph_0DO8dr{YWZpu$Pgv1w1gd1QM*W6+5V1^KK-Vl?*~LaoF|dAUJ!Xf z6IDGlA@|E>VB%&G@X`4*Gj!kpSE&!)4RJ$8<)o+3i`cjW=$KYWs8v4#1waNHPCzO*>0}SEV8;id%LD3 zM4pA)+0WPhEdL&C^*!Hv05r@i^u;dPaL_KV)~Pws^?}TAb)svne#6r8fa|<|N(J>>Kf#>30MeRVK;nnD^^j=HLp&G^VQK1x*YrLn)ya)J&{3-3DAomQ4| z)Rk*-QMS>;w^vWE3M<8`e9+HsKfiid(aK6ry6RSy1(d1oWc_}e?>+7_`jzYZvfK^q zvWmCrx$mtbAMY9JZekZ$tv>VD7qByM!6wMTXSVd=I-V9jAAf#%guU#@X!k{%%(G{* zd#--xeV6I(<%zP?D=)cg>oiVX=H4J#pzO;v?kDvz((7&a5#uJH?e3;}wvk`E*IESz z9(T-rLgKn{%>9Ahb$d>^7psE)FHS7{54LslKkN;`xIif$V zEA+(cO#5+5&pf?uJ@4`Kk+zR*BitWffr!6i=KBs;X%+pDy=dm-M&N}X#E1skXN!m`=Jm2Yq zrp66VKgldxpYisVS2=A;Tlh6zSmUKNT{oqZD9`6j<+oM0yR4ztJNRSMd9XKM@twC} z)SADK2-7pj!CyITO6E7d_e%cZuA|-r^;*QD7AAkKiscZR ztG?|1z0lnh=m=orQ(98ZgbdC3#-m;f16@`N{a(bc+f zi}LEG@k^(S6WoJyrn?a^)Pg>;Nm1aj0jKBXmp76!A3!cvJS+rLRp z*^Zf+LQ4Y{|4U064wB%-4^~;UV337Rx#87Bme>_PR2yHDF$#6UvUC|@WQpW`j~n$J zcO3Qg%$G^eFZ3=txiqmtbAF@bLu-S{yy*$APZTsK^_zIO0piFAx=l6oDJ{AA8G+~V zl_$JWDb$R#iHketf!-9tY2KY5IpOUnyEFAi6!qhml(!L|_@g&UM$e(>cfXU-ClM`B zZqau49|d$2!ci{Z-jfE(K_`*7*UyxG88KgV+@E!b)g7 zOu)u3>M|C`#t%aQsXu{M06ts?7_-$QV?&8vz5?$v`SqGUeH`v2h|7hc{Dv`X(8LuyeI!UpQ=ebZpD1?*y1n+XjTO@k2LS6sWTJ{IjSV;ka+LFl?jM;EnA40g3OK|g_$e*P66BNlnP9T-npxpUm$7 zfkL5f5Cc`JnVyBGwu1(%okLd<)p{21{B0e8Dbx$gaJ054qYA&8r#g9+TzuWk&{?~k zZ}pqp3*n4VJ1o~Xbt>jMfMR5|PoATmYo#Y9oHvGR-g(5I=_ums^Jy-ORYCNH6#r1M zh)>{o-PK6`Z7|bCB&rYH$H!jq*5J7pylH}d=0jW0ZcS&50)|}hQ(7{ekGbG&A`x%A zfV__f__&MS_^=6tihZ;HlKx#m2@2uNxLM?=!ShN~2QRwl^@<{&{4$OIje;nI^FR+i z_M*3$5T_YH8YZ z#5iAz7hUznW>+4A!{1I;ia|*VSw;r@FD*&r&98bJOX@s+6;;F?PlSSR#19#LkfM(l z$>_URy{V!@)1NunYylP>6v9!yHd2DRM@M-0u&V|OJ$8)k{r%!Pn)qB~2kb)0=$Z z@80?XbJgPuyZnNs9fj}_`tIByW+%To7tB^W9G8gZ$x&4K@;EJdjSVhW2vF66pVE?3uj%lvHzHg-wUoH_FQd4; zzku=3dL5#CJ=m7&B2dol^z+CaKtv(DNzZK3GvsxNaPuM8jQU+5Y|n1i6IU%Xd3o)> z_1-Ey&~(-v5QjoWuzJLzE%|`Iz4gQ>&#pf1*e@7%6v8Lenf&d)y+x9nNFP_;;hyW> zYBuVw{mp-f=gqErW0i9hnZU|1}lWs;OT71&|cN1>aMpeBfsyUSx zzi)EzBfHQnPr0tU&HrOphl?7nsYRd*XN*s2$&-6SxorG_e{{uA+j0Ai?x+%l@J8)8 zq-WUtk2gt|Q~#j&-Y1RtOC~>m{$~>|PR`Yb*|Ry*KOO>^Od)(?edm0|TpRA{%1%sV zbLB6{9tz>bPq-^G)Zn&que=tUr&)klk4m*&FfQHOv;?&l#)@C8gEnipf`l7hj7o@z>F{VhBC^;T!jtVvJLu12v6L zY02L@{N4s8jn`Ct(XyBpDscVYP~sA4sHMc!MJ{ zflHL#;emUnm@ySlXdVVXVsHicQ4u=+UR!HS=xt_`Lf0hSoSPie{LFY_-=zB=5t+8ocHe;HjgFjKGU{JIk!YYAn8LU_|= zmr|xYe##6gkUsv${F}|XV=IY5mmqebJbWyvFz{oOo6rB;9L*oF=$X{G;vehKvIil2 z=i%6w6*7@3DZBZ&1>HRKoJ>o#`r>7DQ!Aohz9^$-T76YSlTYVu{q%cuI)(7s54u61 zX?(#4sL~ay-jTOfKR49aJP4NxbN&(RUu5&?s>lEv3OsRF#1?;MhQemkku?9OQF?zg zheG&bC|cof+kI&-Znh*;7?Oo{U^);dlK6raW-||J5hD`);yG0Pd z#8;v1ZLOV$^y z$%#|w^=ajvi04I`FMSEJh%S3?ziY-i=!z7=6|Iu~gt$P~C7_|GNkb_Hq}PmIIZ4rL zhEsF~KQhAN+%XrjF&{lhvvTD zc(luxnog~*_i53~_d6J)kZHi+e`!gTc^1a^EVricNiJjWVUEj}U@GHCNP7W}wR>H@ zXo1DsYub`uA=oJ-hhbT=+yIVr8(2tliP-+Rg)ck+R_f#SImC+Cv&c2{8)T-C5r*Wl zgpKC|f3jG4X%Nv)w=W?g;*dZT&x`+qw-6?eQA&abiNAamN;LBx!NlQ|*x`HOM=n9T5ga9# zyjA&vC|^9^={0)sGcUSpvmU)V%_^4wvWF9VNZXnrvy3gYI{2n^tA+Q+1)>sQGy%Mp zD*zUqy2r^^<{&+>AtWA^xxDKL`v>4s3gI%Kc$7q1Gy**r+MBBq1$_O-$h6L*a8)7g z1qfV|M;F)m5aO0XxHj#cMDPt|D?bN@h=l1@?tRYc58K5NMDRu`aHGu-WFS2 zj8O<5>{HJ1v9ZQr-xv!N6R+ziie6N_{I6Kx7gEU?hs2CKM&cnQhWmqskEF!)clE@9 z|Df=_aXRVF{eJmas+}(kO=p>%n-DbWM|t&Qt9gzmcpHZ}ShyHzvCoqY$oMrduWa z0uAs3s_0Fb|JgqscnY>G5yG4DcPzn6HQ?Ir9-Ur}XN*E#7(@M-Pf#)Z^ZNGWu*Dix z^u9G~Yn%f(g>Vt|X&1^C&G+7Kck>A;M#JW$pkddGE2Mpv;)@skw|KBJZBvq%!?+2c4_$DL8oI$}9P>0~Bv}EvD$`m8+ljo}07bPe@e1*%~&kS!{gm9rVZ(I<*np}$z ztCp^=R&WuOr4YVWD4HCE55KSe#WY`nDC^)#iXEmee603nieXdYi8t;Yflbtq}qJRBUMsGm0sM2SBUOW73 z8eU5Vt>xoI5P8R+f4W>{x49cJCHuNLZd6rj<3L7_8|znlfD7ki*?BeumB{pnrQYt9892NWAw84_-59@KW(ur@$}lfmFC?YWklx=za#08$ zC4V20@U{0jT|BTGW;#Jd()xJ(qW&O1h48T&eM)A!+RWzTg|+lTLg#W-ZpnK;6MQnq zAze0p94~xDBWexX-DM*batdI_;HR|Y?Xv`*E1*JWcvY^&$_T!&wl9v)%<*}6j$gMa zz^^H>1oXbn$WRcl{bWI`6Vcw@l`>yNvI7p2`wnC2Jc^67a!Ta0Ut&}bWjLa z)MvIO&?b@>j!@?BJIKt@xklBpa}oUh`&JDADNAFgkZV@7=-ywZuc$j1yjFkdtA(S?yNz83&u6XMawhev$|flDD=;~LdR zW-6|a%jxO0u7D32SXLfx&kP8{JNdh$d~` z_!C=K6vCCb{@V$>DvxPs&@sCqu<*ka9TPI+45=84PQ>mW1sYPQ)G^;w$W**P(#}5( zV*Qb@Hal7YAf{2U#73A5=AVh=%DpJ^fkr-Qr^ZK8{2s)&=z_BhfBP!ZFLv0(3#e~C zSy0pwLV`lLiaYhMPzL*w#&G`>vr2MfAk%K^X`LJU(#5pa@v-)e9!zD7LU^xz<8bkX zjddv*GB>Tv^^>#tsm8vRqMt%$rf;qaQCoeU7&iWyJ7}1;#Y@CB^%@ zxv0@GeC!Fk7e}TWnX;N7ao7DKu@;}u#85`&P`tO79{*dh3)28Hi57+oSSv3|GY5+d zb%jNjq*B+WK=k-f5g*StcXK8GmraLxXPx_-VOx?y_=MViq{tV;2R5|Y`IV;nzzbT$ zg2gj}Y)(4SfmuAY&{tOsw>4*`IUj=jqY$pl{XCnXQh4$tmz^&wgkC202-X#XqvpRr zX;q}bmFVYNh>n=H$HfPo1pQ9F?MmStni=#P)C`Ho4u%r3QG`TMfiN@Muxhsbo*!xE z%N5iN+1pmun|{40&?a%1;3fz)NHA*147ajI*yeYUp&Ea)xvrEYH*dbR)Tv9+-6?8z8mj{TNoXX*8&|-Fhv3ke8}zL(_sfJ5QeOT zE0;gDM8d;lSZ?RP_xw|Z%`k{5gwH2io|B-tsIR~}uw?eweu-FHAY_JBv*iK;l^YI! z-Z|p9fK&?MjXC|6M7Vh!8gss-LBsNvXh+lrk+x*58#gbkcH8*CWp0Zg5OLKnh3(=S zIdB9M0%#Qcl$LzBL*iWB3hg1#SMIDD{IrEJ3gMs~J~WCjaSs*Y#MB{&U0Q2YA?0(4 zs())=jNrASzkJ=}VW`^_!rMFGtmyG1zPYtenzZxJfFiW^;ok~X>oaF>i* zPy%U9AzWf#tO(CjkE0LiEs?C$$Tq;Z<&O#&`KxVEoEUxYQDXbQ^+YnT<~LEEOmw$J zVjL_Qh2Sn~>x&Vj8J7RX+)LP)qY$o;J{8GNlk9&E83kywyggG!L&KmDE)6!tNzjuf zPZYn@)?mgv$f*v>k!cHjRySXr;IZ&!usswAPjt9MXE=xuK3~jjMwxI%W)Fv(79j%s ziSN4XIjmxP0U?};jTD2o0bi+74%*&c6Kgq(e! zq92_uqp!60RTo9=p8Mgj4&ZzW;rjdj84{F?(RsfP`g%EJs$N<3?)7v9__7YhNIBF2 z#g1NBAevaH?>orrboSNeD|dK=V31^3Pf2AbDJh@d?W8YbMx3;K@+0&*3RqMG zKcyu(rwMehDE+w(ADz2}u|OewuVnfaf@)Ak`zkyI`cHV((mx zwBjqCxR8p}Z_39fpFlrQsI;{L;~SA->@`mft`!@DD5)6O5}l#M6h1@sI)g?7d>#Qz zi_!u14pyu1oE7F~NUKerO}TbvB>W)mL8~p3UFt z>T9Sj%I5pJ!SCRDS6`O8U;Oa%rrq%4y8a{E@5NpkOp^HC?mn0qbOVWC+m2iV)Sj*A ze-`i9-Iu5KnoK`ZxM#B0%QsTYq8Y&$jXQfF=D{B*CV}_t;q$n9!jC(BI&LMnfFZ;F E4-tA^$p8QV delta 55572 zcmbTf2YggT7dL!n?(W{ZcW>H~!a@iM1PCb*It1yxhbAC3KtL1{NP-HM1r<>#q7g<= zX#!CZr6mi}ML@BEy_d&=U9f=)$J1}R?b@ck+A4OTn`mKaCsz`3*H@pPi_v#Qy}u-! zbL}TDhC|c7e8h-k2yT0A$+R0Hh;E2b=bFj*(yqVM+tuaA`4JAF2mb0A?!5bQhcmbf# z+Q5eTOcU*--SV=|p4Z<--`^H?_R#w4g^+edFOU1?ViQbe6PLkyDl+J!QCiKY- zqMZXysCJ>~E$qyfd(Lmw!6NEll@7_PrxyGflcAQ~66FmYm@RTFyBnc@ZWBEt?IC*Z z%LCsWLDfe}&w7(e5A96|EnFf_yPRgTI#ne4i4y&#F|X89&z+A+3O%|)H1^x6ohy!> z|5PLTR8wChcx<3nMT!Vj|3Zu>gkFBDIc@#0Ma+-4gKEYBk>k1@gB&V5Ag* zTXiRP`xy~*M{
H-% zbpIrE`ZZTS7juxb3b)!J$DJZTH-=SfYNof ztu7}|RI;XNb{^M_;~#A-i5g2A%2m>z5*pc1%a%^Y`x677Sombas? zx4UdSR=FO(t|mUDMTJH^s4a6j&3?O5TkQY8z8vbuq5kX)FlitS;<&*& zIGIFsWHQy44JcKnQd8NOn#nY3C7V%unN6K!JL)cbP=DEz zhRI%Zo$O8dvJbUDeeopI84_wqt>JU14YdWF3~#$IyiWsO%q#VE;1yRv)EVPSa^U|D zhz3y~IoO86a;gslA*nqYlhlE-;QJBEpdpk=CUAcUwW>qjD@++L$b{g;xs{+om9A9> zv^E23VnnkkjOr&ZKYtK4wWKyCDoOQ{PL0@U%uW-7(m1-QP8lXz$zrFOPR)T$s`(aI zl6vz~Em03ID)m{VH(sp|YS<+1)jC96wQN+QXyUvsXH`<~3G_mr{ya=+^71 zx8LxjsJFh+>{XGVM4YQu5UDykMwq=AZUUhgZpHS|5PTN~K8m8@VzZ~27;20o#ioZy zq-T@^H!>L^vw)2b3Ih3nGS~~nS#pVUVXKCnj}#XvQK$MIUSOEKU>K^t z@OhAGF)YU^g1rjzmXS-l0)mILNYif88Pi(fCb@by$(XJJZ8gA$M9WO_Occ>ED{*9A zy=IcvtxJyU3Ap2WmYxHXxP6lg4A(&|A$awYkx!YlN0jJdu1Qjqn3N!H;?T{8xWy3j z3^Csjw;JL$L)>nNI}8!z#06etTd0dWO^OyJ{BajMceAsIoqO0>tc!b1ai6%KKbEkw z6i!eq)F|$ zi;cQ?!lW#*iBmRno-GV#89Ps6y^E(f?rC;{TXpda2ez5wS@9f)w(DYtmvY2Tc6OQ6 zR_tb{To)B4b>XsIL`WBV46&CheBKcIOzQ5$@8`q=-0VSJ9P(0sao7+?4Do^?jvC@c zuICuHahyXZOz~1stcNRJ=8-;WidV#|x_HeHuXD|BaJ8qr=*Vey-el)3cHU;^9d_R3 zrrzVw`&`2Zrua~N#Lmb3`3aBFr@Hve5TEnVe_@FKnBq(El_|a!-NH5hOEn(--9C)bupQ{!&oir znKDV%*Jb-;lb(_(hHSuoLqnz-vXLPh8?uQZ(+t@Z(PFJG(@okfGYpvt^pRP*Y^F|~ z)EZn(_kTI5-C+5;j~qUt+pwM!d-fZqTC~=3M5-FxR!eBqz1#3!J-Q7XIGj@ixo6ES z$-hNV#8vU3b+2d*p0rO)i{WGNZ5{ zZ|2N-$zc{Kxd41yGSWhOPit?>#BlV${DSEv*Mk!bfA=exntzM(zoT`GiQp1dydt>6 zA*#j0W_^1$f*bnLA9zoVU1Ps<0^&jK9U!8tW+n|OIj7&5=v-66I^NR$vRttaAGSuv1w-I{lea#T@%|U>$+(0p~ zI(3gNx%DYg9Jo(!Kmxj28$-28$m`RbkzwTT{}D9}v(iTzj%E8TEMEpeSn zdBJ7#yG;g&$%0&YMMbHgnanl2X66?l98oZL)}R^1v+_!&T+b|WR8fB3)EZS|yPBL| zT+-d)GY93D6wR2T_I;#vALiwp-RG5XDHF8}F6@E%z=!TsHWfrj7icMJO5xmslA4X^ zX=A&O?cTKCm1(i-QHMU(ng~-JJ}=wXs$y*g8Q~5Wm!>=$t44gH&2`EE9IhHBP|EV@gwf;;?Hg%Z{gcX#TrY3!&#j49&-FoR`Jt9ks+r}^(QeV@AOP#x3 z)-f&FO18FS8`(DW`)ArPX|%&99iW41h3TuE@Bw{j(U=mm>@r#~zl%TfLUeW=Thmb^xGvgjgRvSeq`%934VSBs8t z))VxCF1uM0OshqoBS!Y*&?_9>%aXliA6+)LWMA3OlKtfXK^<9AS3Irx>>29T9qvf` zWDr#oWd&WMPVd(eIK@7xxic)G$$^#}BnMk`7ElU}foLO#a6+y{f73q}y+sEFbx?E0 zyF2(TdV@~sa;PPT37NU+7ij$`Sf2O8b@;Z#B94p6Ja=e_N&UF9_xV_LF6_gX{D?u5<<`yJRDJ+;aWBS~p zJSGpx*XI={=T%n*ga;|$>eIj>HBFv8wKxbSO-;@NQYK%ORW#+UPO|76 zhE3+l$yks)5mPMDTC~xnXR0Ohfl!AlBWtUyue8*GZR9jAGToBbbCDVB+#u>&@6M3 zZ__`Z`8-R`m$&NjHcQ?v@6ct?k_+TQOWrB((&gQjTm+;MokbT*44@ytbYY;yU`yVE zmPJd^3SVBEdq#$9>7XSSGpzT@`z(1shnC2t3{S2lmkF08Co;gxsSik(N~YV_oWyqofx&nRr(d&W$h)n)NS zKr_*jYviLU`5P@ENt2IZqWTrwls9w6)MV%I-)my-km3DC^cyl*O*^C2_u_E*f+ePi zsj6#NJ+=#|MHO-hYHB0E)EU#J)Ao#v`sARhH{cSJ`K^81-{{L_@Xxtd^#_oz(`a_s?oe z480)#=7BQ`@>TA6S4L^nwW+F8VYFZEJEujem|wNg>fT?qaq7C?wJmCMk*l@(q7sa@ zeRTY6S)@CdKzqQU^EmNN;v{?r4_jL}p{gY` z@*@}f35P;#aCUdFd)Pk6q3vVOvwduPwvU|;zMnTg)aDGqc?$Y@Bdm`23*&+<$Igg7 z5q=%IY>$7XUCCCe1M0!^;e2nyx2W;G1K+a7_f9_hVNb&eU+{T71_;6@%rg6cy^o~Y zJKI}dHNT)u2=1e3T!}&Ul$5@Q>X%b8j-SbT1*NdBRZs)_@Gl^pr{1=p;C>wQyHGb8 z2a)7-8cH|NwKR*y(rlVU#rV2~3TYmd(0sVJ(cN?hJwOX-6)mB)w3N2cG6C6XST0Z| z(hcFor%RXuQ^wHhw*q>|CtR6;py6@yu`H}%Ii;!we`!sE_4ZOzK?5`P5}2CR)Bu4D zUN@P8kS%E+WsS)UQM0|&T+*nr%8$_5F-XV`QHz0Er$i2Jr09(3!4=eU)ZI7+R{osM zh4xYAn2ZXF3{fi-FQ?YX+2#aw*hg)-d@BChRZ#o%;BM-`KI(BFpqw$gHNUjrUP^G$S$En!iT{xjDXqdir}IY`5I| z5Tg3vudjv<^pogS|J^iz+ge9`o#H{K{6MGtzcX`%gOKP)2i;tfBZgdQ;lKKm_3uvP z4oY`GD5s(5&afO;z*RxRS5qzagCla>=>d0$MwV6nfcg`UQxtm@bZt6hgNbF-D92s1 zr=uM>>Q>MgbdpP6cL)Vq@20VB89)Tawe@Iib%c}bxy>;GMtF9!Ib=_7R*pB$9q{fB z(%4CD-EkfSY22i?o;W=m(#tAejdQOBj-$?S1B>I_iwnzXd}IgXO6_LA4A2b=Z4K#-~ z(yjCa-3k9*+6;R16x^q=)olf3-iGbqIZ*ZO!cRLyGuny0wn8Y{D>mRDP)09^=jf;i zR?v%LKOGZ?=(speC&X*?l6aF|1xP2wd-MuKlW#y0c}jZew6y3=nMH3w1pl_|M(;r2 z{k|MdAINF+A^6~rz*T-MSJNkQ6MZV5r7vVT{RaZTujCo}TArsLXopNd&NOd1V4Ff4CY0eM+(8e6j=*GmbOSpNMeWX=)sfn?cJ(9Z{DI z^ekLLt5;P5oeG!G=2w-#D4;~pF7hq}&xvT*ryZsyPK8=IwG{O@)fEXrErVrh7XT`Q zfGx(ADV(iljor+|nx>iiQYzq$PvN zA{9Yh`w6nWMxrr>s)@6mg=5kXgKHn4Tm7iSixjPk^uMXeC5q5R24vN-|In~YxMa~q z=07y45~5m{F0zoOi)M%;oQb1!(LBN5T^HFYmAEwX)=bw&isnKHR7&6&+FqhYxWG+s z`!@ks8X)u+HTvIi^*P^P)c6uZ)B@qZQMKe#5AV8aM5Wds=uq(Vy5&?SJ-u#-rj=7& z-hAusq3Iy)2nAdR==w2Q)h;#zv>k-KCkXrS3c8_;8V;&S`ft37^kZAT?$8b(nimYi z7%X7TW-8i3HclcAqZW*ZR6hU#T^-2fAVe4SfEY>G=Ic`%kxbWO;KyJ_uESi+!_YGq z!NT|JsJ@MkSamc%JjV#J^Z{Mtsqm7^>%R@-$vws?6)tl{ZgAWnA#`&FdNveW+sq1@ zwTB7@fe$FGpxK?%GwX$D4)1@vsc1DB{^BxhtR>h0kz;O%ZbG?4|INk$x@C-(2`(Vu z9zMp+x#tZZ<6(dP9=dh-7@dQ+ac~daUO{(oF*L`Fl`{qnSWig=GjOAkiN0r{N6jf) zWK(<5f(k`zx=FOL``g(8xVo1&+iQ}qajq9_!2n8JI7|}lz`1Ie-qE7H=m2gloBD_x zq`K|Vh$eZNGIh}rSaeM{_B#O(u!=#2WUX_-0#IzzfzY63w_bBGPQ@F00&&^hR;aJH$OfiE%Cj zYCS=<;@okrsR+kzAl=0mDQ%5?v9FYY%rJ+Z;jm5ZAFJ8oaO@F?bUen zEG|n)bjB=pBU5w-k?%>hMK7u=dQ)%FkEV3zmE+JuHI5 z=n*jjvh!=P#f_!c#CZBZOrX!jL^>lT(Vt=pRf?&=y?hZRt{1Ul2I%MwqJg+kWQtj$ zg(v_n6^brmwiqaiaE31y<3x#=B<3R5JjBiyw~57O|I_U_nG8Wa3zniav2@I<>Hgycb81{x*^4HS~DZ(m<$} zeInvGUJ#(?;!%vlKfr2Jl!`v6#f1@CBKjia#yABRiGFCaFJc5BDgrRC z(J6a1dF?fXlTam~^4X-B7t%nc$%E`a9JFd*AooQwL1ixyXzf7Z{fkV{=tf<|C2ECS z*VIm|UoWN}K!UW$S#Go?2*^N}sJYfzYwL=0WgOf{shM%ER#-@DC^iEj=WF>IEGtdi z?ZUG9sGfaCXp_=9OdZ4$)VdTm$;&WQ%c%*%S-3T0YS|jqyD81y<@u_&dDUPc1kn{A ziaD|u27xbRv+R-9><;@#y9D|nQvOB_lI3w>s4Fl^OnLk5qX+oJlUB7itZ;}w4ozeH z4?5xGx{E64A%y*72yLyW8^&$N{H0(X?P%dg416!czoIe zA!?XW)v;9$?1&%ZR|o^@3VLJ|vaaUCYDS3Gl+mtz^eD*9V}p4A^lRbM+$}aa$>u2g zq%$}x%mUwyKn7Ur!*q*t+**Jjmou!3Sgn6Lxb(hk{E&YRu$KxAfA-EoBv-moeHnxyB;oq>V!wvjs_HT%;UbjE@A%q>Bkbtk;Xw z3CNLlfg1P?a3d2z_iUjAqnEurqV`b&CUy&sGi7_|$t%JU+|dGxqhL&5w4rw{>7zuR zn2fe0OJ(6=eam;z8|Hbg+ec6Fl35+1rw3*Fw*vp4;RUg^jQpAYZQ-wN*uKQztP|9# z3KzHtBj2-(4O`iHjvW+m`?tdhBjk=u{|?mA)|Khs$-X-sEOK@jFLOLN+V9HI19}(( zPX+W~_!PFAxAd$ED$nsU7Oo)+8?%GCw$G9OLQQ123N-^bp@(rZtqO3zjl_1&3SwNQ7FekZS?|H6*_0~#5g!Q z0Xz|jSAj^cftNf*&BSRMFW#aFII>L>?^1zyAE@;K-4+xdVgvk$){BpE2>gWhi%;o8 z@i~1Xz7RfKf5(ZhL>lhOJBn{ae_U%{E4~G$eP_@1-I&q!I4&}VdPN!<$w$ZpovACP zA>_k`;$X6cgXuvrUCaQLx|80sF&P9-TqlCg`GkWu0;9lS?3_0OvC>7hn2BmMsA&Bz zW?@HiQ33rZ3J`Kb%J!+f3_R5+@6-WA52uaA>5Wl-5=C63I=U!~=n6EyNUh=W8N}d? zEGWh!1u?vrO~7Ixh$3+`nT@EX*|odbC2)G^$RR$N==Szkvk;oFMVw*|aJ|UJ`(k>$89TRhVD;reIM-v zZ%K2ttnUP$?c-Y9%?Wn6&lRW}!HV=~%lS{G{2Udhx(ZKBLUp~Y;hEV~{=_2MuWTjZ zRCRcht1^-kUb>rzh3C_X4f=FGc#-OY+X{rHf9AT@bb?2|bM23HiY>hC>g*_s57k?@+y*v# zo2AYUfL<(=aiZ01)9nrYE!_t!r@?(uZm)ELFU7d;@H)Z%b=?nood!Qkb+>gn!9Y{D z-^rkww{)L&vbtKiFSzVrMSJ)7Mowhjcz0}ggy&6mPYB0;wAfwdwnyaIt-1Hy4cxq2 zw{@S@q&ohXRZruJb!lC9ROroR?nS!Ov9T-NLCpz%`iT2{IQYS1?mc=W55n@6#q+zT zKw~z=);9{*<5Rhhx?O6~%*d$Fe>S;$q&P#<^tgM!&k25a+I>&F4a%9#d)k}eO?5Jy`IJY|nJvl9d2<3yMu)4XkLDeF8Io zY=S3BeRj!hg@WT54%TrDJuty@)?=5uuu{-s?H*vLRNNmU=QA*fW=TR)#V5nDTt2 z*}Z;oM5j(Yn-TSFrk;J&lNp4D(Q*L}*JlU|Tc3z9nUd)O^dZIs_hlLanmM|t#qqHm zf?AHp$rf*j1dgt4hyeR_*r|)n*%0*%kz|Pa9Fc5@6hkyHL_4|h}QhwMi*@j(T-tgZ-@?DM~)#n zn&KMK$;1Juvmv?|qALfx8KS!`dYIs5dUDZThUl$}J|=bM?`5JdoS^8(!T!2{YIYAX z&=7-kF<2KvOzIb0OGYw~Fgds+n*fK_OhS9nhql@b}eyj;bX`D&9OnfJp zG*aYpg%fo#$;1I3RV_zV;3Be3D6LE}#8ggh4pqBlBA=t$aAF%nOf$rE6UVX{oN|LM zZZxsG6>vWb*_qAJb2wDQp<)h|7|`s4f0JRW^f4spV6K?QqY~Eb;~;DIu^;pz^8yYn z9K`OOT=*_S%;b-|4Y5cU_ZVWa0o6WgChjw7p=i%@bU#sL6kWX8;9e9La( zLTZKDur{)}I=01=;(I`>)CF$uR|%z#+!Wav_8p#N{{iucC02_y7EX8Ychd&*-OYsETCtQYIl;WE$d+6JLQU7z&aDq=Q@C!mYZZ5Jm)g+^>ruRi7J zY&<~95}U;q<$u~UFuu$ZPl~5-OP}mq%Qv&c(}2uYoml#`r(Wj=#8xOJ*%$X=C5&uK zJcE-nxtdL$-%=N)me?kqwQ!kS4(V^R>GSQBX>%+JQMo$vw5Njs)|;E$uF|%8dPYA_ z`z<=o&N1*eRD0F9$l7iUh}fZ4Z}p7Ld_e59#4g&zQM)Z%)gHF6fA52a+Q<>TvoKxd zmZ%URPTr%=mFl%r?laO*9iQNm$!K`8bN_lkif9MCSJG1 z8}z;z!2dWI&vgGmP5`z`Sh}F3T@lv$q}`d(Q=y)sSh3u=mz&e zm9k5VPk0E?_Q4i>P$|OBIl*1J@MtZSQ|?)c6>y^5lhNc#+8y8+S0m>dyQC{JH8mAn z)T6lMK&7ETg~uyUsA0&{rz02QBTqeAhI^0I$UwMgzEV7z^hiG9(BsYihM9m}(ORb> zwRn$bm^vTwbV6lq_WTzF>;4A>>#r2#b(^sF|DwzWT)ps6%-HMs8)c?G?-?FspvR#{ z*Hcei&TN7}p);5$Fqg)`pft&)I{(BL|Utn@S9f2c5V7ZPy9^cIat@vjR?l0@C@G5F}tXbFvUP z`<;yd*sgF#C+zcdXwUN(33v!~ducm0p`A76F|Ed6Ya0z>}`+p{^_(khqMi!t#qXM#tmSY}&>5+hFA#^xOrozCP$_*qJ-UV08=w z!;^Vk7#I#Ysxmy87$1LvyPwlT5GlK=-iJJCEvg&F%|X>5fz7I}B_Ujk7qsn&U2NSU zPePL`#Txw^UNrYYIP)imJef5z*IdGN!)ck-fLbM|3R(;AuE)u_34b!O0EJM)6W zp8i9skqe@?7ziu<5K~r-ioeFFw5aY+iyEVXNgwJA2`Ic>Y2b)wbdW1`M1$AjL^=u= zETbu2jDbvbEG3I^b^~m?!6r5*phYAx2@W>!1azrxs>{_)O>~+973Zdqb1&*(v0~;< zx^BNQNW=X}1&tk}&K~h(V2Tr8@HDxOJHui~56ILjSi_LXz>MKltYHnU`l{D(CIqgp zBQI(T>nZWH$B^q1&Z@Yc{flaKNBcfZ&FL^ndHe;>Wbx3dqn?c1s#d&ctuVRdRv^%vjJkQ{zYS$4CReu}^(1!TaRhO|S^Yl9#O|-2 z7y{hyhU@Xsd$z_C_t4RL;z~ZUD|z>*C#eR0F1IUi0*I){t_%nET80C499F5vJlz4y^kbg-HW6W1 z8es~*MH?PK(7=WTh9wZLF)S|F_|&ywVZnBrFf4A6jPD@Yb6|;M=`C)P(Yh-P%LU-8 z7ov?H!wr9likRY#PRGU5ELO6)aZL+W_VGZWMu#}g@E26TbfX~ELXRajr7I{q8aiR`BL~2Q76GfhF){_1fTp2uAfPOZ4$@g2D#sXgChg}y zDWpBNT``voQ~W;4MW=?2fi~8#5EbpDejzIUFQN)b7Q^DO4)Ngy`YU2^i-#V0={IOx zJ9E|+{o{Hq2rNtdbN#`aD(L3_uI|km)mik1-3>2P*3O43S%^w_{8Ld$%Yhkw$bsi| zN?c2Ei5WX-K0Y=i%@#qpkTfhwgn*{+|UC7rO1R+>s*qP3wLK}0~ zeyKkA6dp4RsPGpsN7|$3C0v*^z<@C#{#^~vf-pP{{|lbjEAV(T0M2dO#eahn{1Whd z1$e#&Jm27g`P&-cG^_?EsT!Pw{{|-|49-8*;9L%)XCc73eW3r2K^e!XBP8Z6{deG+ zcMUbKLed2lq~3VhlNh{xU|Hpf%Q`JW{e}zU->Ikg6GzYUIA>n4`^G5ihQb~%QXDqGT?H>< zteLq=tL8om!&nr?S)0s)PdW=hA6RJMZxza3f@Y$FT0f&!CNV>+QMMXi(J_5Yj0Ed) z&mjBuc=4FQ$Eglh<-Jg-$_z%MXZMZDgpk>UlHQkyw|Fq_ALCzw{w{U;+mm`--CrrG z7A)_Qq>GZJ2d1t%wUY*Ql|JevBdE8GwEN5ZOD!0DRs!z@2vHL#Oz?eHGI%@<9oJ|= zZ>-xYzH6v(*;@_wydJqOL;F#ZeycVNuFIuXy}nF;L8Fo=@DiROO>1kF0M3!EBSJD&-4rYzSp~ zHFeo?!!Z{w`{VWNrU0F$sBZgRaUrBGciC3`dj=V=w8O63PMvKg<3hO? z^=V;{pTDeUgk_E238Td8G@j=(PMWT}c^;a_8<|ez#CT(7IO`h$7Q}C@X3t>=84o$LTmQSbPg+A6qje{>equyz zm3-9jg;pFhdUkT4d0%?RhEb@U%lk(d+-%)@D$1^T`N>NIRDC?$QNJ1lkNRBm2Gof^ ztjN%i1n=tlPE9|y^?p;^E;DjVS=zTi;crPj_(ue0<%r=@KVR#ORz|M(FFVU!Bi0vn z^$~UTv4pxdr^bb@8S2gTIhCy#=lwW5inFJ9UkqbL`x0+f7&B%p^17m&ymKD*HV=>D zrboTYJ$3^}-7U`kl|%GbPWZhSh9P^UHxQb$(d+H#l)U2;@5=BDCVb&tpKZ@@XkUzZ z-sJ=<;?2r%aBp36XPOgR*we%l9E=U2q<-e$CI&CAdxO{fC$w&kS=S}TseNlQ6T747T6EU587M^Bj!m7N2oI0#GAAcBWYaYVepd5?1F zMGhS^*w$2v6S{cGfSD;^I?3O!7~)m-+16BNOVc#5)exuHKh4gYhIotpw+-Z1{aAdii!V&-z<_lS|KaApG@&E;6+2(ME5rYB z&To9UP5ah|tF*7!`Ce!9(_Z37pZH1qti$${`f^A6inAPZjvM<{7r&Zd%6?<8f9KF2 zoZ(Lnoi~GUFYxzY?7$Fpkobo~7umVQpO<~2QevOr{~lSjzyM`uQ`D2TDJrMA4e3GM zQb&{0@JTP4l%|iSNuMq)_9L9q1g1_#@~Hf*vt4R|jK=u)m5_q>g>kB^C1bgYICkQF zGC_RH3AH&Pzz&4p2Vs{g6J1u#yP=`sW3EHi`bW?^V$Geb6qg%;*ZwlHK%lUB%9x`aghVc7=bAlsUN+WRZkWe2|1pTuMAx z5?mw)aO6P#9^{NuRQIli9BjxTTuq!IbJ-ti$YF*YZpaasROx0AMzU!zaAuIamL15; z2g%V~&ls)|GIJ@%a%h|(#{)|wWaih)i6+gI5SY)DdED#ZWJ6Bjf>S+RlkJg%VrsrF zrvcGqoGz!cf4wdtD?cf3(B+LL6n5JfawY&^L_8yBaXSU<6!Pb6h74B9a{!!#0A0#r zLzZyqx$NA;F*mbw3y=IfKL_V?!CQeC@-|)Gu4WgRja5cnD@9FOX4F&b7n&nfa7aYU z(&v7Phehwud~ZGV@SSFJ_4b|SjiKl7GWTf_-r+N*7vz=9Ey@?vTMbQdHBrm!xV)uZ zXX|Rg2476*;9~QfP#@f9&MqC(J5kko+DKKi?l;${FMshhQGGv&$WRNGm<6hIo42>R zVX6797_ajFk_}6@&x(4e)TJiv@FuBGmYH=LvSsZZSk5q-)8ztPF2n)@qGR>x@=jnr za9rK-fcc(PoHq-i=8{>nVWg-|kM`D8H?1%uO&HiU=6~WDwS0xSK)pQ0s-+?yG`qT> z0Qf^_$b;q>A?^u1^^ob3VxxNYVe@{iF3h-QXPS7aXQdhKTLeQqxtQ%rzf(7^G`FX~ z0LL*Ah1JU?3oP7kmTC5U_hO+%}7ZS zb%u6%%)AAkee(KEn+OWS#Gh!5?N-+C*@PN-Qv>{ zD}5nHKBKz6A{twAn^c%DOFk=~Q{6mP9M7a52E5xbW^#wz2`QnkUQq7hs^*JZEx8*Z zSuT25vVub)cJ}b+UUr_B`z*PiLkHM7D7OF!EO|&C#(`l}(TtLk{DNeh!C<61rLbsf zaWhLEfu0S7F!|F8i}I5T=a$Td0qK;Pg~j=ne1Vhv-KMFBR^wTuqw+;d9+Ss)c|v`D z+=_Qu@+GNaL}Z*xzKjmNvD$1KLBF)_6dd`L{8~_Ix8mZtvt|@bPo7bL z-m1d8%{b8vR=zM~#DssN)~+$@1F6RHw$TtOY=x&X}V@unpZO|&8JzG7NJFANJkdjSWtL#LGr*ElZ*0- z=2==4UJA=xYmS8l$5lqGmKLM=QDE%Anx?HCF)X-IfkAn^dSb2Fxt9C^tw2E$7jSFD zqm~wnHTDY)_(v+hCqj38S^v9yf3nR-C6dnj2MC ztZ%wnd51UJrKL6PuTpZ1CMxb$+_HDR?5$n;(S9pbf0H>`gDvwEHPoYZRQEk)UN7pY zt}l3>P``EZbyDr_@?|TnLJU-~vt=Fi$J1tsI(*cd5Zb=goFzn(T7SI}r<%X$ouEp) z`eIbAeMU-X>o#+$P;W^XOuzl8XsX&jXU+?LjcqFiQa(^aJH*zA5&|)T>xT5n5wHIY zUoe4#%W-U}cn&#!EoKo=F+@3nDc-+R&vWVp#Dfa{JN0i0BKaRAur;DnH5*i(!ygh@ z005F4PN-`0cf59hh9OJji^Z%|J=4O;_zzhKLis{D{L@+M)&V~ePc$17S0V<&=J52@QA9zZ{WG~Bk8 ztVMDU%xdr;7r#Y>8;)|)*)h`n-g3xmV(ND-htg*(S>+VT2~n^n3>#GX)1k>-3*WJj zP{x&0{0c1WngCHv z9-i+S2UCbDvEOi!EK>006%9ngFz_{$6xmEL0pg_EdnjvTGg0Wh!X! z8hR@|D-aP1(vY0Uj6h^{15tsf8V%qp5NS6M8HjQk;NrQfDJBpFTY;fw9;`RZD3fD{ zttKD)FyVAkM{w#0m{QnzNAlOmcu&0X@EXbuL_o>-+CcDsl^ex5M^%*@&0nLdhmx&8 zF{qu}8D$h>4`vy80}&c(i}#p$_Mq#l(I#phgWG~K>Jf1B=v*f$-Zb;ZwKHRB>>MwE zfX;NV1}@mBFdkDM52;}b|AZ?@P&e*1ao z?fIXX6BCFzPHd?WABce!(P1jc@dy0J(cS2PA9oQCl+)}$bOp@;M!+j7tGpSIfnvJW zQ&NDTEMX|;vcqtp{R(1HeSj5PlPc&I`ztPAC5?Y{-)9tu!y8~C_LEK0$!2RGtTzTDt>#@byS?mViPo6{)P>)D((GEuj(7fmYx$@*!MAt`z-gm6%2fYNwBgn}NS~(;D1`Jt`ie z$HZz}C2gV&Q2O2|o~KRXByAS&)05&$dPe*J`BMuwe&KiFehap1z(Y?aVA?21IjHhpxg!n%3Uy?+zaE$k6}FdEsQ6Br%yDA zS9GK43oV)cqcx&0VM_Rw)}FrBdeFC8U;0iPLf>n{=tpfFj3{rWv)Tgs)uwyDLSUW% z1BW#9-9mqpMN^TEd!J}(4JIg)d_mkc41zg9Gu)WoOegG&k?8O9cE%_%4GOu18}|+z0*a3#`FqKp4<|0Mr4cKd{JYb^r+{e-IWz7A%Aa zIv2Go=^NJdW%V{4Y6ZAt=6}N=tnH5b*k~9q4*!Sx!M+8z2K;fEV0$q_7bA6XZCq4@ zSuetj)BhrO!r#=z&IRkUQOH*byoAc&zv^|lpo`2Ibk>yPTA(k7CmsQ&`M-J)_=e>>M)5IFD76bB@7C}jD~v# z;vo&!pu2?|v<3GAz{ne8X<^~=YY+@x4J_XYWx&!h(~n!B>J76NE!s!-pfij2(Y^5Q z<4yYhJ+uT?W$FKi#AW|$(_D@Xb2D{B4KQwt55lk7+8^NExr|cw(F$Bv*?Txr>r~K# z{)ZfBLR5+u4<6=?7|Qr7b6nnbZhNbqjtDx1KIBd0O#{Td%qH!3KfU%R-RM4Xg&0~OulVN7S zV}$M7#h=^)Z{HBQ^WJ>|bUG_OT)z)lU_QT&%?S3=dKZ+9US%u9;D)vq`y1Ov#95qU z!xNMc4y~tdaSbF+XH5wuGpxoHm!y zFz2a(4R$fR9=jxmp0o?2#vpBQs@&*Q`6O4#m0Vq~Q)R%1E&nN`!;oZBX*q(hEo3HT zb-bH5R(C9IwKvw>E3;a~jBUeM#k(2jVNR04_C(JKT#W@Y*_6co-of9t4T(Lt7|jR6 zOQP&@lmAnNyJ}XL>EDgda6#ZkIm1{%74Vk9K_(yR0axu47S2=VoAM&qUW#(|H9&PgCT~lFk~rBz}KgNw@ru9 zZ-y8xGGVFM9CnD=;#Lsq<%nGglKdF#`kn{LJpx+$5)@hA0Hyso2phx;qPvu$hqOdb zSxfYibwqEOBKpWqVt^bh2Fg)ln4B(#%a9l?e-dNlIWbON5aV&io?z1h*o26ccs5`X zjw4VYg%)KVLJ`ousf10?57=35q#p&_1^EEzH88833X%Ug=sGqR!E9XAegxS#Z!J2L zi7;95%b&3Y^ZDrxc_-9nd3U%=-X7i^-jvM*RDeOC-jNxw9qEA!Vi0kRV$@{}m_xnm zU8Xqb2Q#I<$cDfpF4H1i%(N-(%u9HQ0XiVTR5n;294-;gZ4>oyJ{8*Da~ABU|66c{2HXN!>SFig0x zqn{kp;82bm3@Nx@rWe6-EI|8f3vTSOF}-1FLCDz>PEH!GJ61KLn!2EU5>J zrW*{kJjZbe(T`I%CZv2gv0q8C4=13BVcOa&2hVC$(BXimf{xfswm;woaX9E=x^S?p z^7Vk5H6m<25-(@?fg6)?gq#9}u&J;(&qvMEuyIYt`B&UPQ-GnfMIqe=Tb}zx2{^&I z;Ph?+hj$COx_Nl*Vm^@cR+#cFurZ#sIA5Ven}za1@34)Hz6L!;gJBW$0~nNzsJuuN zGfLA>VU*?&GdDPPBg9B6!K5O{43-O3eg!Mysla6BUs&hGsbns};ubL_gj@PF=npX? zF%-xE&SNHD+Rx(4`~Qk&oMjdUY0x^kx`>^YeVKYGsh7C}IG z7G^doxFJ;7B|QV$MMDOIsQI(1&jvV*@}7ffsCAt^54HBt3psos^=C1`<>MhE!O>?> zg+m1E1Tj`57NLvxU~2A#uF8Fs37xYJFrBWCY^>ve_Nqs)OZ5nLv4;(htb%wld#h1A z9!z4+(`64AWGa`agT2_tpc4=dhdZG#>|ANXOyFM7M!vx~565L>;mH#|Q6I;74q*1) z?muR0)!5|PQ7l{qV)h73A6HYHSOZJYN5N6As|GX<1a1zvYG#`nR)gHI8svsykWC7R z+Zbdqh~F=UzUf6)#uy1OA?Rc<)iuY=MnT>t9Oa@d=u{cnei8(BD|U=$>^75aMl80v zrC5lQSXzbGr^Nz0X}3lln~ zX9nD$h3~AUs4PCfJJG3FKWN+q+v4=B5WS1f_XJgE2n5}O$|~2}Nq_(?zR%B3Ev*5~ z2Mi4i=`n$b1(Wesjwj#=!&J(lFid*paq12zbj^mMT^3VVhJs&m@CEb=`fzpCGZL;i zw-Xnn57$&hy343rHMmi1s%oZZ#i36hRY4uDFrW{ztHVe2>t@h{uPfo{3ad}zk`wb1B}$4;#u*RC>MW={o)@mRF}kQ zaap`CD(w|94ZE#^Lgf-B=ClyI3o|z^9K09WaXJ{gHFg}+z7N7HA_h#E!|;qpvrAD{ zqI+}jR5-Q~4GLMO#d54^7r4aN#RCXk*(Z<~p7-xG2}@n(k118w1w#%4RSwJYWC^&s$8j>D@ROLh*oc+)9j%hWD_VidFV_4N^N! znze$sP6<=LAxP#UGrvQBP5lb`l((!Xev~BY%Hh{GeZYeWczYZ21HawAqr+UQy8T+P zo9cn~CEw`4#ZZY{RE=q+`C2C^1UeiHlZc=#38l1}G$P=}aVGHz(qZ*(*&m$(?!jf1 zZ`Ua26a({Ay^O)VWo@Q7JWhvmRER#a1Dg;it9+ARW%%4)H}7+=qn++C1R$|6GT_b54GWA!d$VI47+O6pL!EI6f->}h#oR^= zj0Me~O(;@xTsT|RxIjg-616$POsp{^xH=3E$y2~R-qyF^shu_s z4)MzyHt*a3fu8ou8;f|1Y^sd+4qOfn;eC}dD$+XR=eNSR2P_Mh1b$WH9yS)BY`rJ= zh;RaL7w}hSyp7}n?)|G^gR2V$T$_XZ!~)7N+Ll#*cxBw}dm`BrINA%p@Y0*w%KrFO zmjS@hfxyv0G+hp+S#k()bT~aEN6>0Hk~ZW0Xr>CY!D`gU&cJx-LO9LFORXALJyeCO z?jTg)qwS0yh~%2u8Qr*+nq?y^8|2(zBP%m-TdF5-OBks+j!Cerfyta!HDg)iyU0)F z3{M5lg2G-wYN)3Isl9&}RAF_a3#cUmt>C8_@b{D*dKv}@`~xl?x)=EI4Y0caj9g1B zr)f|JYwux|vybJvt^xT4v~>RC@oE}Gz%EaXUy5P@p7+Q4xe*?EE! zHkoW|$Dd`KiUn%^OSr4Bf>6^c;U~_rYKemNz?Smrl@FypGDD*z}p&R4KZ z0ga+>_~Q&i`>jsj>GZvh2k>8S?CTYpl-3AD$x5{LwNh`i^0kQaV2rt2&s4@WzR}T>MSV*o zV?0C(T^2M>oo;~n>e>Y8fH4sbEYXn1`5s!V#I;Kc(cfdzgZ2#YTvwiQjo4QFL2?+#3fKUUGK z2g)iwk3p^lqZwzi@CC3`0+X^Pzd1~1iPjO zv0FD7Wi$kZF;r>;Mbi=4pn{;oN)i-pP#QY%BB6|Rf;O2AK3hUA=3pl~Jktb&(L2wy z32$K~>bWaM7+N~=2&s|Q(hYkgY}sHHvvku=e5+gt=ss~S0)BXe1mtIAVxJ-fH=xty zmB@~TvJ=7UnU9Ad*jCWBA*3dFZwQ{7L53+8B-3<$l&X^spC<9@#4;4p*0VS6Uv?f_}p%wd`HMrz!Psm9*0jO4YkV@M<5F>g3Z* zE}29P(uh`0*XzMoceRv>W5=kkdIzQYv}{R+N-`qBgeV3Jm{~a{zyr824_jR#P9qVg z@&Am|gvV(d7NzNGsW;MS_6{aL8>B?ehMtB9ZVt#hQVaaZSuLgG>j48Qc9n@A5iW3B zX7aUEUSj+a{28Wy1VnDfpmp&~=_6k5GpN6rA38bn^&%zghh{3SQp)fiQBpRRx zry6A!C$R@vr6pErX`^}>2o;xHP%|5WU!l>C&aLBKhJps=xs=n@C)@k^#lq1Un-EYL7xqw+w3r zl%r%Mk3p8s4oz-|9qMjK(1~<-eoh2i6yXR;-ZnJA(&IPK0u-Ilj zwhc0ag&=g)zURC#L#Q+RC*;+lz3cF}4mSi@djilI{lMBn9I6n)G0vAKf4S zGoFtu&+G0ytDogcD{$TsmxTSjsOjK?INNwI5ifa0vEkF5N#(-_I)6xX({Za;+(TYi z2=mQ_vHWaXIl%A4irHEG66aTf#O1%f|Mxu1^ygVumD%E0Q6IalJn;qR^8&i?%@#2q zVRHGAOFumMf>SO3VV(0^mx)p+-?zm%M&^^MM=Xu^zm$d}dTDelGp@YzR_C5Hk(lXo zXS~NnGcW7xxRpbpqleOq$)ANEhDPZPXEF;a%EM?W=pwcT>0N(OxW2U)Do&`>PWa&TGZWt2gX@VF%imHSF!i zyUGVuoYU*)P}zodVJFd)ar7$QLi%``KAyo1GITN1I(ZSlmdQe|Fs`Nd7uVAJi)-oq z#kKNs`go2$@Cf5tc_r@faY!D4`T~(zWtCr~@6{B%hJ0(C(uB|}jOzpP2CKZ0fG-p9 z74p4mmrLc>$hV0;%AB}`XC3))^TY}APOFRwTV3LL zH@)e&h|u4{twauNUcN&RV2IOlP2@#*lQH0cMHeQcP2_zx`8^vB$ia)bXk+<+6-VTx zTp2Ubx^zx%EzZf|<0JC@p99wnVe;vbG5L&_-?7Pud9Dcmh)u>kG+sVRk1Zaf;Lj+^ zaq?j%x|UAN<;pv#WHB8L)FY0*Bp`CrL_a~-M?PSqLv-={5FLGgYokMS@$xAfJ=WL< zk2T_BzY|jcda==gJtGi{$HR^IIA_Jfjad6Y=BLuLc$lC*A}=D`pRMvQ^nISz?blgx zlqrDg!~f&Nhd5MMSH5JEFFRn)cf~1fpp$hTyy^&(`s@5W-4jG}Egh;`ONZ*7kpHsM zk-E7!QiqV6jnRT)4=J7ap-=fp3=s#)bHH<3t@x?yc61u;N`Pd7$bq z|KmZmkxBNI*BY)?MEHT=vnm>dkpJVvQY4B1Qd-_>q`FhF?t*B@(&UvXMM^|wRiiNA+RP+N!Rl3*+K6xIyGfS@o4Noqp~h-%j$CNu>;@^^(1+g!__UwSuTpVM;2|_xX5o=fQuq7DA)0T9C$*wezr>ja9o?Xnt%VKQH9MvDu z2+{Iz8p%Y@2C{}bO)XjGT-9U4`+@X|T{ipATs46W#fPmqJtVjfv`gly?bG%tEmS23 zhgfj4(pxG_bn4IzW_v^L0xGRQ8Ic>+ZLQ>C(YydJuC-Cxs!BUL_|hI3ghkR@s?tH} zh}pIrrz)N3J;;1)EK9`=SLH!}nlxme9g?4|UUC3huC_eZ8Zsw#N-&#Ls}Cz?R3zk+w` z2B6d*>Y>(6R0b*ykp@mlHCGDpRG;!7Dlzm_-vv)?tRUdxZ&k*Pet1U?xxK#6S<|fy zM={9{%6QZ7A!P)sUZNIfb->Gd;3tVV9f_y)P%Fx4{+OQfh>E?qA{Col_?U$o)nWZU znL%4bm;Y3vzAdvk$7CCubiip}ncP3J{F|rLCbqy@bk;Nm(MYu5Q}~rzGP-N>Z$dbv z590vw`*r+=tcHHS0UWw!i*O;i_|`~x4RFXY4~Hm+!gpaRin^BMCOTm}F+p`0LMX>? zh(PG~yZ9}m<9&7(SgRB&Ma_P(t2NLS8)~ zI(%~kS&s-ylCY{K`6*A5b`CK}RbB@^Y>;G{N|`hEv6D;G22oh8aAUQ?1Cvhg+mbH> zHYFNMS*k__qF^?HnW&2M3Vt)q$$d0Ci6#t8%VV*a!S^TPXfY#!wi*1im=T{tCmwuc z6qHa_agMMP`TgH$Fz@i0l5ZE;m?sey2I$F{S*Cy%-!A90@@Tubm&&a+ePoD~hUCgB-lJDP+GC}o4@Sc#TqAUQshS>Wfth)G%xJn{+**Ly zOCL;4`e8~^h>6K?Of|-0W-}f`^hYr&Ud3lMYaokjac|B#469#4v2Vo9LNCiEm?K3( ztVkY~qVl@o77w~S$^>dLuyOw!^T}11AXpza zguDU)JBqO^UH@dE+6mS>d?Z#7F!-pUV4ee?sQ2+;Pb9nn5vo8tczi+>B0FjhwxQd9qbHSAhsvy$OP>;14UEQC8`v+fvFs zdba*&`0q(6Se}OGmXty(J3_Y4L$J{Q;5W^N-h-f-@ZB6Y1xb#Oa4S!q=K$nW(*-GPLdv~5OFZ-L#_8(8#sGt@?EV{wMAcA_hw zQh52%Ot3j`E!46yP-=9&)giL*60-Vh+h~88j&5{ESK6~t&&bGwcUe5F(#U?Q7d_1g_<28 zsV1|!0xZtbn}b%YnvT*1vp$p)2`e72faH^~QVkyq3M+g~P8{xl>@)!7j5`KV3_$w6Q6e3Xdq)O)dQk+$~QAGmpN4$VD49cJ&!QoS%xXT0uugbGKmJfX-SrK_c<{XtFUp$fx;Lqhp* z>vCSd!^mWZ*Kd4|tsWUlcfj>a|5SO1-CXNPl_!u4mFG9{+OyFUG@|wbkp1;&2&leC zgB-#v6t;Z^;b|Nao?M|GJ45SgT`>9nSc)Q8H(YFLCMHF5I!rn0PB(;|3sSX^wS@zl z%DbfedJqgJG0`Saj8lf0M4<=TA)o|-ol;NeZ$Ft3Iw3wibHF#3d?c>(R;ieuAnQJljz;#(V}|9Fso4Ay_(=vX$L8pFyQmg9z@MoXzplOW;H zSjz)g7^H^B7vhA5!;&Cm{CkKhyZC~-jW-Uuzw zcguK~8NX9-dKQKnXo@-5l|nBfZ^Mq`ee^ai`DJL#3P5q5AL}O6Y@PBnI)R2TyZj_> zeWnAn4tWSVD~f9i$R1d^N1(M@w4)O3tSEv64;%FD|5MSh>-lsbWGJqBkYyGhBZ~N`$?TG6vF1LK>R0u(~a^*mBjOLVs7Vf~wMY6vck>kW|0Dk8oI_g193!%9OrPLf3 zY{fJL;f)NAm`d-0;P9gB5Jq-*Ed!?=gKO~3aEy^c-PA5{RDHD?6NoPalC@kKNyksM zE`B%OQ!JAjM|<$D=RPS6S|(MG_O6!vJP?c;qxjWBl3!Pos^1Y*lW6rWoTP!8;`;6W zrBE-1;v0vtJUhQy9n0GhJ@+&o(|td+1q=ZL*d;xQE`ebYBLO1`4_qHP5uaWxdjH8q z9Jb6b2d@S^8#ar6#vUq-+B(CEg1SFknvTbrk}w)^V(06aoQy`HV$u9QZxpzeQjQhv z5d-kEl5qL8nz_erBM z;2DF{2URt4$$u$iNHL@jQcU7D7JZ3@`Vd{3A#EQ^ar>l4KrS#2!UrIodLBJ(m zBTk^hxg1e6D4d`6y(A<3k-UIJ#o#eszvhqB3n&uZu~+zU(A;m%#Ia+XWPU6H zg(S)?zU=-fJbg#ob2PR*v0i;pn{lBHOcp&38uQbl$3bgy$n_BehhVY1N@q8G@^g4Tp&{Z%%1>eOvk@Ar zCQn2sN_~_sfbQudo-kgBz|&Af+JoL=h9H9u?S$M|Zi?j2V@^R)ROu*QwI;S4VDoRz zmQy;2iy)c-&n|6|YjZsGFEDb}7X{FnW$O+7Yq>er9H^mxjZQCc9W5PRnOZ+a&IqJ7 zZ$=wFDz`wG1#7r1H#Fi%LV7Jg5`2CST1rdx<8(Cly0P+$p>I| zATo(#!6It`xfP=De1J?aAFV-}n97#(xMBg%a+?s()H@26-w3iSw+*rkqg%2rlc0V) zq;{Z7q@OA<#SH0-_B3LrV6F~=>Zb!&JkUTjg*mkJa|Lo`4JsK!BOOBTK2W&K07Zv6BiV&=eNY!3y&0NtWT-)SlV$#CVxbZ=va*O3x+3^db~R_mM`}eIg`*rgt>L@ zKIsW)-kDXimmHQoJx6p7qg+Ww0sLzL9y0ga zax4g0SXS`_z82u*h~EUy(|D-w3I0&uGr`q>!KLZ|UI{bNHXET53gljp-9!wBNi~{b z(lH8eZe~HuMxwVPs0~}O&vGdCM4|A4B6khv9zG-Gg7Y4Ixya!LZ@C|46%o+qXK>1c ztbp?H2-*P9Gh?#zCQi>HwLoY+OGGFPMJPlBl2S;=eJPQK*nE_Ym7DvdE@YSrpv7Wn zJxI>JCHWz@E_4@I>gFwx-nLLkvq>2dxrtbk!rbr`EJr5pm7b+RG0x-%urB#<05_f| zZk>ngx++d9dRK->e7jv~3d{6nQnHc_YyPIV?=XiiO3@0jf?j9{azaZ{890dl9lpso z49$d=qJk^LfXT!0i!$P%d%G1m={E8&1o9zC3DAPQ;23u=3xO;T3M3YJbg8ObcS@ug zEmtFRbIdrr#6^N6@vKlcVM|m~La11x3Jt#~%Hv{50mOULSRpjvk@&e!ojPU`H2hk$ zM_OEuJP^Gl@CU~PpJA!Mg%t>l&}rmVR_IfpRLB9Fw#F zmiwa+W`~3|j6tcxS#^DFzFZ!w zOWk(7Aq%M!az$0e3CrWSQloLr`bbQ11XRDg03Y z_0cp}nA8O{sX+H_;=P291ZtvcF))|0moiCIEhZMiOw2*1gmqA_ON9x=K}6t25z3&D zY>}h{bLikppVY1#I5c|_)Wlu`(I$f{jIc3}5P}|y3b8>5_~0%u0?V#qR#!EpVPPr7 zhEj^M67V&W zLN*kNENzviu|~2z6<2Ap&t-d*N!o^x$vkAX9Rrmjrq~%p3zZ6zJ|13##^tDowu+OLG_b(_QGqNy>ns!~RXN!3PJr#`{Q+_@)JKHW0;ifwkU_BwO&fI_1`QxuJH`fg zX{Dwnf2;CIbbEB3Ry3v7C>Hul+9$n$G0v*pQeMJ7iNX8gZVBtktKnO-TWU+14zoXY ze5)EgYCe7_@&fE1Nf^P?12}7?^=QKz@V(JO23sEK{X9R$?F`R_*fGx;Avdwuu?Ize zWF;lOSBr1C6qo#@2C|8JXKzz?`F!M^@RI-g%^1Uw>%@Gq_Gq^J zUupyP!xyTje8oX^i$lc7`&8{^72X%VP%Td3-EdlcTsJC;KdX+77NHCNQvcAcj^^A_ z<8*}8chq6M1;X8Y*DO`s`#x)sE7mN$tA@E+M+$^*r@F>w32%p2T^CetRCH`wgU?%` zKW-Hfb}RVyT`q6=s2#4JdV+g)yLRZ7NfY+Cn#OXgsEmtu_?iW`*aW(z1UtI5aOmnC-7yjj1W)`9GcU&KN zIE&q9c5VJ}GVY5{=2z#x<8+rKbHHamo*n!?2KDb-=xN1OPPc_6XS?Is(F1sA@{79e z{rdTy=Nq|u$B2wy=P@Wy-K4KAtJaB@0R=n_@T;Rrop|dG7*B83W9KyS9h5J95f`exc zxCgt0_diG7W1Iq^?>FxCkvvl`7X7}!iA*^yD}A3vvEtrZIy+O-X)kYg$*o%4R7{h` z?MbhZDoHg`g?ZcAT#t)g>#aG;pZ?vQnJS3yiS|@;3-7n7o?rYn;%UR)?pxWS5>F;O z(bnTEznAN|Z4(UK>FDXwPjwv!2DeOcgBmtW6Nbb@dHTeZAyT z-t@ew@{;>Gw{u)`h<tPLFG9pvL#?-=1ZbOpNlikY2T;P$iiLo{Fch4J2-W)5D=;{F)wge9Sq zi!wyk%&rd6JmqmyyiZYd@426d5a6Y$d5Sv(`hk1lAzUXD|PBju-Dda3jVO zbG<9Xt@;O6YUg2}CLME1rKSK za_?Dhx}MYn=e*rrBB@1Jy!j5n%l9|EJ#+@@%f7@2(e8Vge0ZLLtAILA-(Z{YJ`?FH z^zt;fr#$y1mg=!h%pY;(h2q)iO=fqaJ)ZLBX}%GMObjx>Ub^7RcCCVM^%%Mi5~EV~ z;{{(+(?kL$T=d1qMS=B6qaJNg&`nmNV5bI6FU=xoKA_E->CkDT4G2`=yc^Iz+@T*{ z>2jL~+4_sVYW(#LwjWeX3qgfFx7=Kf-MZ-0co{IyC11R9ErO-Ako}qAiIePJ)9MZ# z2(clT8=@^qr6n~@JjTkR$Yi$pQjpjIBKD1)gI3sL*^(=`2F|$x@E^GXoYkrtEB?!;CX?pa^HsH^J?MqWwKZ_q>>-`n zl%XftwT>i_YiL04F*1nmVuP;&o$|kb^u&^$en}!%U_kdQ73j;K^Q}CK zqmPwYxpo!29xV%jS?)DG@kPmB#aU4W@DubuR=4cB~;NtCseN1yMWg@Pw% zEd1W=^J~8DTw$=3>qzeO=fU`KZ2EOyqyQ}@(C(8#P?Di~DTQ6?{-F~=ot!a+Wv7F& z;#jxeeUZHIo^Q~xZaau2xd#mV5BMpF9?RbT9hFMd*{8o#tz8tLtV_`>@eiNg%Vz!o z(6(Cys>i1P;foT&xP*dl+~dJ3^X^w;XRo53sV08D(0N}G*d*7^kapw7tk+GSpY3QO z*J8^W%N9d6xdu+HQSnQrB#|>#x~WAdZWQa$Lbh-gd1gPT5x4L%DvDf|A;IROd8CyE zMUgDwuV7{y{sq>jjSYeP}%;sFGCPNO~LQH!h=`2Wfx1j>C=*^BxUuWAT{%%h{2L?A&bPp>JBeY9083>u7Lsd z_cKJiE?agB%o5cL*PYu^Aui;Ms^MjJ@RqL^N9sQzEwHGSB#~=jpqIw7Zu>fm5}AD) zC1S~BQ||c^{)_u_d!B06O~GSsaE%NkazzaG!&oJfx$fvnWkA8ZSGNF{Tw??NH}!c6 z%-aB0cYjl%i+*Qq<7VcA=7y_p!2hzLz(0sf;@J_<7u=q`L-p4#1T(ROyMR%du!eU5 z`?-4z*%# z7pwW$+vne|AxY$nWM;FXd%gz*Gu!X^;v;4S@S?h`#bt(de8lSJp{s`WLJ^TOYRm12 zG37}movklc>@4p->YMb8(^56Q8fCnDffM?QV41B%wIQA4jFMo9eBlGFHtQA{;o|7LP{*Cx)E-7SWQc5tzsS(^wgrf&E!h0-3kvzlChG9;G~jDYXb zl6aOEp|ubU&WQknyBcv4Y&(94;A0egDo+GgL}V7*dxOVZ;p=r~qmcZGZFDpJlOmapFuP+i5iWWyW1-qQ3BWAF)dvO@dy)44Y z+?Ic+gy;}UR0-#nvxn?jtU(PD$>sr5qdCX`Ib(@MJ;!6PVJjj$Y%@|2rELC=kwej7VhTtii~|1naO?z^?A}`dH`OAnZGWoq1Kj zJg=iF5^OcB?ltd z%cZ6`mbE`3nN>KnSRNrZFl>4Sm?hW75Fz&C$_R1J9uG|%HL~qPQy-o@lF5JjB6;NG z4I7f9+hZ04XB>7+KUx_%mW_5ICnc1V*-jMQ_g|72X!@xEB5A!pR>QLph7 z*p5vm7rXzO$;Q@r^hkU2>)sxNZl0V`B=){8BAsw+Nu2(*afu(E059Z>J?!Ln1?uvy zW*f`)Am==H`5xr%nSBIHVFwQ)*~#zeX$yocgz)2m5Q3ZC^J!`l$&4~PCOXQ*Z73_bh*DU?7%dsC z#B3M+IB!)Djd(@%2$Y#!F?1MC~imynXF$_kn!;p+^>-cE}8~G*cYWmalzU4{i$c1=p)D(2iv`}!|yxn`Bc4qe^gMS*{Ix7op}uSU7AjYYXrBJ7P=6mZ76 z5Wy7YjY9ztY-yuuGuvXEL(Z71^RI^RZOl9n_|!80SbD2fA1srR>uF$M=Krx2KgMwl zj+&#`RnX^?FeSF&hlVr(gELOUb6B!p>na3ek{=9y^s7kF`v{od{u@5_`-77686n)c zpl9)-Sz|qNp@yirpUZmtlW$PnQqvVY9 zvuP5WQXQlbGlAm^s>c&RwRWt4Wwiw@Y870PZZ2A!@C$e&XPl@tj}!PC+MxM!e5%Oe z;yDA?$4e49<4}I0U*KO$u;880bc=-@OVn$Bc#3KJJJ3DkjOzydn}j0S*aJ;14&GS^ zacWZyTn}m!LU6N1)q-vLWrEEp6tMkW%}zFa5K`nifXm{2LRdG55NfcN)wO7z+pT$< z>S4E@toy1|;^Tt}wJq1|@BPa?cMHb2#LDO zAcZY|%;I7PMuC}(E0_jwg*1f_%;N;Y#z`iXT^(n!vAsEZh4y&9=TqJN*hPmkc5w$L zhDAz9Mv5ehtYI<;4q7N+!!Kcw+WKjvI4e2W6c+(O?!h)CYssQ8k08J_XR!sBXuoVt zV%8Lp;ssSL1%Ma^z|LKGuI~~9W7MQ;5cH$Ixw-98&E#-jWazK7md5E7ir~%VSo@VzN2tIIx5P7hLoj7UnuvImJOl>8EjVD3~W>)=6 zIum3rViG+CKw_8wzk)J`-c!haz^c-jwf{Sye``T0y4(1pv>k>v8wDE5uh@Bd{RNf zkD&d@)d|%L$>^t31>ZTfH1V!58&XS)kNEdYD|_y5tG{0OG>Itv-k{sRSGpyMT(*Id zI!i=hI|5ceJ5gKHQYb>yy$b2^4fJK0D#IS=m=!&6h$u*(DHul^Hwn)POp14b1@!A(D z=;O&56%QVr&Z?&+vh8*CA;G`b5AZ+PUXsWeD{)S22-U+T`Be*huniO&(ck&ucZp9z zi;^>T%iRZ4q^4|MCgwQ2pw?xgiqjt!!N(8Vee8awEk%7IaATm7I;`;@}JdW(nq#S`Ov_RaH_*#N+-0~}b3O>PiODL~=A z4|#ZXhDXIM>|h9Zf`AJoJ5^ousziG{OU%+Dd7ehS{qfhYV2Vu6SRlX1A^apZFDocF zYqCJ=2U7?Z&n8(>!ZfaBVCJYJ-D=iJgrixjn5tr1w?-J=p1&GW_Cet!I|I%^?Yma-)juEx$59z>rjeX z+~ND|X>9v^yOTYerI%*E&g&D8gC%ku4Y86|gvUDmqTRt}v<&k0U=#4rdvzs7CRpV7NeL+0Dzo76NYWOE7le?L?L9c&N9B%Q%N15kOy z!*({qxMp%Q&CIU(99}V!;0O(y0byiR2%$zY1xHPPZ|>CE*fWHyWys+eKcP0^Sp7ch z|NJP7m&q9iQq|fLRty{70g2Vk#x#tme)YusmJZY-IpcEE(T-vG^RsomKrhA*#yp9f zu^CjK#2U7M7ARVW+ctlg;;^u}Ep?q_cp~1w`sdR`q~MG38fU8VhT5tUdt*Q0XN!WI z&;sSO_B4g~*~S|V8{6Chs*RW72L!AAa~SMCV4V7qL9ZTvCPk9S8MV{W%VGF^XeLRt z=&IWrC{A+5@%kVS^W|w#?CerU9Q&)CPMyzoU+`5tl@sNagjl zRU7y7)xsi?e&cGsL%Y34pF|VQ|T*kLVsSL{vAtbW1 zk2|%n_OO%?hIR-;I5FSpVv`1f%w<%kXwmlwYt9IPwJ{4-stQCOD!QSY+&l_2>&h&9SN#s9b#}1ah+6R(kgDpO1MeLx#x0 zr5Z@xU;@8rxx>zu5%qPl9-a_X=&d9R)bHcC2PPf)2D0B4w zp4$>7i5!G9^jBJP`%6M?zzJTuJMq9BR6jXmQ~&iEVMVc|bX7wqqT1NfykOP5lm`wo z{|ZM45@cyd{hOc%$QgCPbPM~uofgZ6`SHRw4K+kHk8cB757#GH0z24Fb+PSjv`Efe zU2WJ$F6any#?kiS#$ou5w&-)HbXZDT&`N2c!@8?(HX18*0ydjqU*?A5`B#lcTC{>E z;8zJgqg@D|%sRHyOTF?|FzjoD3X+B*zjYA!!7IWd*0 zM~V~IQbVV|$h9-1wH#AD;%$f36)((eIKi$RLC|?jz_Px@4QZ=7 z21{`(!5;rH1ans=Cb;o+0>Y&8I^kPttT0m`!tRZdblin1)Oh=J~n$?k{$37vSX7`Abct2$ehjn1DY zeK&up)l7+$aC>0YrnRuUh_Ld+ae!4Fg=0$Q(Z3R?}7v< zdx5^)N_S!wFgUE?-Jt)vLa>gbP3A;u`Vc; z%^NF$QS8cYkhJ}c9bF>%_SeJy_!G-dxs`I`>h&|Ry+%c#;8wra(G`8UT+WzK;Zlu&tAn#WdMn0hRZHO$xAa;*4wzZ&O7 ziIX#y&io|8N@L~u!SX(qkM?n@DLdC4U0FrG-s{cnvV3(R%9fl_1I4u_)Mz&M3wRu7PQx1E4(90`lvLm}Ar1oL(fFqqsXvwA(D>8MO94RWhaEYo_1@Z%UvbcImy z?2Yl=A3qyIW^%?Q)>*wnkxXpcSa&SDLMc&mGr!xge=96Tk~8*XT?$2{WhMnHF9kaO zPuKsf_y+PyE-fUtq=P@1#-cZTMs+D!?h2WW6SxRpjP<-T0V(WV8ok3iq1DRVLb> zA)?jDR$16%Ge;lfWwXLQ=!f8}TW%BE8s~{-gZt!ZaeGbRbpr5BM_y?%PZzPD^%ZYwl)az?S+`MAI@27{cf;Ux=; z1}9*`0cWhUHggEIF&F+n@1Ag6K#h_!GJp0(K@e8fCf1@q)Gb#a1^vOdwk8CN726I3 ztF+Iq+K@in6vDT&+x@Y^P53Y0-7sQ4+7CIDUFfg0r0Y%{e}GnbF0C*>|i}lE@kJJ@iXLt%hqLAtpRC2?LQC`zZ>>QHnoNOLOsv;nWaVtEHv9Rdt0=7vgTdRLwt~Xh!BCHs*e7#sgX6FmN6uJ>T{;nJ z8oLR;1ceDhK%s0P1xJ&OeKirhpMp;gW#@)MGhV|F(Ve@8B6#{Z3XWyRUPY0Uei1Xh zaRmEdJi*f0n?o@_q0+9jsIS_k4VX%(Ekxw%!w`AmToHNslU^$uJ1ofJT!IZP7A!7b ziDEiF4E4+_=M2FbJ|ke>;d%kIUVnVoM0B9!j8(jGNr;R$td7?Y2bsQm1&plp8Lag6 zSt?KG9wEGQG=#v7^hMQ&oPe zYa_Eyg(4-h&#r)r)@1I%m&baJ02jx8(Gf;ywM8LprQoD1Jb0DcXTdh7&&jTgfb3B( zGI++JGylBI+xmJiP7OBTVU#ytE1&o<$i?3jsSJ{$jQyEjfvHBmF + + diff --git a/settings/repository/net.sf/sam-1.58.1057.xml b/settings/repository/net.sf/sam-1.58.1057.xml deleted file mode 100644 index 4f0dfe44e3..0000000000 --- a/settings/repository/net.sf/sam-1.58.1057.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/settings/repository/net.sf/sam-1.58.1057.jar b/settings/repository/net.sf/sam-1.59.1066.jar similarity index 96% rename from settings/repository/net.sf/sam-1.58.1057.jar rename to settings/repository/net.sf/sam-1.59.1066.jar index 804e21b6165869fa1dff4e622ddf3e7dac8f0f7c..8380da8644ecd4d4480b1c47276c4c91d99985d4 100644 GIT binary patch delta 8787 zcmZV@33yCb_q{hW?_DFyge3>O4amzfKxc`Q~k9_fZmpzgs?OE1+y%yWFbxiyAeV1MD5cw10aN7BqaWrY% zv{o)DzaoXbha6`9nGPn%QD(}s8{}A-kN%^)Z{nk5)!oWRV{G?4_-IJTk4+it?#{ig z)ZwFs1D_~-6r8eJ<)gjBGsIlik-;*b8#U^_g^yktch}5ENpEM?=c6-oHhc2XtekrS zo3?yO7e4p2s+kWTMg8)LpBJ@x(cp<;>9QnIXbAB>a^P<3(_G=j$2BumX0aY%k+)m; z@Pt)<)9OKatCYzJxBMmPmOq4rX}_|-RGSQkD=ohEGm%7J z+DeY(e&~B*!hjrO4NUn2CDX6j=-4!_Twje(qd*-8FKtmC5hBZo*Tu zDoJJ)21J{EA#SUzfyZXq&0byOssB|ZTX+=pXC-ny3uAda_N8MrK9W@9V;>XZ3rnjl zs@|zW4)Wz{h98yB6Gr&j<_iZRA|)vzlG#ovJ>at=nuk86N^Ts<$qOFHW`&WPY&qK| zdrQ(}Z*Ge1v0UH&QHY2B^Fw*6H#dB^oAR|AXN>e$el_u-Em)Zp;6=mrzpJ0$9f+Am zAsoUwD-9s#xUA`A9hBGkLh9a~m3d;gCQkWYglt-}(vD|@z9U6xDxCgdxblo}9hb%` z2_~jv%&GVZt#K1j2(|yH)Q6x6idFABUWq4+pl?<9#wHk33So@k_DZ=f#4IK%TzV+i z(NCp0H9VT31ZiADgSkq9a7fQ|>fM{A_>0-|If}?t{nT>h<+_~BZ=I46&4&fYl~rQ4 z!*|L#A=mw~@_}$I`ym53=ufthPhdqsoFY+|*yJX1&iCdkxV zWLscEQ+F{O6Jjzq;F2bEFx?gwj_72X?Z;;?4K%%AV(VHK>$5G&Q<9=QS%QPDmD#2z zykhDoLZ3O#lqhoN^arMuVr|~aFlF*>tN$_2G%}k@?dNG;EiAZUGykeF1;>sLcrgx* zkCT}Kd-4<;lqM^ly0^bMUQoXpU``O?HUybR@rAN4|8v3;H1Jxh=0Tdio&qbi$MH(QtE_u)X9RbDnUx>wNP=5r)`B=2;?b^jvdWUYqpWtIWr( z+>QO4%p=5X&<^tm;qvUA=Cxv>f^>7ajnh_rZ9d+H57$06ivp$}cw)ZUgwK{mSZ+7w z!@@+%KZ31rnB{a`mX7ykPtf;8ND_sH)JyyA=|nGh({eI`bG@Bqxhzab%C(FZ^`yMW zGDO&DEwcoOh&27g@^u|1HtpN4anlq@qR@tAdGaMWNN;t}vOok{ue7uk>3I93rN4!- zoful~)2WdpQ3z*jo~JBs`lPd#q(|axC<+a^jDEr#;7WwKUzE?wqu^H!8FOlD|J^BV)LA%#BxG+?H!fvz28pOT~LD=P@}+)If^e-CTb1zu8Eq5keb{_>n?`*{k6}z zLmMC6oN^S4<53kRjnpFaj3HXJ2-Mh-+UH_j!cw)73R{fqN%?0Mnjn# z!nUbeFl^X{Wi_XdR!6U%tSu3`CQjEPx-bJ*#}-Ew+9ava28G?VKuF!Ex#?f+(sl|h zvyNz@Wzc6H)ous}$Dh;ch)kP)QJdnGD+*_>ELng5n1r>8LL2IeKa36|K6;lR;w3W*AGbTZ*5J`X zp}UhJ5~|D4&u34}MBg4v)(YKyBFXP!(KrgkJAvHueyJp0Gt;&Y>@!FQ zW8(+sXOMQRG=l9t5|HkLzVS{^R=iETW2C*FvlYaF=KRAZ-Hh)bIHtad?V*c_I^(3n z0WPEXDANcR)6_8&r=_O8JC$)aapb+rawJdluol9EQz8QMMG0svPQN@1LT=1}D0LqIV33&<}_| z{4$FK5~}30m-aV@vV7tzw}y}pNL!}4|E=Kjg;lwC9tTYi?l&N+HZi zNWHsLMM?eP!W`nuj3`*V_4<2$l0?BMHTV}sn%~%@LW{W~jSLqAgXHi2vETV86w>En zOlSr{?p$;{uLfrWpyCO3=#B~QZZLHEaD^eGs#8A1*q1xMqRb{fjMQFnU9(pp>PVfp?auHU{)saV?wvC*!z>ayChL)Vi39=Dl)MO zvNiZU6P>78%>^`Hg946iapY|4d5GIm#BrnNkw(mpsz+rz|BDGpA=n`IUI`}+EtTD% z*AC*w=v|kn=W76|;PkoG77O6L${PabJ9J0SN8SC7)*=`+pYWR=q*Gk8Dkshz_RV)J z%2|qA`hel`7m$c>8ei9aOOl!yOwNBOOZkuG2#8&T5?Dm$E+7Hser}2cVFj`eWVk76 zi|0%Am|o@X<#Fvh^&Ux8&EuswLZfOv;dCG}Wn4A(~k@xn9(3e%ME=;tFJdHIdM&tXGFA;!RR1I8>S z-IxnAeUsnljY~tJgMp|n99m3zvb_Ayfc=5{&_@c!v#BH#Di@P5s2CwzAZdvsU}KhG zYVF9PI3LJcCf9)iCvGRjoyg(1m^{S?JeMjeJYC{gK=&*pIJBG**z@jqyJ|(=M1Lg# z15F{qV61f=Cr{3|y2w{apILgO#X)O1)Z#Q}9QqJ?c9bKJOVoSH&(`NQuR8GNYa{4ug4l6rsc(QcN zM)h6&p>Qejv1hz*h3Qx*IqD;d`Y?#2LN5?F7=)+EiEKyQE>gTlLyoUZv9g1PvqtA2 zYH)}P71zw9!NeTl)DH)Pl*V2 z!ygD;_y;#(E}b5h;hff>CKK&npF!i5>x>trx;l?hEnlOuPKxcduF(&ZUy&t=f-xNN zp4cBRCf=|iU$MdQJck`~^RTR3u?b{cmN}pn@+w6hYao!eWh8`Y*|KGDM+JK|3dTdH zPXMRQ|3>kE*S3oaFtc}CxB9pNDYP*J#~#G#sS}5mk;Y8C;Tgj~{#PSmN_7{r4Vo=? zENc96w6kM$Z6rOVnY4kI_C>vjiYXY=c*-EbuwgJ>z8-8t26{F$+mmo|0v_oUaJM?Y zqRbn^IfLEyXFX_c7YaUowqkRCY?~+;qw(=Of__7557!!1yg#XUM0H2FXyGS_Qy*u1!EuF6}(nsA50e;2CPPv8;gX52BAuJDBHfgv*aSK z1qEX_{B(y!g2GQN{%~cr<392PC7A<1aUf{+Gbx5=FoWU09zf!_OLB9-H!%$08sf$_ z*xnmEzWNJq#S|JDI7%ulFyO2O&(S?5D@<^O2H>G+I5!N?E(RFFQkjH~@MN3Ab$J6uFKH&^Z6K}T%q-OgBR2@aTkpE3Ohkzk+8N9ZhujUM2Q%0E z+Qruh2ABf&7|yRK^I9=e))?-Ts_qc8S+&6Btq!%F3elXd2L(mkVbq&@P__K4egEvy z$w%GsnS+AyEL`vbu=olkXuuvqq01&wMxyHW z$rz941ci>y0MTXt`6p(QFI4|*u|nP^Mb zHehcNECksto*v%ihG!FkaWCJ%u8XB|uvznj(jteneN@_2n?UAeqbgcUxa-zpL>>qh z$chV?d%cPsyL~i8TEl81*A_B#4Yp|Qd5DyS+ZZJDw0Z$DYo6Cb81v^5a@ z-9$(o-2|JbVNqILIKNedG&y_YUz@S5reI9XW$j&D0gQoracVOfn$=w(pKT@$`Fz+G zod3I*m``hp5h_bo-2k79Rha0!V9%js34H`v#THSlPE3Av-XzR43dR#+Nq-o<4Ntkz z_waJ@_5jU;*}~dP*Gu#BDcW-gXUbcQ`(ZXZ%jhqk+C66qmJABUlpZqDsl-;x*PB~0 z7rj%RIBaomp%pE0NmvAaxV+@5~-d=hOHz0)sCpq1OSFQ_& zio05vtKS`e=fcpFxU3ZV8W>vq;bMq}k>w6|Z*51T-}uvobc5uFxbWR~h^%!z#Xnza z$lorbt+3X?V0f4}*zh2b(2&3IuYzfKsW-5^6ii@DFiCI~iWKROfGj*_DD*VYc8g@< zn?tv9yg}nDKTn*!GoG%FQt=XwkZfRR0bb=~6iXcEOFuS2F$SWqAh(=!V>#`5@PEEW z9qk2i6}WLuCldUpe}&LIJ@{T}3TmZb?6=!Ow+hEo$D9gG{Bwh-fM5tsQO)2n82>?0 giM5`m>lq|4#itTp1xIPb3%qv{Uv^7wT12Fz0RsaA1 delta 8765 zcmZ8G30PIt*5{mi?m36~qJYe^pooI=fFy#JCJJeki7A$pLn&x#O6o-{hnVE5O9W>e z5EansnkiB_r1V7Xr9BNp8|*g+eCkWH@Lzkl7ms}3`rNL0t-bcz>+D;5EwtuZsB7v7 zgX|~>njoxwW5b|Tu8}Tl@ZZw6`@d!t1i`v0ag)-oQ}@{3eOLY7H{x9+A=dsU<7!6c zdogWFa*)F^R1A@4lEpMJN+vP~)`|T^w)$Rr-@sNcDK3ugg!=L)i(dj)K_D;~YX215 zTQVFyA!@$nZwWFu95`%R=gLIJ23_sMR`J8GILI@I>Y&k&C2H+JU|Olda9^aGCAJzg zAwXoSjEVP*Z1rm9U74+tr{%V1t8)u?xUp4E!GAe5yJST?v)0Yv~~q7={Eygx(7%zeBFtT|NH6l zyQ8sa2!xWPc|caMB$aU$2Xl#H z;<5FMm68v~{!k$C!m<9VM0&xAF?es3%w5>B{FJngWBYwCo#%R8o1`4>d%-p7i$07u z*~c)19R%xYf5TF~&HjOgM3JWZ?MStIeFs6P?*PFym~K;lgNwB&$Z)X(lb_btaEJT# zdYs_{j%^xk=xLzx`}%uUM!5+>lpCopA1EGYZ~?D8gR6D)D~4z{nx7wLKbWj~2?Bvm z;NDU826MH<*}lp&IPlVF%rUItdosmr$Yn>%dTWVcLOydQ#Z4~d5!^J%zwy8?c*|yv z9p@`2aj`A_@d*$&wiiNduIp2~1t6XhjhR>dqPxWBdHvA=H z-yl8ATK8DK70H&>Va7Wh-3d0yWsc)flOPZXgDw6BFK7nM1Z6RzgZ0XAW4(#8nfo_* zuXV!KiV#TItjEV0tDKm+_uewz3uC6IuQWDs-;#@r8T_QzR2W}%V7yNIjJ^t6cK*b8 zl81Evh_RkEKkLDIV=rENEe*yLBi-QBW2-&m!UcgqD2R1N5BTX@qmy;!Ipc_MrmgOQ zv6mNJ!V95FFnBpDJma06l|T6D^7d3_Fzc|=PvJSL_fzsX5+!3;APYj3AtJT??cha+ z?&GyWARZ>Ys(M3BjM5&CMk;Prhj2yaF}QbCcA1zjN8**Kd|RTDlux5+4%Z(Yd?Cm| z5D4_AdMu_gH!x?TnushVhO2pPzVaQHja{r9=IyIxyE4|BDNn9b5*2zhK6}{kFcGyS zI?zsLNqy51Tuvzt*5k*N(LCRNXB3SWR^$cc6VQ9NOJvg&3X`inBvK($*@%ZE#q#gJvc-5eRYdWGhScRqjp=UAZ!bT?iUn4*!;EX4lX@4H*5MM zwIY;O+=@;2z7gz=V`t@Jur4Q*@=>efenacwzH3ZxKQo?F$uQ@_kjuXG_N4C zk9IsRutD-a!jul@=4tJ_q>gCQN`|2$eUeM=Asc~c8$bSAk-LHj#4*QA?`yu&3CWUW zinQF3-N0hj+65Ew+|(aS_TzyjFjyy^s8H_4D6ckd-MFda9%Dl8Bq%j&&OtHq?@urhtDQ(ufwQMVhoH9YS(6ANXy7=BE+#XG0H1 zL-ktCON@o!9IY3%m~uPd;udU61p3+Skwm^Uk;n&IOp`pIbgR~msjkaG)x$DvBxlP9 zZ<(GVM}L6i_-QsWPN*MfY&L}gLi%bUBRyFd1e>;M&eVsi^R`7*VGR@Ls`KHU0>*x} z&;jBX@+8eRbswJ`F9-y}bnI`G5cVJ_B&=jVo`3v8>unrC354ia?=%utAIQL48=43| zsl&46yROf}8H#{jwg0b-ztbYko2u50^E%c4{@Zf*?_e-5!tEeVK=C4syr>nizHn9( z-Jr81I@6?4O-XO~-^;}G6VQk6Y=kct<3XiD0~hn4;xZN? z5wg>oZ2N8yga{pb1k^9qdebdXZrlr3lUehi7W0J3pH}({0)ZeM>!CEl>S>9+u0Xe3t~q;H zcJEX8^+Q1K;O1F2b|)|+dl0GhU8@iM?BXH_1R{0XtDrU)dm`PKKXNgWe@cmbd+54e z)S{p6GepnGjZ^=Tv11bGtkbijf{8Uh78P*YVcSq5)c9~;?PaWG0s%UqyStwfs$0Ti z`)l&hZ(6a#6VP`JwTAH*Hye#Gum-<=+75Yi9y&kr#8V6zc^dn+12ds@YP8c_;BX!{ zW**=5i?48(31FA9|5fA`JYf88e~RvK<^hJ~y;O4!B@wlK$~~KgHGhf%h<}M;Fkl(> z2%6T#OR+VLbe055?jBFrzf@~Sl}s8|?NW^+4gq~yynQ4AR-G0-A=yXLU|0^eQ)2d( ztK+%;zBt7p_}M)r%}Ii8A8Jvo9>;%(=HBazDzc9m{Q*A>6)6AE)(iI#sxpun2!QJ! zY937N2_fVs*a!nYvbDI>k2GKLEtvk1Hjr^vA;PdJO@bq7QV=X($CJ`9I`-%xywC}B z(RtU9j#`s5q>kV=9-T>8W^3B1%h2YT$w;kPsYJk(oi>}h3Bhv;5?rQAzA$PEid`z_ zhIC)0IzvmdP@;K_=m+?P%JwUSBrc-q%}3|4WC_ITByP-=K$|CZr`?w(dA?Wbn=j&) z5a_Gpb%grm+Vd<2jw`UDDs$n>kHw3shA zK)H?m^IyhvuEdi;pqq~UQ~_m&=0eF6OobRw>k8X8x~@d~btR0Ur37=ddnLa3`6!R^k%n5?NmD)XZi3NU)^v>TiS@R>@^BEuW zSi{o}gm%V0O+bHLKKaq6F9=eb$o73~^Zf?Vx9A$vS947=!HgOkEiNBxE8fQd*+B+x z$jZkiOV3a~!GGw_@RpX1G92|cDA15^6TD0qTmmUWAe1ez1s7d__%T5g-v!D(;z#K} zb^g*G++hOxMwSkn3bcM~moKBRG%5}@7ieKn7>5Rf3vC8t3sLcz(NwV~1YR<_LKe;~ zw1(Q=4xYVZjIgB;=R{JJaEWM`ooRqonTE~~Tx8c=g!uX#1H7~fFT1ykY$tRv!M|qa z_|{y58*Da{B-kY{6OJoOIEQ_Vdalc#sjpyvBcKm9!40z4YMyYm5qIHqu`Owfi!tsd z6Qk7>YthW%W`aKHP0?#w@Hr%K6|aMxJ6|3kVZ9U3-!CJ4DZ8aOTY=O+u$xWT!)4~b z*vq*+_6h<$bah(d-zHPPO6y23NK486sgJSD2&<5CtF?xHUQ z^g;LUV&^xt-PNOi{35C!x~*#9bWD~kxf7yMWxYf!qLHxAE8VN?+gQ3QhR+~j(>x{mQD zl*tO5UTrIr=GAC>-!?+)0Cy^o=Cj5|i&=v-*S&-m4l~#ADQDKblj1mTwdWY( z1|keOehM`DDLxRq!Dc-cjTmYfz8mD&@x-{)FBDd7#*= z9c6UF7jdqC$My?D=HS}{0X@H865;oT)FcHTc-PuuFWZC;|9d8(MbOtsxVeezsyu#a z+Iwi1z);=3zc`=pcZZbCS}1J*yY9NAy^Hn;^wzP4K=Ed62#ww2+NIYuZ~T!Cp^uK~ z>UL`1GU~cB#O_cOII_d$-@q;CUwj?+uZ}!E;$f!x+8#iky<75*5q^LA916c|u@%!_ zM3?g!Li2{r5k_|ygii=GfRxtljiO?KE7%lb}*$Fmn=h@6RF8b~|#zUZ^F0M0hb~`@u z5Pt?wd%Wy<{H>HgFCEjvNT#&gK-Cq}qf{rb?yvEc*F!O&nj+0HG3QuRh zwq%BikBbQUyJC3?Osd2Su70`VLH(xuZSTy_-VQ@w<^2EPcs_fV=1=*XzCEyT7WN1N z`rG7{2~Y8#L4LLyw|Akh?PuC)&g6l*9eu~lpRxAra;z`{Lv)Fr0~wXt7;5mpw=4IUo&)#GmbY^yKQ^- zC813}!DyRLsGaHe9+T@vHF#`nzY?(0)%`x8Q`!>}YP11tD;Cw@ rR-79{6a;|N%bFwY$n0F4BeVp(->R|cWvH4vc-CrOO5R)?AcX%1$8$zW diff --git a/settings/repository/net.sf/sam-1.59.1066.xml b/settings/repository/net.sf/sam-1.59.1066.xml new file mode 100644 index 0000000000..75a327daa3 --- /dev/null +++ b/settings/repository/net.sf/sam-1.59.1066.xml @@ -0,0 +1,3 @@ + + + From c5320ef1af7a9f65ef9715121a0e5e246e1b44a3 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Tue, 10 Jan 2012 12:14:16 -0500 Subject: [PATCH 039/356] Resolving changes in integration test during merge --- .../gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java index d89c9ef6f8..7179680bdf 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java @@ -294,7 +294,7 @@ public void testWithIndelAllelesPassedIn4() { WalkerTest.WalkerTestSpec spec4 = new WalkerTest.WalkerTestSpec( baseCommandIndelsb37 + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "ALL.wgs.union_v2_chr20_100_110K.20101123.indels.sites.vcf -I " + validationDataLocation + "phase1_GBR_realigned.chr20.100K-110K.bam -o %s -L 20:100,000-110,000", 1, - Arrays.asList("69bfc9bec43a4fdd85dda3b947e6a98e")); + Arrays.asList("fcd590a55f5fec2a9b7e628187d6b8a8")); executeTest("test MultiSample Phase1 indels with complicated records", spec4); } From 25d0d53d880a210b134fbec9b56ef8c16cdd4c5f Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Tue, 10 Jan 2012 12:38:47 -0500 Subject: [PATCH 040/356] Moving the approximate summing of log10 vals to MathUtils; keeping the more efficient implementation of fast rounding. --- .../genotyper/ExactAFCalculationModel.java | 59 +------------------ .../gatk/walkers/genotyper/UGBoundAF.java | 2 +- .../broadinstitute/sting/utils/MathUtils.java | 57 +++++++++++++++--- 3 files changed, 53 insertions(+), 65 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java index 295d3f9f01..8da72ef7a7 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java @@ -70,59 +70,6 @@ private static final ArrayList getGLs(GenotypesContext GLs) { return genotypeLikelihoods; } - final static double approximateLog10SumLog10(double[] vals) { - - final int maxElementIndex = MathUtils.maxElementIndex(vals); - double approxSum = vals[maxElementIndex]; - if ( approxSum == Double.NEGATIVE_INFINITY ) - return approxSum; - - for ( int i = 0; i < vals.length; i++ ) { - if ( i == maxElementIndex || vals[i] == Double.NEGATIVE_INFINITY ) - continue; - - final double diff = approxSum - vals[i]; - if ( diff < MathUtils.MAX_JACOBIAN_TOLERANCE ) { - final int ind = fastRound(diff / MathUtils.JACOBIAN_LOG_TABLE_STEP); // hard rounding - approxSum += MathUtils.jacobianLogTable[ind]; - } - } - - return approxSum; - } - - final static double approximateLog10SumLog10(double small, double big) { - // make sure small is really the smaller value - if ( small > big ) { - final double t = big; - big = small; - small = t; - } - - if ( small == Double.NEGATIVE_INFINITY || big == Double.NEGATIVE_INFINITY ) - return big; - - final double diff = big - small; - if ( diff >= MathUtils.MAX_JACOBIAN_TOLERANCE ) - return big; - - // OK, so |y-x| < tol: we use the following identity then: - // we need to compute log10(10^x + 10^y) - // By Jacobian logarithm identity, this is equal to - // max(x,y) + log10(1+10^-abs(x-y)) - // we compute the second term as a table lookup with integer quantization - // we have pre-stored correction for 0,0.1,0.2,... 10.0 - final int ind = fastRound(diff / MathUtils.JACOBIAN_LOG_TABLE_STEP); // hard rounding - return big + MathUtils.jacobianLogTable[ind]; - } - - // A fast implementation of the Math.round() method. This method does not perform - // under/overflow checking, so this shouldn't be used in the general case (but is fine - // here because we already make those checks before calling in to the rounding). - final static int fastRound(double d) { - return (d > 0) ? (int)(d + 0.5d) : (int)(d - 0.5d); - } - // ------------------------------------------------------------------------------------- // // Multi-allelic implementation. @@ -403,7 +350,7 @@ private static void computeLofK(final ExactACset set, } } - final double log10Max = approximateLog10SumLog10(log10ConformationLikelihoods); + final double log10Max = MathUtils.approximateLog10SumLog10(log10ConformationLikelihoods); // finally, update the L(j,k) value set.log10Likelihoods[j] = log10Max - logDenominator; @@ -427,10 +374,10 @@ private static void computeLofK(final ExactACset set, // update the likelihoods/posteriors vectors which are collapsed views of each of the various ACs for ( int i = 0; i < set.ACcounts.getCounts().length; i++ ) { int AC = set.ACcounts.getCounts()[i]; - result.log10AlleleFrequencyLikelihoods[i][AC] = approximateLog10SumLog10(result.log10AlleleFrequencyLikelihoods[i][AC], log10LofK); + result.log10AlleleFrequencyLikelihoods[i][AC] = MathUtils.approximateLog10SumLog10(result.log10AlleleFrequencyLikelihoods[i][AC], log10LofK); final double prior = log10AlleleFrequencyPriors[nonRefAlleles-1][AC]; - result.log10AlleleFrequencyPosteriors[i][AC] = approximateLog10SumLog10(result.log10AlleleFrequencyPosteriors[i][AC], log10LofK + prior); + result.log10AlleleFrequencyPosteriors[i][AC] = MathUtils.approximateLog10SumLog10(result.log10AlleleFrequencyPosteriors[i][AC], log10LofK + prior); } } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UGBoundAF.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UGBoundAF.java index e40054c9f7..99d55bc698 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UGBoundAF.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UGBoundAF.java @@ -204,6 +204,6 @@ private double simpAux(double[] likelihoods, double a,double b,double eps,double return Math.log10(s_2 + (s_2 - s)/15.0); } - return ExactAFCalculationModel.approximateLog10SumLog10(simpAux(likelihoods,a,c,eps/2,s_l,fa,fc,fd,cap-1),simpAux(likelihoods, c, b, eps / 2, s_r, fc, fb, fe, cap - 1)); + return MathUtils.approximateLog10SumLog10(simpAux(likelihoods,a,c,eps/2,s_l,fa,fc,fd,cap-1),simpAux(likelihoods, c, b, eps / 2, s_r, fc, fb, fe, cap - 1)); } } diff --git a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java index 737f4bb5f3..5ffd634cc3 100644 --- a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java @@ -56,17 +56,58 @@ public class MathUtils { private MathUtils() { } - @Requires({"d > 0.0"}) - public static int fastPositiveRound(double d) { - return (int) (d + 0.5); + // A fast implementation of the Math.round() method. This method does not perform + // under/overflow checking, so this shouldn't be used in the general case (but is fine + // if one is already make those checks before calling in to the rounding). + public static int fastRound(double d) { + return (d > 0) ? (int)(d + 0.5d) : (int)(d - 0.5d); } - public static int fastRound(double d) { - if (d > 0.0) { - return fastPositiveRound(d); - } else { - return -1 * fastPositiveRound(-1 * d); + public static double approximateLog10SumLog10(double[] vals) { + + final int maxElementIndex = MathUtils.maxElementIndex(vals); + double approxSum = vals[maxElementIndex]; + if ( approxSum == Double.NEGATIVE_INFINITY ) + return approxSum; + + for ( int i = 0; i < vals.length; i++ ) { + if ( i == maxElementIndex || vals[i] == Double.NEGATIVE_INFINITY ) + continue; + + final double diff = approxSum - vals[i]; + if ( diff < MathUtils.MAX_JACOBIAN_TOLERANCE ) { + // See notes from the 2-inout implementation below + final int ind = fastRound(diff / MathUtils.JACOBIAN_LOG_TABLE_STEP); // hard rounding + approxSum += MathUtils.jacobianLogTable[ind]; + } + } + + return approxSum; + } + + public static double approximateLog10SumLog10(double small, double big) { + // make sure small is really the smaller value + if ( small > big ) { + final double t = big; + big = small; + small = t; } + + if ( small == Double.NEGATIVE_INFINITY || big == Double.NEGATIVE_INFINITY ) + return big; + + final double diff = big - small; + if ( diff >= MathUtils.MAX_JACOBIAN_TOLERANCE ) + return big; + + // OK, so |y-x| < tol: we use the following identity then: + // we need to compute log10(10^x + 10^y) + // By Jacobian logarithm identity, this is equal to + // max(x,y) + log10(1+10^-abs(x-y)) + // we compute the second term as a table lookup with integer quantization + // we have pre-stored correction for 0,0.1,0.2,... 10.0 + final int ind = fastRound(diff / MathUtils.JACOBIAN_LOG_TABLE_STEP); // hard rounding + return big + MathUtils.jacobianLogTable[ind]; } public static double sum(Collection numbers) { From 5bf960deb81d0c4d350c8d2d89280067e157d922 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Tue, 10 Jan 2012 12:38:35 -0500 Subject: [PATCH 041/356] adding dbsnp to indel VQSR --- .../queue/qscripts/MethodsDevelopmentCallingPipeline.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala index b50bf3d674..2f0715ae90 100755 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala @@ -299,8 +299,9 @@ class MethodsDevelopmentCallingPipeline extends QScript { } class indelRecal(t: Target) extends VQSRBase(t) with UNIVERSAL_GATK_ARGS { - this.input :+= t.rawIndelVCF - this.resource :+= new TaggedFile( indelGoldStandardCallset, "known=true,training=true,truth=true,prior=12.0" ) + this.input :+= t.rawIndelVCF + this.resource :+= new TaggedFile(indelGoldStandardCallset, "known=false,training=true,truth=true,prior=12.0" ) + this.resource :+= new TaggedFile( t.dbsnpFile, "known=true,prior=2.0" ) this.use_annotation ++= List("QD", "HaplotypeScore", "ReadPosRankSum", "FS") if(t.nSamples >= 10) this.use_annotation ++= List("InbreedingCoeff") // InbreedingCoeff is a population-wide statistic that requires at least 10 samples to calculate From aae61767c6574937efcc77a5e2d6e8e0cc76a5ba Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Tue, 10 Jan 2012 17:32:30 -0500 Subject: [PATCH 042/356] queueJobReport now compresses PDF when running R 2.13+. Updated PostCallingQC.scala's VE and R to include missense to silent ratio and plot. --- .../broadinstitute/sting/queue/util/queueJobReport.R | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/public/R/scripts/org/broadinstitute/sting/queue/util/queueJobReport.R b/public/R/scripts/org/broadinstitute/sting/queue/util/queueJobReport.R index d5ee3626f4..ae340e688d 100644 --- a/public/R/scripts/org/broadinstitute/sting/queue/util/queueJobReport.R +++ b/public/R/scripts/org/broadinstitute/sting/queue/util/queueJobReport.R @@ -1,6 +1,7 @@ library(gsalib) -require("ggplot2") -require("gplots") +library(ggplot2) +library(gplots) +library(tools) # # Standard command line switch. Can we loaded interactively for development @@ -201,4 +202,7 @@ for ( group in gatkReportData ) { if ( ! is.na(outputPDF) ) { dev.off() -} + if (exists("compactPDF")) { + compactPDF(outputPDF) + } +} From 77a03c97097625c54b00055438dacdc25d1e162d Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Tue, 10 Jan 2012 16:30:38 -0500 Subject: [PATCH 043/356] Patching special case in the adaptor clipping * if the adaptor boundary is more than MAXIMUM_ADAPTOR_SIZE bases away from the read, then let's not clip anything and consider the fragment to be undetermined for this read pair. * updated md5's accordingly --- .../sting/utils/sam/ReadUtils.java | 12 ++++--- .../VariantAnnotatorIntegrationTest.java | 6 ++-- .../CallableLociWalkerIntegrationTest.java | 2 +- .../DepthOfCoverageIntegrationTest.java | 32 +++++++++---------- .../UnifiedGenotyperIntegrationTest.java | 4 +-- .../RecalibrationWalkersIntegrationTest.java | 12 +++---- .../sting/utils/sam/ReadUtilsUnitTest.java | 21 ++++++++++++ 7 files changed, 57 insertions(+), 32 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java b/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java index 7fa2f6230b..cc0b1ae673 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java @@ -186,17 +186,21 @@ public static boolean isBaseInsideAdaptor(final GATKSAMRecord read, long basePos * @return the reference coordinate for the adaptor boundary (effectively the first base IN the adaptor, closest to the read. NULL if the read is unmapped or the mate is mapped to another contig. */ public static Integer getAdaptorBoundary(final SAMRecord read) { + final int MAXIMUM_ADAPTOR_LENGTH = 8; final int insertSize = Math.abs(read.getInferredInsertSize()); // the inferred insert size can be negative if the mate is mapped before the read (so we take the absolute value) - if (insertSize == 0 || read.getReadUnmappedFlag()) // no adaptors in reads with mates in another - return null; // chromosome or unmapped pairs - - int adaptorBoundary; // the reference coordinate for the adaptor boundary (effectively the first base IN the adaptor, closest to the read) + if (insertSize == 0 || read.getReadUnmappedFlag()) // no adaptors in reads with mates in another chromosome or unmapped pairs + return null; + + Integer adaptorBoundary; // the reference coordinate for the adaptor boundary (effectively the first base IN the adaptor, closest to the read) if (read.getReadNegativeStrandFlag()) adaptorBoundary = read.getMateAlignmentStart() - 1; // case 1 (see header) else adaptorBoundary = read.getAlignmentStart() + insertSize + 1; // case 2 (see header) + if ( (adaptorBoundary < read.getAlignmentStart() - MAXIMUM_ADAPTOR_LENGTH) || (adaptorBoundary > read.getAlignmentEnd() + MAXIMUM_ADAPTOR_LENGTH) ) + adaptorBoundary = null; // we are being conservative by not allowing the adaptor boundary to go beyond what we belive is the maximum size of an adaptor + return adaptorBoundary; } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java index 0aec946638..174a46bdd5 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java @@ -32,7 +32,7 @@ public void testHasAnnotsNotAsking2() { public void testHasAnnotsAsking1() { WalkerTestSpec spec = new WalkerTestSpec( baseTestString() + " -G Standard --variant:VCF3 " + validationDataLocation + "vcfexample2.vcf -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -L 1:10,020,000-10,021,000", 1, - Arrays.asList("e70eb5f80c93e366dcbe3cf684c154e4")); + Arrays.asList("604328867fc9aaf3e71fa0f9ca2ba5c9")); executeTest("test file has annotations, asking for annotations, #1", spec); } @@ -66,7 +66,7 @@ public void testNoAnnotsNotAsking2() { public void testNoAnnotsAsking1() { WalkerTestSpec spec = new WalkerTestSpec( baseTestString() + " -G Standard --variant:VCF3 " + validationDataLocation + "vcfexample2empty.vcf -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -L 1:10,020,000-10,021,000", 1, - Arrays.asList("1e52761fdff73a5361b5eb0a6e5d9dad")); + Arrays.asList("bbde8c92d27ad2a7ec1ff2d095d459eb")); executeTest("test file doesn't have annotations, asking for annotations, #1", spec); } @@ -82,7 +82,7 @@ public void testNoAnnotsAsking2() { public void testExcludeAnnotations() { WalkerTestSpec spec = new WalkerTestSpec( baseTestString() + " -G Standard -XA FisherStrand -XA ReadPosRankSumTest --variant:VCF3 " + validationDataLocation + "vcfexample2empty.vcf -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -L 1:10,020,000-10,021,000", 1, - Arrays.asList("bb4eebfaffc230cb8a31e62e7b53a300")); + Arrays.asList("8ec9f79cab84f26d8250f00d99d18aac")); executeTest("test exclude annotations", spec); } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/coverage/CallableLociWalkerIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/coverage/CallableLociWalkerIntegrationTest.java index 02332b64e9..c9e91e6646 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/coverage/CallableLociWalkerIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/coverage/CallableLociWalkerIntegrationTest.java @@ -62,7 +62,7 @@ public void testCallableLociWalker2() { public void testCallableLociWalker3() { String gatk_args = commonArgs + " -format BED -L 1:10,000,000-11,000,000 -minDepth 10 -maxDepth 100 --minBaseQuality 10 --minMappingQuality 20 -summary %s"; WalkerTestSpec spec = new WalkerTestSpec(gatk_args, 2, - Arrays.asList("4496551d4493857e5153d8172965e527", "b0667e31af9aec02eaf73ca73ec16937")); + Arrays.asList("b7d26a470ef906590249f2fa45fd6bdd", "da431d393f7c2b2b3e27556b86c1dbc7")); executeTest("formatBed lots of arguments", spec); } } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageIntegrationTest.java index f2f72978f2..1c58346b4f 100644 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageIntegrationTest.java @@ -55,25 +55,25 @@ public void testBaseOutputNoFiltering() { spec.setOutputFileLocation(baseOutputFile); // now add the expected files that get generated - spec.addAuxFile("2f072fd8b41b5ac1108797f89376c797", baseOutputFile); - spec.addAuxFile("d17ac7cc0b58ba801d2b0727a363d615", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".library_cumulative_coverage_counts")); - spec.addAuxFile("c05190c9e6239cdb1cd486edcbc23505", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".library_cumulative_coverage_proportions")); + spec.addAuxFile("0f9603eb1ca4a26828e82d8c8f4991f6", baseOutputFile); + spec.addAuxFile("51e6c09a307654f43811af35238fb179", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".library_cumulative_coverage_counts")); + spec.addAuxFile("229b9b5bc2141c86dbc69c8acc9eba6a", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".library_cumulative_coverage_proportions")); spec.addAuxFile("9cd395f47b329b9dd00ad024fcac9929", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".library_interval_statistics")); - spec.addAuxFile("c94a52b4e73a7995319e0b570c80d2f7", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".library_interval_summary")); - spec.addAuxFile("1970a44efb7ace4e51a37f0bd2dc84d1", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".library_statistics")); - spec.addAuxFile("c321c542be25359d2e26d45cbeb6d7ab", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".library_summary")); - spec.addAuxFile("9023cc8939777d515cd2895919a99688", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".read_group_cumulative_coverage_counts")); - spec.addAuxFile("3597b69e90742c5dd7c83fbc74d079f3", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".read_group_cumulative_coverage_proportions")); + spec.addAuxFile("e69ee59f447816c025c09a56e321cef8", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".library_interval_summary")); + spec.addAuxFile("fa054b665d1ae537ada719da7713e11b", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".library_statistics")); + spec.addAuxFile("28dec9383b3a323a5ce7d96d62712917", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".library_summary")); + spec.addAuxFile("a836b92ac17b8ff9788e2aaa9116b5d4", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".read_group_cumulative_coverage_counts")); + spec.addAuxFile("d32a8c425fadcc4c048bd8b48d0f61e5", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".read_group_cumulative_coverage_proportions")); spec.addAuxFile("7b9d0e93bf5b5313995be7010ef1f528", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".read_group_interval_statistics")); - spec.addAuxFile("1a6ea3aa759fb154ccc4e171ebca9d02", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".read_group_interval_summary")); - spec.addAuxFile("b492644ff06b4ffb044d5075cd168abf", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".read_group_statistics")); - spec.addAuxFile("77cef87dc4083a7b60b7a7b38b4c0bd8", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".read_group_summary")); - spec.addAuxFile("8e1adbe37b98bb2271ba13932d5c947f", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".sample_cumulative_coverage_counts")); - spec.addAuxFile("761d2f9daf2ebaf43abf65c8fd2fcd05", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".sample_cumulative_coverage_proportions")); + spec.addAuxFile("4656c8797696cf5ef0cdc5971271236a", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".read_group_interval_summary")); + spec.addAuxFile("6f1d7f2120a4ac524c6026498f45295a", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".read_group_statistics")); + spec.addAuxFile("69c424bca013159942337b67fdf31ff8", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".read_group_summary")); + spec.addAuxFile("6909d50a7da337cd294828b32b945eb8", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".sample_cumulative_coverage_counts")); + spec.addAuxFile("a395dafde101971d2b9e5ddb6cd4b7d0", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".sample_cumulative_coverage_proportions")); spec.addAuxFile("df0ba76e0e6082c0d29fcfd68efc6b77", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".sample_interval_statistics")); - spec.addAuxFile("0582b4681dbc02ece2dfe2752dcfd228", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".sample_interval_summary")); - spec.addAuxFile("0685214965bf1863f7ce8de2e38af060", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".sample_statistics")); - spec.addAuxFile("7a0cd8a5ebaaa82621fd3b5aed9c32fe", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".sample_summary")); + spec.addAuxFile("185b910e499c08a8b88dd3ed1ac9e8ec", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".sample_interval_summary")); + spec.addAuxFile("d5d11b686689467b5a8836f0a07f447d", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".sample_statistics")); + spec.addAuxFile("ad1a2775a31b1634daf64e691676bb96", createTempFileFromBase(baseOutputFile.getAbsolutePath()+".sample_summary")); execute("testBaseOutputNoFiltering",spec); } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java index 7179680bdf..32fa8679e4 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java @@ -189,7 +189,7 @@ public void testMultiTechnologies() { " -o %s" + " -L 1:10,000,000-10,100,000", 1, - Arrays.asList("2b2729414ae855d390e7940956745bce")); + Arrays.asList("f0fbe472f155baf594b1eeb58166edef")); executeTest(String.format("test multiple technologies"), spec); } @@ -208,7 +208,7 @@ public void testCallingWithBAQ() { " -L 1:10,000,000-10,100,000" + " -baq CALCULATE_AS_NECESSARY", 1, - Arrays.asList("95c6120efb92e5a325a5cec7d77c2dab")); + Arrays.asList("8c87c749a7bb5a76ed8504d4ec254272")); executeTest(String.format("test calling with BAQ"), spec); } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationWalkersIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationWalkersIntegrationTest.java index 65de6697b3..b53daaf397 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationWalkersIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationWalkersIntegrationTest.java @@ -34,8 +34,8 @@ public Object[][] createCCTestData() { new CCTest( validationDataLocation + "NA12892.SLX.SRP000031.2009_06.selected.bam", "ab4940a16ab990181bd8368c76b23853" ); new CCTest( validationDataLocation + "NA19240.chr1.BFAST.SOLID.bam", "17d4b8001c982a70185e344929cf3941"); - new CCTest( validationDataLocation + "NA12873.454.SRP000031.2009_06.chr1.10_20mb.bam", "36c0c467b6245c2c6c4e9c956443a154" ); - new CCTest( validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.allTechs.bam", "955a8fa2ddb2b04c406766ccd9ac45cc" ); + new CCTest( validationDataLocation + "NA12873.454.SRP000031.2009_06.chr1.10_20mb.bam", "714e65d6cb51ae32221a77ce84cbbcdc" ); + new CCTest( validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.allTechs.bam", "932f0063abb2a23c22ec992ef8d36aa5" ); return CCTest.getTests(CCTest.class); } @@ -91,8 +91,8 @@ public String toString() { public Object[][] createTRTestData() { new TRTest( validationDataLocation + "NA12892.SLX.SRP000031.2009_06.selected.bam", "0b7123ae9f4155484b68e4a4f96c5504" ); new TRTest( validationDataLocation + "NA19240.chr1.BFAST.SOLID.bam", "d04cf1f6df486e45226ebfbf93a188a5"); - new TRTest( validationDataLocation + "NA12873.454.SRP000031.2009_06.chr1.10_20mb.bam", "b2f4757bc47cf23bd9a09f756c250787" ); - new TRTest( validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.allTechs.bam", "502c7df4d4923c4d078b014bf78bed34" ); + new TRTest( validationDataLocation + "NA12873.454.SRP000031.2009_06.chr1.10_20mb.bam", "74314e5562c1a65547bb0edaacffe602" ); + new TRTest( validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.allTechs.bam", "41c2f82f7789421f3690ed3c35b8f2e4" ); return TRTest.getTests(TRTest.class); } @@ -291,7 +291,7 @@ public void testCountCovariatesVCFPlusDBsnp() { @Test public void testCountCovariatesNoIndex() { HashMap e = new HashMap(); - e.put( validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.allTechs.noindex.bam", "828d247c6e8ef5ebdf3603dc0ce79f61" ); + e.put( validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.allTechs.noindex.bam", "aac7df368ca589dc0a66d5bd9ad007e3" ); for ( Map.Entry entry : e.entrySet() ) { String bam = entry.getKey(); @@ -317,7 +317,7 @@ public void testCountCovariatesNoIndex() { @Test(dependsOnMethods = "testCountCovariatesNoIndex") public void testTableRecalibratorNoIndex() { HashMap e = new HashMap(); - e.put( validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.allTechs.noindex.bam", "991f093a0e610df235d28ada418ebf33" ); + e.put( validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.allTechs.noindex.bam", "02249d9933481052df75c58a2a1a8e63" ); for ( Map.Entry entry : e.entrySet() ) { String bam = entry.getKey(); diff --git a/public/java/test/org/broadinstitute/sting/utils/sam/ReadUtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/sam/ReadUtilsUnitTest.java index b9f831028b..367f6294df 100755 --- a/public/java/test/org/broadinstitute/sting/utils/sam/ReadUtilsUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/sam/ReadUtilsUnitTest.java @@ -103,10 +103,31 @@ public void testGetAdaptorBoundary() { read.setReadNegativeStrandFlag(false); boundary = ReadUtils.getAdaptorBoundary(read); Assert.assertNull(boundary); + read.setInferredInsertSize(10); // Test case 6: read is unmapped read.setReadUnmappedFlag(true); boundary = ReadUtils.getAdaptorBoundary(read); Assert.assertNull(boundary); + read.setReadUnmappedFlag(false); + + // Test case 7: reads don't overlap and look like this: + // <--------| + // |------> + // first read: + myStart = 980; + read.setAlignmentStart(myStart); + read.setInferredInsertSize(20); + read.setReadNegativeStrandFlag(true); + boundary = ReadUtils.getAdaptorBoundary(read); + Assert.assertNull(boundary); + + // second read: + myStart = 1000; + read.setAlignmentStart(myStart); + read.setMateAlignmentStart(980); + read.setReadNegativeStrandFlag(false); + boundary = ReadUtils.getAdaptorBoundary(read); + Assert.assertNull(boundary); } } From 423d4ac2d3db1b0824ce7c7a7b8fbec8de15d698 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Wed, 11 Jan 2012 17:47:08 -0500 Subject: [PATCH 044/356] Quick fix to CalibrateGenotypeLikelihoods we were using an old check for no calls that doesn't work anymore. From 410a340ef5ee55dc1463d9b9f339763b9cef9a94 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Thu, 12 Jan 2012 02:04:03 -0500 Subject: [PATCH 045/356] Swapping the iteration order to run over AF conformations and then samples instead of the reverse minimizes calls to HashMap.get; instead of it being O(n) since we called it for each sample it's now O(1). Runtime on T2D GENES test set is reduced by 5-10%. More optimizations to follow. --- .../genotyper/ExactAFCalculationModel.java | 79 ++++++++++--------- 1 file changed, 43 insertions(+), 36 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java index 8da72ef7a7..e54ab1feff 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java @@ -35,7 +35,7 @@ public class ExactAFCalculationModel extends AlleleFrequencyCalculationModel { - private final static boolean DEBUG = false; + // private final static boolean DEBUG = false; private final static double MAX_LOG10_ERROR_TO_STOP_EARLY = 6; // we want the calculation to be accurate to 1 / 10^6 @@ -199,8 +199,8 @@ private static double calculateAlleleCountConformation(final ExactACset set, final double[][] log10AlleleFrequencyPriors, final AlleleFrequencyCalculationResult result) { - if ( DEBUG ) - System.out.printf(" *** computing LofK for set=%s%n", set.ACcounts); + //if ( DEBUG ) + // System.out.printf(" *** computing LofK for set=%s%n", set.ACcounts); // compute the log10Likelihoods computeLofK(set, genotypeLikelihoods, indexesToACset, log10AlleleFrequencyPriors, result); @@ -209,8 +209,8 @@ private static double calculateAlleleCountConformation(final ExactACset set, if ( !preserveData ) { for ( ExactACcounts index : set.dependentACsetsToDelete ) { indexesToACset.put(index, null); - if ( DEBUG ) - System.out.printf(" *** removing used set=%s after seeing final dependent set=%s%n", index, set.ACcounts); + //if ( DEBUG ) + // System.out.printf(" *** removing used set=%s after seeing final dependent set=%s%n", index, set.ACcounts); } } @@ -218,8 +218,8 @@ private static double calculateAlleleCountConformation(final ExactACset set, // can we abort early because the log10Likelihoods are so small? if ( log10LofK < maxLog10L - MAX_LOG10_ERROR_TO_STOP_EARLY ) { - if ( DEBUG ) - System.out.printf(" *** breaking early set=%s log10L=%.2f maxLog10L=%.2f%n", set.ACcounts, log10LofK, maxLog10L); + //if ( DEBUG ) + // System.out.printf(" *** breaking early set=%s log10L=%.2f maxLog10L=%.2f%n", set.ACcounts, log10LofK, maxLog10L); // no reason to keep this data around because nothing depends on it if ( !preserveData ) @@ -262,8 +262,8 @@ private static double calculateAlleleCountConformation(final ExactACset set, // if the last dependent set was not at the back of the queue (i.e. not just added), then we need to iterate // over all the dependent sets to find the last one in the queue (otherwise it will be cleaned up too early) if ( !preserveData && lastSet == null ) { - if ( DEBUG ) - System.out.printf(" *** iterating over dependent sets for set=%s%n", set.ACcounts); + //if ( DEBUG ) + // System.out.printf(" *** iterating over dependent sets for set=%s%n", set.ACcounts); lastSet = determineLastDependentSetInQueue(set.ACcounts, ACqueue); } if ( lastSet != null ) @@ -323,36 +323,43 @@ private static void computeLofK(final ExactACset set, else { // all possible likelihoods for a given cell from which to choose the max final int numPaths = set.ACsetIndexToPLIndex.size() + 1; - final double[] log10ConformationLikelihoods = new double[numPaths]; // TODO can be created just once, since you initialize it - - for ( int j = 1; j < set.log10Likelihoods.length; j++ ) { - final double[] gl = genotypeLikelihoods.get(j); - final double logDenominator = MathUtils.log10Cache[2*j] + MathUtils.log10Cache[2*j-1]; - - // initialize - for ( int i = 0; i < numPaths; i++ ) - // TODO -- Arrays.fill? - // todo -- is this even necessary? Why not have as else below? - log10ConformationLikelihoods[i] = Double.NEGATIVE_INFINITY; - - // deal with the AA case first - if ( totalK < 2*j-1 ) - log10ConformationLikelihoods[0] = MathUtils.log10Cache[2*j-totalK] + MathUtils.log10Cache[2*j-totalK-1] + set.log10Likelihoods[j-1] + gl[HOM_REF_INDEX]; - - // deal with the other possible conformations now - if ( totalK <= 2*j ) { // skip impossible conformations - int conformationIndex = 1; - for ( Map.Entry mapping : set.ACsetIndexToPLIndex.entrySet() ) { - if ( DEBUG ) - System.out.printf(" *** evaluating set=%s which depends on set=%s%n", set.ACcounts, mapping.getKey()); - log10ConformationLikelihoods[conformationIndex++] = - determineCoefficient(mapping.getValue(), j, set.ACcounts.getCounts(), totalK) + indexesToACset.get(mapping.getKey()).log10Likelihoods[j-1] + gl[mapping.getValue()]; + final double[][] log10ConformationLikelihoods = new double[set.log10Likelihoods.length][numPaths]; // TODO can be created just once, since you initialize it + // initialize + for ( int i = 0; i < set.log10Likelihoods.length; i++ ) + for ( int j = 0; j < numPaths; j++ ) + // TODO -- Arrays.fill? + // todo -- is this even necessary? Why not have as else below? + log10ConformationLikelihoods[i][j] = Double.NEGATIVE_INFINITY; + + // deal with the non-AA possible conformations + int conformationIndex = 1; + for ( Map.Entry mapping : set.ACsetIndexToPLIndex.entrySet() ) { + //if ( DEBUG ) + // System.out.printf(" *** evaluating set=%s which depends on set=%s%n", set.ACcounts, mapping.getKey()); + + ExactACset dependent = indexesToACset.get(mapping.getKey()); + + for ( int j = 1; j < set.log10Likelihoods.length; j++ ) { + final double[] gl = genotypeLikelihoods.get(j); + + if ( totalK <= 2*j ) { // skip impossible conformations + log10ConformationLikelihoods[j][conformationIndex] = + determineCoefficient(mapping.getValue(), j, set.ACcounts.getCounts(), totalK) + dependent.log10Likelihoods[j-1] + gl[mapping.getValue()]; } } - final double log10Max = MathUtils.approximateLog10SumLog10(log10ConformationLikelihoods); + conformationIndex++; + } - // finally, update the L(j,k) value + // finally, deal with the AA case (which depends on previous cells in this column) and then update the L(j,k) value + for ( int j = 1; j < set.log10Likelihoods.length; j++ ) { + final double[] gl = genotypeLikelihoods.get(j); + + if ( totalK < 2*j-1 ) + log10ConformationLikelihoods[j][0] = MathUtils.log10Cache[2*j-totalK] + MathUtils.log10Cache[2*j-totalK-1] + set.log10Likelihoods[j-1] + gl[HOM_REF_INDEX]; + + final double logDenominator = MathUtils.log10Cache[2*j] + MathUtils.log10Cache[2*j-1]; + final double log10Max = MathUtils.approximateLog10SumLog10(log10ConformationLikelihoods[j]); set.log10Likelihoods[j] = log10Max - logDenominator; } } @@ -523,7 +530,7 @@ public int linearExact(GenotypesContext GLs, lastK = k; maxLog10L = Math.max(maxLog10L, log10LofK); if ( log10LofK < maxLog10L - MAX_LOG10_ERROR_TO_STOP_EARLY ) { - if ( DEBUG ) System.out.printf(" *** breaking early k=%d log10L=%.2f maxLog10L=%.2f%n", k, log10LofK, maxLog10L); + //if ( DEBUG ) System.out.printf(" *** breaking early k=%d log10L=%.2f maxLog10L=%.2f%n", k, log10LofK, maxLog10L); done = true; } From f5f5ed5dcdb96752d37d00450cb0fcfc8369bf62 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Thu, 12 Jan 2012 08:50:03 -0500 Subject: [PATCH 046/356] Don't initialize the cell conformation values (use an else in the loop instead) as per Mark's TODO --- .../genotyper/ExactAFCalculationModel.java | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java index e54ab1feff..986b2a8000 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java @@ -324,12 +324,6 @@ private static void computeLofK(final ExactACset set, // all possible likelihoods for a given cell from which to choose the max final int numPaths = set.ACsetIndexToPLIndex.size() + 1; final double[][] log10ConformationLikelihoods = new double[set.log10Likelihoods.length][numPaths]; // TODO can be created just once, since you initialize it - // initialize - for ( int i = 0; i < set.log10Likelihoods.length; i++ ) - for ( int j = 0; j < numPaths; j++ ) - // TODO -- Arrays.fill? - // todo -- is this even necessary? Why not have as else below? - log10ConformationLikelihoods[i][j] = Double.NEGATIVE_INFINITY; // deal with the non-AA possible conformations int conformationIndex = 1; @@ -340,12 +334,14 @@ private static void computeLofK(final ExactACset set, ExactACset dependent = indexesToACset.get(mapping.getKey()); for ( int j = 1; j < set.log10Likelihoods.length; j++ ) { - final double[] gl = genotypeLikelihoods.get(j); if ( totalK <= 2*j ) { // skip impossible conformations - log10ConformationLikelihoods[j][conformationIndex] = + final double[] gl = genotypeLikelihoods.get(j); + log10ConformationLikelihoods[j][conformationIndex] = determineCoefficient(mapping.getValue(), j, set.ACcounts.getCounts(), totalK) + dependent.log10Likelihoods[j-1] + gl[mapping.getValue()]; - } + } else { + log10ConformationLikelihoods[j][conformationIndex] = Double.NEGATIVE_INFINITY; + } } conformationIndex++; @@ -353,10 +349,13 @@ private static void computeLofK(final ExactACset set, // finally, deal with the AA case (which depends on previous cells in this column) and then update the L(j,k) value for ( int j = 1; j < set.log10Likelihoods.length; j++ ) { - final double[] gl = genotypeLikelihoods.get(j); - if ( totalK < 2*j-1 ) + if ( totalK < 2*j-1 ) { + final double[] gl = genotypeLikelihoods.get(j); log10ConformationLikelihoods[j][0] = MathUtils.log10Cache[2*j-totalK] + MathUtils.log10Cache[2*j-totalK-1] + set.log10Likelihoods[j-1] + gl[HOM_REF_INDEX]; + } else { + log10ConformationLikelihoods[j][0] = Double.NEGATIVE_INFINITY; + } final double logDenominator = MathUtils.log10Cache[2*j] + MathUtils.log10Cache[2*j-1]; final double log10Max = MathUtils.approximateLog10SumLog10(log10ConformationLikelihoods[j]); From e7fe9910f7d8e3df40f2848231dfadf301f09424 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Thu, 12 Jan 2012 10:27:10 -0500 Subject: [PATCH 047/356] Create the temp storage for calculating cell values just once as per Mark's TODO --- .../genotyper/ExactAFCalculationModel.java | 31 +++++++++++-------- .../broadinstitute/sting/utils/MathUtils.java | 26 +++++++++++----- 2 files changed, 37 insertions(+), 20 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java index 986b2a8000..1594c92cb0 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java @@ -177,12 +177,18 @@ public static void linearExactMultiAllelic(final GenotypesContext GLs, ACqueue.add(zeroSet); indexesToACset.put(zeroSet.ACcounts, zeroSet); + // optimization: create the temporary storage for computing L(j,k) just once + final int maxPossibleDependencies = numAlternateAlleles + (numAlternateAlleles * (numAlternateAlleles + 1) / 2) + 1; + final double[][] tempLog10ConformationLikelihoods = new double[numSamples+1][maxPossibleDependencies]; + for ( int i = 0; i < maxPossibleDependencies; i++ ) + tempLog10ConformationLikelihoods[0][i] = Double.NEGATIVE_INFINITY; + // keep processing while we have AC conformations that need to be calculated double maxLog10L = Double.NEGATIVE_INFINITY; while ( !ACqueue.isEmpty() ) { // compute log10Likelihoods final ExactACset set = ACqueue.remove(); - final double log10LofKs = calculateAlleleCountConformation(set, genotypeLikelihoods, maxLog10L, numChr, preserveData, ACqueue, indexesToACset, log10AlleleFrequencyPriors, result); + final double log10LofKs = calculateAlleleCountConformation(set, genotypeLikelihoods, maxLog10L, numChr, preserveData, ACqueue, indexesToACset, log10AlleleFrequencyPriors, result, tempLog10ConformationLikelihoods); // adjust max likelihood seen if needed maxLog10L = Math.max(maxLog10L, log10LofKs); @@ -197,13 +203,14 @@ private static double calculateAlleleCountConformation(final ExactACset set, final Queue ACqueue, final HashMap indexesToACset, final double[][] log10AlleleFrequencyPriors, - final AlleleFrequencyCalculationResult result) { + final AlleleFrequencyCalculationResult result, + final double[][] tempLog10ConformationLikelihoods) { //if ( DEBUG ) // System.out.printf(" *** computing LofK for set=%s%n", set.ACcounts); // compute the log10Likelihoods - computeLofK(set, genotypeLikelihoods, indexesToACset, log10AlleleFrequencyPriors, result); + computeLofK(set, genotypeLikelihoods, indexesToACset, log10AlleleFrequencyPriors, result, tempLog10ConformationLikelihoods); // clean up memory if ( !preserveData ) { @@ -309,7 +316,8 @@ private static void computeLofK(final ExactACset set, final ArrayList genotypeLikelihoods, final HashMap indexesToACset, final double[][] log10AlleleFrequencyPriors, - final AlleleFrequencyCalculationResult result) { + final AlleleFrequencyCalculationResult result, + final double[][] tempLog10ConformationLikelihoods) { set.log10Likelihoods[0] = 0.0; // the zero case final int totalK = set.getACsum(); @@ -321,10 +329,6 @@ private static void computeLofK(final ExactACset set, } // k > 0 for at least one k else { - // all possible likelihoods for a given cell from which to choose the max - final int numPaths = set.ACsetIndexToPLIndex.size() + 1; - final double[][] log10ConformationLikelihoods = new double[set.log10Likelihoods.length][numPaths]; // TODO can be created just once, since you initialize it - // deal with the non-AA possible conformations int conformationIndex = 1; for ( Map.Entry mapping : set.ACsetIndexToPLIndex.entrySet() ) { @@ -337,10 +341,10 @@ private static void computeLofK(final ExactACset set, if ( totalK <= 2*j ) { // skip impossible conformations final double[] gl = genotypeLikelihoods.get(j); - log10ConformationLikelihoods[j][conformationIndex] = + tempLog10ConformationLikelihoods[j][conformationIndex] = determineCoefficient(mapping.getValue(), j, set.ACcounts.getCounts(), totalK) + dependent.log10Likelihoods[j-1] + gl[mapping.getValue()]; } else { - log10ConformationLikelihoods[j][conformationIndex] = Double.NEGATIVE_INFINITY; + tempLog10ConformationLikelihoods[j][conformationIndex] = Double.NEGATIVE_INFINITY; } } @@ -348,17 +352,18 @@ private static void computeLofK(final ExactACset set, } // finally, deal with the AA case (which depends on previous cells in this column) and then update the L(j,k) value + final int numPaths = set.ACsetIndexToPLIndex.size() + 1; for ( int j = 1; j < set.log10Likelihoods.length; j++ ) { if ( totalK < 2*j-1 ) { final double[] gl = genotypeLikelihoods.get(j); - log10ConformationLikelihoods[j][0] = MathUtils.log10Cache[2*j-totalK] + MathUtils.log10Cache[2*j-totalK-1] + set.log10Likelihoods[j-1] + gl[HOM_REF_INDEX]; + tempLog10ConformationLikelihoods[j][0] = MathUtils.log10Cache[2*j-totalK] + MathUtils.log10Cache[2*j-totalK-1] + set.log10Likelihoods[j-1] + gl[HOM_REF_INDEX]; } else { - log10ConformationLikelihoods[j][0] = Double.NEGATIVE_INFINITY; + tempLog10ConformationLikelihoods[j][0] = Double.NEGATIVE_INFINITY; } final double logDenominator = MathUtils.log10Cache[2*j] + MathUtils.log10Cache[2*j-1]; - final double log10Max = MathUtils.approximateLog10SumLog10(log10ConformationLikelihoods[j]); + final double log10Max = MathUtils.approximateLog10SumLog10(tempLog10ConformationLikelihoods[j], numPaths); set.log10Likelihoods[j] = log10Max - logDenominator; } } diff --git a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java index 5ffd634cc3..408faf61a1 100644 --- a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java @@ -63,14 +63,18 @@ public static int fastRound(double d) { return (d > 0) ? (int)(d + 0.5d) : (int)(d - 0.5d); } - public static double approximateLog10SumLog10(double[] vals) { + public static double approximateLog10SumLog10(final double[] vals) { + return approximateLog10SumLog10(vals, vals.length); + } + + public static double approximateLog10SumLog10(final double[] vals, final int endIndex) { - final int maxElementIndex = MathUtils.maxElementIndex(vals); + final int maxElementIndex = MathUtils.maxElementIndex(vals, endIndex); double approxSum = vals[maxElementIndex]; if ( approxSum == Double.NEGATIVE_INFINITY ) return approxSum; - for ( int i = 0; i < vals.length; i++ ) { + for ( int i = 0; i < endIndex; i++ ) { if ( i == maxElementIndex || vals[i] == Double.NEGATIVE_INFINITY ) continue; @@ -582,11 +586,15 @@ public static double[] normalizeFromLog10(List array) { return normalizeFromLog10(array, false); } - public static int maxElementIndex(double[] array) { + public static int maxElementIndex(final double[] array) { + return maxElementIndex(array, array.length); + } + + public static int maxElementIndex(final double[] array, final int endIndex) { if (array == null) throw new IllegalArgumentException("Array cannot be null!"); int maxI = -1; - for (int i = 0; i < array.length; i++) { + for (int i = 0; i < endIndex; i++) { if (maxI == -1 || array[i] > array[maxI]) maxI = i; } @@ -594,11 +602,15 @@ public static int maxElementIndex(double[] array) { return maxI; } - public static int maxElementIndex(int[] array) { + public static int maxElementIndex(final int[] array) { + return maxElementIndex(array, array.length); + } + + public static int maxElementIndex(final int[] array, int endIndex) { if (array == null) throw new IllegalArgumentException("Array cannot be null!"); int maxI = -1; - for (int i = 0; i < array.length; i++) { + for (int i = 0; i < endIndex; i++) { if (maxI == -1 || array[i] > array[maxI]) maxI = i; } From 28aa3535013ff99b1317a026c520697c1e0cd428 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Thu, 12 Jan 2012 15:07:11 -0500 Subject: [PATCH 048/356] Added "unbiased" downsampling parameter to PrintReads * also cleaned up and updated part of the unit tests for print reads. Needs a more thorough cleaning. --- .../sting/gatk/walkers/PrintReadsWalker.java | 35 ++++++++++++++++--- .../walkers/PrintReadsWalkerUnitTest.java | 34 +++++------------- 2 files changed, 39 insertions(+), 30 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/PrintReadsWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/PrintReadsWalker.java index ac69738d3d..f029b768e8 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/PrintReadsWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/PrintReadsWalker.java @@ -30,11 +30,13 @@ import net.sf.samtools.SAMRecord; import org.broadinstitute.sting.commandline.Argument; import org.broadinstitute.sting.commandline.Output; +import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.utils.SampleUtils; import org.broadinstitute.sting.utils.baq.BAQ; import java.io.File; import java.util.Collection; +import java.util.Random; import java.util.Set; import java.util.TreeSet; @@ -70,12 +72,21 @@ * -I input2.bam \ * --read_filter MappingQualityZero * + * // Prints the first 2000 reads in the BAM file * java -Xmx2g -jar GenomeAnalysisTK.jar \ * -R ref.fasta \ * -T PrintReads \ * -o output.bam \ * -I input.bam \ * -n 2000 + * + * // Downsamples BAM file to 25% + * java -Xmx2g -jar GenomeAnalysisTK.jar \ + * -R ref.fasta \ + * -T PrintReads \ + * -o output.bam \ + * -I input.bam \ + * -ds 0.25 * * */ @@ -95,9 +106,18 @@ public class PrintReadsWalker extends ReadWalker { @Argument(fullName = "platform", shortName = "platform", doc="Exclude all reads with this platform from the output", required = false) String platform = null; + /** + * Only prints the first n reads of the file + */ @Argument(fullName = "number", shortName = "n", doc="Print the first n reads from the file, discarding the rest", required = false) int nReadsToPrint = -1; + /** + * Downsamples the bam file by the given ratio, printing only approximately the given percentage of reads. The downsampling is balanced (over the entire coverage) + */ + @Argument(fullName = "downsample_coverage", shortName = "ds", doc="Downsample BAM to desired coverage", required = false) + public double downsampleRatio = 1.0; + /** * Only reads from samples listed in the provided file(s) will be included in the output. */ @@ -112,6 +132,8 @@ public class PrintReadsWalker extends ReadWalker { private TreeSet samplesToChoose = new TreeSet(); private boolean SAMPLES_SPECIFIED = false; + + Random random; /** * The initialize function. @@ -132,13 +154,15 @@ public void initialize() { if(!samplesToChoose.isEmpty()) { SAMPLES_SPECIFIED = true; } + + random = GenomeAnalysisEngine.getRandomGenerator(); } /** * The reads filter function. * - * @param ref the reference bases that correspond to our read, if a reference was provided + * @param ref the reference bases that correspond to our read, if a reference was provided * @param read the read itself, as a SAMRecord * @return true if the read passes the filter, false if it doesn't */ @@ -188,12 +212,13 @@ else if (nReadsToPrint > 0) { * @return the read itself */ public SAMRecord map( ReferenceContext ref, GATKSAMRecord read, ReadMetaDataTracker metaDataTracker ) { - return read; + return (random.nextDouble() < downsampleRatio) ? read : null; } /** * reduceInit is called once before any calls to the map function. We use it here to setup the output * bam file, if it was specified on the command line + * * @return SAMFileWriter, set to the BAM output file if the command line option was set, null otherwise */ public SAMFileWriter reduceInit() { @@ -202,12 +227,14 @@ public SAMFileWriter reduceInit() { /** * given a read and a output location, reduce by emitting the read - * @param read the read itself + * + * @param read the read itself * @param output the output source * @return the SAMFileWriter, so that the next reduce can emit to the same source */ public SAMFileWriter reduce( SAMRecord read, SAMFileWriter output ) { - output.addAlignment(read); + if (read != null) + output.addAlignment(read); return output; } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/PrintReadsWalkerUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/PrintReadsWalkerUnitTest.java index 8cd10048aa..484641981b 100644 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/PrintReadsWalkerUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/PrintReadsWalkerUnitTest.java @@ -60,19 +60,23 @@ public class PrintReadsWalkerUnitTest extends BaseTest { private ReferenceContext bases = null; //private ReferenceContext ref = new ReferenceContext() + PrintReadsWalker walker; + ArtificialSAMFileWriter writer; + @BeforeMethod public void before() { trav = new ArtificialReadsTraversal(); readTotal = ( ( trav.endingChr - trav.startingChr ) + 1 ) * trav.readsPerChr + trav.unMappedReads; + + walker = new PrintReadsWalker(); + writer = new ArtificialSAMFileWriter(); + walker.out = writer; + walker.initialize(); } /** test that we get out the same number of reads we put in */ @Test public void testReadCount() { - PrintReadsWalker walker = new PrintReadsWalker(); - ArtificialSAMFileWriter writer = new ArtificialSAMFileWriter(); - walker.out = writer; - trav.traverse(walker, null, writer); assertEquals(writer.getRecords().size(), readTotal); } @@ -80,10 +84,6 @@ public void testReadCount() { /** test that we're ok with a null read */ @Test public void testNullRead() { - PrintReadsWalker walker = new PrintReadsWalker(); - ArtificialSAMFileWriter writer = new ArtificialSAMFileWriter(); - walker.out = writer; - SAMRecord rec = walker.map(bases, null, null); assertTrue(rec == null); } @@ -91,10 +91,6 @@ public void testNullRead() { /** tes that we get the read we put into the map function */ @Test public void testReturnRead() { - PrintReadsWalker walker = new PrintReadsWalker(); - ArtificialSAMFileWriter writer = new ArtificialSAMFileWriter(); - walker.out = writer; - SAMFileHeader head = ArtificialSAMUtils.createArtificialSamHeader(3,1,1000); GATKSAMRecord rec = ArtificialSAMUtils.createArtificialRead(head, "FakeRead", 1, 1, 50); SAMRecord ret = walker.map(bases, rec, null); @@ -102,20 +98,6 @@ public void testReturnRead() { assertTrue(ret.getReadName().equals(rec.getReadName())); } - /** test that the read makes it to the output source */ - @Test - public void testReducingRead() { - PrintReadsWalker walker = new PrintReadsWalker(); - ArtificialSAMFileWriter writer = new ArtificialSAMFileWriter(); - walker.out = writer; - - SAMFileHeader head = ArtificialSAMUtils.createArtificialSamHeader(3,1,1000); - SAMRecord rec = ArtificialSAMUtils.createArtificialRead(head, "FakeRead", 1, 1, 50); - SAMRecord ret = walker.map(bases, null,null); - walker.reduce(ret,writer); - - assertTrue(writer.getRecords().size() == 1); - } } From b06074d6e74f576f549a6a00411cc026640183d1 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 13 Jan 2012 09:24:13 -0500 Subject: [PATCH 049/356] Updated SortingVCFWriterBase to use PriorityBlockingQueue so that the class is thread-safe -- Uses PriorityBlockingQueue instead of PriorityQueue -- synchronized keywords added to all key functions that modify internal state Note that this hasn't been tested extensivesly. Based on report: http://getsatisfaction.com/gsa/topics/missing_loci_output_in_multi_thread_mode_when_implement_sortingvcfwriterbase?utm_content=topic_link&utm_medium=email&utm_source=new_topic --- .../codecs/vcf/SortingVCFWriterBase.java | 100 ++++++++++++------ 1 file changed, 65 insertions(+), 35 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/SortingVCFWriterBase.java b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/SortingVCFWriterBase.java index c299511db5..84ecc7fcd2 100755 --- a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/SortingVCFWriterBase.java +++ b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/SortingVCFWriterBase.java @@ -27,10 +27,8 @@ import org.broadinstitute.sting.utils.variantcontext.VariantContext; -import java.util.Comparator; -import java.util.PriorityQueue; -import java.util.Set; -import java.util.TreeSet; +import java.util.*; +import java.util.concurrent.PriorityBlockingQueue; /** * This class writes VCF files, allowing records to be passed in unsorted. @@ -39,20 +37,26 @@ public abstract class SortingVCFWriterBase implements VCFWriter { // The VCFWriter to which to actually write the sorted VCF records - private VCFWriter innerWriter = null; + private final VCFWriter innerWriter; // the current queue of un-emitted records - private PriorityQueue queue = null; + private final Queue queue; // The locus until which we are permitted to write out (inclusive) protected Integer mostUpstreamWritableLoc; protected static final int BEFORE_MOST_UPSTREAM_LOC = 0; // No real locus index is <= 0 // The set of chromosomes already passed over and to which it is forbidden to return - private Set finishedChromosomes = null; + private final Set finishedChromosomes; // Should we call innerWriter.close() in close() - private boolean takeOwnershipOfInner; + private final boolean takeOwnershipOfInner; + + // -------------------------------------------------------------------------------- + // + // Constructors + // + // -------------------------------------------------------------------------------- /** * create a local-sorting VCF writer, given an inner VCF writer to write to @@ -62,16 +66,27 @@ public abstract class SortingVCFWriterBase implements VCFWriter { */ public SortingVCFWriterBase(VCFWriter innerWriter, boolean takeOwnershipOfInner) { this.innerWriter = innerWriter; - this.queue = new PriorityQueue(50, new VariantContextComparator()); - this.mostUpstreamWritableLoc = BEFORE_MOST_UPSTREAM_LOC; this.finishedChromosomes = new TreeSet(); this.takeOwnershipOfInner = takeOwnershipOfInner; + + // has to be PriorityBlockingQueue to be thread-safe + // see http://getsatisfaction.com/gsa/topics/missing_loci_output_in_multi_thread_mode_when_implement_sortingvcfwriterbase?utm_content=topic_link&utm_medium=email&utm_source=new_topic + this.queue = new PriorityBlockingQueue(50, new VariantContextComparator()); + + this.mostUpstreamWritableLoc = BEFORE_MOST_UPSTREAM_LOC; } public SortingVCFWriterBase(VCFWriter innerWriter) { this(innerWriter, false); // by default, don't own inner } + // -------------------------------------------------------------------------------- + // + // public interface functions + // + // -------------------------------------------------------------------------------- + + @Override public void writeHeader(VCFHeader header) { innerWriter.writeHeader(header); } @@ -79,6 +94,7 @@ public void writeHeader(VCFHeader header) { /** * attempt to close the VCF file; we need to flush the queue first */ + @Override public void close() { stopWaitingToSort(); @@ -86,27 +102,14 @@ public void close() { innerWriter.close(); } - private void stopWaitingToSort() { - emitRecords(true); - mostUpstreamWritableLoc = BEFORE_MOST_UPSTREAM_LOC; - } - - protected void emitSafeRecords() { - emitRecords(false); - } - - protected void noteCurrentRecord(VariantContext vc) { - // did the user break the contract by giving a record too late? - if (mostUpstreamWritableLoc != null && vc.getStart() < mostUpstreamWritableLoc) // went too far back, since may have already written anything that is <= mostUpstreamWritableLoc - throw new IllegalArgumentException("Permitted to write any record upstream of position " + mostUpstreamWritableLoc + ", but a record at " + vc.getChr() + ":" + vc.getStart() + " was just added."); - } /** * add a record to the file * * @param vc the Variant Context object */ - public void add(VariantContext vc) { + @Override + public synchronized void add(VariantContext vc) { /* Note that the code below does not prevent the successive add()-ing of: (chr1, 10), (chr20, 200), (chr15, 100) since there is no implicit ordering of chromosomes: */ @@ -125,7 +128,43 @@ public void add(VariantContext vc) { emitSafeRecords(); } - private void emitRecords(boolean emitUnsafe) { + /** + * Gets a string representation of this object. + * @return a string representation of this object + */ + @Override + public String toString() { + return getClass().getName(); + } + + // -------------------------------------------------------------------------------- + // + // protected interface functions for subclasses to use + // + // -------------------------------------------------------------------------------- + + private synchronized void stopWaitingToSort() { + emitRecords(true); + mostUpstreamWritableLoc = BEFORE_MOST_UPSTREAM_LOC; + } + + protected synchronized void emitSafeRecords() { + emitRecords(false); + } + + protected void noteCurrentRecord(VariantContext vc) { + // did the user break the contract by giving a record too late? + if (mostUpstreamWritableLoc != null && vc.getStart() < mostUpstreamWritableLoc) // went too far back, since may have already written anything that is <= mostUpstreamWritableLoc + throw new IllegalArgumentException("Permitted to write any record upstream of position " + mostUpstreamWritableLoc + ", but a record at " + vc.getChr() + ":" + vc.getStart() + " was just added."); + } + + // -------------------------------------------------------------------------------- + // + // private implementation functions + // + // -------------------------------------------------------------------------------- + + private synchronized void emitRecords(boolean emitUnsafe) { while (!queue.isEmpty()) { VCFRecord firstRec = queue.peek(); @@ -140,15 +179,6 @@ private void emitRecords(boolean emitUnsafe) { } } - /** - * Gets a string representation of this object. - * @return a string representation of this object - */ - @Override - public String toString() { - return getClass().getName(); - } - private static class VariantContextComparator implements Comparator { public int compare(VCFRecord r1, VCFRecord r2) { return r1.vc.getStart() - r2.vc.getStart(); From ca48f04fb895ec599c60b5c0c34064a8caab280b Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Fri, 13 Jan 2012 16:30:04 -0500 Subject: [PATCH 050/356] Better handling in pre QC R scripts for older projects (whole_exome_agilent_designed_120) that came out before some metrics were added to Picard. PCT_PF_READS was plotted with a plot title for PCT_PF_ALIGNED_READS. Now plotting both metrics separately. From 3110a8b69d649911f3e08f74d1566c5289bfba6b Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Fri, 13 Jan 2012 16:31:30 -0500 Subject: [PATCH 051/356] Genotype likelihoods calibration tool refactored * automatically generates pdf with all the plots * new and updated documentation * R script now lives in the classpath (under private) From 3a9d9789aebce3582e1c0c193929d9519bf7e2cc Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Fri, 13 Jan 2012 16:56:57 -0500 Subject: [PATCH 052/356] Removing old scripts for genotype accuracy From cec7107762937832f4014f3df0830a45da4757d9 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Sat, 14 Jan 2012 12:13:15 -0500 Subject: [PATCH 053/356] Better location for the downsampling of reads in PrintReads * using the filter() instead of map() makes for a cleaner walker. * renaming the unit tests to make more sense with the other unit and integration tests --- .../sting/gatk/walkers/PrintReadsWalker.java | 19 +++++++++---------- ...rUnitTest.java => PrintReadsUnitTest.java} | 15 +++++++-------- 2 files changed, 16 insertions(+), 18 deletions(-) rename public/java/test/org/broadinstitute/sting/gatk/walkers/{PrintReadsWalkerUnitTest.java => PrintReadsUnitTest.java} (97%) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/PrintReadsWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/PrintReadsWalker.java index f029b768e8..0702b08c13 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/PrintReadsWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/PrintReadsWalker.java @@ -31,8 +31,11 @@ import org.broadinstitute.sting.commandline.Argument; import org.broadinstitute.sting.commandline.Output; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; +import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.gatk.refdata.ReadMetaDataTracker; import org.broadinstitute.sting.utils.SampleUtils; import org.broadinstitute.sting.utils.baq.BAQ; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import java.io.File; import java.util.Collection; @@ -40,10 +43,6 @@ import java.util.Set; import java.util.TreeSet; -import org.broadinstitute.sting.gatk.contexts.ReferenceContext; -import org.broadinstitute.sting.gatk.refdata.ReadMetaDataTracker; -import org.broadinstitute.sting.utils.sam.GATKSAMRecord; - /** * Renders, in SAM/BAM format, all reads from the input data set in the order in which they appear in the input file. * @@ -201,18 +200,19 @@ else if (nReadsToPrint > 0) { nReadsToPrint--; // n > 0 means there are still reads to be printed. } - return true; - } + // if downsample option is turned off (= 1) then don't waste time getting the next random number. + return (downsampleRatio == 1 || random.nextDouble() < downsampleRatio); + } /** * The reads map function. * - * @param ref the reference bases that correspond to our read, if a reference was provided + * @param ref the reference bases that correspond to our read, if a reference was provided * @param read the read itself, as a SAMRecord * @return the read itself */ public SAMRecord map( ReferenceContext ref, GATKSAMRecord read, ReadMetaDataTracker metaDataTracker ) { - return (random.nextDouble() < downsampleRatio) ? read : null; + return read; } /** @@ -233,8 +233,7 @@ public SAMFileWriter reduceInit() { * @return the SAMFileWriter, so that the next reduce can emit to the same source */ public SAMFileWriter reduce( SAMRecord read, SAMFileWriter output ) { - if (read != null) - output.addAlignment(read); + output.addAlignment(read); return output; } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/PrintReadsWalkerUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/PrintReadsUnitTest.java similarity index 97% rename from public/java/test/org/broadinstitute/sting/gatk/walkers/PrintReadsWalkerUnitTest.java rename to public/java/test/org/broadinstitute/sting/gatk/walkers/PrintReadsUnitTest.java index 484641981b..0fcaad3bf6 100644 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/PrintReadsWalkerUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/PrintReadsUnitTest.java @@ -1,20 +1,19 @@ package org.broadinstitute.sting.gatk.walkers; +import net.sf.samtools.SAMFileHeader; +import net.sf.samtools.SAMRecord; import org.broadinstitute.sting.BaseTest; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; import org.broadinstitute.sting.utils.sam.ArtificialReadsTraversal; import org.broadinstitute.sting.utils.sam.ArtificialSAMFileWriter; import org.broadinstitute.sting.utils.sam.ArtificialSAMUtils; -import net.sf.samtools.SAMRecord; -import net.sf.samtools.SAMFileHeader; - -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertTrue; - import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + /* * Copyright (c) 2009 The Broad Institute @@ -44,11 +43,11 @@ /** * @author aaron *

- * Class PrintReadsWalkerUnitTest + * Class PrintReadsUnitTest *

* This tests the print reads walker, using the artificial reads traversal */ -public class PrintReadsWalkerUnitTest extends BaseTest { +public class PrintReadsUnitTest extends BaseTest { /** * our private fake reads traversal. This traversal seeds the From 8272c8bd263f94047a1e9f47eb36f1db28e16fe0 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Sat, 14 Jan 2012 14:10:19 -0500 Subject: [PATCH 054/356] Added exceptions to CGL walker * Assert that a user provided a VCF not some other type of ROD * Assert that the VCF has samples * Assert that the samples in the BAM exist in the VCF * Warn the user if not all samples in the BAM are present in the VCF From 61f82f138fcec3293f85864459ffa8c245d69c7f Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Mon, 16 Jan 2012 09:30:48 -0500 Subject: [PATCH 055/356] Extract a high-level GATK version from the SVN / GIT full version numbers in analyzeRunReports -- Maps SVN versions 1.0.5988 for example to 0.5, 1.0.6134 to 0.6, etc -- Maps GIT versions 1.x-XXX to 1.x Used in tableau analyses From 8ddac9a06f2db1a9a90c28b837fbaa82683c8fa0 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Mon, 16 Jan 2012 09:33:05 -0500 Subject: [PATCH 056/356] Don't show individual jobs in queueStatus for gsaadm, just count From aa8a885a5baff1adebd2b4cbe172e9217735edaa Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Mon, 16 Jan 2012 09:33:41 -0500 Subject: [PATCH 057/356] Generalizing forest.R analysis script -- Support for N tree analyses -- Testing of NA omit and roughfix options -- Misc. analyses and refactoring From cde224746f1c4ce58862ef9891582eca671f2d71 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Tue, 17 Jan 2012 13:51:05 -0500 Subject: [PATCH 058/356] Bait Redesign supports baits that overlap, by picking only the start of intervals. CalibrateGenotypeLikelihoods supports using an external VCF as input for genotype likelihoods. Currently can be a per-sample VCF, but has un-implemented methods for allowing a read-group VCF to be used. Removed the old constrained genotyping code from UGE -- the trellis calculated is exactly the same as that done in the MLE AC estimate; so we should just re-use that one. --- .../genotyper/UnifiedGenotyperEngine.java | 167 ------------------ 1 file changed, 167 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java index 5d73e8d289..ee5aed3e59 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java @@ -858,171 +858,4 @@ public static GenotypesContext assignGenotypes(final VariantContext vc, return calls; } - - /** - * @param vc variant context with genotype likelihoods - * @param allelesToUse bit vector describing which alternate alleles from the vc are okay to use - * @param exactAC integer array describing the AC from the exact model for the corresponding alleles - * @return genotypes - */ - public static GenotypesContext constrainedAssignGenotypes(VariantContext vc, boolean[] allelesToUse, int[] exactAC ) { - - final GenotypesContext GLs = vc.getGenotypes(); - - // samples - final List sampleIndices = GLs.getSampleNamesOrderedByName(); - - // we need to determine which of the alternate alleles (and hence the likelihoods) to use and carry forward - final int numOriginalAltAlleles = allelesToUse.length; - final List newAlleles = new ArrayList(numOriginalAltAlleles+1); - newAlleles.add(vc.getReference()); - final HashMap alleleIndexMap = new HashMap(); // need this for skipping dimensions - int[] alleleCount = new int[exactAC.length]; - for ( int i = 0; i < numOriginalAltAlleles; i++ ) { - if ( allelesToUse[i] ) { - newAlleles.add(vc.getAlternateAllele(i)); - alleleIndexMap.put(vc.getAlternateAllele(i),i); - alleleCount[i] = exactAC[i]; - } else { - alleleCount[i] = 0; - } - } - final List newAltAlleles = newAlleles.subList(1,newAlleles.size()); - final int numNewAltAlleles = newAltAlleles.size(); - ArrayList likelihoodIndexesToUse = null; - - // an optimization: if we are supposed to use all (or none in the case of a ref call) of the alleles, - // then we can keep the PLs as is; otherwise, we determine which ones to keep - final int[][] PLcache; - if ( numNewAltAlleles != numOriginalAltAlleles && numNewAltAlleles > 0 ) { - likelihoodIndexesToUse = new ArrayList(30); - PLcache = PLIndexToAlleleIndex[numOriginalAltAlleles]; - - for ( int PLindex = 0; PLindex < PLcache.length; PLindex++ ) { - int[] alleles = PLcache[PLindex]; - // consider this entry only if both of the alleles are good - if ( (alleles[0] == 0 || allelesToUse[alleles[0] - 1]) && (alleles[1] == 0 || allelesToUse[alleles[1] - 1]) ) - likelihoodIndexesToUse.add(PLindex); - } - } else { - PLcache = PLIndexToAlleleIndex[numOriginalAltAlleles]; - } - - // set up the trellis dimensions - // SAMPLE x alt 1 x alt 2 x alt 3 - // todo -- check that exactAC has alt counts at [1],[2],[3] (and not [0],[1],[2]) - double[][][][] transitionTrellis = new double[sampleIndices.size()+1][exactAC[1]][exactAC[2]][exactAC[3]]; - // N x AC1 x AC2 x AC3; worst performance in multi-allelic where all alleles are moderate frequency - // capped at the MLE ACs* - // todo -- there's an optimization: not all states in the rectangular matrix will be reached, in fact - // todo -- for tT[0] we only care about tT[0][0][0][0], and for tT[1], only combinations of 0,1,2. - int idx = 1; // index of which sample we're on - int prevMaxState = 0; // the maximum state (e.g. AC) reached by the previous sample. Symmetric. (AC capping handled by logic in loop) - // iterate over each sample - for ( String sample : sampleIndices ) { - // push the likelihoods into the next possible states, that is to say - // L[state] = L[prev state] + L[genotype getting into state] - // iterate over each previous state, by dimension - // and contribute the likelihoods for transitions to this state - double[][][] prevState = transitionTrellis[idx-1]; - double[][][] thisState = transitionTrellis[idx]; - Genotype genotype = GLs.get(sample); - if ( genotype.isNoCall() || genotype.isFiltered() ) { - thisState = prevState.clone(); - } else { - double[] likelihoods = genotype.getLikelihoods().getAsVector(); - int dim1min = Math.max(0, alleleCount[0]-2*(sampleIndices.size()-idx+1)); - int dim1max = Math.min(prevMaxState,alleleCount[0]); - int dim2min = Math.max(0,alleleCount[1]-2*(sampleIndices.size()-idx+1)); - int dim2max = Math.min(prevMaxState,alleleCount[1]); - int dim3min = Math.max(0,alleleCount[2]-2*(sampleIndices.size()-idx+1)); - int dim3max = Math.min(prevMaxState,alleleCount[2]); - // cue annoying nested for loop - for ( int a1 = dim1min ; a1 <= dim1max; a1++ ) { - for ( int a2 = dim2min; a2 <= dim2max; a2++ ) { - for ( int a3 = dim3min; a3 <= dim3max; a3++ ) { - double base = prevState[a1][a2][a3]; - for ( int likIdx : likelihoodIndexesToUse ) { - int[] offsets = calculateOffsets(PLcache[likIdx]); - thisState[a1+offsets[1]][a2+offsets[2]][a3+offsets[3]] = base + likelihoods[likIdx]; - } - } - } - } - prevMaxState += 2; - } - idx++; - } - - // after all that pain, we have a fully calculated trellis. Now just march backwards from the EAC state and - // assign genotypes along the greedy path - - GenotypesContext calls = GenotypesContext.create(sampleIndices.size()); - int[] state = alleleCount; - for ( String sample : Utils.reverse(sampleIndices) ) { - --idx; - // the next state will be the maximum achievable state - Genotype g = GLs.get(sample); - if ( g.isNoCall() || ! g.hasLikelihoods() ) { - calls.add(g); - continue; - } - - // subset to the new likelihoods. These are not used except for subsetting in the context iself. - // i.e. they are not a part of the calculation. - final double[] originalLikelihoods = GLs.get(sample).getLikelihoods().getAsVector(); - double[] newLikelihoods; - if ( likelihoodIndexesToUse == null ) { - newLikelihoods = originalLikelihoods; - } else { - newLikelihoods = new double[likelihoodIndexesToUse.size()]; - int newIndex = 0; - for ( int oldIndex : likelihoodIndexesToUse ) - newLikelihoods[newIndex++] = originalLikelihoods[oldIndex]; - - // might need to re-normalize - newLikelihoods = MathUtils.normalizeFromLog10(newLikelihoods, false, true); - } - - // todo -- alter this. For ease of programming, likelihood indeces are - // todo -- used to iterate over achievable states. - double max = Double.NEGATIVE_INFINITY; - int[] bestState = null; - int[] bestAlleles = null; - int bestLikIdx = -1; - for ( int likIdx : likelihoodIndexesToUse ) { - int[] offsets = calculateOffsets(PLcache[likIdx]); - double val = transitionTrellis[idx-1][state[0]-offsets[0]][state[1]-offsets[1]][state[2]-offsets[2]]; - if ( val > max ) { - max = val; - bestState = new int[] { state[0]-offsets[0],state[1]-offsets[1],state[2]-offsets[2]}; - bestAlleles = PLcache[likIdx]; - bestLikIdx = likIdx; - } - } - state = bestState; - List gtAlleles = new ArrayList(2); - gtAlleles.add(newAlleles.get(bestAlleles[0])); - gtAlleles.add(newAlleles.get(bestAlleles[1])); - - final double qual = numNewAltAlleles == 0 ? Genotype.NO_LOG10_PERROR : GenotypeLikelihoods.getQualFromLikelihoods(bestLikIdx, newLikelihoods); - Map attrs = new HashMap(g.getAttributes()); - if ( numNewAltAlleles == 0 ) - attrs.remove(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY); - else - attrs.put(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY, GenotypeLikelihoods.fromLog10Likelihoods(newLikelihoods)); - calls.add(new Genotype(sample, gtAlleles, qual, null, attrs, false)); - - } - return calls; - } - - private static int[] calculateOffsets(int[] alleleIndeces) { - int[] offsets = new int[4]; - for ( int i = 0; i < alleleIndeces.length; i++ ) { - offsets[alleleIndeces[i]]++; - } - - return offsets; - } } From 284a8e9ddc960c957ef958c3f49de4e382b19b2c Mon Sep 17 00:00:00 2001 From: Menachem Fromer Date: Tue, 17 Jan 2012 14:24:41 -0500 Subject: [PATCH 059/356] Fixed to match recent minor updates by Khalid and Eric From ae259f81cc46dbb4966b52c1affe73951ea8494d Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Tue, 17 Jan 2012 14:39:27 -0500 Subject: [PATCH 060/356] Bug fixing for merging of read fragments when one fragment contained an indel --- .../variantrecalibration/VariantRecalibratorEngine.java | 3 +-- .../java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibratorEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibratorEngine.java index 6d2ac643ba..378765051a 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibratorEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibratorEngine.java @@ -27,7 +27,6 @@ import org.apache.log4j.Logger; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; -import org.broadinstitute.sting.utils.exceptions.UserException; import java.util.List; @@ -126,7 +125,7 @@ private void variationalBayesExpectationMaximization( final GaussianMixtureModel iteration++; model.maximizationStep( data ); currentChangeInMixtureCoefficients = model.normalizePMixtureLog10(); - model.expectationStep(data); + model.expectationStep( data ); if( iteration % 5 == 0 ) { // cut down on the number of output lines so that users can read the warning messages logger.info("Finished iteration " + iteration + ". \tCurrent change in mixture coefficients = " + String.format("%.5f", currentChangeInMixtureCoefficients)); } diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java b/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java index cc0b1ae673..d1e3ce26b4 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java @@ -608,7 +608,7 @@ else if (cigarElement.getOperator() != CigarOperator.HARD_CLIP && cigarElement.g * Example: Locus => {read1, read2, ..., readN} * * - Second: readToLocusMap -- a HashMap that describes for each read what loci it contributes to the coverage. - * Note: Locus is a boolean array, indexed from 0 (= startLocation) to N (= stopLocation), with true meaning it contributes to the coverage. + * Note: Locus is a boolean array, indexed from 0 (= startLocation) to N (= stopLocation), with value==true meaning it contributes to the coverage. * Example: Read => {true, true, false, ... false} * * @param readList the list of reads to generate the association mappings From 2390449f0fbc7abd75f11bd813f6193ea1156af8 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Tue, 17 Jan 2012 14:42:48 -0500 Subject: [PATCH 061/356] Local and S3 archiving scripts now push data to MySQL as well From 41d70abe4e852de5cf3da11b80c8b5a0f509bbb3 Mon Sep 17 00:00:00 2001 From: Matt Hanna Date: Tue, 17 Jan 2012 14:47:53 -0500 Subject: [PATCH 062/356] At chartl's request, add the bwa aln -N and bwa aln -m parameters to the bindings. --- public/c/bwa/build_linux.sh | 2 +- public/c/bwa/bwa_gateway.cpp | 2 ++ public/c/bwa/bwa_gateway.h | 2 ++ ...tute_sting_alignment_bwa_c_BWACAligner.cpp | 36 +++++++++++++++++++ .../sting/alignment/bwa/BWAConfiguration.java | 10 ++++++ 5 files changed, 51 insertions(+), 1 deletion(-) diff --git a/public/c/bwa/build_linux.sh b/public/c/bwa/build_linux.sh index b3631a28df..8683bb3772 100755 --- a/public/c/bwa/build_linux.sh +++ b/public/c/bwa/build_linux.sh @@ -1,5 +1,5 @@ #!/bin/sh -export BWA_HOME="/humgen/gsa-scr1/hanna/src/bwa-trunk/bwa" +export BWA_HOME="/humgen/gsa-scr1/hanna/src/bio-bwa/bwa" export JAVA_INCLUDE="/broad/tools/Linux/x86_64/pkgs/jdk_1.6.0_12/include -I/broad/tools/Linux/x86_64/pkgs/jdk_1.6.0_12/include/linux" export TARGET_LIB="libbwa.so" export EXTRA_LIBS="-lc -lz -lstdc++ -lpthread" diff --git a/public/c/bwa/bwa_gateway.cpp b/public/c/bwa/bwa_gateway.cpp index 00f5aa5bcd..088ee43bf9 100644 --- a/public/c/bwa/bwa_gateway.cpp +++ b/public/c/bwa/bwa_gateway.cpp @@ -233,6 +233,8 @@ void BWA::set_disallow_indel_within_range(int indel_range) { options.indel_end_s void BWA::set_mismatch_penalty(int penalty) { options.s_mm = penalty; } void BWA::set_gap_open_penalty(int penalty) { options.s_gapo = penalty; } void BWA::set_gap_extension_penalty(int penalty) { options.s_gape = penalty; } +void BWA::set_mode_nonstop() { options.mode |= BWA_MODE_NONSTOP; options.max_top2 = 0x7fffffff; } +void BWA::set_max_entries_in_queue(int max_entries) { options.max_entries = max_entries; } /** * Create a sequence with a set of reasonable initial defaults. diff --git a/public/c/bwa/bwa_gateway.h b/public/c/bwa/bwa_gateway.h index 2d26ec6509..62756ec2a1 100644 --- a/public/c/bwa/bwa_gateway.h +++ b/public/c/bwa/bwa_gateway.h @@ -60,6 +60,8 @@ class BWA { void set_mismatch_penalty(int penalty); void set_gap_open_penalty(int penalty); void set_gap_extension_penalty(int penalty); + void set_mode_nonstop(); + void set_max_entries_in_queue(int max_entries); // Perform the alignment Alignment* generate_single_alignment(const char* bases, diff --git a/public/c/bwa/org_broadinstitute_sting_alignment_bwa_c_BWACAligner.cpp b/public/c/bwa/org_broadinstitute_sting_alignment_bwa_c_BWACAligner.cpp index 1ccbef0d41..90d70d4a1b 100644 --- a/public/c/bwa/org_broadinstitute_sting_alignment_bwa_c_BWACAligner.cpp +++ b/public/c/bwa/org_broadinstitute_sting_alignment_bwa_c_BWACAligner.cpp @@ -8,11 +8,13 @@ #include "bwa_gateway.h" #include "org_broadinstitute_sting_alignment_bwa_c_BWACAligner.h" +typedef void (BWA::*boolean_setter)(); typedef void (BWA::*int_setter)(int value); typedef void (BWA::*float_setter)(float value); static jobject convert_to_java_alignment(JNIEnv* env, const jbyte* read_bases, const jsize read_length, const Alignment& alignment); static jstring get_configuration_file(JNIEnv* env, jobject configuration, const char* field_name); +static void set_boolean_configuration_param(JNIEnv* env, jobject configuration, const char* field_name, BWA* bwa, boolean_setter setter); static void set_int_configuration_param(JNIEnv* env, jobject configuration, const char* field_name, BWA* bwa, int_setter setter); static void set_float_configuration_param(JNIEnv* env, jobject configuration, const char* field_name, BWA* bwa, float_setter setter); static void throw_config_value_exception(JNIEnv* env, const char* field_name, const char* message); @@ -100,6 +102,10 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_alignment_bwa_c_BWACAligner if(env->ExceptionCheck()) return; set_int_configuration_param(env, configuration, "gapExtensionPenalty", bwa, &BWA::set_gap_extension_penalty); if(env->ExceptionCheck()) return; + set_boolean_configuration_param(env, configuration, "nonStopMode", bwa, &BWA::set_mode_nonstop); + if(env->ExceptionCheck()) return; + set_int_configuration_param(env, configuration, "maxEntriesInQueue", bwa, &BWA::set_max_entries_in_queue); + if(env->ExceptionCheck()) return; } JNIEXPORT jobjectArray JNICALL Java_org_broadinstitute_sting_alignment_bwa_c_BWACAligner_getPaths(JNIEnv *env, jobject instance, jlong java_bwa, jbyteArray java_bases) @@ -357,6 +363,36 @@ static jstring get_configuration_file(JNIEnv* env, jobject configuration, const return path; } +static void set_boolean_configuration_param(JNIEnv* env, jobject configuration, const char* field_name, BWA* bwa, boolean_setter setter) { + jclass configuration_class = env->GetObjectClass(configuration); + if(configuration_class == NULL) return; + + jfieldID configuration_field = env->GetFieldID(configuration_class, field_name, "Ljava/lang/Boolean;"); + if(configuration_field == NULL) return; + + jobject boxed_value = env->GetObjectField(configuration,configuration_field); + if(env->ExceptionCheck()) return; + + if(boxed_value != NULL) { + jclass boolean_box_class = env->FindClass("java/lang/Boolean"); + if(boolean_box_class == NULL) return; + + jmethodID boolean_extractor = env->GetMethodID(boolean_box_class,"booleanValue", "()Z"); + if(boolean_extractor == NULL) return; + + jboolean value = env->CallBooleanMethod(boxed_value,boolean_extractor); + if(env->ExceptionCheck()) return; + + if(value) + (bwa->*setter)(); + + env->DeleteLocalRef(boolean_box_class); + } + + env->DeleteLocalRef(boxed_value); + env->DeleteLocalRef(configuration_class); +} + static void set_int_configuration_param(JNIEnv* env, jobject configuration, const char* field_name, BWA* bwa, int_setter setter) { jclass configuration_class = env->GetObjectClass(configuration); if(configuration_class == NULL) return; diff --git a/public/java/src/org/broadinstitute/sting/alignment/bwa/BWAConfiguration.java b/public/java/src/org/broadinstitute/sting/alignment/bwa/BWAConfiguration.java index 73441cb6a4..e453c7f8a9 100644 --- a/public/java/src/org/broadinstitute/sting/alignment/bwa/BWAConfiguration.java +++ b/public/java/src/org/broadinstitute/sting/alignment/bwa/BWAConfiguration.java @@ -41,4 +41,14 @@ public class BWAConfiguration { * What is the scoring penalty for a gap extension? */ public Integer gapExtensionPenalty = null; + + /** + * Enter bwa's 'non-stop' mode (equivalent to bwa aln -N parameter). + */ + public Boolean nonStopMode = false; + + /** + * Set the max queue size that bwa will use when searching for matches (equivalent to bwa aln -m parameter). + */ + public Integer maxEntriesInQueue = null; } From 75f87db468e27b51d124bf854eea7c7b30284d9f Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Tue, 17 Jan 2012 15:02:45 -0500 Subject: [PATCH 063/356] Replacing Mills file with new gold standard indel set in the resource bundle for release with v1.5 --- .../sting/queue/qscripts/GATKResourcesBundle.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala index 8c9063c293..22ac524536 100755 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala @@ -134,8 +134,8 @@ class GATKResourcesBundle extends QScript { addResource(new Resource("/humgen/1kg/processing/official_release/phase1/ALL.wgs.VQSR_consensus_biallelic.20101123.indels.sites.vcf", "1000G_biallelic.indels", b37, true, false)) - addResource(new Resource("/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/Mills_Devine_Indels_2011/ALL.wgs.indels_mills_devine_hg19_leftAligned_collapsed_double_hit.sites.vcf", - "Mills_Devine_2hit.indels", b37, true, true)) + addResource(new Resource("/humgen/gsa-hpprojects/GATK/data/Comparisons/Unvalidated/GoldStandardIndel/gold.standard.indel.MillsAnd1000G.b37.vcf", + "Mills_and_1000G_gold_standard.indels", b37, true, true)) // // example call set for wiki tutorial From ff2fc514ae3c1344765faabd77c42a85580f58c2 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Tue, 17 Jan 2012 15:12:49 -0500 Subject: [PATCH 064/356] Updated plots to CGL walker a few updates on the CalibrateGenotypeLikelihoods walker output * Fixed ggplot2 issue with dataset with poor coverage * Added jitter as default geometry * Dropped the cut by technology from the graphs From 8b0ddf0aaf5462986e405de2b966f146ac00a880 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Tue, 17 Jan 2012 16:13:13 -0500 Subject: [PATCH 065/356] Adding notes to CountCovariates docs about using interval lists as database of known variation --- .../gatk/walkers/recalibration/CountCovariatesWalker.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java index 88a9668cce..bdf25419f2 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java @@ -155,7 +155,9 @@ public class CountCovariatesWalker extends LocusWalker> knownSites = Collections.emptyList(); From 816dcf9616c3074ceae8fa3ac26fc597b53541dd Mon Sep 17 00:00:00 2001 From: Menachem Fromer Date: Tue, 17 Jan 2012 16:35:16 -0500 Subject: [PATCH 066/356] Finally got around to adding support for Eric's fix to permit annotation exclusion by VariantAnnotator From f2b0575deeb833b808a1aca04028da7733467621 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Mon, 16 Jan 2012 14:07:42 -0500 Subject: [PATCH 067/356] Detect unreasonably large allele strings (>2^16) and throw an error -- samtools can emit alleles where the ref is 42M Ns and this caused the GATK (via tribble) to hang in several places. -- Tribble was updated so we actually could read the line properly (rev. to 51 here). -- Still the parsing algorithms in the GATK aren't happy with such a long allele. Instead of optimizing the code around an improper use case I put in a limit of 2^16 bp for any allele, and throw a meaningful exception when encountered. --- .../utils/codecs/vcf/AbstractVCFCodec.java | 16 +++++++++++----- .../{tribble-46.jar => tribble-51.jar} | Bin 301252 -> 304586 bytes .../{tribble-46.xml => tribble-51.xml} | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) rename settings/repository/org.broad/{tribble-46.jar => tribble-51.jar} (87%) rename settings/repository/org.broad/{tribble-46.xml => tribble-51.xml} (51%) diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java index e44c10f1f2..43c878b2fa 100755 --- a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java +++ b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java @@ -18,6 +18,7 @@ public abstract class AbstractVCFCodec implements FeatureCodec, NameAwareCodec { + public final static int MAX_EXPLICIT_ALLELE_SIZE = (int)Math.pow(2, 16); protected final static Logger log = Logger.getLogger(VCFCodec.class); protected final static int NUM_STANDARD_FIELDS = 8; // INFO is the 8th column @@ -252,7 +253,7 @@ public Feature decode(String line) { // if we have don't have a header, or we have a header with no genotyping data check that we have eight columns. Otherwise check that we have nine (normal colummns + genotyping data) if (( (header == null || !header.hasGenotypingData()) && nParts != NUM_STANDARD_FIELDS) || - (header != null && header.hasGenotypingData() && nParts != (NUM_STANDARD_FIELDS + 1)) ) + (header != null && header.hasGenotypingData() && nParts != (NUM_STANDARD_FIELDS + 1)) ) throw new UserException.MalformedVCF("there aren't enough columns for line " + line + " (we expected " + (header == null ? NUM_STANDARD_FIELDS : NUM_STANDARD_FIELDS + 1) + " tokens, and saw " + nParts + " )", lineNo); @@ -518,8 +519,11 @@ protected static List parseAlleles(String ref, String alts, int lineNo) * @param lineNo the line number for this record */ private static void checkAllele(String allele, boolean isRef, int lineNo) { - if ( allele == null || allele.length() == 0 ) - generateException("Empty alleles are not permitted in VCF records", lineNo); + if ( allele == null || allele.length() == 0 ) + generateException("Empty alleles are not permitted in VCF records", lineNo); + + if ( allele.length() > MAX_EXPLICIT_ALLELE_SIZE ) + generateException(String.format("Allele detected with length %d, exceeding max size %d. Please remove this from the VCF file before continuing", allele.length(), MAX_EXPLICIT_ALLELE_SIZE), lineNo); if ( isSymbolicAllele(allele) ) { if ( isRef ) { @@ -572,12 +576,13 @@ protected static boolean isSingleNucleotideEvent(List alleles) { public static int computeForwardClipping(List unclippedAlleles, String ref) { boolean clipping = true; + final byte ref0 = ref.getBytes()[0]; for ( Allele a : unclippedAlleles ) { if ( a.isSymbolic() ) continue; - if ( a.length() < 1 || (a.getBases()[0] != ref.getBytes()[0]) ) { + if ( a.length() < 1 || (a.getBases()[0] != ref0) ) { clipping = false; break; } @@ -589,6 +594,7 @@ public static int computeForwardClipping(List unclippedAlleles, String r protected static int computeReverseClipping(List unclippedAlleles, String ref, int forwardClipping, int lineNo) { int clipping = 0; boolean stillClipping = true; + final byte[] refBytes = ref.getBytes(); while ( stillClipping ) { for ( Allele a : unclippedAlleles ) { @@ -604,7 +610,7 @@ protected static int computeReverseClipping(List unclippedAlleles, Strin stillClipping = false; else if ( ref.length() == clipping ) generateException("bad alleles encountered", lineNo); - else if ( a.getBases()[a.length()-clipping-1] != ref.getBytes()[ref.length()-clipping-1] ) + else if ( a.getBases()[a.length()-clipping-1] != refBytes[ref.length()-clipping-1] ) stillClipping = false; } if ( stillClipping ) diff --git a/settings/repository/org.broad/tribble-46.jar b/settings/repository/org.broad/tribble-51.jar similarity index 87% rename from settings/repository/org.broad/tribble-46.jar rename to settings/repository/org.broad/tribble-51.jar index 401fcfc3a92c558e6975e80ed948831447bb6bc2..04121804f22936341a786a08d97a58fc4293f595 100644 GIT binary patch delta 19167 zcmaic34ByV@_)S}c{7ujYeGU!l5l5o5+Fdh5-`Y-0|JN<6#@hZV8RhZ1RR1Kf+E3D zT0Bvs?4lwV$#DF{6WL`w&|Og!5p;JwRsoOoT3r6$di^GupugXLKA(QCUw3tNb#--h zRdpw??sXsgi#uZcI8ApEIf!1Y%PZO%(aZ5X-bX?|E#5k`Hm@LiMqc0Sz61MZ_sJ{k z+b_R=TJPe83m4R+jj5ZNlM(I;ceS_Yc3V$$(XIDS-L1DTP06w{E#nnKlSNbYi?V2H z$T(Hzb{VS_nagwzkzg(JO}Aav82*D@wjz3m-6FNq2sNC0+C5?X-OUc5}n`tRqwFqw0V|`8eVZQ14E;*#S{3&+s z&qE$pEdu^J^V?ND45A)}h~5qYE>uLfJU+a?Asd_?bwamtzWw9!tR9vOd?u$`2oa|| z?m0oS6%%mn=E<--CxiO+F2El1^DJ%BCEH;ta}jQ8@MC zM?Tl|bx?#0?|%Hz-$8DE7Z@}^A^nj^S4-2?a%c!0B+zkzSO*a71_m6eTKS zAc{&PbdB#2a#ClCCxsG3@oXh2HW7uP_#}D{RY{a=RV|vWBzuFaQUG9DU}4mSQt_oS zS$HNOKbka900>WJvY$ddDf`ISYr!y%2-FgtLN^~#7@a1@6^E9Lv8zQT=a^27x}j91 z?&zt9mwGZSD9FW|DGc#EN5)eYL_2L~Eqs!;5F4L^dQoq?@pz1xe|d*UuW~yy%}>r2 za#>$tx^8`$Au=AeySxyhj0U33R+cF~=*#h@k^Sh>|fcR9$%pm+24Jucctf^+v8w4ICZ zbJG3v0GB-IB=7z75Pv_+h<0${Bm8)j@jqrza~M5NPZ+e*pg%b1k1kBeE{1%Pfu3S) zPdjKgzn|e2dkorZ(6a`$bX@8RnPckeY8H=VZNh6b%39sfF(#DERlIO{rTdCckMi{| z>n0w#V6Tdl;8ii0mtR<*B^rteKdEuQs|=j8YG&5Yswu6XsZc>+y&Jxqb%xJZ*4y`e z*>c}IrKw@AhWhz6btCJS)-{N9p~m^{Dj(^p09(OKaLae9e2Ex&QA_q!RP^mWx2Ael z&EnG9x|$`{FrNck6pC1k;?aRBU$iIQeGRCbiQ1a zpnX@YB9mUFBPP8@M-BRuNq?qeCjEt8S18q&UY?p~(i`-qLHkU4i|S1}PH&s^4(I;L zkH68oCY_-7OnRSAnp8{k#I#z?(qYVBXXtY#eLx3A`d;j9Yu;C5e0e9j4N<65$>PQJ zi_;cXFJ4kxHz!Rpld1a9gkt!}gjV?2q%(97ODJbX%`B77(p-~@s8}4o)0PI`=*mpd zx0|i_!h&Q&M181wd`l{Oy6xQ~NJ;y`KTR`27u3|vX_(vcYGsY;bnu+Qc57i?M@`+VB~xk}=0dgQVnMP7PXDSlF= zx*LQN7itJ9)`0FRL{6$E2Mwkn_+lJGZ5(3&$*|!bKWU!KR`TlmD5M>eRGewhD(S$| z)xstfXSj_GZD|+{2R?|Lfi0!(g+&qwGct-BqH@u z=uQ}NN;lv!TFG%Ag(!P6-Og5W-AA#R?l3E3X1Yyewo@4Q7d}PV6BYS9MU-V`?Wf3L zDIvh3?8(yd3-v5Le?LV*i-$}v2#F0T2u%rTrm(CKE-eT>MBeDC1U)t+Hq=k;s`^l*nP~h< zZiA*M|D~=5U9Bh=si#3zR~dATevuMTRDz-yCHiMFVq?(jRJv9QE#=_ND^ri5ZkNmc z9YHHnw<%Y^?(1b+uaX23*^7P07Ikq3*dwb8Mu;qKnkv~-X#{td!Pm@)C=On(EC<^ z-3<$78@>eu=RQcJ2I*icY_dHt&Q(K76inAR zYjRlKOr)9ClrX1r2yBKPy9BdFH)s~-zlOODwH6f;atSTD6otqs2F(U~Y&4Qr&||C{ zp<6U*lC^7?r-C#LO9AUW)$ zGDfTcl~U-CFK1SiD?h;z1qG8aB`Yc{Kfx*U39#Oho*;njBwa6XB{&Z!I1cZia;+dN zHVj{n(#WI)XKWZFonBzZnh6g3L!VU;?#+%hW5Zbi#kyL_V`Fo!bzRy?p=({r+Az%R zAy&|GfsciiP4p~=&;qM^AEi+%7P%jM-47NVz|oH>y^dK!n+Ypd=HxvY~xHP&QYN9`uelJ=JWl@2(oI|VnVkuU@4Y&#@ zmZI8augB0!5E~an0YWeZ%g~@36-DN!_|m4vdY7R;9rBQXbW@vz48tt5gt)LRWV*3X zvM-mEH-aR^)iy2{lpZumR58}b&9=1tpL?2@B7+bGZ)z)SL`A=kDCcqo@qgmMU{+!{ z(4zQk)ITedB$a7ZWq3-3Qn>BN8dT~}X|2=)DPhaCoRY@QJa z4TyIlEg*673bm_m$8o2xey39+Sh+JNPUdz$zsZga3_0?-o$LL^A7n@3KO@jG?}Y2F z6E@O_6HDw!#=4U;WzP52ss0jIkB?rGWVL+x#n-YY--qAj%gX5QZ?-wH<%h-DR-~fT zckE6dl&mjA?7dd$BGTSbLd5u+^-%x7?#hE9Ry+U648?BfpOU4-bhDWB%EG4Be}R9T zvc*dql_as}CM8l_zd{cc-jS-)3tuOlu607duj9w{EnOw$7%>_xH zZO|NpY8|w|g#}jU!m6t`Xd%Nb;%18tTH>Gv7u?gO-0ns$T*lQm@nbn-z1g4@2Ca0! zk&A~Sg(sIrs|{*2XpKR)8no7+b^hY%aZ?fZQVgWRXJ~29+fbmLa z7k_Ux3I4Yzo~#T{&ZWCd>{z2r4jY*;2PT+Me`t0$-6P$EXC^Dl9J#d3#HKb{jPC+W z%pGy{Rmu~-yS9uC4Vk&1eo2jZWS15vj;>aEhyhcSIx%vj5-*Iam5pAGLufFMyCH56 zg4Ky3S*w&|00a#!-E=n(+A0Sjcsrq3*u+<0ZxFo<47_&^hAFOivi6bc$=nB{+>_}i z9Y%Q;8?6T@RHgFlqZGqmFfpxa%Vt5|8ZLb~HeRMXSQ}Sk$X%nSd4pKqWgLvAks$@aBv#lDE(xOg(S1 zD$l7sqWgi9N0f1hoK+Zwt86E^%MX!R%Yzp)SffC$k~Z6t?dV8dx{5x~(;8}%7B^}JMyvFTe|JrJm+293N( z5eC${!ZKk~Is$L*IviYezea|Rz1y-WBo)$Y`IY)^yGH5ag^bwEuEd*nmoRQt$`g<& zwKA>!_);x) zFd^`e3=5g`>e#Yp-%x>FTm~=}r7V0g3f0q2jx`1)EU0C`t1P$BSD*!!2A+&z956A4 zI1jv??c~A$JMXk!Azt`8PopY`HA-|4+BlmamJ+Omh*3IlVMW`R-?J77(FBi_;P`#i zSt*tIgwo8{s1sMHFntqM*fVapRUQRz6uOU+k$Kd6>BLToD?LQXRqhm&!l#OWaCLcM zFM0@rKo5n4Na8G0R_RP(N&m2@gpy3#tKeYqdnzn1Rv9Yb=E7uiLZ8pYcJJ&E(Nl{2JFitfmiwT8({x41 zQw4vI8OkJk|5`Oixj9U3W&c^M?33HBcy6OI*xJcf7ruF954L(vx#FcAN|rxslk%e6 zfBhHkQeF(V_R8?nf4KP*SjeA5h@#JxzW#4^C_hF>v|l}=6sO5-pX18k?0%gmlnl2l zE%;nHW|N?Que>h-#F?L!L9#yO7o|Q`W}6b!>yl*ln|^Aj?19aI0*eaeZ+&hl#VnL! zq|5b5gbUp`X3Rt}pDjegdsc9qQ(M ziJIhf;%gm05Zv5Aw{hi0gEsMz0%DsUy2C}A=}xZO!Vg5{w(?GVmq9)U2RLKt9)q?S z9N|o$dtC^+AjY|w?&IA3PUIdi=s_2(Xs?4F;x_`FskFnvVa`-~6s3=GrrE`DPHu!A zb|Tp6f(8RR563(`biadkIXLL);rgcxdfK4fh?k9+JZe-)<+O^cN-9T{R!o^TrhH^& zN%6#zX`_lqPO7Mc78s*Y_(%i+8>;JMoRbx$igSa~G36zrN;>!PBgLDs!aGahegv0B{@qUQCXN~!nb*mA1^5sn}#Eink8u* zWy?uRt3)6)ZKwtDvUp^yni}>B`wy?0bc7B_yWz}OwXZgY#`@#NsS{L~Pl?c}>|GA9 z*5p8Z4}xY1d=JL=kf86OfZ`Y!|2PK5Kd{4kgE!V2oUz^<-QXX`TciPxet~%$zIYqM zej-kksgeClt!mz0JaCdSq1*#!77okR!Y5p&Ke_=r}h4bn_li2~8-e%eB7c8HkH_02NzKAfz|Y zLMT<6H^j!MSTPmV8rul;#l7)X5`qv`*wt*TNE9+&IRtbL4H~AajV^Wqd9R&<7VP73 zv;{JN*5I1biXDZ`j32cjz-_2IL}i8giX)xRYMh7Xiq!FHH!li+ zY!Prom_&Wl){ z&h(y$Eu|~DoqGojTD9UUpj$*tPzR3X!ASe)R_uFg;ef7#TyDUAu_z3{AF3qkEpN3l822jiPCr34Nk2-c?+f zuZD}UJ5=et`>)=ieq$Q~WlyQQ>_-1Npr+aOM(WF|VZ(Y}SLaEGU80IQu7(F)=-|5v z{&(M0vqLPyAn}#`2VTeOemz1oF4y7(pp!-YMtI}nOH`+S;@{LaA}qk&3&wu%0~q!L zi5DaEc=6(=YNCJor)sEeInDdK`ms$x*?ILrc~`-I?mP8_ZDsxGNA;|2;ynGU`dOwm zaB**j)k7KbfO{YzAY!?#hGj^RScf4MfkNmZCJX5E}5X6v9V=L z(T>?{_;H#RYxf*AOS7#Yf8t`TN=8Wh%a&^+Z0GOdVH%k!$wvhv^VS#8QZl- zdq^suKCGRT{cd_ed))@T=7?5fSiJdS)&Tb|SWLSj#g;@pMVvXMMT_%W!NdoTXimTD z4ed!8(eWQRrM($tSz39wq+k77I1wknz|Jvft_$j? zmOtk4V}2MG%?&Orl6tN{xMLA5=EoBLM6^TWXon`!-q4c9Gwez(T4k{57DuzW#LJJ> zVbn-#_`zWg7cc_`23<0(chCkG-9{T33k~LQ4WHP6bIr2qg_EjhEI=f>NxXGl`$BAv)+5Ey3tCsVLXijt)YZ%yV;NO5YnF&E z`AUKb!_Z)}a4ov`Rlk)e%x|lE)&Xt+aG{{*&jEPjpMizRwkM1_1GbSq(J+J!4>fEIb zwF`1bS1*}6v8KV~({>XP*oVSFpb71URy*hswtTvZkH5i`OHuP&2Us7S#;~J!&HS-)usiq3b6sbJc`;-6eK($3bb{Q7wjpHeE%x zy*M3zc^6_ozCYrGwr_@-Dh__DZ5B;WYCS|t54BVr-K|xGAi7dhEw;X^4H3r;HBlV- zP8&Bk8M4E>JscC&<-h46gd#%qncD3a8*biv> zLuVIYzZn1nXdq0WfGw3~8G87nGz+0L@puy&h{Q%8BYfd|Ez8RY+$|33i-OK_dxo8%a2XX9W6UzTJpFAtIA{^lN_>)fIVod;NC~siN@U>@ z8|0F)iRd4+#2~(u2a)4!BgYBb=W=q`3YNo~2oiZCv&hLz#_4DHQsQr~>5h zVK^TTDl8mfR``UisxUBo927+&6Z#F z>+(jRSO!b?rl98T4pPQKmH|VR<1S3i&zMn`kuirTuF6lH$Kl39=mCmXX^Q6nB05yc z;UWcwW;!xS3eLEa%W&9}vXjEgpQTiHS~j9q={ql-vf6c3h+{=r&=-B-C*j^8aLa({ z3UGNPqA#mZ>qY$<)U5`cjgYG~Sc+>Qbn7g-SseR8sjwfS0XiB67<)_!5J-6v?7*VC z4&xxU=)731@Vc#{Ocb%Y=uY~te2_fNpy`*;5K(T<8t|E6zRmcBoWB5}1Aa$<5q8&dJduy{S%YwvV{r8$Qap$N16AkHYyVIdW{Q?a@C(W_h$zkY%@Ix9_^(s=mj$_Z@W7K_5&t=tF}}Ip`x7*5he| zK6YU{I^&?TjP(=l;!_9x-9i6w&_7-D8GX*w;StRb_%jarf{FgpLI2|SR|b8}?{fy7 z7kA|9^P^1!tFD+fX-rv3#pFp)IAg@WbM>eEkLT&(it?PlrH?*UaeZyj1&|D4M0{Vp zML8u_W$80SPCvc>i0f&Z33q%B8$VM_tiY)zytC^#rZmfhk68_C%`%AUY8rAU*3X>J z=Fn{V#ttEUOW%n-E+sD9gvb7GuKA%Y?$qgVtxNciY@5xdpZM{!@9oM&0qHX%`STZ( z{zDf{IE-_|qLXTRU+2h*va;gxD>xSQEB)7`OSm1UD5jz+nyKhY2#x}l;-(y`8>r4y zLg}Kot-qe$&rlpD-9;-+#mP!Hys~EDg5_y^eK3uqR=+Cn_b!-9m}2rH+*BfP-uO{} zy=#}*N|dQYb0Y-@Thpo9P~4^xqjWOim{*CE0{!OD+1zn~I9sXXm@}|yI%5P%EC{MB znK;SnBhFMhgP<=9^q!&dCgSCDMBD(qo0f$N%Y^7__LKF z{S>v2bd*F_9U^y?R~E}l8g~dLPVe*Aq3S6KZz}3#E7tA zEJg&O-Oz(<*~6?w*SuY1KTC;+CA|E|IPcPr0srS7=Vm%XPA zpa%fJK}jG0+BxX9P+g4>d6gFgT>~agwTKEFUCN2(t@S30CeAHKPDPV$2!$*JHEn!L zznpI?uw)65__{XImt z1Q4k+&^P2qsqnaa1Ch5Bvz8G}pFB)_keU&#?AifK4fln(BD0Lkh6T+!GZZrqZu6{X zMFB@nfu0qWfmp-zR?6j@kPp$pEdGoQX{EeE+=T2?l@RLA_fy|OBQQIB)36_KdENc- z41E3XALwr%6?iPb{WJiYYHlDueIE@BqHqxKK^hD2au>?{z&3(+Qd*(o5DmfFlTz#* z3J_aL{50$Zj5mHeI(m+jqm;|*nQ!+f9q~N?(fe3$x zsL)`0rCyI{Fk#K|j3hyKq+!kVz#{WN$VUQp9Np-J0$YZShVL0IN6`ud5Le+Yp%;kO z0M%-oP&X9e5_e*$_hVA-yJ5*B7PCV4461Pr%g zHbTcKIwI6nSSM3}d;@YJmcVCF z$VEzm!0VUjU2+(NOBss|P{S9f__T}7&D(#b~AW>*bn_7AR^f9s$%#Xu)_x;o>AYSfbIrF$PJHisx zZmdj4LP2D;D~KxqSbK8cZ4{P( za#moh7!V%M?!WZ+mc$2lf=ewn&Fx3gSQnSm`fL!bA-_ZGHf*S=iBpkIxY5$S56AvaA26N3RA%p4n;$bF z6Z&h^lV7bg8ZT(0F?eAU9*dXqnst8Uw8uAG>?mFly!AT!_7nuzU?z#}Tj&V(Cq|3dgZ&ybZV$fczVROz&d%ejja4Qn43* zVK9nL!N@w@HZeI6FE2!nkyJn0PC`>9$_QJP{n>4-$IGey`oykw!KWM!Aak z-$cCm?}dhAYp*2g=yv&?C3>bEk{(#5zizvRjtP2nlyn$Vs`cxo>n6N2^ntSQ&YVoTrz| zT;_azg3PU+uUE+2U+3%NWv?Vv^aL|N9fNWWUK~^wRh-N-&^yrNyI6u1DbIgE%G8)$F)NT0YJ3{ zeo^#waldW8;`~m0-&8Dbum>~y{qJH^KmaF>s04n^jq?J~5#nfr-q{+ux_sMzZ^sD- zCmA-D@Xd|Kci6a-EK20G4PA=SJh4YSvQ$sbx_qqbmRH^Xc`Om9j-xtlwXj9~W5yA6 z)WYG*c*zvqZnU|3mv2b=!#JMj#IftZuetG|Hzm0F=SZ?WfLx8pCwc8cHg0ZQVuLPp z;?Cw1%L60+pXC_VHYCtLKnS?^;(-S9@~wnB8q zspI+^+?Sye#hG-~A?7v)Iy{In^+11Fcj0EehlQrYt>O-y9lu(F1}+&(;!e;0tmsh~ zQJgvw7dl6rU7@FnXI9wbSY7a8$V6;PoID*+e=(0irRo+bD?!)V1yyIcy z-3%pE+`Lk^Pj)vk(5XcMplESoWnf%ian&`e0#$~{U4o{^eD=fybqwDv@Z+T&bGOZ> zdlctyR$7NUYJpaGH#b(cH;Wb@tO}5EVHL=zc(4s+ipX4JGot+-8(BA41L&F?W1f`o z$M-^({&-6O*9*4**SLeSu4p^%1G~I|x->89nm&+q7mni(u2&vcerK_5Ww^O#g5bUk z;MlerRrh?`hCWe@Uu~0m)OhFf+d_yqbrj>7|CR7(z%6m=UQE$*Xe4#k`E@`zTo;dQ z)H{hhj~f0PcK$NqqMn>e#EJh_Ch%)+T-n|%T}=N4Q+L4|nDtv4f%iQADAvXsFJ5aC z=Tlraet1w_USGwqm@bL3^wBkC762~oxx|V!0lc@fs=B+as;l@2Qx&9s2UbY5xYX?s znYY@K)Zv0(U;ubGH!f_4?<_vJ)#l|tA9{G$dDL_2DF17wN&Hch@t2}!fM51~;Z~4! zU^>?&{f2{a-FHV7#Sn2C)}i+cvV}O)pt{BCwR)+;H{c$xvuwgxq}0Df5? zn8m0K0NCUlH7WNmbCw;dCT7nZ_xDNc98?RMG`dv^2QQu>X z({9(}CCgXb4m8CMSyyx%k}oh{7Op^Besc&!{#Zw6^<(%15phR=nl5($O|>lEmZf<_ z7mpSy?ptO{aQg)qj(N8Knj1$KTBy(lHze+~dtUea`*+`t5pd#zvB0mn@#NG1c%)c3 zNsAQYHV34udNW9PzFJZ?ehU7gW!L5ac?Y@f_Ia}I{ANAZ8sO=;$^DOnLn@GPYTWi~ zZfvfTpqJ)rF42rGEck>qpWl%T6OB{HU3T|M3IEicdb|a?>$!}-e2=BasiW+Kt(Blf zH{u#Hk66xwXA3Btu(hqu6c^UxCdR5Qfgx;WfLRYnfD1m{pLlOepz+sSckGe2I;Y5N z3e*)hq3-_2WL^COnkm+x<4LwP8o^#i%)T9DnhjC4AF3tcufzF_j-R1qk_?5q3Ei z1dz4W9`y~$u37P54yTTW>AANUK3zl%)FQ+(5N;9Be$D7|dvVHp4DaDPM-ucO*ZDG6 zM-g%A$k4LS7<7b%u)`gsw)y0Bp869*brom6(VToG$(_z#O1IXt$6p0TP8|*J{7_w- zy({2{ru+0nZDePwC{3Ih42JL^gb5nXV?wqr|9Ny?e?vA18IGx4Rah{lALo~J}1vA?=5b_yjeqS zJ9I@q9WB$^z7}k}Jcwm!mx$QJ*Z$Mbu55Zs)5Y<7Y-}CQVLLFoUw^y7W#_Ui>J}j# z&RIJ&%`N;*Q=Geoil2ntYtOLE&7VWIIAwHb`q+fP7)o#k{KYoALE@&P;x^dCoH}my z?^Ff%)n5ek;RJy}KYE>Axp0au*Na8@1y{+fxbH=rIx2Yo4R!-?DOP+dY@rPz5%&rL yKh~fL^4U*8)D4~av03y5@6fZJ>0s*2R zHV{b=6w9-V%6mS$Pe7ldPkA8vP!Qq!pPjwQz3BISH@}&kopR>PnKP&D+^*EkUgPw1_sne9^?Q_SqXp!5hd0uXd7JXZ`&TE_Aq|2;X z(@N8O&KTb`Gtv|3sTt4XHhomru%Ev*u%G^Pkz*GQ4-lrVszw^uRMp6^-kK^MFqa8c z%64~D-45Sjd7Zkl$jeUMlGt)*L|Uz`PN_8EH>b5E`KnVlw$3I!it%Xp=!9om=|rt` z`O`=}(cipjeYs+Z5q`KJqq8%rtU)I|misO|v8C|=jfhiczsP9h-<27E*wf^g1N6?cLlSaW<^Myyv5dI5n_G6!)|I>jp;qj(MmNO)_b+n;Oy-53;Eq(&!G( zrnzW3KWDgTri*5IXg1C9&|I2l(tHm^(gGJPbkQR2awoqp_Rtbq%B5u{E$3>Vn^sU6 z*WAUWyE$9Qa8_~MY7aGuq)PY013O?u%y5m@8;rlb-&Pia0szIW5jVU`m{AJ zfAP=_CJ<<1(kB1Kp(*~UTAHpcYVKdsFHO&+EGc@4xB`nBQxl6?QEQ7HAiqVM=|P}S z-=ZxvPEgpqIVI!o5M+*leSy*5IyV{8ycf;ChfqWdVb6Gf9jUdlzuQoX4uHN%^r=G4N05jW+l0{ zQ+&49Z5OO;uLp%~6ebR4XFW^$QSwD}N)6k?75uK}_2Y+;ca$unxQ!8SvH-TN z$ag>OqCFH(y1jR6sU9|=9yp)@_P#OpDhEW)rM4(_2HAV#tv|J)!KfKa`81shX%2Ov zGU`hA(yf4Y8xBHGz#IUmy$GwMedzrJ4WTL;N++p^KA;iwC5@zS$Tx~E(ir-U#?f^e zFHD*sVrjCdO;bcBO|y4@CFDmLnV7OA@~ud1sU7kPAlx1Zm;l>_I@qlzfowAx7>62N-XS*)QDT-%@iMcq&e!|Gn7Tc|sj>sC-D zm!cYqYZNW6Q)82Q2$Ug3ZYwgWr*Vx^QPmw)aU%8#nTfSQ+Fk&vj!aWL`N$Y?lorJ0 zOn-s^t~n(}^(pB?eVxH=@YK*x6G21mkqhmS>VRm0Wuk9CtbxZsULePr6<=Tt&=s^l zCnntC@9>-$A2X-b#>|Ov+XXNj4s*U+Jv=!vp1dS?OwLhCAC?mnk(cCA<)kq1s5f4|PjP^2A#Vw75-Vd}NZ_c`+umiOP+SjE{;Bk2m9e z5tWqTKy$B*xUrAIS4N1Hme_aWILi~olf58Eh?Y?VEvH&gICa1f4d^bc>uxZ^O6=8Y zx*gm%iq_C1S_^uvL+gFCk=BD@HbC@lqTOiQM-STc+6qd{1+@l|uaGqvrD1p>MS-Dj z1FIH7V1ZE8%<8h2{vG&stIV4QyS6Nu$BYW58lyEtAFypjPm5ZC{2X&oH|C%hU?I89(U_$#$1o)~NqYRd}_k0?FiSWu~x<05S9 zX2;vJ|L7V?#HqP$-87${czF8UL6NrACNA&&wI3|fel98Q5Q+ZcNnZcj(&)qCW&K^s zU@BVkjin5x-hb44V0XBG>K4c5TC`=HQ~I*}AJKLnd24^Io=?2p9`d2Za8lWW&i&CeC5 z!c?(mOX0Lvz0q34)wk(4ZO_7U{{<6s%9RM?OfRjo*;Ms8|jrV{hCvd;g2%1QfxHQ>=YzmjAnskSYrg?BYVOG|q z8Qgm&XS2BXY?ELZ&2_PDnM@0~$3ioT7MXOXNsCQdV$xDsutaMsFBOT)vV5dyB)hz$ zr)((_@kR%TFZuZuJzXX**Q26M9>zu}&;_RQY^l~>HW(#7HGG&wiqYbqk+}rp$2Qy8 zjJrNsr0Th}Le?1r3#?2QjuDSXEVFRh@3z3GE2TIm;$?Y%t#$^#ud=8vb?;upaJvP zRu`XF;DdIy!*X%7gl$P(#=eE2wlCXZqF+|d5z?}l0Kj&LN$_ zY-dsT754XQ^!NsSzC*8Xv9ITV-S<$-KSCCthYtq_xNjeKTg}kyPf3cBKhG-;Ekc(kyK>Bxafb_tV#bg8n>BD5mR=Sqg9|ceZBPEQk zKx|*NK`EUZp;m*Ia43mJHYVL*F{u4%g>j6T=Qw$bym3`j>qWe2Hcc5V5aD_dULFUS zfM&TB-Pl;pu|Z-`Rt-A#ii8$4QgcT3u)*JkF-*nxBJVwes=QASXxC zXs|^xp7Y2ZtlgGJ6nr`HT;U#Y&mLX+az$Nv@LnTA{%}dSs|HLGdzB)n`gN+1!WP^y zKUFUN9mj-I0SoSQIjcZK%8i{xbk*n6M1&*t|1(=Gj!+6<@-lHm=_;AAS#(g9#hb+y zrJ|}v?+`CVs@jalMJuK9s)~+@F6pXv=V#)d4%h`}M5fn9m*G8wP;+O(0JjyPz#%!aM_&-}L?}I() zBm6TH5@q>rtzOl)eYH_4sKYwBm5oXLM?Yz{5tb{bW^4g+RVVWlET=4vPvp@u zH%iMnLxduWR&Z9v* zTyhsZ;DI^iH)*p;4|>3*TRfCV5+i)Hm5bZl7`)x2hghjDbJ4^6+`-vS7wzI_xr=u5 zbB~9eejbDP$0H^^>VZfHxGdVo*)|vLchO^9MEe1g9yjR;_~d&T{t7Mjge*v}ZWg3Ycaxs7z~zWKjHO~b7Lv+C4_aVySfYVI zgdKS(L60u2MM4+{A&TX!!CFhb2UW?8E=G!cd9W5=y9XWR(a%|Qj0Re;j)z%voSv6I z-J+%D7+nYDr(5&_o#5<6Sgz^Qrpzdvn~s3agr@1~gAmq9?_~FQNwyxMrA55V=E^G; z{SA{;t8ck(h?a_ZZ>u^mL>s7;@=k{SU_uLvt&N4IV5vHzS{7<*mWIX*1%O8 zfvJJ7vp$V}g?KN-nns~EmfY7!6Yh*gaQ8Drv)Wlf%hvdy@UrE>ky^W6yqE;6o?v)J zz+a7q`&^US&=I=f*{eDQr z$H4vvu)9xSJD$XXo}$I{G>oM~Sn4yt`!Mi*7Q1@{y{pi26o1db3_V6a0M<`*!p2|} zShWGYf&HHX!f(X(O~r|{FvnXyx&yq*HudWWGIhsNyJ3>)D1~FXM`#9^#RdKFZ!kcZU( z!_c`Jo{imd8)?;W0^4^dCR&W)8Z8O2D+;PMj@4Q^n(~cet))`4*Ne5SajLj|o|cuW zif?YzIywe`Yo~Tdd7AQt0|AwN^?>%1qtY)_Y6qOIJ6_e&9Sh>+lbWf#+p3cvX;YOq z$^bHD`KM6qH=(xhQ!UrAC-#4-B|8Sk>*uwu3gIg2g64S3RTr*kJJgqrs*TsRj~shw zs-~ZHAoO?X=dzXMkl#vgp;k1rmHxHUJ~UsyKSR}?8>ydhYTqf*-*%ASHdT)|Y=h&= zgdwe8gCp}=r1Uj6>d2R->K?hTr4e0qX@>r~v+U=W>P5=SuS!~}cXevVZqlP8Y#^@= zZ2xWoPIE$-Jat}6l<95sFqyVjkFR>_VZD(v{U^Kicb&C9wpZWNP~qSIE&Wr4Y0lgF zNvG?z_w`a|zS58NOAfI{p3~opv8|sLhof3FgjnRHte*j$bu6GOa38i(kM_YV$)Hhg zh|AFk6?q^%#_~%sXC)DQhCL96rQ9&lgF^`KeG-2b()qKH&Yy*J_TVA*ID-eTvj?y9 z$040&GXQ)c(rGT2=0!kiF5ry0mhGXrb>PfJxvb(OplfyUDc2Krw)mgdw$o`aUjZU_x?`*-#|sBM&N zpS=&YvLFgD!&+L&iUlQ@fI!1S_!@2NX#+lY%$zf+89winOpuZ1gj?Elzj00X%72YC zq8eEcDHykj`U#4iTC$*|+5CA^rZu~*WbWkKOYuGJ0gL>!S(f~v_mVB1L8Rg3KlIwN zyrq^Vt?T;za_~g0fjo6Zc(x=6)BnPW+KJJ#=1iF}@Ai54urwH7m+(a!rCjHJdj5IP1ya5)eJjJEbXn;avIx&_=>eN*ZJ0p2wT<;ZH3 zBeC-x0H0kqMww>>L)bu$3nSH6Gh_Fl86zA3!Nd3qf(np!0S5nN)K*t0l_!s!8h(?Y)q#MbtYdB&&4#R~13qC4@A)wP+fi^i?j1b0~{1+1n zmJ5urEJWv^X{N*Snei6`f)@)xVh>$!81-#-Q+C=M@a|mLjPrtqd;he+Oc=N99^`=V z$M9sfYxN9;KSv3IdS^XHwFOMIR1^{gy+lcUurbMqwDirv7ruQr&e~n;2pCBMwq?2x z@*xF4m{9dVB7*&q3hOwXmZD`bB*zlSilvYd%W<-OHfro@Gyo$GgiD~Iw*@=9Il#FA zVX$qH^2x4C_|^rPJ@f|TKTMXPCX8sln=-x!5gIzXTdym9tqgY+)ITH)w;~!}Po{sbyHi3@ zNEyE(d^Zj!r;c{`p|Blp6r5~pc&qMhZTz4|o`$eHgFQYAL;5SZqP;OW_FGW#9O(BQ zEZgrv$sgpE_C}$NMf0yyFTMfG_YFCFnMjf=_8N)4x33>Nf#v6kL^t?vrw1x&7njO8 z+s)aYup7>g-}Z7=fiQvx(>`i)1i?f5J(%z@6+7VfCp=ICPkMmVQzkv_fpve7vqK(? zeTD%XHtAW`;k$e;I>LoY&Z<0YO*5=xE;`PgpXXsOxafq7UUbn*E_#_OUg4&{ap_eT zy~ZP7_rThIle4!td)uUUTy)X{PJfrP_gwToPkYKkf2V(N=>r%2(?uVeBj_WOK6cS3 z9!Ts@P5R6O0sOg(zF?^T;wip#(Pu4O{Fv*RMMzj!5S4DR=Mhee&Cj9^$P5MaAD=-d=kL644jB#>p zq0u^H1dX)d{YF4Xrdhok;l6jR^!X`tA!}@6_c)7^gCU% z=nuLMD_}k@n50|qK<=>MS^Q7_w~LVhfhH`te|K6!qbrCE4Jw^A?auV^GbgB!$z`Eq zd6qCl7-!*@FojEY>uS`iJ6^ag;o(7|sRe&_v?(GiVTni!e(P9ycUNOE7IuA0H={%R zco78z26exEFc7#bqAd{v1O{|78imJN!b=n6(r!lmx^bcwJpNmz%pE^-#*EVO^GYYA zPn!giE1@RWxPwWDoZ0ca6keAA~w;4IUiKtW_ZW>NueZb6w!<>ba*cA1j z!}<+4-6^(j$f9T}974_hKKUZ-8Wz37_&bUl-%311*)j1|6oIcQc(aP2UJ<{J#O;qG zWS}Ox=s3kVwXs#?#fO!+BNTR=YH=lQiNv#nPS{Q+Ld~^vIMxhH9J!Lptsy+x0JpXX z0kwnKkdJsrM@ZmKkhg_4t}K;-kAo|hc;S4vAQH~&bl^P$?f5WsjfR*s(+8hbSwQ*d z1fFpgHU(f<7Uw=giN`4k!Z_K>mqVaNQV|2Mq`E-Yz*|}n>MAnM*eX@l3kR|{Y|uV7 zW-8K^Ta8&RT;$Pwqahdr_LdcCcK`#nayo!&YX0=#`7`?E;tTb53d=oM-Jw1@uzs@< zM28ycd2LWb5KbKme1^eV9B$*3-k3X_XtehX02Fr+?|;dEeAO}GpYE(GgE zP?ZYn7I^Y#Gt1imcNJ@n1n|_!p3hZ-#Uepd|r78p00X zY7#xNyo~^eErG_d^wlnkM9J4URvajY8CXe8xFxeMH+K%&b+n{nzfsM*o)gogl5*M@ zIWd{>hEZHexoyJY!*4t%|M`7G#FxGrTah`!)^^Oll)z2`f zF2e=>jh2fzx=SR}-6Dfliac5+y5pV-i=c_nT)zSIQuK2H#N#l;Mq^{=k_)ag$|kM0FO;k{=^i%V?!7{(EkbyA4IE^3ZKE@wvlt4`N;^U>w%;_H zOv)>}57XRluPTAkSjdDs=yGWt#5KDn_bE{%WBbA4Q9Du`^_Rj>9sP&xxs>Z`IWq*G{EgMJw-2B(9*KdYnj|1twMd<}m zN!|IAuBh30C4discaOmkdmcvG3qf6Pw?Q=QcPC<4g|{6}=S;;ew~)kUs4H5RPdsRH-`3`A}YjNUsT_ zG>YCt;NvZr0<d-(V+X;c5DKbtvp3}k6 zMr@27$h_$3ezzB-@(L%A`MsW8C1x1)lshCF&otWE5y-8p&;I)tL_9eu2dH?q(Lg@+ zh!MW;>6u2j!T%Q^56?H+R{v*!yfoj)S3%F_3yh|!G;V>>N|m-RFgmKzXA6v>s?=(s z(NmSm78?Cj>4SyFKvin8$mp+1OBNXeRO$Uiff1>98dFr=n(ET!JBn%Se0H` zY)n$6+$Bb-Ds5R}%vPm{rN(4cTDa60q)P8DHF~I0;xc2DDlJ+T7<+b^al5K(x7--6 zN}HD(g*MqjF6hf|mK)7v#VW&_&c5cxsHSDhake=%!Y&4W6=h$H^mmz?=wDG9BTug| zoKXE2J|kJ%IR-7+vi=GOM#vSp002BI%8pDy&%B%gy@ggw$Hxme|2%2Ur8zzxHNT3o z?hE~YUy&0u;e{2L@N=Jyif>JV-C`qNGYZ12`Xz?WcX{awE0zjMiahBvtUpbe9OwzF|9qD=`=q$rNR>Rr{|Tx8(bGIn+9}_Q8d>m_(d7!V>sZl$F20P#kV8xElbPzQ#3ae&0VV zx%zIyt2XvwZc2XFp3mnC0L~fLi!zO}qThN~5vDV8f~3#oe_h zRy$zcFcRwLV)r;@2f?f;E4!@vZw$l5m7(_smX>l4VEtNOo?T}o%fhW9Ob)##&^C?R z-pN#L@5jJXJy6~D61VltQEeAmBW4nPufrQ5*BnD|DDJQN^Q}iY;oiVJi|+-jZKG5h zuHw{IbbN=amlmt)jmv;qd4Vux?Uzq;|1LIixo7`Q3!ykTbqncL|CEASxJyJy-+j)ZIoA7` z+uw^N;?yOi|L-3L^}l$ZgGIj5k?V;R1Mdoe_fa$aeVEQ@LXUg!o%)cqH+2ejJIt?OoY7u~fXGPh6 zrYJCHHW>}(>JzlIc_p%(t8?waYibvKvN;0RDv&}sy_el9LtgY7F>-_7NVofc z9^T@Kk>CJM999qfD#{9;V5oNgdUn^IH-35GStu?}g982VtSCGGoa!3$mX<45YKN5{n=3pkbx4Wzb^$R|nEzif{^L;8b~Nvr$hLJZRKW ztY|NL?RyL*#s2s&cL0iNd-6f-+#5A*t!?J8SCIU-qtBg$70#(u$O?Y@K*1{B5>RvN zwg61R*Qz>3*JHQ*S98hFw*;p7f!jy@yQW4RVsr_eK2 za3N^q>ECh4n_tC9mJl|o3r_uS-fF09PHjR^J@a$W$Q#l~RLf4;3Ve?J8r0VG41#7}`SJ`s@lR)P?dzi>?tn-Khy-wd>R;M1mAC3{HP~*IbwEJermU|4Z4flojePV1w zB2JkhaIb82usO9CHwM=42%x%i2Tc)s&Xng#O zn#N3fnIV@@LXcT@zNW8RuIP$`SOe{ChIZjfFfud4Ap?Kk>5zKsmd&%r;+$~`?0vBCII`X<$?9BCQBGre3Z&4f@&Ikmpix_xEMxXSx2NqklGFk1xEs|ubyW3J znT91JwmOG>%+)mcI79@eP}?a7%7MYW)WmSfFLyhV)%Q?L*b9)7oEnD^qCo@h-`XZu z^@qkkaZsbi4(9yvQBHa&uq-z1^`lJ;Ik2^n`X^op`2-m<>rZM(ry5SL{=E1nqw>dG z8+=3gz+Q)Y8%1Q_kB?x)DU^GEzs2sGCQlE6Gdtw9 zZ~h)UI!>X`t`2Ypq)jlw5pWb7sVAKgbqe*gdg diff --git a/settings/repository/org.broad/tribble-46.xml b/settings/repository/org.broad/tribble-51.xml similarity index 51% rename from settings/repository/org.broad/tribble-46.xml rename to settings/repository/org.broad/tribble-51.xml index bb8df5c876..b38fc4bdba 100644 --- a/settings/repository/org.broad/tribble-46.xml +++ b/settings/repository/org.broad/tribble-51.xml @@ -1,3 +1,3 @@ - + From 62801e430a409ed9857bb310911a2d0e65c8df11 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Mon, 16 Jan 2012 18:49:58 -0500 Subject: [PATCH 068/356] Bugfix for unnecessary optimization -- don't cache the ref bytes --- .../sting/utils/codecs/vcf/AbstractVCFCodec.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java index 43c878b2fa..836ba22bf9 100755 --- a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java +++ b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java @@ -522,7 +522,7 @@ private static void checkAllele(String allele, boolean isRef, int lineNo) { if ( allele == null || allele.length() == 0 ) generateException("Empty alleles are not permitted in VCF records", lineNo); - if ( allele.length() > MAX_EXPLICIT_ALLELE_SIZE ) + if ( MAX_EXPLICIT_ALLELE_SIZE != -1 && allele.length() > MAX_EXPLICIT_ALLELE_SIZE ) generateException(String.format("Allele detected with length %d, exceeding max size %d. Please remove this from the VCF file before continuing", allele.length(), MAX_EXPLICIT_ALLELE_SIZE), lineNo); if ( isSymbolicAllele(allele) ) { @@ -576,13 +576,12 @@ protected static boolean isSingleNucleotideEvent(List alleles) { public static int computeForwardClipping(List unclippedAlleles, String ref) { boolean clipping = true; - final byte ref0 = ref.getBytes()[0]; for ( Allele a : unclippedAlleles ) { if ( a.isSymbolic() ) continue; - if ( a.length() < 1 || (a.getBases()[0] != ref0) ) { + if ( a.length() < 1 || (a.getBases()[0] != ref.getBytes()[0]) ) { clipping = false; break; } @@ -594,7 +593,6 @@ public static int computeForwardClipping(List unclippedAlleles, String r protected static int computeReverseClipping(List unclippedAlleles, String ref, int forwardClipping, int lineNo) { int clipping = 0; boolean stillClipping = true; - final byte[] refBytes = ref.getBytes(); while ( stillClipping ) { for ( Allele a : unclippedAlleles ) { @@ -610,7 +608,7 @@ protected static int computeReverseClipping(List unclippedAlleles, Strin stillClipping = false; else if ( ref.length() == clipping ) generateException("bad alleles encountered", lineNo); - else if ( a.getBases()[a.length()-clipping-1] != refBytes[ref.length()-clipping-1] ) + else if ( a.getBases()[a.length()-clipping-1] != ref.getBytes()[ref.length()-clipping-1] ) stillClipping = false; } if ( stillClipping ) From b0560f9440054ebe893e0a7209756f085f399f16 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Tue, 17 Jan 2012 15:56:49 -0500 Subject: [PATCH 069/356] Rev. tribble to fix BED codec bug in tribble 51 --- .../{tribble-51.jar => tribble-53.jar} | Bin 304586 -> 304182 bytes .../{tribble-51.xml => tribble-53.xml} | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename settings/repository/org.broad/{tribble-51.jar => tribble-53.jar} (92%) rename settings/repository/org.broad/{tribble-51.xml => tribble-53.xml} (51%) diff --git a/settings/repository/org.broad/tribble-51.jar b/settings/repository/org.broad/tribble-53.jar similarity index 92% rename from settings/repository/org.broad/tribble-51.jar rename to settings/repository/org.broad/tribble-53.jar index 04121804f22936341a786a08d97a58fc4293f595..02865df435a3e43dffaad91132c34afaec455d81 100644 GIT binary patch delta 8387 zcmahu30xIb_vg-BnD-t7JRd5X2#C0_sGykQ2Bs~pm}#OSxPYcCZvS)c%tNsMGW_n$d1t@po_p?@;p8dTp%X6u zjJ_&&0N4QR`AKX2Td6jDblM~z7b4~3k60CHMkGt5pH%~yO(nn>&TmV^aG!pP1YXh> zG6}@lyGfR+iF%NUDe(Wp#LR6_ZZbD*_@D`lYkb2b<%HcdF{2{ZsR3vqE%t?RuXa!Y zI;i6M2sOlL*Cs|3%;UaQJ)*mtG6l^o;Al(L#2k;Cpfw<2sB=XaEV_}gCfU!(Np?sV zCnpaxffHR{Fs)AO@uVh6Uq19HmoxVCO|p@unxeBWRMhn4tlGFfupO%wd7afrBR!+J z{XF=>N8kq^8&fg{uuqJ)GRE6aLjatCCQxl84oZ-outu%G=>j>LOHXnJ=*Fr+f$np5 zNPw9Rh=W-Uh=ETpH~1_d@KB;q0so^`-tSmJ=CuuOyH z4rl-?Y_QS>tBCbE`n}o#MNmx88V%MGdYv5%SWlP@cIah53H=tNsFaW!>9&dVZ`NQ7 zp|=ud8;NeWK^Z;EiTu0<6&h4(u%l|~W78E!A9M$uKr6-%kIyjrZA`9ue@Hyz$n=6y zGqhQwfbqHcgY!pE&mWkVHzPNnf!eC|qlWW|p%3kZR2`C_6Wx+^coC-Q&=%V1&;z>a z@Dl9OVK+=;z+!aR16d4w@~4j;JBfidX8O~!X5>!SVI*V;)`tZ-HF#Nvy|7P%{W`n? z2Mku+62+3B3gw$I03BY1gE|}nj|Q*l@H)Jq!(n*S2p!hE&0Fv`daJ_`7^lNKa8!qP z33`ui@52W=9E0OJoPZB?7zNpa<*SX1{euIdbT|p8bf|{YIz`FyV0F6L{#*l&#$k2oKpxq#_Sl@{Nl>1it$bGVS{!9U#Uyx+gfW8+4aaxMU<% z6)~zSbCc0c&;sx6(Agw3+5uXk*+ewg7S*O8a~DX5H0X(cz2RYa1RjSzFbw)a7W9K$ z7zp_=1m?g{SOQPLb1(wdB4s0nzYM7rkPSOv9A?au@G?wV|*8 zG`I#c;CGk}w_pxaU@p^PA#;J7MZqE&fg+5{5{zLQ&cKGMHFT&aLOU>84?-`54kD(v zq@Kj>`4GBkM@6KTbxZhRQ53X)DJ~C&haetN1PAN z^b`z0$QS3i4`-W!<_BTCk|0nI({6)HgU8q%Xr{qn#%@6i4IUq=!4Q56LJ`#)Q9-Q1 z4bU0|Vbq6Wh|CphBiF(Wz=I&>{smhhr)?$+uT7R=kZB2b#3wDc!tR$`EhWvAQM{8z z-r=||nDn#@>+0xAO4L1cm-%Yu%hvxzi5WSCb{6utH)f9M<=^~?1gBqesNI?H8Zh==pp&&z`fnV)$$Oux_`mmuCrjlj@pSjaFdyPekx2e z&hHM&@VQ^|T`6g1nvW^yOvkD|-O7^xyRhg~SdwOt5tQokKTEZ~KS!lhf|SB}<>Sm0 zcn0H0<&xp{jf2Fr(0fe}9`99|xknT_@2kQmH1unLK-$)m({A_l2cXeT9@y=k38|Vq z<1zmLBD3Es{uqEee}GM}Ni4+38SgSy<_1>z77nz@4r17fuXO^9^pf@xF{@;(*=>AT z_HiI#MDKx0jN$y;Zd~4Pmn`4!Z`)V`5yw}VeaP-J-b7F5X_%`(Ht_Eiy)Y-BD^^|Mk1lwW`ctuGwXm zlU@=dYV6FnPAf=JDY!k)?2GQ)ogo`PPktgh??XyFkF;kd(Q_j&0rcD@p){ z8%Lsu9?!x>;4&6yT+{;HB@T#&k#KPB52_n{tqjr%@XXmg%qpRvOnmO?-6Rf33ugEEdC7iHK3y zH9RNMMzOuN)lfu@NwHwxutuV6pc0I{ZDEZVv`w2G3`jf*_Kgq_m-r6Y4}xPSXejO{0>`qb(O*F`_y#BREmp!Gaijf& zC)jm)hR2zf5Ko-~+QgnC0XGCet#MN*C>1QXsES6{N-^St|9x`|nuI zZXos#tf7BGIQ%7kAInCBIgWwGcLJ9OhRJUp758-|buhc|&t;vNzj!#8wT}te2o4OWwo^VR#4jU%9h)pj zY|Uk^7rexJxn<;#|5?Ne-sxlzBH#^(9S zEjxYzJJgu=yNFuPx=K{`diIx%P@a)n*-?K9MeSxOff5?_2J0Cmp{<{@3qBHh<_e2* z$qM;)Kuz^*Jo#vJGI#sbf3h&ozkg=S{UquS`)fkmS_i z`RZ_aaCkx%s}K6iu#H-)I?Y6wwnZIlCda4c>KW6~7c10?SmF}{K2$%KO5#tdr%huw z&#HNv#Ett@{l%Q)@Nd+!PKitSQ+1kVd-(Ez1`?MM%y*b`{4G;KndhZTuYUC_RxKJ`sS^-6(|B{ScL(>~x$kN2OQytuXLu(Wy&s?9JtWX} zCT}Z&(KC69ircaxaUX7VQ_(0&&0BY}che4qqAn#CWJK+D-YLjrSW^Fb0gGuvVg zpTj3g%yKXA+Z>)NFBcm>z zY0xX6_o^%`A8Vw~2r&wBo#Ju6y-v_Hq0xJ&&tBN(~vVc zA)_x-8mhIQuj({rfM?phtFqAHHqI9$nD!{ouN9!ZvSrc{_lm}nAl<$v1Q1gD!wEm8 zuUK!SCpTPI?kn7d=rw66Dnu_N_*S; zT<_l$4e>6?9;?kbH7rzYT5Q?aw-_~by;`fuCjPUSx0L3$JddP`uSC)k-dZBE zmLTE_iHI=X`8r7KT4KfEb)uACC)Nlf=lei$%x$Gh9XIYs@6m`E$do@3}Ti!_NyYm!XT zStqlJU0%`=A_a6XNk65qP!YJ?TKalDa~=0FPo6OOsf8BfmRnZlEJsz-b4?QV0Kqa= zCkZ>`NfWzf0lHPzgK5Hlg_Rr5>xbsv#72`wk9t;WCe{k+BD)iIwR2HODlJ}tN*|e8 zhu$NNIm8!U`gNion^#MBiPV)CTq=i;u0)c19g$+h3dG8cDM&r@!G3&cqS3aV7jH|} zcyVQ=wUBu&vx_I;s~3&>$#r-$X^j^(%a|@FC=HHUMFw9YQd^OI1Ou>dm8ELv?8CqQ z8j1BA2Yt)(UX_KO*N8t|7MjGX=V2CBcp49SS@2m|c5wMv_+vJ}V3VoNj8lvX(cO32`BE?$xt80FA-*SX&gE ztOe^(wR5c!E;gcRDKz5KRt1?TK%;v-p}Z}o&^zmR(!Es=e~GY(5|NIRP=-lFw5Tz- zQxqFKN~*o+o3v*H?i?EC%buybvatU_QXu)8N!Ca2{<`i}JTYkCN2A)Svhe1+Cad#9 zB~cWtw<7Xp*1<3K*#T(O_c-#DrMeoYnX$nNF57@!#$7N;JvQ)&dy^^sl(6BlsTNzk z0((f{!aY)oTyE)W*n6+NUX9-tXwZjc?^Rjo`=zNWrvy(E+9NAVP)qY4OsrE?8c`VCs5f(2R57%?#bh1d>$cIRGf2eqGRM%*1WO2~H z2)tKi;j5j9tGx)@VtI0VBTm2faKg3{dmmN92VYI{*yVe$UDv`N)!e0`u%zq##LCV8Vp z`{%h!&g-qSZk8PjK%=!+DLyL;HGG|?CRF^2)YI(YKD|-6!af`M+2anx?4^`j9 z`(yv^f$)?i=q!!IdWu(`s4HAljnnM3&05+1+fe)PGu}AhH%GC*ry3wOdKvZ-!+RGj zhP3Uxf%wf!b8JW2>;Jw-Yc2EoDMGLM{T|vb3VPy}s+IeY7`Ab4PPc}PKJAM|>*h8L zC@p?z$@*#8xQl4i&!c^f?nwoRiDlL_+-0coiBO9c6^^CkT$z;}myxElh!xJ~@%W4^ zw+_iJ<%qr5(!}m<&2`~dYVG4ue?^Mj*d5WR-da!uJ{) z^;7lZHpHJI=T!SD`?n;ykYxEyueW0SW@}E=1*_xRb1^7|H~HIg$M)$`RE`ePpnt7+ zugbzQQ>2@j>{I~O!=hyp?PwiZif>>(1_AyX4SClj delta 8941 zcmaJm2Ur!?(sOnXx3Ls2f`CX7D@{Z(ia{l6o)H9$F$SXoBK8WFSOH^+ih?e35DO@z zsIf$%*Yb=SV~G{>VvKoFAuloJkI__gPmCuBpGIMB}w}YG3o$V1OIo3OCiMsmY{XJ(@bs{3W=_;UppRLLi|^2;_Zzsd;JuJAUzV}y5&@SX{_Tc926;FLQ# z(feH5E)(qL_dT3ruMs{l!agI^)Qo&NN3x`#hNu@VtVXe1)^`*QP`9!eZ`bkJnPakZ zk|s>e&J`>l;u19rhDH*yU)}X;YjsFzjOshA^@|6fw*uWDo?o6*zy=2u=nAn4^n(Ni z4#8msj=(_zBvgTa!gvBMGjlStCJ``>&Y3nRH#_C1wP}@zxd^IxT?Sxa7}?P;kp77U?RH`L%r3EYWZGbeY#)N#R@Ke!(MR^V&6p}y!s2NDPbs`W{M z>UU+~tRY|WQZtf$g(76k400>Cjy$HD?j4sD(cYNxG^XMBeh3x}k1L*^VCFp@y z3*G^I!3cg3hmfMtK~H!dpSX;8E<>F^BCrRicLy1-!9XDdXTn2an`U)WSgh2l}Ic0q_qg-=UWS?YG#AofDGbNh9|db612#Y= zRKj?ug3*{Fj4Tvwyo{E`f(zPq9)=)fKt=c9{7_u}RaB}ocqsnH zCtx+gF!B^y8{suV9z#1Lq`hi{;q)>1;Z%Q|@*>Ut3r24*^!n>Cg8LrXqr_h10Z=ap z-HQ`iz-U`o*d$n}t~x<~kJ6Y)OVrQ>d4|fp!Vf_k0%?OzgLj10d;l!v9B>tY5@A&k z*Z}@v5eL9+m^QxwTq5=^k;%Iar(5N1;SXeQ96bW<(yZ~F-H}l65Xcf+!d)E&=x0o^20S^a5yluHn|m7*&eO%E0U7m4 z?ouO+Me$hGG*@AU@vKGn%J1{5P+xQM`>0l^JyCcR@RZ~vT7XErM&VO;bxlS;w~oUi zJY8e_%$^Ykp`&^$V-N{bBQslx{22&VI_cFf;7oJ{!3wp#v1848q)r#k3!d!9E2c?)#mZ zzTAI1MWFJ`--!{;1DZoq(ZRfy>_neVLyx$_W|)r5E~sJ&CLj;!U|5A|7>fbj9+p5Z z@|tkjg&625U`CaP!YssE^l&#c;HFs6{pjOi80AKoO-w|47^bF<_D-LJ>rwQ#AC5O1 zJJ1U?;yPKv4?wsT12OXNJ8@B%tyL_GVc2ZN+gb}OG+Twd}@oZoGx782HPeKn$2b^+~XwI=SAW9>3+MJ`D3v%j;*e z+7WVd`((}MH*+*oU#cG?T1s{Ax7~z+KYaVKFtBF#FZay4rCkK(>ZAFRJ74Dv^$CrP zMC&41LWb-~ z2g67|iK57CcK>_g$2zVe!L~WW$ytJj#Bv2zz)A&EE>XdZO=Mdi$^mjGC{)wV1{;j9 znm`YAZ5!O*5#%iscB~49S+QD~?ak_jNG&YQ@}VO!Emq#f?K8CdF*aBRMAGnd0;7tO+kF~ye|Z^QC0(vx3 z>PkdKLYFyyek`F#YQ;vTOQQ1GCZtOnyac^Vt`r_1pe{vHyk=DHCTX82m8`>dUB&F} z(yyBS!}m)&H8RslDMa%}#Cgf2>Gs@JX_6?7oHdNqU6%|kop!0ur2)~Oox)`1BbkNV zkUFyqUrGKfuLLXQq?3}_HtEMSKLXJ$fu%6UV^*tMWJ6rW`iNcfz^F?ClsiMwtFpjQp*J zJkecV6C&KV^yrr*tFVvdqbD0&Df_eIFUlTlcNz7zZ5b}F*BH8G$mcaD&Wp=28libF z@ilr;Q>Wz!YSq+^o=nr9jpaaXGxk(bbJI@4c4-P7WRUT57Ym$8y9-C9&7{2ruyH2s zB!G)EsmK`SGKKWdW$OX{rEj&!&R~5I09BUOtCT z6f?iufy_5(wwU?!4Vo!{$hma90Ltdlu>v6TXpR78&%?0cvG~nAIz#~7=hFcKcym4- zDS-dX*SV4w&{xDv)dHF(fQAJ$!SRuc4OmED6~Wf{-HN@xkoF741^N`bM-?o>R+kU{ zN^SpD7bIn~6N{*?`g5U0y_IEPH)c^qu)H_7DC!)3MdYKCogVMi1@8D=W)Tf%MT@9a zkZwLVsFxl`t z+EPkfp=-9nlU3yDOEuxcj~DizC;XJqwUk}SqwR!e9&-MPrD{S&YqhS-h4sqU*`2yr zJ2&2|3u-Hox)ZC&r&{sbosS~k*{c?u3W!KeBvNx(`=ZLdxpZ#Y#yj2sd_*`&xhWV} zrve%%6iF>WDV}FgN)+2zpo`wYC%0U}$so1&C58Q3plf%3ko(2g(K~$bp5FeeE?D^m zSA_LhOxp@y+A;hd;CSF}=ypr3zY+M1%8n))8|j`725$It&JfSy7wFoxd=s_%{u?eu zjrzu$O?p$`lzfgzzO5$2G*>fjo7v$vb&gLtV!0jR!#tPh_VnnnXLFSqfDh-mNcv3| z(@Z`3gON?Ob2QQHZ`&LHge43k{_OA)`>IQD)m&}1bSVuJ{*GLVlfN_wQk-;SKFf4n zlGJl+RIKKF;JdZsS6whJl7+0G{wyPgxUi3x>hx1CCY8ovw(#McF+ICz^v6uZ#4dFv z1{SbP*C~7%E`KOi!|zy1yx8W|x;&>>1jp(k@?O*E?XL`4PTd^)eV=W(S$sI_YZ#}| zACE_U9riVLJ%7)xA0}$_KgSbA=p#I}W;v?5WU@n)wR8%;vD{v+Z}uaO{(6Q3?ZXDV zrK3i^g{a{R9VmrWBI=p23~?CA!_upS(}zA2m-%s*PVL2T=fk;6y95vz3ZGghJa^}qg|$gyo_ zjGx_k3qz8RZcdW3o^qy8cDVufrCp&eFJmpr>rGj~8rqx%TO~I(w@@$G4+lS{0YO*~ zaPUuI_Fr|uf<}5@7A;F2tPYt(7~C&JdG|wgd8_p_8nhZ`Q+qpB@nzFy<68m`<{(}G z?V|bJx%>=g`D>`RXj$1$tMx@5qc~FHz;s}{*XtNjWxJ>Fg*pg*-d}?&PZtvbEIk!f zjb*y&<|0HbPUo}%tnQ#>VbP-`1MA6jS(zMvD1$Xyi}>;)>MFL*r_FB9KKL&VAHACB zU0v{V2J2i*BiX2Y^kPGi9>O1uzMFQdB>*3to#+c^a{Asv>eZlm^_Q`s<70>u)!|~U zKpb=5CV8`4#k$njuXV^zLs~wZwRpIh(trq`lzTh{8v?c)1>w<)UP zuoTG5>vZP?u0wqNfhMdeJ0(vx*^V#Z_@Yl7_#k%tl+>Kvwd0A3_?in%@DA1~9QDx` z8mjIKsac~`p>V`?_CW~^VdfG&5S^NQC+AEMq|uI^U#l98jXvNTc;m0Sp!_Bm{ep1$ z?#TncdVoki_zSN6S6y)MCr;Il)!mg$Z0cRf@Jy8%-^ItP5YKNuoKxfJJ&iZxI)>VN z>veA)UC+JwUx6wur9nc0*|mSnT!z({59d@}{8SSX{Y0`pv-a`KSAHwP?B;`iUbp|M z3r0vRqzu!z_`Vd(u9fO*-z~+p6HFSa@=M9LF#`rNL`8JvC_k2OkNYA%n<-|yuv?6_ z7TVseIyvfjtigOdXI~TV)dd6MxPbm5;wF?K@AcRD1ZHI!4H3LU!YW+xZ%h8yC4LBL z{8bkm&E%wktfD72_dXkRmBTlnlvQ}laJ_m3Q45`I8`$*?dKCFBzEc>0p9}eLJ^*(w(3n!@%L)rC*E6I?IV$10L_^^VR!3pr zMm{@utv1{6AL7e?DA%=q%28uBX{hwgXi(556!_9c9Y1O#;?Heug5QiNVbIabc_h6p z03RKl4BA+w(Jy=(Mg49UW!{9M5-m_NrVax0W_-{tkuxlDBbu$eWx_viN4bS%k;KMna27T`!ZN?hDLj@hqj$Rb< z+G*S!KAex?puaSx;}7sN3D-(25_+J%_7hxy4`=!q1G`d*VNkpryQ6eAuCZM)Er5=BK&0vGK;u2kB9TaXXJ(gM<1NaEXcZzf1y`C|u zooo2&JD5LwILB$#US$^upIOrbqN85A1$$I7`_!vWb)i zyA;1FJnn6*z6zV_x@_zQ##HIsHWx9H-!zR0-{HN)j+HH||MqvRj(lkUbZOgGT`;tf V)X1Jp!NL< - + From 9770250b7292984c1e410d4b4b2cb075b12139b1 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Tue, 17 Jan 2012 17:28:58 -0500 Subject: [PATCH 070/356] Fix for Amy W - evidently binding defaults are not null but an unbound object, which caused the improper branch to be entered into. From 0c7865fdb57fc44708fcd43b24e508b7c6260bba Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Tue, 17 Jan 2012 19:18:04 -0500 Subject: [PATCH 071/356] UnitTest for reverseAlleleClipping -- No code modified yet, just implementing a unit test to ensure correctness of the existing code --- .../utils/codecs/vcf/AbstractVCFCodec.java | 1 + .../utils/codecs/vcf/VCFCodecUnitTest.java | 91 +++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 public/java/test/org/broadinstitute/sting/utils/codecs/vcf/VCFCodecUnitTest.java diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java index 836ba22bf9..4726b4e001 100755 --- a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java +++ b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java @@ -617,6 +617,7 @@ else if ( a.getBases()[a.length()-clipping-1] != ref.getBytes()[ref.length()-cli return clipping; } + /** * clip the alleles, based on the reference * diff --git a/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/VCFCodecUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/VCFCodecUnitTest.java new file mode 100644 index 0000000000..7681ed7d1a --- /dev/null +++ b/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/VCFCodecUnitTest.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +// our package +package org.broadinstitute.sting.utils.codecs.vcf; + + +// the imports for unit testing. + + +import org.broadinstitute.sting.BaseTest; +import org.broadinstitute.sting.utils.variantcontext.*; +import org.testng.Assert; +import org.testng.annotations.BeforeSuite; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.util.*; + + +public class VCFCodecUnitTest extends BaseTest { + + // -------------------------------------------------------------------------------- + // + // Provider + // + // -------------------------------------------------------------------------------- + + private class AlleleClippingTestProvider extends TestDataProvider { + final String ref; + final List alleles = new ArrayList(); + final int expectedClip; + + private AlleleClippingTestProvider(final int expectedClip, final String ref, final String ... alleles) { + super(AlleleClippingTestProvider.class); + this.ref = ref; + for ( final String allele : alleles ) + this.alleles.add(Allele.create(allele)); + this.expectedClip = expectedClip; + } + + @Override + public String toString() { + return String.format("ref=%s allele=%s reverse clip %d", ref, alleles, expectedClip); + } + } + + @DataProvider(name = "AlleleClippingTestProvider") + public Object[][] MakeAlleleClippingTest() { + // pair clipping + new AlleleClippingTestProvider(0, "ATT", "CCG"); + new AlleleClippingTestProvider(1, "ATT", "CCT"); + new AlleleClippingTestProvider(2, "ATT", "CTT"); + new AlleleClippingTestProvider(2, "ATT", "ATT"); // cannot completely clip allele + + // triplets + new AlleleClippingTestProvider(0, "ATT", "CTT", "CGG"); + new AlleleClippingTestProvider(1, "ATT", "CTT", "CGT"); // the T can go + new AlleleClippingTestProvider(2, "ATT", "CTT", "CTT"); // both Ts can go + + return AlleleClippingTestProvider.getTests(AlleleClippingTestProvider.class); + } + + + @Test(dataProvider = "AlleleClippingTestProvider") + public void TestAlleleClipping(AlleleClippingTestProvider cfg) { + int result = AbstractVCFCodec.computeReverseClipping(cfg.alleles, cfg.ref, 0, 1); + Assert.assertEquals(result, cfg.expectedClip); + } +} \ No newline at end of file From 763c81d52002cb3e6af5310951b49bc03fde54ef Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Tue, 17 Jan 2012 21:06:02 -0500 Subject: [PATCH 072/356] No longer enforce MAX_ALLELE_SIZE in VCF codec -- Instead issue a warning when a large (>1MB) record is encountered -- Optimized ref.getBytes()[i] => (byte)ref.charAt(i), which avoids an implicit O(n) allocation each iteration through computeReverseClipping() --- .../sting/utils/codecs/vcf/AbstractVCFCodec.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java index 4726b4e001..1bdee802b4 100755 --- a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java +++ b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java @@ -18,7 +18,7 @@ public abstract class AbstractVCFCodec implements FeatureCodec, NameAwareCodec { - public final static int MAX_EXPLICIT_ALLELE_SIZE = (int)Math.pow(2, 16); + public final static int MAX_ALLELE_SIZE_BEFORE_WARNING = (int)Math.pow(2, 20); protected final static Logger log = Logger.getLogger(VCFCodec.class); protected final static int NUM_STANDARD_FIELDS = 8; // INFO is the 8th column @@ -522,8 +522,8 @@ private static void checkAllele(String allele, boolean isRef, int lineNo) { if ( allele == null || allele.length() == 0 ) generateException("Empty alleles are not permitted in VCF records", lineNo); - if ( MAX_EXPLICIT_ALLELE_SIZE != -1 && allele.length() > MAX_EXPLICIT_ALLELE_SIZE ) - generateException(String.format("Allele detected with length %d, exceeding max size %d. Please remove this from the VCF file before continuing", allele.length(), MAX_EXPLICIT_ALLELE_SIZE), lineNo); + if ( MAX_ALLELE_SIZE_BEFORE_WARNING != -1 && allele.length() > MAX_ALLELE_SIZE_BEFORE_WARNING ) + log.warn(String.format("Allele detected with length %d exceeding max size %d at approximately line %d, likely resulting in degraded VCF processing performance", allele.length(), MAX_ALLELE_SIZE_BEFORE_WARNING, lineNo)); if ( isSymbolicAllele(allele) ) { if ( isRef ) { @@ -576,12 +576,13 @@ protected static boolean isSingleNucleotideEvent(List alleles) { public static int computeForwardClipping(List unclippedAlleles, String ref) { boolean clipping = true; + final byte ref0 = (byte)ref.charAt(0); for ( Allele a : unclippedAlleles ) { if ( a.isSymbolic() ) continue; - if ( a.length() < 1 || (a.getBases()[0] != ref.getBytes()[0]) ) { + if ( a.length() < 1 || (a.getBases()[0] != ref0) ) { clipping = false; break; } @@ -608,7 +609,7 @@ protected static int computeReverseClipping(List unclippedAlleles, Strin stillClipping = false; else if ( ref.length() == clipping ) generateException("bad alleles encountered", lineNo); - else if ( a.getBases()[a.length()-clipping-1] != ref.getBytes()[ref.length()-clipping-1] ) + else if ( a.getBases()[a.length()-clipping-1] != ((byte)ref.charAt(ref.length()-clipping-1)) ) stillClipping = false; } if ( stillClipping ) From b52db515991ed937c6ee55f6c7fdaf204ce8313f Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Wed, 18 Jan 2012 08:26:49 -0500 Subject: [PATCH 073/356] Don't try to write log to a non-existant file From 5bd1a458794c38cc98a2ae0980d54241a4eb0e33 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Wed, 18 Jan 2012 08:27:15 -0500 Subject: [PATCH 074/356] Usability improvements to analyzeRunReports -- Print out the name / db of SQL server, not a python connection object -- Print out the ID, not a python objects, of XML record that fails to convert From 9e77facda530a097291f16ca3c26a152abf36a5a Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Wed, 18 Jan 2012 08:28:47 -0500 Subject: [PATCH 075/356] More analyses for random forest test script forest.R From 11982b5a34e20e350cd1e8d04b27c67610d5a143 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Wed, 18 Jan 2012 09:42:41 -0500 Subject: [PATCH 076/356] We no longer calculate the population-level TDT statistic if there are fewer than 5 trios with full genotype likelihood information. When there is a high degree of missingness the results are skewed or in the worst case come out as NaN. --- .../sting/gatk/traversals/TraverseActiveRegions.java | 4 ++-- .../walkers/annotator/TransmissionDisequilibriumTest.java | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java index 01bfe396ae..384affcb78 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java +++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java @@ -147,13 +147,13 @@ private T processActiveRegion( final ActiveRegion activeRegion, final LinkedHash ActiveRegion bestRegion = activeRegion; for( final ActiveRegion otherRegionToTest : workQueue ) { if( otherRegionToTest.getLocation().sizeOfOverlap(readLoc) >= maxOverlap ) { - maxOverlap = otherRegionToTest.getLocation().sizeOfOverlap(readLoc); + maxOverlap = otherRegionToTest.getLocation().sizeOfOverlap( readLoc ); bestRegion = otherRegionToTest; } } bestRegion.add( (GATKSAMRecord) read, true ); - // The read is also added to all other region in which it overlaps but marked as non-primary + // The read is also added to all other regions in which it overlaps but marked as non-primary if( !bestRegion.equals(activeRegion) ) { activeRegion.add( (GATKSAMRecord) read, false ); } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java index ecdde1e4fe..43d5f0b287 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java @@ -8,7 +8,6 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ExperimentalAnnotation; import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation; import org.broadinstitute.sting.utils.MathUtils; -import org.broadinstitute.sting.utils.MendelianViolation; import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType; import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine; import org.broadinstitute.sting.utils.exceptions.UserException; @@ -18,7 +17,7 @@ /** * Created by IntelliJ IDEA. - * User: rpoplin + * User: rpoplin, lfran * Date: 11/14/11 */ @@ -28,6 +27,7 @@ public class TransmissionDisequilibriumTest extends InfoFieldAnnotation implemen private final static int REF = 0; private final static int HET = 1; private final static int HOM = 2; + private final static int MIN_NUM_VALID_TRIOS = 5; // don't calculate this population-level statistic if there are less than X trios with full genotype likelihood information public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc) { if ( trios == null ) { @@ -50,7 +50,9 @@ public Map annotate(RefMetaDataTracker tracker, AnnotatorCompati } } - toRet.put("TDT", calculateTDT( vc, triosToTest )); + if( triosToTest.size() >= MIN_NUM_VALID_TRIOS ) { + toRet.put("TDT", calculateTDT( vc, triosToTest )); + } return toRet; } From 60024e0d7b08907ef4ffe4781c398ceaf1b51b89 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Wed, 18 Jan 2012 09:52:50 -0500 Subject: [PATCH 077/356] updating TDT integration test --- .../gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java index 174a46bdd5..14f7457b82 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java @@ -171,7 +171,7 @@ public void testSnpEffAnnotationsUnsupportedVersion() { @Test public void testTDTAnnotation() { - final String MD5 = "204e67536a17af7eaa6bf0a910818997"; + final String MD5 = "0aedd760e8099f0b95d53a41bdcd793e"; WalkerTestSpec spec = new WalkerTestSpec( "-T VariantAnnotator -R " + b37KGReference + " -A TransmissionDisequilibriumTest --variant:vcf " + validationDataLocation + "ug.random50000.subset300bp.chr1.family.vcf" + " -L " + validationDataLocation + "ug.random50000.subset300bp.chr1.family.vcf -NO_HEADER -ped " + validationDataLocation + "ug.random50000.family.ped -o %s", 1, From 2eb45340e1b198cb58b581a63e9b4b49e29a19a1 Mon Sep 17 00:00:00 2001 From: Guillermo del Angel Date: Wed, 18 Jan 2012 20:54:10 -0500 Subject: [PATCH 078/356] Initial, raw, mostly untested version of new pool caller that also does allele discovery. Still needs debugging/refining. Main modification is that there is a new operation mode, set by argument -ALLELE_DISCOVERY_MODE, which if true will determine optimal alt allele at each computable site and will compute AC distribution on it. Current implementation is not working yet if there's more than one pool and it will only output biallelic sites, no functionality for true multi-allelics yet --- .../broadinstitute/sting/utils/MathUtils.java | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java index 5ffd634cc3..5c952b13ad 100644 --- a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java @@ -1557,4 +1557,48 @@ public static Object[] arrayShuffle(Object[] array) { } return shuffled; } + + /** + * Vector operations + */ + public static double[] vectorSum(double v1[], double v2[]) { + if (v1.length != v2.length) + throw new UserException("BUG: vectors v1, v2 of different size in vectorSum()"); + + double result[] = new double[v1.length]; + for (int k=0; k < v1.length; k++) + result[k] = v1[k]+v2[k]; + + return result; + } + + public static double[] scalarTimesIntVector(double a, int[] v1) { + + double result[] = new double[v1.length]; + for (int k=0; k < v1.length; k++) + result[k] = a*v1[k]; + + return result; + } + + public static double dotProduct(double v1[], double v2[]) { + if (v1.length != v2.length) + throw new UserException("BUG: vectors v1, v2 of different size in vectorSum()"); + + double result = 0.0; + for (int k=0; k < v1.length; k++) + result += v1[k]*v2[k]; + + return result; + + } + + public static double[] vectorLog10(double v1[]) { + double result[] = new double[v1.length]; + for (int k=0; k < v1.length; k++) + result[k] = Math.log10(v1[k]); + + return result; + + } } From b0b0cd9aefe32b781df1c6a00ceb81566bf4d3b4 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Sat, 14 Jan 2012 21:58:49 -0500 Subject: [PATCH 079/356] Conforming to the guru's recommendation on library usage ;-) thanks Khalid. From ab8f499bc32982d000d5b9e291659d56927d9bfc Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Wed, 18 Jan 2012 22:04:51 -0500 Subject: [PATCH 080/356] Annotate with FS even for filtered sites --- .../sting/gatk/walkers/annotator/FisherStrand.java | 2 +- .../walkers/genotyper/UnifiedGenotyperIntegrationTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java index c4025a25c2..7d728dd5ec 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java @@ -54,7 +54,7 @@ public class FisherStrand extends InfoFieldAnnotation implements StandardAnnotat private static final double MIN_PVALUE = 1E-320; public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc) { - if ( ! vc.isVariant() || vc.isFiltered() ) + if ( !vc.isVariant() ) return null; int[][] table; diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java index 32fa8679e4..5cdf12f1bd 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java @@ -44,7 +44,7 @@ public void testWithAllelesPassedIn1() { public void testWithAllelesPassedIn2() { WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec( baseCommand + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1, - Arrays.asList("43e7a17d95b1a0cf72e669657794d802")); + Arrays.asList("1899bdb956c62bbcbf160b18cd3aea60")); executeTest("test MultiSample Pilot2 with alleles passed in and emitting all sites", spec2); } @@ -275,7 +275,7 @@ public void testWithIndelAllelesPassedIn2() { baseCommandIndels + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "indelAllelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,100,000", 1, - Arrays.asList("36ce53ae4319718ad9c8ae391deebc8c")); + Arrays.asList("320f61c87253aba77d6dc782242cba8b")); executeTest("test MultiSample Pilot2 indels with alleles passed in and emitting all sites", spec2); } From cf9b1d350a373fde820b4d97b2df3f321646601a Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 19 Jan 2012 00:20:49 -0500 Subject: [PATCH 081/356] Some minor changes to in-process functions that nobody else uses. CGL now properly ignores no-calls for external VCFs. --- .../sting/queue/library/ipf/vcf/VCFExtractIntervals.scala | 1 + .../sting/queue/library/ipf/vcf/VCFExtractSamples.scala | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractIntervals.scala b/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractIntervals.scala index 3935c2138d..e661f1bb57 100755 --- a/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractIntervals.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractIntervals.scala @@ -44,6 +44,7 @@ class VCFExtractIntervals(inVCF: File, outList: File, useFilterSites: Boolean) e } } } + out.printf("%s%n",cur) out.close } diff --git a/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractSamples.scala b/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractSamples.scala index 54e5411429..3179c327f9 100755 --- a/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractSamples.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractSamples.scala @@ -6,7 +6,7 @@ import collection.JavaConversions._ import org.broadinstitute.sting.commandline._ import java.io.{PrintWriter, PrintStream, File} -class VCFExtractSamples(inVCF: File, outVCF: File, samples: List[String]) extends InProcessFunction { +class VCFExtractSamples(inVCF: File, outVCF: File, samples: List[String]) extends InProcessFunction { def this(in: File, out: File, samples: File) = this(in,out, (new XReadLines(samples)).readLines.toList) @Input(doc="VCF from which to extract samples") var inputVCF : File = inVCF From 9946853039d2370d043d284f61356e9eaf66be1f Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 19 Jan 2012 00:25:22 -0500 Subject: [PATCH 082/356] Remove duplicated line --- .../sting/queue/library/ipf/vcf/VCFExtractIntervals.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractIntervals.scala b/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractIntervals.scala index e661f1bb57..a7b324a2b1 100755 --- a/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractIntervals.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractIntervals.scala @@ -39,8 +39,6 @@ class VCFExtractIntervals(inVCF: File, outList: File, useFilterSites: Boolean) e if ( elems.hasNext ) { prev = cur cur = elems.next - } else { - out.printf("%s%n",cur) } } } From 1e037a0ecf0c40b6ed472dbb5786825e8e9882d2 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 19 Jan 2012 00:33:08 -0500 Subject: [PATCH 083/356] Ensure second-to-last line printed --- .../sting/queue/library/ipf/vcf/VCFExtractIntervals.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractIntervals.scala b/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractIntervals.scala index a7b324a2b1..03d31a217c 100755 --- a/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractIntervals.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractIntervals.scala @@ -42,6 +42,7 @@ class VCFExtractIntervals(inVCF: File, outList: File, useFilterSites: Boolean) e } } } + out.printf("%s%n",prev) out.printf("%s%n",cur) out.close From 39e6df5aa9d1008a7318fa900137b0c5efa5546d Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 19 Jan 2012 00:51:28 -0500 Subject: [PATCH 084/356] Fix edge case for very small VCFs --- .../library/ipf/vcf/VCFExtractIntervals.scala | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractIntervals.scala b/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractIntervals.scala index 03d31a217c..3c7cd0a2d5 100755 --- a/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractIntervals.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/library/ipf/vcf/VCFExtractIntervals.scala @@ -26,24 +26,24 @@ class VCFExtractIntervals(inVCF: File, outList: File, useFilterSites: Boolean) e var cur : String = null if ( elems.hasNext ) { cur = elems.next - } else { - out.printf("%s%n",prev) - } - while ( elems.hasNext ) { - out.printf("%s%n",prev) - while ( cur.equals(prev) && elems.hasNext && !cur.equals("") ) { - cur = elems.next - } - - if ( ! cur.equals(prev) ) { - if ( elems.hasNext ) { - prev = cur + while ( elems.hasNext ) { + out.printf("%s%n",prev) + while ( cur.equals(prev) && elems.hasNext && !cur.equals("") ) { cur = elems.next } + + if ( ! cur.equals(prev) ) { + if ( elems.hasNext ) { + prev = cur + cur = elems.next + } + } } + out.printf("%s%n",prev) + out.printf("%s%n",cur) + } else { + out.printf("%s%n",prev) } - out.printf("%s%n",prev) - out.printf("%s%n",cur) out.close } From d1c8c3854154d06f81aba169dd5612eee814dfdf Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 19 Jan 2012 02:04:04 -0500 Subject: [PATCH 085/356] A QScript to generate a VQSR of union sites for T2D, using a broad set and a union site set as input. From ecdd07b74872277586c8c2a416cd52bc4909be1e Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Thu, 19 Jan 2012 09:31:22 -0500 Subject: [PATCH 086/356] updating HaplotypeCaller integration test From 7f3ad25b0109e91a82cf41378c33830f169fb705 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 19 Jan 2012 10:54:48 -0500 Subject: [PATCH 087/356] Adding a mode to VariantFiltration to invalidate previously-applied filters to allow complete re-filtering of a VCF. T2D VQSR: re-calling now done with appropriate quality settings and using BAQ. --- .../gatk/walkers/filters/VariantFiltrationWalker.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/filters/VariantFiltrationWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/filters/VariantFiltrationWalker.java index 8278dbab76..7eb6fad542 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/filters/VariantFiltrationWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/filters/VariantFiltrationWalker.java @@ -139,6 +139,12 @@ public class VariantFiltrationWalker extends RodWalker { @Argument(fullName="missingValuesInExpressionsShouldEvaluateAsFailing", doc="When evaluating the JEXL expressions, missing values should be considered failing the expression", required=false) protected Boolean FAIL_MISSING_VALUES = false; + /** + * Invalidate previous filters applied to the VariantContext, applying only the filters here + */ + @Argument(fullName="invalidatePreviousFilters",doc="Remove previous filters applied to the VCF",required=false) + boolean invalidatePrevious = false; + // JEXL expressions for the filters List filterExps; List genotypeFilterExps; @@ -215,6 +221,9 @@ public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentCo for ( VariantContext vc : VCs ) { + if ( invalidatePrevious ) { + vc = (new VariantContextBuilder(vc)).filters(new HashSet()).make(); + } // filter based on previous mask position if ( previousMaskPosition != null && // we saw a previous mask site previousMaskPosition.getContig().equals(vc.getChr()) && // it's on the same contig From 98f8431b0774d8aa4d848c890a9e1f082a2a86da Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 19 Jan 2012 12:36:30 -0500 Subject: [PATCH 088/356] Right. Forgot the = true. If only there were some way to silently commit this OH WAIT From ced6775de3af5b5df967ea96ada5c73300aa3bc1 Mon Sep 17 00:00:00 2001 From: Aaron McKenna Date: Wed, 18 Jan 2012 16:34:42 -0500 Subject: [PATCH 089/356] Changes to allow for external tests Changes to the build script that allow the external directory to have tests. This means groups like CGA don't have to reinvent the wheel on testing, and can instead use the GATKs unit and integration tests. Signed-off-by: David Roazen --- build.xml | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/build.xml b/build.xml index dbdafa3d96..ffb564cd21 100644 --- a/build.xml +++ b/build.xml @@ -281,6 +281,10 @@ + + + + @@ -331,7 +335,7 @@ - + @@ -761,6 +765,7 @@ + @@ -811,7 +816,23 @@ - + + + + + + + + + + + + + + + + + @@ -852,6 +873,7 @@ + @@ -934,6 +956,9 @@ + + + From 6e30d715cf69ef46c158ae7cc2730e026c7362d9 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 19 Jan 2012 15:01:50 -0500 Subject: [PATCH 090/356] Minor changes to T2D VQSR. Adding in a small walker for multiplying likelihoods for generation of a consensus panel. From 066da80a3d9a83ec5e59cd3e545c6c2bf102a625 Mon Sep 17 00:00:00 2001 From: Menachem Fromer Date: Thu, 19 Jan 2012 18:19:58 -0500 Subject: [PATCH 091/356] Added KEEP_UNCONDTIONAL option which permits even sites with only filtered records to be included as unfiltered sites in the output --- .../sting/utils/variantcontext/VariantContextUtils.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java index c9a4965c1c..39045ea212 100755 --- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java @@ -464,7 +464,11 @@ public enum FilteredRecordMergeType { /** * Requires all records present at site to be unfiltered. VCF files that don't contain the record don't influence this. */ - KEEP_IF_ALL_UNFILTERED + KEEP_IF_ALL_UNFILTERED, + /** + * If any record is present at this site (regardless of possibly being filtered), then all such records are kept and the filters are reset. + */ + KEEP_UNCONDITIONAL } /** @@ -635,7 +639,7 @@ public static VariantContext simpleMerge(final GenomeLocParser genomeLocParser, } // if at least one record was unfiltered and we want a union, clear all of the filters - if ( filteredRecordMergeType == FilteredRecordMergeType.KEEP_IF_ANY_UNFILTERED && nFiltered != VCs.size() ) + if ( (filteredRecordMergeType == FilteredRecordMergeType.KEEP_IF_ANY_UNFILTERED && nFiltered != VCs.size()) || filteredRecordMergeType == FilteredRecordMergeType.KEEP_UNCONDITIONAL ) filters.clear(); From 253d6483e1f2add27e1c8f182250cff3cddc23d3 Mon Sep 17 00:00:00 2001 From: Menachem Fromer Date: Thu, 19 Jan 2012 18:21:22 -0500 Subject: [PATCH 092/356] Updated Batch-merge to retain ALL sites in input (SNPs, indels, regardless of their filtering status), and also optionally go back to the BAMs to perform VariantAnnotation From 0644b750897ab92690ec4abb0f86fd743fd352a2 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 19 Jan 2012 18:30:32 -0500 Subject: [PATCH 093/356] Remove attribute data from VariantContext and genotypes. From ed5302667b8c252fad3e98928f8a75e2ca589b63 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 19 Jan 2012 18:44:39 -0500 Subject: [PATCH 094/356] Oops. Let's actually retain the genotype likelihoods. From 72cd0a24507fa8d3f4e7fa5e0cc488e841a0301c Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 19 Jan 2012 18:52:06 -0500 Subject: [PATCH 095/356] And do it conditional on having likelihoods in the first place From b9f7103d099badd7078560419c5f6dc5b444f158 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 19 Jan 2012 19:41:43 -0500 Subject: [PATCH 096/356] Fix edge case where DP annotations (format) were creeping in From cd38110b7bfb0fb7143b6edb24f23b0d02f6be53 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 19 Jan 2012 20:11:20 -0500 Subject: [PATCH 097/356] GQs are not always purged with this method of modifying attributes. To drop them, create the Genotype anew. From ace93330688e2a55b1d6d81f2d4e4bba21698cb6 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Thu, 19 Jan 2012 22:05:08 -0500 Subject: [PATCH 098/356] Active region walkers can now see the reads in a buffer around thier active reigons. This buffer size is specified as a walker annotation. Intervals are internally extended by this buffer size so that the extra reads make their way through the traversal engine but the walker author only needs to see the original interval. Also, several corner case bug fixes in active region traversal. --- .../sting/gatk/GenomeAnalysisEngine.java | 12 ++- .../gatk/executive/LinearMicroScheduler.java | 2 +- .../traversals/TraverseActiveRegions.java | 87 ++++++++++++------- .../gatk/walkers/ActiveRegionExtension.java | 19 ++++ .../gatk/walkers/ActiveRegionWalker.java | 29 ++++++- .../gatk/walkers/annotator/FisherStrand.java | 1 - .../broadinstitute/sting/utils/GenomeLoc.java | 2 +- .../sting/utils/GenomeLocSortedSet.java | 15 ++++ .../utils/activeregion/ActiveRegion.java | 29 ++++--- 9 files changed, 148 insertions(+), 48 deletions(-) create mode 100644 public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionExtension.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java b/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java index ede8e93406..6140d543a9 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java @@ -443,14 +443,22 @@ protected Iterable getShardStrategy(SAMDataSource readsDataSource, Refere if(!readsDataSource.hasIndex() && intervals != null && !argCollection.allowIntervalsWithUnindexedBAM) throw new UserException.CommandLineException("Cannot perform interval processing when reads are present but no index is available."); - if(walker instanceof LocusWalker || walker instanceof ActiveRegionWalker) { + if(walker instanceof LocusWalker) { if (readsDataSource.getSortOrder() != SAMFileHeader.SortOrder.coordinate) throw new UserException.MissortedBAM(SAMFileHeader.SortOrder.coordinate, "Locus walkers can only traverse coordinate-sorted data. Please resort your input BAM file(s) or set the Sort Order tag in the header appropriately."); if(intervals == null) return readsDataSource.createShardIteratorOverMappedReads(referenceDataSource.getReference().getSequenceDictionary(),new LocusShardBalancer()); else return readsDataSource.createShardIteratorOverIntervals(intervals,new LocusShardBalancer()); - } + } + else if(walker instanceof ActiveRegionWalker) { + if (readsDataSource.getSortOrder() != SAMFileHeader.SortOrder.coordinate) + throw new UserException.MissortedBAM(SAMFileHeader.SortOrder.coordinate, "Active region walkers can only traverse coordinate-sorted data. Please resort your input BAM file(s) or set the Sort Order tag in the header appropriately."); + if(intervals == null) + return readsDataSource.createShardIteratorOverMappedReads(referenceDataSource.getReference().getSequenceDictionary(),new LocusShardBalancer()); + else + return readsDataSource.createShardIteratorOverIntervals(((ActiveRegionWalker)walker).extendIntervals(intervals, this.genomeLocParser, this.getReferenceDataSource().getReference()), new LocusShardBalancer()); + } else if(walker instanceof ReadWalker || walker instanceof ReadPairWalker || walker instanceof DuplicateWalker) { // Apply special validation to read pair walkers. if(walker instanceof ReadPairWalker) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java index 774b532f34..16487054bc 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java +++ b/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java @@ -77,7 +77,7 @@ public Object execute(Walker walker, Iterable shardStrategy) { done = walker.isDone(); } - // Special function call to empty out the work queue. Ugly for now but will be cleaned up when we push this functionality more into the engine + // Special function call to empty out the work queue. Ugly for now but will be cleaned up when we eventually push this functionality more into the engine if( traversalEngine instanceof TraverseActiveRegions ) { final Object result = ((TraverseActiveRegions) traversalEngine).endTraversal(walker, accumulator.getReduceInit()); accumulator.accumulate(null, result); // Assumes only used with StandardAccumulator diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java index 384affcb78..b18d1ceb9b 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java +++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java @@ -7,10 +7,9 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext; import org.broadinstitute.sting.gatk.datasources.providers.*; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; -import org.broadinstitute.sting.gatk.walkers.ActiveRegionWalker; -import org.broadinstitute.sting.gatk.walkers.DataSource; -import org.broadinstitute.sting.gatk.walkers.Walker; +import org.broadinstitute.sting.gatk.walkers.*; import org.broadinstitute.sting.utils.GenomeLoc; +import org.broadinstitute.sting.utils.GenomeLocSortedSet; import org.broadinstitute.sting.utils.activeregion.ActiveRegion; import org.broadinstitute.sting.utils.pileup.PileupElement; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; @@ -46,13 +45,15 @@ public T traverse( final ActiveRegionWalker walker, T sum) { logger.debug(String.format("TraverseActiveRegion.traverse: Shard is %s", dataProvider)); - LocusView locusView = getLocusView( walker, dataProvider ); + final LocusView locusView = getLocusView( walker, dataProvider ); + final GenomeLocSortedSet initialIntervals = engine.getIntervals(); - int minStart = Integer.MAX_VALUE; final LocusReferenceView referenceView = new LocusReferenceView( walker, dataProvider ); + final int activeRegionExtension = walker.getClass().getAnnotation(ActiveRegionExtension.class).extension(); if ( locusView.hasNext() ) { // trivial optimization to avoid unnecessary processing when there's nothing here at all + int minStart = Integer.MAX_VALUE; final ArrayList isActiveList = new ArrayList(); //ReferenceOrderedView referenceOrderedDataView = new ReferenceOrderedView( dataProvider ); @@ -90,9 +91,11 @@ public T traverse( final ActiveRegionWalker walker, final RefMetaDataTracker tracker = referenceOrderedDataView.getReferenceOrderedDataAtLocus(locus.getLocation(), refContext); // Call the walkers isActive function for this locus and add them to the list to be integrated later - final boolean isActive = walker.isActive( tracker, refContext, locus ); - isActiveList.add( new ActiveRegion(location, isActive, engine.getGenomeLocParser()) ); - + if( initialIntervals.overlaps(location) ) { + final boolean isActive = walker.isActive( tracker, refContext, locus ); + isActiveList.add( new ActiveRegion(location, isActive, engine.getGenomeLocParser(), activeRegionExtension ) ); + } + // Grab all the previously unseen reads from this pileup and add them to the massive read list for( final PileupElement p : locus.getBasePileup() ) { final SAMRecord read = p.getRead(); @@ -101,11 +104,20 @@ public T traverse( final ActiveRegionWalker walker, } } - // If this is the last pileup for this shard then need to calculate the minimum alignment start so that - // we know which active regions in the work queue are now safe to process + // If this is the last pileup for this shard then need to first do a special walker.isActive() call + // and then calculate the minimum alignment start so that we know which active regions in the work queue are now safe to process if( !locusView.hasNext() ) { + // Call the walkers isActive function for this locus and add them to the list to be integrated later + if( initialIntervals.overlaps(location) ) { + final boolean isActive = walker.isActive( tracker, refContext, locus ); + isActiveList.add( new ActiveRegion(location, isActive, engine.getGenomeLocParser(), activeRegionExtension ) ); + } + for( final PileupElement p : locus.getBasePileup() ) { final SAMRecord read = p.getRead(); + if( !myReads.contains(read) ) { + myReads.add(read); + } if( read.getAlignmentStart() < minStart ) { minStart = read.getAlignmentStart(); } } } @@ -117,11 +129,14 @@ public T traverse( final ActiveRegionWalker walker, final ArrayList activeRegions = integrateActiveList( isActiveList ); logger.debug("Integrated " + isActiveList.size() + " isActive calls into " + activeRegions.size() + " regions." ); workQueue.addAll( activeRegions ); - } - while( workQueue.peek().getLocation().getStop() < minStart ) { - final ActiveRegion activeRegion = workQueue.remove(); - sum = processActiveRegion( activeRegion, myReads, workQueue, sum, walker ); + // Since we've sufficiently past this point (or this contig!) in the workQueue we can unload those regions and process them + if( !workQueue.isEmpty() ) { + while( workQueue.peek().getExtendedLoc().getStop() < minStart || !workQueue.peek().getExtendedLoc().getContig().equals(dataProvider.getLocus().getContig()) ) { + final ActiveRegion activeRegion = workQueue.remove(); + sum = processActiveRegion( activeRegion, myReads, workQueue, sum, walker ); + } + } } return sum; @@ -158,16 +173,18 @@ private T processActiveRegion( final ActiveRegion activeRegion, final LinkedHash activeRegion.add( (GATKSAMRecord) read, false ); } for( final ActiveRegion otherRegionToTest : workQueue ) { - if( !bestRegion.equals(otherRegionToTest) && otherRegionToTest.getLocation().overlapsP( readLoc ) ) { + if( !bestRegion.equals(otherRegionToTest) && otherRegionToTest.getExtendedLoc().overlapsP( readLoc ) ) { activeRegion.add( (GATKSAMRecord) read, false ); } } placedReads.add( read ); + } else if( activeRegion.getExtendedLoc().overlapsP( readLoc ) ) { + activeRegion.add( (GATKSAMRecord) read, false ); } } reads.removeAll( placedReads ); // remove all the reads which have been placed into their active region - logger.debug(">> Map call with " + activeRegion.getReads().size() + " " + (activeRegion.isActive ? "active" : "inactive") + " reads @ " + activeRegion.getLocation() + " with full extent: " + activeRegion.getReferenceLocation()); + logger.debug(">> Map call with " + activeRegion.getReads().size() + " " + (activeRegion.isActive ? "active" : "inactive") + " reads @ " + activeRegion.getLocation() + " with full extent: " + activeRegion.getReferenceLoc()); final M x = walker.map( activeRegion, null ); // BUGBUG: tracker needs to be filled in and passed to the walker return walker.reduce( x, sum ); } @@ -178,8 +195,8 @@ private T processActiveRegion( final ActiveRegion activeRegion, final LinkedHash * @param dataProvider Data which which to drive the locus view. * @return A view of the locus data, where one iteration of the locus view maps to one iteration of the traversal. */ - private LocusView getLocusView( Walker walker, LocusShardDataProvider dataProvider ) { - DataSource dataSource = WalkerManager.getWalkerDataSource(walker); + private LocusView getLocusView( final Walker walker, final LocusShardDataProvider dataProvider ) { + final DataSource dataSource = WalkerManager.getWalkerDataSource(walker); if( dataSource == DataSource.READS ) return new CoveredLocusView(dataProvider); else if( dataSource == DataSource.REFERENCE ) //|| ! GenomeAnalysisEngine.instance.getArguments().enableRodWalkers ) @@ -193,21 +210,29 @@ else if( dataSource == DataSource.REFERENCE_ORDERED_DATA ) // integrate active regions into contiguous chunks based on active status private ArrayList integrateActiveList( final ArrayList activeList ) { final ArrayList returnList = new ArrayList(); - ActiveRegion prevLocus = activeList.remove(0); - ActiveRegion startLocus = prevLocus; - for( final ActiveRegion thisLocus : activeList ) { - if( prevLocus.isActive != thisLocus.isActive ) { + if( activeList.size() == 0 ) { + return returnList; + } else if( activeList.size() == 1 ) { + returnList.add( new ActiveRegion( engine.getGenomeLocParser().createGenomeLoc(activeList.get(0).getLocation().getContig(), activeList.get(0).getLocation().getStart(), activeList.get(0).getLocation().getStart()), + activeList.get(0).isActive, engine.getGenomeLocParser(), activeList.get(0).getExtension() ) ); + return returnList; + } else { + ActiveRegion prevLocus = activeList.get(0); + ActiveRegion startLocus = prevLocus; + for( final ActiveRegion thisLocus : activeList ) { + if( prevLocus.isActive != thisLocus.isActive || !prevLocus.getLocation().contiguousP( thisLocus.getLocation() ) ) { + returnList.add( new ActiveRegion( engine.getGenomeLocParser().createGenomeLoc(startLocus.getLocation().getContig(), startLocus.getLocation().getStart(), prevLocus.getLocation().getStart()), + prevLocus.isActive, engine.getGenomeLocParser(), startLocus.getExtension() ) ); + startLocus = thisLocus; + } + prevLocus = thisLocus; + } + // output the last region if necessary + if( startLocus != prevLocus ) { returnList.add( new ActiveRegion( engine.getGenomeLocParser().createGenomeLoc(startLocus.getLocation().getContig(), startLocus.getLocation().getStart(), prevLocus.getLocation().getStart()), - prevLocus.isActive, engine.getGenomeLocParser() ) ); - startLocus = thisLocus; + prevLocus.isActive, engine.getGenomeLocParser(), startLocus.getExtension() ) ); } - prevLocus = thisLocus; - } - // output the last region if necessary - if( startLocus != prevLocus ) { - returnList.add( new ActiveRegion( engine.getGenomeLocParser().createGenomeLoc(startLocus.getLocation().getContig(), startLocus.getLocation().getStart(), prevLocus.getLocation().getStart()), - prevLocus.isActive, engine.getGenomeLocParser() ) ); + return returnList; } - return returnList; } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionExtension.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionExtension.java new file mode 100644 index 0000000000..bb007893c9 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionExtension.java @@ -0,0 +1,19 @@ +package org.broadinstitute.sting.gatk.walkers; + +import java.lang.annotation.Documented; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Describes the size of the buffer region that is added to each active region when pulling in covered reads. + * User: rpoplin + * Date: 1/18/12 + */ +@Documented +@Inherited +@Retention(RetentionPolicy.RUNTIME) + +public @interface ActiveRegionExtension { + public int extension() default 0; +} diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java index d2891c959a..8405f762da 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java @@ -1,13 +1,26 @@ package org.broadinstitute.sting.gatk.walkers; +import net.sf.picard.reference.IndexedFastaSequenceFile; import org.broadinstitute.sting.gatk.contexts.AlignmentContext; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.gatk.filters.DuplicateReadFilter; +import org.broadinstitute.sting.gatk.filters.FailsVendorQualityCheckFilter; +import org.broadinstitute.sting.gatk.filters.NotPrimaryAlignmentFilter; +import org.broadinstitute.sting.gatk.filters.UnmappedReadFilter; import org.broadinstitute.sting.gatk.refdata.ReadMetaDataTracker; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; +import org.broadinstitute.sting.utils.GenomeLoc; +import org.broadinstitute.sting.utils.GenomeLocParser; +import org.broadinstitute.sting.utils.GenomeLocSortedSet; import org.broadinstitute.sting.utils.activeregion.ActiveRegion; +import org.broadinstitute.sting.utils.interval.IntervalMergingRule; +import org.broadinstitute.sting.utils.interval.IntervalUtils; + +import java.util.ArrayList; +import java.util.List; /** - * Created by IntelliJ IDEA. + * Base class for all the Active Region Walkers. * User: rpoplin * Date: 12/7/11 */ @@ -15,7 +28,10 @@ @By(DataSource.READS) @Requires({DataSource.READS, DataSource.REFERENCE_BASES}) @PartitionBy(PartitionType.READ) +@ActiveRegionExtension(extension=50) +@ReadFilters({UnmappedReadFilter.class, NotPrimaryAlignmentFilter.class, DuplicateReadFilter.class, FailsVendorQualityCheckFilter.class}) public abstract class ActiveRegionWalker extends Walker { + // Do we actually want to operate on the context? public boolean filter(final RefMetaDataTracker tracker, final ReferenceContext ref, final AlignmentContext context) { return true; // We are keeping all the reads @@ -26,4 +42,15 @@ public boolean filter(final RefMetaDataTracker tracker, final ReferenceContext r // Map over the ActiveRegion public abstract MapType map(final ActiveRegion activeRegion, final ReadMetaDataTracker metaDataTracker); + + public final GenomeLocSortedSet extendIntervals( final GenomeLocSortedSet intervals, final GenomeLocParser genomeLocParser, IndexedFastaSequenceFile reference ) { + final int activeRegionExtension = this.getClass().getAnnotation(ActiveRegionExtension.class).extension(); + final List allIntervals = new ArrayList(); + for( final GenomeLoc interval : intervals.toList() ) { + final int start = Math.max( 1, interval.getStart() - activeRegionExtension ); + final int stop = Math.min( reference.getSequenceDictionary().getSequence(interval.getContig()).getSequenceLength(), interval.getStop() + activeRegionExtension ); + allIntervals.add( genomeLocParser.createGenomeLoc(interval.getContig(), start, stop) ); + } + return IntervalUtils.sortAndMergeIntervals(genomeLocParser, allIntervals, IntervalMergingRule.ALL); + } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java index c4025a25c2..cb9bc103f9 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java @@ -73,7 +73,6 @@ else if (vc.isIndel() || vc.isMixed()) { if ( pvalue == null ) return null; - // use Math.abs to prevent -0's Map map = new HashMap(); map.put(FS, String.format("%.3f", QualityUtils.phredScaleErrorRate(pvalue))); return map; diff --git a/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java b/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java index 6941b888b5..ad10b61e7a 100644 --- a/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java +++ b/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java @@ -467,6 +467,6 @@ private final static double overlapPercent(final GenomeLoc gl1, final GenomeLoc } public long sizeOfOverlap( final GenomeLoc that ) { - return ( this.overlapsP(that) ? Math.min( getStop(), that.getStop() ) - Math.max( getStart(), that.getStart() ) : 0L ); + return ( this.overlapsP(that) ? Math.min( getStop(), that.getStop() ) - Math.max( getStart(), that.getStart() ) + 1L : 0L ); } } diff --git a/public/java/src/org/broadinstitute/sting/utils/GenomeLocSortedSet.java b/public/java/src/org/broadinstitute/sting/utils/GenomeLocSortedSet.java index 26be0e59ef..d11adf9e3a 100755 --- a/public/java/src/org/broadinstitute/sting/utils/GenomeLocSortedSet.java +++ b/public/java/src/org/broadinstitute/sting/utils/GenomeLocSortedSet.java @@ -127,6 +127,21 @@ public boolean isEmpty() { return mArray.isEmpty(); } + /** + * Determine if the given loc overlaps any loc in the sorted set + * + * @param loc the location to test + * @return + */ + public boolean overlaps(final GenomeLoc loc) { + for(final GenomeLoc e : mArray) { + if(e.overlapsP(loc)) { + return true; + } + } + return false; + } + /** * add a genomeLoc to the collection, simply inserting in order into the set * diff --git a/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRegion.java b/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRegion.java index e8908480c6..abf74469fa 100644 --- a/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRegion.java +++ b/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRegion.java @@ -18,21 +18,25 @@ public class ActiveRegion implements HasGenomeLocation { private final ArrayList reads = new ArrayList(); private byte[] reference = null; - private final GenomeLoc loc; - private GenomeLoc referenceLoc = null; + private final GenomeLoc activeRegionLoc; + private final GenomeLoc extendedLoc; + private final int extension; + private GenomeLoc fullExtentReferenceLoc = null; private final GenomeLocParser genomeLocParser; public final boolean isActive; - public ActiveRegion( final GenomeLoc loc, final boolean isActive, final GenomeLocParser genomeLocParser ) { - this.loc = loc; + public ActiveRegion( final GenomeLoc activeRegionLoc, final boolean isActive, final GenomeLocParser genomeLocParser, final int extension ) { + this.activeRegionLoc = activeRegionLoc; this.isActive = isActive; this.genomeLocParser = genomeLocParser; - referenceLoc = loc; + this.extension = extension; + extendedLoc = genomeLocParser.createGenomeLoc(activeRegionLoc.getContig(), activeRegionLoc.getStart() - extension, activeRegionLoc.getStop() + extension); + fullExtentReferenceLoc = extendedLoc; } - // add each read to the bin and extend the reference genome loc if needed + // add each read to the bin and extend the reference genome activeRegionLoc if needed public void add( final GATKSAMRecord read, final boolean isPrimaryRegion ) { - referenceLoc = referenceLoc.union( genomeLocParser.createGenomeLoc( read ) ); + fullExtentReferenceLoc = fullExtentReferenceLoc.union( genomeLocParser.createGenomeLoc( read ) ); reads.add( new ActiveRead(read, isPrimaryRegion) ); } @@ -41,15 +45,18 @@ public void add( final GATKSAMRecord read, final boolean isPrimaryRegion ) { public byte[] getReference( final IndexedFastaSequenceFile referenceReader ) { // set up the reference if we haven't done so yet if ( reference == null ) { - reference = referenceReader.getSubsequenceAt(referenceLoc.getContig(), referenceLoc.getStart(), referenceLoc.getStop()).getBases(); + reference = referenceReader.getSubsequenceAt(fullExtentReferenceLoc.getContig(), fullExtentReferenceLoc.getStart(), fullExtentReferenceLoc.getStop()).getBases(); } return reference; } - public GenomeLoc getLocation() { return loc; } - - public GenomeLoc getReferenceLocation() { return referenceLoc; } + @Override + public GenomeLoc getLocation() { return activeRegionLoc; } + public GenomeLoc getExtendedLoc() { return extendedLoc; } + public GenomeLoc getReferenceLoc() { return fullExtentReferenceLoc; } + + public int getExtension() { return extension; } public int size() { return reads.size(); } } \ No newline at end of file From e245cde47f7a0d53cca9d4abed0515101462792e Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Fri, 20 Jan 2012 12:48:32 -0500 Subject: [PATCH 099/356] A new beagle script for generating a reference panel from lowpass, exome, and chip data. This is for T2D, but potentially useful. From 7c6a9471e88fa7a24f43dc47e70baeb063f15901 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Fri, 20 Jan 2012 13:20:13 -0500 Subject: [PATCH 100/356] After ensuring MultiplyLikelihoods does what I want it to do, add a quick and simple integration test to ensure I don't break it. From b902d778ca42b7e399aa0b54ea5044b6d2d6f9c3 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Fri, 20 Jan 2012 13:22:46 -0500 Subject: [PATCH 101/356] . From f3564bbf43cc353221349fba3b035f06d3c4f603 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Fri, 20 Jan 2012 13:25:11 -0500 Subject: [PATCH 102/356] Ugh. Darn intelliJ not telling me I was missing an import statement. From 9b4f6afa212f838504f002469e080ebe274ab59f Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Fri, 20 Jan 2012 23:07:59 -0500 Subject: [PATCH 103/356] Alterations to scripts for better performance. Grid search now expands the sens/spec tradeoff (90 was far too aggressive against hapmap chr20), and 20 max gaussians was too many, and caused errors. For consensus genotypes: remember to gunzip the beagle outputs before converting to VCF. Also, beagle can in fact create 'null' alleles in certain circumstances. I'm not sure what exactly those circumstances are, but those sites should be ignored. When it does, all alleles apear to be set to null, so this should not affect the actual phasing in the output VCF. --- .../sting/gatk/walkers/beagle/BeagleOutputToVCFWalker.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/beagle/BeagleOutputToVCFWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/beagle/BeagleOutputToVCFWalker.java index f827856be9..4f115b46e3 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/beagle/BeagleOutputToVCFWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/beagle/BeagleOutputToVCFWalker.java @@ -241,6 +241,11 @@ public Integer map( RefMetaDataTracker tracker, ReferenceContext ref, AlignmentC String alleleA = beagleGenotypePairs.get(0); String alleleB = beagleGenotypePairs.get(1); + if ( alleleA.equals("null") || alleleB.equals("null") ) { + logger.warn("Beagle produced 'null' alleles at location "+ref.getLocus().toString()+". Ignoring."); + return 0; + } + // Beagle always produces genotype strings based on the strings we input in the likelihood file. String refString = vc_input.getReference().getDisplayString(); if (refString.length() == 0) // ref was null From 3b1aad4f1731125d67b51e6681a812f4c2a59a6d Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Fri, 20 Jan 2012 23:43:51 -0500 Subject: [PATCH 104/356] After a minor and abject freakout, alter the T2D script to seek out truth sensitivities between 80 and 100, rather than between 0.8 and 1. Also, don't consider a genotype "changed by beagle" if the initial genotype is a no-call. --- .../gatk/walkers/beagle/BeagleOutputToVCFWalker.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/beagle/BeagleOutputToVCFWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/beagle/BeagleOutputToVCFWalker.java index 4f115b46e3..ec67563dcd 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/beagle/BeagleOutputToVCFWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/beagle/BeagleOutputToVCFWalker.java @@ -320,8 +320,7 @@ else if (originalAlleleB.isReference()) og = a1+"/"+a2; // See if Beagle switched genotypes - if (!((bglAlleleA.equals(originalAlleleA) && bglAlleleB.equals(originalAlleleB) || - (bglAlleleA.equals(originalAlleleB) && bglAlleleB.equals(originalAlleleA))))){ + if (! originalAlleleA.equals(Allele.NO_CALL) && beagleSwitchedGenotypes(bglAlleleA,originalAlleleA,bglAlleleB,originalAlleleB)){ originalAttributes.put("OG",og); numGenotypesChangedByBeagle++; } @@ -364,6 +363,11 @@ else if (originalAlleleB.isReference()) return 1; } + private boolean beagleSwitchedGenotypes(Allele bglAlleleA, Allele originalAlleleA, Allele bglAlleleB, Allele originalAlleleB) { + return !((bglAlleleA.equals(originalAlleleA) && bglAlleleB.equals(originalAlleleB) || + (bglAlleleA.equals(originalAlleleB) && bglAlleleB.equals(originalAlleleA)))); + } + public Integer reduceInit() { return 0; // Nothing to do here } From 4d6312d4ea1e22f53649c4c38fd0458753effe0f Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Sun, 22 Jan 2012 14:31:01 -0500 Subject: [PATCH 105/356] HaplotypeCaller is now an ActiveRegionWalker. --- .../traversals/TraverseActiveRegions.java | 24 +++---- .../gatk/walkers/ActiveRegionWalker.java | 4 ++ .../sting/utils/activeregion/ActiveRead.java | 19 ------ .../utils/activeregion/ActiveRegion.java | 24 +++---- .../sting/utils/fragments/FragmentUtils.java | 65 +++++++++++++++++++ 5 files changed, 95 insertions(+), 41 deletions(-) delete mode 100644 public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRead.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java index b18d1ceb9b..ebfcc0c29b 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java +++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java @@ -46,7 +46,7 @@ public T traverse( final ActiveRegionWalker walker, logger.debug(String.format("TraverseActiveRegion.traverse: Shard is %s", dataProvider)); final LocusView locusView = getLocusView( walker, dataProvider ); - final GenomeLocSortedSet initialIntervals = engine.getIntervals(); + final GenomeLocSortedSet initialIntervals = engine.getIntervals(); // BUGBUG: unfortunate inefficiency that needs to be removed final LocusReferenceView referenceView = new LocusReferenceView( walker, dataProvider ); final int activeRegionExtension = walker.getClass().getAnnotation(ActiveRegionExtension.class).extension(); @@ -166,20 +166,22 @@ private T processActiveRegion( final ActiveRegion activeRegion, final LinkedHash bestRegion = otherRegionToTest; } } - bestRegion.add( (GATKSAMRecord) read, true ); + bestRegion.add( (GATKSAMRecord) read ); // The read is also added to all other regions in which it overlaps but marked as non-primary - if( !bestRegion.equals(activeRegion) ) { - activeRegion.add( (GATKSAMRecord) read, false ); - } - for( final ActiveRegion otherRegionToTest : workQueue ) { - if( !bestRegion.equals(otherRegionToTest) && otherRegionToTest.getExtendedLoc().overlapsP( readLoc ) ) { - activeRegion.add( (GATKSAMRecord) read, false ); + if( walker.wantsNonPrimaryReads() ) { + if( !bestRegion.equals(activeRegion) ) { + activeRegion.add( (GATKSAMRecord) read ); + } + for( final ActiveRegion otherRegionToTest : workQueue ) { + if( !bestRegion.equals(otherRegionToTest) && otherRegionToTest.getExtendedLoc().overlapsP( readLoc ) ) { + activeRegion.add( (GATKSAMRecord) read ); + } } } placedReads.add( read ); - } else if( activeRegion.getExtendedLoc().overlapsP( readLoc ) ) { - activeRegion.add( (GATKSAMRecord) read, false ); + } else if( activeRegion.getExtendedLoc().overlapsP( readLoc ) && walker.wantsNonPrimaryReads() ) { + activeRegion.add( (GATKSAMRecord) read ); } } reads.removeAll( placedReads ); // remove all the reads which have been placed into their active region @@ -207,7 +209,7 @@ else if( dataSource == DataSource.REFERENCE_ORDERED_DATA ) throw new UnsupportedOperationException("Unsupported traversal type: " + dataSource); } - // integrate active regions into contiguous chunks based on active status + // integrate active regions into contiguous chunks with identical active status private ArrayList integrateActiveList( final ArrayList activeList ) { final ArrayList returnList = new ArrayList(); if( activeList.size() == 0 ) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java index 8405f762da..d7e170d739 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java @@ -37,6 +37,10 @@ public boolean filter(final RefMetaDataTracker tracker, final ReferenceContext r return true; // We are keeping all the reads } + public boolean wantsNonPrimaryReads() { + return false; + } + // Determine active status over the AlignmentContext public abstract boolean isActive(final RefMetaDataTracker tracker, final ReferenceContext ref, final AlignmentContext context); diff --git a/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRead.java b/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRead.java deleted file mode 100644 index 8d08a29b6e..0000000000 --- a/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRead.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.broadinstitute.sting.utils.activeregion; - -import org.broadinstitute.sting.utils.sam.GATKSAMRecord; - -/** - * Created by IntelliJ IDEA. - * User: rpoplin - * Date: 1/4/12 - */ - -public class ActiveRead { - final public GATKSAMRecord read; - final public boolean isPrimaryRegion; - - ActiveRead( final GATKSAMRecord read, final boolean isPrimaryRegion ) { - this.read = read; - this.isPrimaryRegion = isPrimaryRegion; - } -} diff --git a/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRegion.java b/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRegion.java index abf74469fa..6279e0061b 100644 --- a/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRegion.java +++ b/public/java/src/org/broadinstitute/sting/utils/activeregion/ActiveRegion.java @@ -16,8 +16,7 @@ public class ActiveRegion implements HasGenomeLocation { - private final ArrayList reads = new ArrayList(); - private byte[] reference = null; + private final ArrayList reads = new ArrayList(); private final GenomeLoc activeRegionLoc; private final GenomeLoc extendedLoc; private final int extension; @@ -35,28 +34,31 @@ public ActiveRegion( final GenomeLoc activeRegionLoc, final boolean isActive, fi } // add each read to the bin and extend the reference genome activeRegionLoc if needed - public void add( final GATKSAMRecord read, final boolean isPrimaryRegion ) { + public void add( final GATKSAMRecord read ) { fullExtentReferenceLoc = fullExtentReferenceLoc.union( genomeLocParser.createGenomeLoc( read ) ); - reads.add( new ActiveRead(read, isPrimaryRegion) ); + reads.add( read ); } - public ArrayList getReads() { return reads; } + public ArrayList getReads() { return reads; } public byte[] getReference( final IndexedFastaSequenceFile referenceReader ) { - // set up the reference if we haven't done so yet - if ( reference == null ) { - reference = referenceReader.getSubsequenceAt(fullExtentReferenceLoc.getContig(), fullExtentReferenceLoc.getStart(), fullExtentReferenceLoc.getStop()).getBases(); - } + return getReference( referenceReader, 0 ); + } - return reference; + public byte[] getReference( final IndexedFastaSequenceFile referenceReader, final int padding ) { + return referenceReader.getSubsequenceAt( fullExtentReferenceLoc.getContig(), + Math.max(1, fullExtentReferenceLoc.getStart() - padding), + Math.min(referenceReader.getSequenceDictionary().getSequence(fullExtentReferenceLoc.getContig()).getSequenceLength(), fullExtentReferenceLoc.getStop() + padding) ).getBases(); } @Override public GenomeLoc getLocation() { return activeRegionLoc; } - public GenomeLoc getExtendedLoc() { return extendedLoc; } public GenomeLoc getReferenceLoc() { return fullExtentReferenceLoc; } public int getExtension() { return extension; } public int size() { return reads.size(); } + public void clearReads() { reads.clear(); } + public void remove( final GATKSAMRecord read ) { reads.remove( read ); } + public void removeAll( final ArrayList readsToRemove ) { reads.removeAll( readsToRemove ); } } \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/utils/fragments/FragmentUtils.java b/public/java/src/org/broadinstitute/sting/utils/fragments/FragmentUtils.java index e5500ca213..68bf6dce8e 100644 --- a/public/java/src/org/broadinstitute/sting/utils/fragments/FragmentUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/fragments/FragmentUtils.java @@ -1,9 +1,15 @@ package org.broadinstitute.sting.utils.fragments; +import net.sf.samtools.Cigar; +import net.sf.samtools.CigarElement; +import net.sf.samtools.CigarOperator; import net.sf.samtools.SAMRecord; +import org.broadinstitute.sting.utils.collections.Pair; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.pileup.PileupElement; import org.broadinstitute.sting.utils.pileup.ReadBackedPileup; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; +import org.broadinstitute.sting.utils.sam.ReadUtils; import java.util.*; @@ -121,4 +127,63 @@ public final static FragmentCollection create(List return create(reads, reads.size(), SamRecordGetter); } + public final static List mergeOverlappingPairedFragments( List overlappingPair ) { + final byte MIN_QUAL_BAD_OVERLAP = 16; + if( overlappingPair.size() != 2 ) { throw new ReviewedStingException("Found overlapping pair with " + overlappingPair.size() + " reads, but expecting exactly 2."); } + + GATKSAMRecord firstRead = overlappingPair.get(0); + GATKSAMRecord secondRead = overlappingPair.get(1); + if( !(secondRead.getUnclippedStart() <= firstRead.getUnclippedEnd() && secondRead.getUnclippedStart() >= firstRead.getUnclippedStart() && secondRead.getUnclippedEnd() >= firstRead.getUnclippedEnd()) ) { + firstRead = overlappingPair.get(1); + secondRead = overlappingPair.get(0); + } + if( !(secondRead.getUnclippedStart() <= firstRead.getUnclippedEnd() && secondRead.getUnclippedStart() >= firstRead.getUnclippedStart() && secondRead.getUnclippedEnd() >= firstRead.getUnclippedEnd()) ) { + return overlappingPair; // can't merge them, yet: AAAAAAAAAAA-BBBBBBBBBBB-AAAAAAAAAAAAAA, B is contained entirely inside A + } + if( firstRead.getCigarString().contains("I") || firstRead.getCigarString().contains("D") || secondRead.getCigarString().contains("I") || secondRead.getCigarString().contains("D") ) { + return overlappingPair; // fragments contain indels so don't merge them + } + + final Pair pair = ReadUtils.getReadCoordinateForReferenceCoordinate(firstRead, secondRead.getSoftStart()); + + final int firstReadStop = ( pair.getSecond() ? pair.getFirst() + 1 : pair.getFirst() ); + final int numBases = firstReadStop + secondRead.getReadLength(); + final byte[] bases = new byte[numBases]; + final byte[] quals = new byte[numBases]; + final byte[] firstReadBases = firstRead.getReadBases(); + final byte[] firstReadQuals = firstRead.getBaseQualities(); + final byte[] secondReadBases = secondRead.getReadBases(); + final byte[] secondReadQuals = secondRead.getBaseQualities(); + for(int iii = 0; iii < firstReadStop; iii++) { + bases[iii] = firstReadBases[iii]; + quals[iii] = firstReadQuals[iii]; + } + for(int iii = firstReadStop; iii < firstRead.getReadLength(); iii++) { + if( firstReadQuals[iii] > MIN_QUAL_BAD_OVERLAP && secondReadQuals[iii-firstReadStop] > MIN_QUAL_BAD_OVERLAP && firstReadBases[iii] != secondReadBases[iii-firstReadStop] ) { + return overlappingPair;// high qual bases don't match exactly, probably indel in only one of the fragments, so don't merge them + } + bases[iii] = ( firstReadQuals[iii] > secondReadQuals[iii-firstReadStop] ? firstReadBases[iii] : secondReadBases[iii-firstReadStop] ); + quals[iii] = ( firstReadQuals[iii] > secondReadQuals[iii-firstReadStop] ? firstReadQuals[iii] : secondReadQuals[iii-firstReadStop] ); + } + for(int iii = firstRead.getReadLength(); iii < numBases; iii++) { + bases[iii] = secondReadBases[iii-firstReadStop]; + quals[iii] = secondReadQuals[iii-firstReadStop]; + } + + final GATKSAMRecord returnRead = new GATKSAMRecord(firstRead.getHeader()); + returnRead.setAlignmentStart(firstRead.getUnclippedStart()); + returnRead.setReadBases( bases ); + returnRead.setBaseQualities( quals ); + returnRead.setReadGroup( firstRead.getReadGroup() ); + returnRead.setReferenceName( firstRead.getReferenceName() ); + final CigarElement c = new CigarElement(bases.length, CigarOperator.M); + final ArrayList cList = new ArrayList(); + cList.add(c); + returnRead.setCigar( new Cigar( cList )); + returnRead.setMappingQuality( firstRead.getMappingQuality() ); + + final ArrayList returnList = new ArrayList(); + returnList.add(returnRead); + return returnList; + } } From 4a08e8ca6ecaffdd92623be3283faaa4aa881d01 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Mon, 23 Jan 2012 08:25:34 -0500 Subject: [PATCH 106/356] Minor tweaks to T2D-related qscripts. Replacing old md5s from the BeagleIntegrationTest. All differences boiled down either to the accounting of genotypes changed (./. --> 0/0 is no longer a "changed" genotype, and original genotypes that were ./. are represented as OG=. rather than OG=./. .) This is somewhat of an arbitrary decision, and is negotiable. I could see treating GT:PL ./.:. differently from GT:PL .:0,3,6 but am not sure the worth of doing so. --- .../sting/gatk/walkers/beagle/BeagleIntegrationTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/beagle/BeagleIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/beagle/BeagleIntegrationTest.java index 1a01ef8e8e..9aae1f0ae9 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/beagle/BeagleIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/beagle/BeagleIntegrationTest.java @@ -41,7 +41,7 @@ public void testBeagleOutput() { "--beagleR2:BEAGLE " + beagleValidationDataLocation + "inttestbgl.r2 " + "--beagleProbs:BEAGLE " + beagleValidationDataLocation + "inttestbgl.gprobs " + "--beaglePhased:BEAGLE " + beagleValidationDataLocation + "inttestbgl.phased " + - "-o %s -NO_HEADER", 1, Arrays.asList("b445d280fd8fee1eeb4aacb3f5a54847")); + "-o %s -NO_HEADER", 1, Arrays.asList("6d0f213918e3b9ea33bc2f8a51a462f1")); executeTest("test BeagleOutputToVCF", spec); } @@ -72,7 +72,7 @@ public void testBeagleOutput2() { "--beagleR2:beagle /humgen/gsa-hpprojects/GATK/data/Validation_Data/EUR_beagle_in_test.r2 "+ "--beagleProbs:beagle /humgen/gsa-hpprojects/GATK/data/Validation_Data/EUR_beagle_in_test.gprobs.bgl "+ "--beaglePhased:beagle /humgen/gsa-hpprojects/GATK/data/Validation_Data/EUR_beagle_in_test.phased.bgl "+ - "-L 20:1-70000 -o %s -NO_HEADER ",1,Arrays.asList("51a57ea565176edd96d907906914b0ee")); + "-L 20:1-70000 -o %s -NO_HEADER ",1,Arrays.asList("ddbf490f1d9f37cc79fe414c8d40886f")); executeTest("testBeagleChangesSitesToRef",spec); } From 966387ca0bf7d40ea65ac23c860df7221eaad569 Mon Sep 17 00:00:00 2001 From: Guillermo del Angel Date: Mon, 23 Jan 2012 09:22:31 -0500 Subject: [PATCH 107/356] Next intermediate commit in the pool caller. Lots of bug fixes and now we can emit true vcf's with calls in discovery mode (still of unknown quality) - old validation mode is temporarily broken,will be fixed in next refactoring. --- .../broadinstitute/sting/utils/MathUtils.java | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java index 2da54ca424..2f2dbd47e4 100644 --- a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java @@ -1572,34 +1572,37 @@ public static Object[] arrayShuffle(Object[] array) { /** * Vector operations + * @param v1 first numerical array + * @param v2 second numerical array + * @return a new array with the elements added */ - public static double[] vectorSum(double v1[], double v2[]) { + public static Double[] vectorSum(E v1[], E v2[]) { if (v1.length != v2.length) throw new UserException("BUG: vectors v1, v2 of different size in vectorSum()"); - double result[] = new double[v1.length]; + Double[] result = new Double[v1.length]; for (int k=0; k < v1.length; k++) - result[k] = v1[k]+v2[k]; + result[k] = v1[k].doubleValue()+v2[k].doubleValue(); return result; } - public static double[] scalarTimesIntVector(double a, int[] v1) { + public static Double[] scalarTimesVector(E a, E[] v1) { - double result[] = new double[v1.length]; + Double result[] = new Double[v1.length]; for (int k=0; k < v1.length; k++) - result[k] = a*v1[k]; + result[k] = a.doubleValue()*v1[k].doubleValue(); return result; } - public static double dotProduct(double v1[], double v2[]) { + public static Double dotProduct(E[] v1, E[] v2) { if (v1.length != v2.length) throw new UserException("BUG: vectors v1, v2 of different size in vectorSum()"); - double result = 0.0; + Double result = 0.0; for (int k=0; k < v1.length; k++) - result += v1[k]*v2[k]; + result += v1[k].doubleValue() *v2[k].doubleValue(); return result; @@ -1613,4 +1616,15 @@ public static double[] vectorLog10(double v1[]) { return result; } + + // todo - silly overloading, just because Java can't unbox/box arrays of primitive types, and we can't do generics with primitive types! + public static Double[] vectorLog10(Double v1[]) { + Double result[] = new Double[v1.length]; + for (int k=0; k < v1.length; k++) + result[k] = Math.log10(v1[k]); + + return result; + + } + } From 798596257b76d51730b8948c9e35624faad8bb44 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Mon, 23 Jan 2012 10:50:16 -0500 Subject: [PATCH 108/356] Enable the Genotype Phasing Evaluator. Because it didn't have the same argument structure as the base class, update2 of VariantEvaluator was being called, rather than update2 of the actual module. --- .../varianteval/evaluators/GenotypePhasingEvaluator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/GenotypePhasingEvaluator.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/GenotypePhasingEvaluator.java index ea12ada484..07cd95997f 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/GenotypePhasingEvaluator.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/GenotypePhasingEvaluator.java @@ -80,6 +80,10 @@ public String toString() { return getName() + ": "; } + public String update2(VariantContext eval, VariantContext comp, RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) { + return update2(eval,comp,tracker,ref,context,null); + } + public String update2(VariantContext eval, VariantContext comp, RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context, NewEvaluationContext group) { //public String update2(VariantContext eval, VariantContext comp, RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context, VariantEvalWalker.EvaluationContext group) { Reasons interesting = new Reasons(); From 15c0c294c1793d171774bb66a1b3b351583667c0 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Mon, 23 Jan 2012 14:51:24 -0500 Subject: [PATCH 109/356] Adding in this walker to try to debug the 0-byte ref bases From 3392d67c1a9eeec876473239abdee812de87080a Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Mon, 23 Jan 2012 15:10:03 -0500 Subject: [PATCH 110/356] Maybe a switch to reference bases will fix this From cc4ba7372f4164afc6a0f6c76f2e1e1eb30de8e9 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Mon, 23 Jan 2012 15:18:59 -0500 Subject: [PATCH 111/356] Why is reference_bases even an option anymore? From c18beadbdb4b864d4e04d453a6edebac7ab0818a Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Mon, 23 Jan 2012 16:17:04 -0500 Subject: [PATCH 112/356] Device files like /dev/null are now tracked as special by Queue and are not used to generate .out file paths, scattered into a temporary directory, gathered, deleted, etc. Attempted workaround for xdr_resourceInfoReq unsatisfied link during loading of libbat.so. --- .../sting/jna/lsf/v7_0_6/LibBat.java | 24 ++++--- .../gatk/ArgumentDefinitionField.java | 6 +- .../sting/utils/io/IOUtils.java | 19 +++++- .../sting/utils/io/IOUtilsUnitTest.java | 36 ++++++++++ .../qscripts/examples/DevNullOutput.scala | 49 ++++++++++++++ .../sting/queue/function/QFunction.scala | 8 ++- .../ScatterGatherableFunction.scala | 10 +-- .../examples/DevNullOutputPipelineTest.scala | 67 +++++++++++++++++++ .../ExampleUnifiedGenotyperPipelineTest.scala | 1 + 9 files changed, 199 insertions(+), 21 deletions(-) create mode 100644 public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/DevNullOutput.scala create mode 100644 public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/DevNullOutputPipelineTest.scala diff --git a/public/java/src/org/broadinstitute/sting/jna/lsf/v7_0_6/LibBat.java b/public/java/src/org/broadinstitute/sting/jna/lsf/v7_0_6/LibBat.java index d7b34a2530..f948a9bcfa 100644 --- a/public/java/src/org/broadinstitute/sting/jna/lsf/v7_0_6/LibBat.java +++ b/public/java/src/org/broadinstitute/sting/jna/lsf/v7_0_6/LibBat.java @@ -71,6 +71,14 @@ automatically autoRead(), and the API user will have to pass the public class LibBat { static { + // via Platform LSF Configuration Reference, by default quiet the BSUB output. + if ("Y".equals(System.getProperty("BSUB_QUIET", "Y"))) + LibC.setenv("BSUB_QUIET", "Y", 1); + String lsfLibDir = System.getenv("LSF_LIBDIR"); + if (lsfLibDir != null) { + NativeLibrary.addSearchPath("lsf", lsfLibDir); + NativeLibrary.addSearchPath("bat", lsfLibDir); + } /* LSF 7.0.6 on the mac is missing the unsatisfied exported symbol for environ which was removed on MacOS X 10.5+. nm $LSF_LIBDIR/liblsf.dylib | grep environ @@ -79,16 +87,14 @@ public class LibBat { */ if (Platform.isMac()) NativeLibrary.getInstance("environhack"); - String lsfLibDir = System.getenv("LSF_LIBDIR"); - if (lsfLibDir != null) { - NativeLibrary.addSearchPath("lsf", lsfLibDir); - NativeLibrary.addSearchPath("bat", lsfLibDir); - } - NativeLibrary.getInstance("lsf"); - // via Platform LSF Configuration Reference, by default quiet the BSUB output. - if ("Y".equals(System.getProperty("BSUB_QUIET", "Y"))) - LibC.setenv("BSUB_QUIET", "Y", 1); + NativeLibrary liblsf = NativeLibrary.getInstance("lsf"); Native.register("bat"); + // HACK: Running into a weird error: + // java.lang.UnsatisfiedLinkError: Unable to load library 'bat': <$LSF_LIBDIR>/libbat.so: undefined symbol: xdr_resourceInfoReq + // This function is very clearly unsatisfied by running 'nm $LSF_LIBDIR/libbat.so | grep xdr_resourceInfoReq' but is + // found in liblsf.so when running 'nm $LSF_LIBDIR/liblsf.so | grep xdr_resourceInfoReq'. For now holding on to a reference + // to the LSF lib just in case this is a problem with the NativeLibrary's internal WeakReferences and the library being unloaded? + liblsf.getFunction("xdr_resourceInfoReq").getName(); } // Via support@platform.com: diff --git a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java index 71640c66a6..00a6ac1ae8 100644 --- a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java +++ b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java @@ -468,7 +468,7 @@ public VCFWriterIndexArgumentField(ArgumentDefinition originalArgumentDefinition } @Override protected String getFreezeFields() { return String.format( - ("if (%2$s != null)%n" + + ("if (%2$s != null && !org.broadinstitute.sting.utils.io.IOUtils.isSpecialFile(%2$s))%n" + " if (!org.broadinstitute.sting.gatk.io.stubs.VCFWriterArgumentTypeDescriptor.isCompressed(%2$s.getPath))%n" + " %1$s = new File(%2$s.getPath + \"%3$s\")%n"), auxFieldName, originalFieldName, Tribble.STANDARD_INDEX_EXTENSION); @@ -481,7 +481,7 @@ public SAMFileWriterIndexArgumentField(ArgumentDefinition originalArgumentDefini } @Override protected String getFreezeFields() { return String.format( - ("if (%2$s != null)%n" + + ("if (%2$s != null && !org.broadinstitute.sting.utils.io.IOUtils.isSpecialFile(%2$s))%n" + " if (!%3$s)%n" + " %1$s = new File(%2$s.getPath.stripSuffix(\".bam\") + \"%4$s\")%n"), auxFieldName, originalFieldName, SAMFileWriterArgumentTypeDescriptor.DISABLE_INDEXING_FULLNAME, BAMIndex.BAMIndexSuffix); @@ -494,7 +494,7 @@ public SAMFileWriterMD5ArgumentField(ArgumentDefinition originalArgumentDefiniti } @Override protected String getFreezeFields() { return String.format( - ("if (%2$s != null)%n" + + ("if (%2$s != null && !org.broadinstitute.sting.utils.io.IOUtils.isSpecialFile(%2$s))%n" + " if (%3$s)%n" + " %1$s = new File(%2$s.getPath + \"%4$s\")%n"), auxFieldName, originalFieldName, SAMFileWriterArgumentTypeDescriptor.ENABLE_MD5_FULLNAME, ".md5"); diff --git a/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java b/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java index b3fdb93d30..a5ba857efd 100644 --- a/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -37,6 +37,7 @@ public class IOUtils { private static Logger logger = Logger.getLogger(IOUtils.class); + private static final File DEV_DIR = new File("/dev"); /** * Checks if the temp directory has been setup and throws an exception if they user hasn't set it correctly. @@ -301,12 +302,17 @@ public static List tail(File file, int count) throws IOException { } /** - * Tries to delete a file. Emits a warning if the file was unable to be deleted. + * Tries to delete a file. Emits a warning if the file + * is not a special file and was unable to be deleted. * * @param file File to delete. * @return true if the file was deleted. */ public static boolean tryDelete(File file) { + if (isSpecialFile(file)) { + logger.debug("Not trying to delete " + file); + return false; + } boolean deleted = FileUtils.deleteQuietly(file); if (deleted) logger.debug("Deleted " + file); @@ -385,4 +391,13 @@ public static LineIterator lineIterator(File file) { } } + + /** + * Returns true if the file is a special file. + * @param file File path to check. + * @return true if the file is a special file. + */ + public static boolean isSpecialFile(File file) { + return file != null && (file.getAbsolutePath().startsWith("/dev/") || file.equals(DEV_DIR)); + } } diff --git a/public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java index 4caf7f485c..757e6efdf0 100644 --- a/public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.utils.io; import org.apache.commons.io.FileUtils; @@ -194,4 +218,16 @@ public void testResourceProperties() { Assert.assertEquals(resource.getPath(), "foo"); Assert.assertEquals(resource.getRelativeClass(), Resource.class); } + + @Test + public void testIsSpecialFile() { + Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev"))); + Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev/null"))); + Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev/full"))); + Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev/stdout"))); + Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev/stderr"))); + Assert.assertFalse(IOUtils.isSpecialFile(null)); + Assert.assertFalse(IOUtils.isSpecialFile(new File("/home/user/my.file"))); + Assert.assertFalse(IOUtils.isSpecialFile(new File("/devfake/null"))); + } } diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/DevNullOutput.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/DevNullOutput.scala new file mode 100644 index 0000000000..d891ebaafd --- /dev/null +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/DevNullOutput.scala @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.queue.qscripts.examples + +import org.broadinstitute.sting.queue.QScript +import org.broadinstitute.sting.queue.extensions.gatk._ + +/** + * Script used for testing output to /dev/null + */ +class DevNullOutput extends QScript { + @Input(doc="The reference file for the bam files.", shortName="R") + var referenceFile: File = _ + + @Input(doc="Bam file to genotype.", shortName="I") + var bamFile: File = _ + + def script() { + val genotyper = new UnifiedGenotyper + genotyper.reference_sequence = referenceFile + genotyper.memoryLimit = 2 + genotyper.scatterCount = 3 + genotyper.input_file :+= bamFile + genotyper.out = "/dev/null" + add(genotyper) + } +} diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala index dee1acfacc..7d9debbdc6 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala @@ -163,7 +163,9 @@ trait QFunction extends Logging with QJobReport { * Returns prefixes for hidden done/fail files. * @return prefixes. */ - private def statusPrefixes = statusPaths.map(file => file.getParentFile + "/." + file.getName) + private def statusPrefixes = statusPaths. + filter(file => !IOUtils.isSpecialFile(file)). + map(file => file.getParentFile + "/." + file.getName) /** * Returns the output files for this function. @@ -236,7 +238,7 @@ trait QFunction extends Logging with QJobReport { * Deletes the output files and all the status files for this function. */ def deleteOutputs() { - outputs.foreach(file => IOUtils.tryDelete(file)) + outputs.filter(file => !IOUtils.isSpecialFile(file)).foreach(file => IOUtils.tryDelete(file)) doneOutputs.foreach(file => IOUtils.tryDelete(file)) failOutputs.foreach(file => IOUtils.tryDelete(file)) } @@ -346,7 +348,7 @@ trait QFunction extends Logging with QJobReport { if (jobOutputFile == null) { jobOutputFile = firstOutput match { - case file: File => new File(file.getParentFile, file.getName + ".out") + case file: File if (!IOUtils.isSpecialFile(file)) => new File(file.getParentFile, file.getName + ".out") case _ => new File(jobName + ".out") } } diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala index 921928bce0..4578f0e826 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala @@ -134,8 +134,10 @@ trait ScatterGatherableFunction extends CommandLineFunction { var gatherOutputs = ListMap.empty[ArgumentSource, File] var gatherAddOrder = numClones + 2 - // Only track fields that will have a value - val outputFieldsWithValues = this.outputFields.filter(hasFieldValue(_)) + // Only track fields that will have an output file + val outputFieldsWithValues = this.outputFields. + filter(hasFieldValue(_)). + filter(gatherField => !IOUtils.isSpecialFile(getFieldFile(gatherField))) for (gatherField <- outputFieldsWithValues) { gatherOutputs += gatherField -> getFieldFile(gatherField) @@ -175,9 +177,9 @@ trait ScatterGatherableFunction extends CommandLineFunction { cloneFunction.analysisName = this.analysisName cloneFunction.cloneIndex = i cloneFunction.commandDirectory = this.scatterGatherTempDir(dirFormat.format(i)) - cloneFunction.jobOutputFile = new File(this.jobOutputFile.getName) + cloneFunction.jobOutputFile = if (IOUtils.isSpecialFile(this.jobOutputFile)) this.jobOutputFile else new File(this.jobOutputFile.getName) if (this.jobErrorFile != null) - cloneFunction.jobErrorFile = new File(this.jobErrorFile.getName) + cloneFunction.jobErrorFile = if (IOUtils.isSpecialFile(this.jobErrorFile)) this.jobErrorFile else new File(this.jobErrorFile.getName) cloneFunction.addOrder = this.addOrder :+ (i+1) cloneFunction.isIntermediate = true diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/DevNullOutputPipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/DevNullOutputPipelineTest.scala new file mode 100644 index 0000000000..9bb287ac44 --- /dev/null +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/DevNullOutputPipelineTest.scala @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.queue.pipeline.examples + +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +import org.testng.annotations.Test +import org.broadinstitute.sting.queue.pipeline.{PipelineTest, PipelineTestSpec} +import org.broadinstitute.sting.BaseTest + +class DevNullOutputPipelineTest { + @Test + def testDevNullOutput() { + val spec = new PipelineTestSpec + spec.name = "devnulloutput" + spec.args = Array( + " -S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/DevNullOutput.scala", + " -R " + BaseTest.testDir + "exampleFASTA.fasta", + " -I " + BaseTest.testDir + "exampleBAM.bam").mkString + spec.jobRunners = PipelineTest.allJobRunners + PipelineTest.executeTest(spec) + } +} diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleUnifiedGenotyperPipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleUnifiedGenotyperPipelineTest.scala index d50673a1a9..f598402af5 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleUnifiedGenotyperPipelineTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleUnifiedGenotyperPipelineTest.scala @@ -39,6 +39,7 @@ class ExampleUnifiedGenotyperPipelineTest { " -I " + BaseTest.testDir + "exampleBAM.bam", " -filter QD", " -filterExpression 'QD < 2.0'").mkString + spec.jobRunners = PipelineTest.allJobRunners PipelineTest.executeTest(spec) } } From 0ec6f86c2117da594cba193b7c1fa9f09623ad3f Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sat, 21 Jan 2012 14:08:25 -0500 Subject: [PATCH 113/356] Tests for event length, combined snps and indels. Partial infrastructure to train and eval trees. From bb203ccf0aa548930cc993ef1881ca41a1616498 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Mon, 23 Jan 2012 10:51:07 -0500 Subject: [PATCH 114/356] combined analyses of snps and indels. From 4b17fc3cc1f5a29ad5a1ce0991808af961e935a3 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Mon, 23 Jan 2012 12:06:46 -0500 Subject: [PATCH 115/356] Parallel implementation of random forest training. Very cool (and easy) example of parallel processing in R From 10bc26079d2bb813b12c2e4104a3998db5886d28 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Mon, 23 Jan 2012 12:11:38 -0500 Subject: [PATCH 116/356] bugfix to actually run correct python script From 1f620c79e6d8c378d62a9f11e4acf987a2c2bc89 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Mon, 23 Jan 2012 12:12:00 -0500 Subject: [PATCH 117/356] Add busers and bugroup information to queueStatus From ceca7e0b37a70bef02713404796da8e10de08872 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Mon, 23 Jan 2012 12:12:38 -0500 Subject: [PATCH 118/356] Bugfix to now separate completed, sting and user exceptions. Added dry run mode From 1172517abb861fac82388cab0c400f9791d8f989 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Mon, 23 Jan 2012 16:02:53 -0500 Subject: [PATCH 119/356] Bugfix for version parsing. -- Now maps anything that doesn't exactly fit our git / svn schemes to unknown -- Added max records and specific id options From b6c816fe12fd9ebba37e9ef5f10e9e516d75fe24 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Mon, 23 Jan 2012 17:04:12 -0500 Subject: [PATCH 120/356] Turn off unnecessary printing in analyzeRunReports From 2bb9525e7f9e505393651a2c47cfcb37d3dae075 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Mon, 23 Jan 2012 17:56:27 -0500 Subject: [PATCH 121/356] Don't set base qualities if fastQ is provided * Pacbio Processing pipeline now works with the new fastQ files outputted by the Pacbio instrument --- .../queue/qscripts/PacbioProcessingPipeline.scala | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala index 2f954713e9..5cbea8ac46 100755 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala @@ -1,7 +1,6 @@ package org.broadinstitute.sting.queue.qscripts import org.broadinstitute.sting.queue.QScript -import org.broadinstitute.sting.queue.extensions.gatk._ import org.broadinstitute.sting.queue.util.QScriptUtils import net.sf.samtools.SAMFileHeader.SortOrder import org.broadinstitute.sting.utils.exceptions.UserException @@ -60,12 +59,15 @@ class PacbioProcessingPipeline extends QScript { for (file: File <- fileList) { var USE_BWA: Boolean = false + var resetQuals: Boolean = true if (file.endsWith(".fasta") || file.endsWith(".fq")) { if (bwaPath == null) { throw new UserException("You provided a fasta/fastq file but didn't provide the path for BWA"); } USE_BWA = true + if (file.endsWith(".fq")) + resetQuals = false } // FASTA -> BAM steps @@ -99,7 +101,7 @@ class PacbioProcessingPipeline extends QScript { add(cov(bam, recalFile1), recal(bam, recalFile1, recalBam), - cov(recalBam, recalFile2), + cov(recalBam, recalFile2, resetQuals), analyzeCovariates(recalFile1, path1), analyzeCovariates(recalFile2, path2)) } @@ -158,8 +160,9 @@ class PacbioProcessingPipeline extends QScript { this.jobName = queueLogDir + outBam + ".rg" } - case class cov (inBam: File, outRecalFile: File) extends CountCovariates with CommandLineGATKArgs { - this.DBQ = dbq + case class cov (inBam: File, outRecalFile: File, resetQuals: Boolean) extends CountCovariates with CommandLineGATKArgs { + if (resetQuals) + this.DBQ = dbq this.knownSites :+= dbSNP this.covariate ++= List("ReadGroupCovariate", "QualityScoreCovariate", "CycleCovariate", "DinucCovariate") this.input_file :+= inBam From 945cf038895d930705bfa80cf80ab8e2c0de6743 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Mon, 23 Jan 2012 21:46:45 -0500 Subject: [PATCH 122/356] IntelliJ ate my import! --- .../sting/queue/qscripts/PacbioProcessingPipeline.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala index 5cbea8ac46..d5f7512e4c 100755 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala @@ -6,6 +6,7 @@ import net.sf.samtools.SAMFileHeader.SortOrder import org.broadinstitute.sting.utils.exceptions.UserException import org.broadinstitute.sting.commandline.Hidden import org.broadinstitute.sting.queue.extensions.picard.{ReorderSam, SortSam, AddOrReplaceReadGroups} +import org.broadinstitute.sting.queue.extensions.gatk._ /** * Created by IntelliJ IDEA. @@ -99,9 +100,9 @@ class PacbioProcessingPipeline extends QScript { val bam = if (BLASR_BAM) {mqBAM} else {bamBase} - add(cov(bam, recalFile1), + add(cov(bam, recalFile1, resetQuals), recal(bam, recalFile1, recalBam), - cov(recalBam, recalFile2, resetQuals), + cov(recalBam, recalFile2, false), analyzeCovariates(recalFile1, path1), analyzeCovariates(recalFile2, path2)) } From 0a3172a9f1365cc9ed049173c468e1142de97396 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Tue, 24 Jan 2012 10:53:37 -0500 Subject: [PATCH 123/356] Fix for ref 0 bases for Chris -- Disturbingly, fixing this bug doesn't actually cause an test failures. -- Wrote a new QCRefWalker to actually check in detail that the reference bases coming into the RefWalker are all correct when comparing against a clean uncached load of the contig bases directly. -- However, I cannot run this tool due to some kind of weird BAM error -- sending this on to Matt --- .../sting/gatk/walkers/qc/QCRefWalker.java | 124 ++++++++++++++++++ .../CachingIndexedFastaSequenceFile.java | 2 +- 2 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 public/java/src/org/broadinstitute/sting/gatk/walkers/qc/QCRefWalker.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/QCRefWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/QCRefWalker.java new file mode 100644 index 0000000000..bddf27d84b --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/QCRefWalker.java @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.walkers.qc; + +import net.sf.picard.reference.IndexedFastaSequenceFile; +import net.sf.picard.reference.ReferenceSequence; +import net.sf.samtools.SAMSequenceRecord; +import org.broad.tribble.Feature; +import org.broadinstitute.sting.commandline.Argument; +import org.broadinstitute.sting.commandline.Input; +import org.broadinstitute.sting.commandline.Output; +import org.broadinstitute.sting.commandline.RodBinding; +import org.broadinstitute.sting.gatk.contexts.AlignmentContext; +import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; +import org.broadinstitute.sting.gatk.walkers.RefWalker; +import org.broadinstitute.sting.utils.BaseUtils; +import org.broadinstitute.sting.utils.collections.ExpandingArrayList; +import org.broadinstitute.sting.utils.collections.Pair; +import org.broadinstitute.sting.utils.exceptions.StingException; + +import java.io.PrintStream; +import java.util.Collections; +import java.util.List; + +/** + * Prints out counts of the number of reference ordered data objects encountered. + * + * + *

Input

+ *

+ * One reference file only. And optionally -L intervals + *

+ * + *

Output

+ *

+ * If ok, nothing, else will throw an exception at the site where there's been a problem + *

+ * + *

Examples

+ * 
+ * java -Xmx2g -jar GenomeAnalysisTK.jar \
+ *   -R ref.fasta \
+ *   -T QCRefWalker
+ *
+ * + */ +public class QCRefWalker extends RefWalker { + @Output + public PrintStream out; + + String contigName = ""; + int contigStart, contigEnd; + IndexedFastaSequenceFile uncachedRef; + byte[] uncachedBases; + + @Override + public void initialize() { + super.initialize(); //To change body of overridden methods use File | Settings | File Templates. + uncachedRef = getToolkit().getReferenceDataSource().getReference(); + } + + private final void throwError(ReferenceContext ref, String message) { + throw new StingException(String.format("Site %s failed: %s", ref, message)); + } + + public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) { + final String locusContigName = ref.getLocus().getContig(); + if ( ! locusContigName.equals(contigName) ) { + contigName = locusContigName; + ReferenceSequence refSeq = uncachedRef.getSequence(contigName); + contigStart = 1; + contigEnd = contigStart + refSeq.length(); + uncachedBases = uncachedRef.getSubsequenceAt(contigName, contigStart, contigEnd).getBases(); + logger.warn(String.format("Loading contig %s (%d-%d)", contigName, contigStart, contigEnd)); + } + + final byte refBase = ref.getBase(); + if (! ( BaseUtils.isRegularBase(refBase) || BaseUtils.isNBase(refBase) ) ) + throwError(ref, String.format("Refbase isn't a regular base (%d %c)", refBase, (char)refBase)); + + // check bases are equal + final int pos = (int)context.getPosition() - contigStart; + if ( pos > contigEnd ) + throwError(ref, String.format("off contig (len=%d)", contigEnd)); + final byte uncachedBase = uncachedBases[pos]; + + if ( uncachedBase != refBase ) + throwError(ref, String.format("Provided refBase (%d %c) not equal to uncached one (%d %c)", + refBase, (char)refBase, uncachedBase, (char)uncachedBase)); + + return 1; + } + + public Integer reduceInit() { + return 0; + } + + public Integer reduce(Integer one, Integer sum) { + return one + sum; + } +} \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/utils/fasta/CachingIndexedFastaSequenceFile.java b/public/java/src/org/broadinstitute/sting/utils/fasta/CachingIndexedFastaSequenceFile.java index 43ef4aa741..44b586bcd2 100644 --- a/public/java/src/org/broadinstitute/sting/utils/fasta/CachingIndexedFastaSequenceFile.java +++ b/public/java/src/org/broadinstitute/sting/utils/fasta/CachingIndexedFastaSequenceFile.java @@ -167,7 +167,7 @@ public ReferenceSequence getSubsequenceAt( String contig, long start, long stop if ( start < myCache.start || stop > myCache.stop || myCache.seq == null || myCache.seq.getContigIndex() != contigInfo.getSequenceIndex() ) { cacheMisses++; myCache.start = Math.max(start - cacheMissBackup, 0); - myCache.stop = Math.min(myCache.start + cacheSize, contigInfo.getSequenceLength()); + myCache.stop = Math.min(start + cacheSize + cacheMissBackup, contigInfo.getSequenceLength()); myCache.seq = super.getSubsequenceAt(contig, myCache.start, myCache.stop); //System.out.printf("New cache at %s %d-%d%n", contig, cacheStart, cacheStop); } else { From 7c7ca0d799256c484fa65ded9e1f2c04585c761f Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Tue, 24 Jan 2012 10:59:25 -0500 Subject: [PATCH 124/356] fixing bug with fastq extension * PPP only recognized .fasta and .fq, failing when the user provided a .fastq file. Fixed. --- .../org/broadinstitute/sting/queue/util/QScriptUtils.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala b/public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala index 5d76f39ed7..1529d99518 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala @@ -45,9 +45,12 @@ object QScriptUtils { * to have empty lines and comment lines (lines starting with #). */ def createSeqFromFile(in: File):Seq[File] = { - // If the file provided ends with .bam, .fasta or .fq, it is not a bam list, we treat it as a single file. + // If the file provided ends with .bam, .fasta, fastq or .fq, it is not a bam list, we treat it as a single file. // and return a list with only this file. - if (in.toString.endsWith(".bam") || in.toString.endsWith(".fasta") || in.toString.endsWith(".fq")) + if (in.toString.toUpperCase.endsWith(".BAM") || + in.toString.toUpperCase.endsWith(".FASTA") || + in.toString.toUpperCase.endsWith(".FQ") || + in.toString.toUpperCase.endsWith("FASTQ") ) return Seq(in) var list: Seq[File] = Seq() From 8aaca83e9147041d41c18a1de7f4698a599e15a1 Mon Sep 17 00:00:00 2001 From: Guillermo del Angel Date: Tue, 24 Jan 2012 11:45:19 -0500 Subject: [PATCH 125/356] Next intermediate iteration on pool caller: start using UnifiedArgumentCollection in order to have similar syntax to UG, several numerical bug fixes, add more logging to vcf (not done yet) From c312bd59600c941afc99b7708e156765d62ce3be Mon Sep 17 00:00:00 2001 From: Matt Hanna Date: Tue, 24 Jan 2012 15:30:04 -0500 Subject: [PATCH 126/356] Weirdly, PicardException inherits from SAMException, which means that our specialty code for reporting malformed BAMs was actually misreporting any error that happened in the Picard layer as a BAM ERROR. Specifically changing PicardException to report as a ReviewedStingException; we might want to change it in the future. I'll followup with the Picard team to make sure they really, really want PicardException to inherit from SAMException. --- .../org/broadinstitute/sting/gatk/CommandLineGATK.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java b/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java index b4d337d8df..9c59ffe9a9 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java +++ b/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java @@ -25,6 +25,8 @@ package org.broadinstitute.sting.gatk; +import net.sf.picard.PicardException; +import net.sf.samtools.SAMException; import org.broad.tribble.TribbleException; import org.broadinstitute.sting.commandline.Argument; import org.broadinstitute.sting.commandline.ArgumentCollection; @@ -95,7 +97,11 @@ public static void main(String[] argv) { // We can generate Tribble Exceptions in weird places when e.g. VCF genotype fields are // lazy loaded, so they aren't caught elsewhere and made into User Exceptions exitSystemWithUserError(e); - } catch (net.sf.samtools.SAMException e) { + } catch(PicardException e) { + // TODO: Should Picard exceptions be, in general, UserExceptions or ReviewedStingExceptions? + exitSystemWithError(e); + } + catch (SAMException e) { checkForTooManyOpenFilesProblem(e.getMessage()); exitSystemWithSamError(e); } catch (Throwable t) { From ffd61f4c1cdbf95dafc14976a375d31cc00e40f9 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Tue, 17 Jan 2012 18:56:50 -0500 Subject: [PATCH 127/356] Refactor the Pileup Element with regards to indels Eric reported this bug due to the reduced reads failing with an index out of bounds on what we thought was a deletion, but turned out to be a read starting with insertion. * Refactored PileupElement to distinguish clearly between deletions and read starting with insertion * Modified ExtendedEventPileup to correctly distinguish elements with deletion when creating new pileups * Refactored most of the lazyLoadNextAlignment() function of the LocusIteratorByState for clarity and to create clear separation between what is a pileup with a deletion and what's not one. Got rid of many useless if statements. * Changed the way LocusIteratorByState creates extended event pileups to differentiate between insertions in the beginning of the read and deletions. * Every deletion now has an offset (start of the event) * Fixed bug when LocusITeratorByState found a read starting with insertion that happened to be a reduced read. * Separated the definitions of deletion/insertion (in the beginning of the read) in all UG annotations (and the annotator engine). * Pileup depth of coverage for a deleted base will now return the average coverage around the deletion. * Indel ReadPositionRankSum test now uses the deletion true offset from the read, changed all appropriate md5's * The extra pileup elements now properly read by the Indel mode of the UG made any subsequent call have a different random number and therefore all RankSum tests have slightly different values (in the 10^-3 range). Updated all appropriate md5s after extremely careful inspection -- Thanks Ryan! phew! --- .../datasources/providers/AllLocusView.java | 38 +- .../gatk/iterators/LocusIteratorByState.java | 425 ++--- .../annotator/BaseQualityRankSumTest.java | 8 +- .../gatk/walkers/annotator/FisherStrand.java | 4 +- .../walkers/annotator/HaplotypeScore.java | 161 +- .../gatk/walkers/annotator/RankSumTest.java | 52 +- .../walkers/annotator/ReadPosRankSumTest.java | 73 +- .../DiploidSNPGenotypeLikelihoods.java | 2 +- ...elGenotypeLikelihoodsCalculationModel.java | 140 +- ...NPGenotypeLikelihoodsCalculationModel.java | 2 +- .../pileup/AbstractReadBackedPileup.java | 522 +++--- .../pileup/ExtendedEventPileupElement.java | 106 +- .../sting/utils/pileup/PileupElement.java | 84 +- .../ReadBackedExtendedEventPileupImpl.java | 113 +- .../utils/pileup/ReadBackedPileupImpl.java | 27 +- .../sting/utils/sam/AlignmentUtils.java | 1519 +++++++++-------- .../sting/utils/sam/ArtificialSAMUtils.java | 87 +- .../org/broadinstitute/sting/BaseTest.java | 13 +- .../UnifiedGenotyperIntegrationTest.java | 8 +- .../sting/utils/sam/ReadUtilsUnitTest.java | 8 +- 20 files changed, 1784 insertions(+), 1608 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/providers/AllLocusView.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/providers/AllLocusView.java index a6731ee184..d1a2e7519b 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/providers/AllLocusView.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/providers/AllLocusView.java @@ -23,7 +23,7 @@ */ /** - * A LocusView over which the user can iterate. + * A LocusView over which the user can iterate. */ public class AllLocusView extends LocusView { @@ -47,12 +47,13 @@ public class AllLocusView extends LocusView { /** * Create a new queue of locus contexts. + * * @param provider */ - public AllLocusView(LocusShardDataProvider provider) { - super( provider ); + public AllLocusView(LocusShardDataProvider provider) { + super(provider); // Seed the state tracking members with the first possible seek position and the first possible locus context. - locusIterator = new GenomeLocusIterator(genomeLocParser,provider.getLocus()); + locusIterator = new GenomeLocusIterator(genomeLocParser, provider.getLocus()); } public boolean hasNext() { @@ -63,7 +64,7 @@ public boolean hasNext() { public AlignmentContext next() { advance(); - if(nextPosition == null) + if (nextPosition == null) throw new NoSuchElementException("No next is available in the all locus view"); // Flag to the iterator that no data is waiting in the queue to be processed. @@ -72,7 +73,7 @@ public AlignmentContext next() { AlignmentContext currentLocus; // If actual data is present, return it. Otherwise, return empty data. - if( nextLocus != null && nextLocus.getLocation().equals(nextPosition) ) + if (nextLocus != null && nextLocus.getLocation().equals(nextPosition)) currentLocus = nextLocus; else currentLocus = createEmptyLocus(nextPosition); @@ -82,15 +83,15 @@ public AlignmentContext next() { private void advance() { // Already at the next element? Don't move forward. - if(atNextElement) + if (atNextElement) return; // Out of elements? - if(nextPosition == null && !locusIterator.hasNext()) - return; + if (nextPosition == null && !locusIterator.hasNext()) + return; // If nextLocus has been consumed, clear it out to make room for the next incoming locus. - if(nextPosition != null && nextLocus != null && !nextLocus.getLocation().isPast(nextPosition)) { + if (nextPosition != null && nextLocus != null && !nextLocus.getLocation().isPast(nextPosition)) { nextLocus = null; // Determine the next locus. The trick is that we may have more than one alignment context at the same @@ -98,9 +99,9 @@ private void advance() { // is still at the current position, we do not increment current position and wait for next call to next() to return // that context. If we know that next context is past the current position, we are done with current // position - if(hasNextLocus()) { + if (hasNextLocus()) { nextLocus = nextLocus(); - if(nextPosition.equals(nextLocus.getLocation())) { + if (nextPosition.equals(nextLocus.getLocation())) { atNextElement = true; return; } @@ -108,7 +109,7 @@ private void advance() { } // No elements left in queue? Clear out the position state tracker and return. - if(!locusIterator.hasNext()) { + if (!locusIterator.hasNext()) { nextPosition = null; return; } @@ -119,9 +120,9 @@ private void advance() { // Crank the iterator to (if possible) or past the next context. Be careful not to hold a reference to nextLocus // while using the hasNextLocus() / nextLocus() machinery; this will cause us to use more memory than is optimal. - while(nextLocus == null || nextLocus.getLocation().isBefore(nextPosition)) { + while (nextLocus == null || nextLocus.getLocation().isBefore(nextPosition)) { nextLocus = null; - if(!hasNextLocus()) + if (!hasNextLocus()) break; nextLocus = nextLocus(); } @@ -129,12 +130,15 @@ private void advance() { /** * Creates a blank locus context at the specified location. + * * @param site Site at which to create the blank locus context. * @return empty context. */ private final static List EMPTY_PILEUP_READS = Collections.emptyList(); private final static List EMPTY_PILEUP_OFFSETS = Collections.emptyList(); - private AlignmentContext createEmptyLocus( GenomeLoc site ) { - return new AlignmentContext(site,new ReadBackedPileupImpl(site, EMPTY_PILEUP_READS, EMPTY_PILEUP_OFFSETS)); + private final static List EMPTY_DELETION_STATUS = Collections.emptyList(); + + private AlignmentContext createEmptyLocus(GenomeLoc site) { + return new AlignmentContext(site, new ReadBackedPileupImpl(site, EMPTY_PILEUP_READS, EMPTY_PILEUP_OFFSETS)); } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java b/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java index 75e787e05a..f1ffa121b1 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java +++ b/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java @@ -49,9 +49,13 @@ import java.util.*; -/** Iterator that traverses a SAM File, accumulating information on a per-locus basis */ +/** + * Iterator that traverses a SAM File, accumulating information on a per-locus basis + */ public class LocusIteratorByState extends LocusIterator { - /** our log, which we want to capture anything from this class */ + /** + * our log, which we want to capture anything from this class + */ private static Logger logger = Logger.getLogger(LocusIteratorByState.class); // ----------------------------------------------------------------------------------------------------------------- @@ -92,12 +96,14 @@ static private class SAMRecordState { boolean generateExtendedEvents = true; // should we generate an additional, special pile for indels between the ref bases? // the only purpose of this flag is to shield away a few additional lines of code // when extended piles are not needed, it may not be even worth it... - byte[] insertedBases = null; // remember full inserted sequence if we are generating piles of extended events (indels) - int eventLength = -1; // will be set to the length of insertion/deletion if we are generating piles of extended events - byte eventDelayedFlag = 0; // will be set to non-0 if there was an event (indel) right before the + + byte[] insertedBases = null; // remember full inserted sequence if we are generating piles of extended events (indels) + int eventLength = -1; // will be set to the length of insertion/deletion if we are generating piles of extended events + byte eventDelayedFlag = 0; // will be set to non-0 if there was an event (indel) right before the // current base on the ref. We use a counter-like variable here since clearing the indel event is // delayed by one base, so we need to remember how long ago we have seen the actual event - int eventStart = -1; // where on the read the extended event starts (i.e. the last position on the read prior to the + + int eventStart = -1; // where on the read the extended event starts (i.e. the last position on the read prior to the // event, or -1 if alignment starts with an insertion); this one is easy to recompute on the fly, // we cache it here mainly for convenience @@ -111,23 +117,31 @@ public SAMRecordState(SAMRecord read, boolean extended) { //System.out.printf("Creating a SAMRecordState: %s%n", this); } - public SAMRecord getRead() { return read; } + public SAMRecord getRead() { + return read; + } /** * What is our current offset in the read's bases that aligns us with the reference genome? * * @return */ - public int getReadOffset() { return readOffset; } + public int getReadOffset() { + return readOffset; + } /** * What is the current offset w.r.t. the alignment state that aligns us to the readOffset? * * @return */ - public int getGenomeOffset() { return genomeOffset; } + public int getGenomeOffset() { + return genomeOffset; + } - public int getGenomePosition() { return read.getAlignmentStart() + getGenomeOffset(); } + public int getGenomePosition() { + return read.getAlignmentStart() + getGenomeOffset(); + } public GenomeLoc getLocation(GenomeLocParser genomeLocParser) { return genomeLocParser.createGenomeLoc(read.getReferenceName(), getGenomePosition()); @@ -137,19 +151,26 @@ public CigarOperator getCurrentCigarOperator() { return curElement.getOperator(); } - /** Returns true if we just stepped over insertion/into a deletion prior to the last return from stepForwardOnGenome. + /** + * Returns true if we just stepped over insertion/into a deletion prior to the last return from stepForwardOnGenome. * * @return */ public boolean hadIndel() { - return ( eventLength > 0 ); + return (eventLength > 0); } - public int getEventLength() { return eventLength; } + public int getEventLength() { + return eventLength; + } - public byte[] getEventBases() { return insertedBases; } + public byte[] getEventBases() { + return insertedBases; + } - public int getReadEventStartOffset() { return eventStart; } + public int getReadEventStartOffset() { + return eventStart; + } public String toString() { return String.format("%s ro=%d go=%d co=%d cec=%d %s", read.getReadName(), readOffset, genomeOffset, cigarOffset, cigarElementCounter, curElement); @@ -160,9 +181,9 @@ public CigarOperator stepForwardOnGenome() { // (-1 if we did not process a single base yet); this can be last matching base, or last base of an insertion - if ( curElement == null || ++cigarElementCounter > curElement.getLength() ) { + if (curElement == null || ++cigarElementCounter > curElement.getLength()) { cigarOffset++; - if ( cigarOffset < nCigarElements ) { + if (cigarOffset < nCigarElements) { curElement = cigar.getCigarElement(cigarOffset); cigarElementCounter = 0; // next line: guards against cigar elements of length 0; when new cigar element is retrieved, @@ -174,15 +195,15 @@ public CigarOperator stepForwardOnGenome() { // current offset of this read is the next ref base after the end of the indel. This position will // model a point on the reference somewhere after the end of the read. genomeOffset++; // extended events need that. Logically, it's legal to advance the genomic offset here: - // we do step forward on the ref, and by returning null we also indicate that we are past the read end. + // we do step forward on the ref, and by returning null we also indicate that we are past the read end. - if ( generateExtendedEvents && eventDelayedFlag > 0 ) { + if (generateExtendedEvents && eventDelayedFlag > 0) { // if we had an indel right before the read ended (i.e. insertion was the last cigar element), // we keep it until next reference base; then we discard it and this will allow the LocusIterator to // finally discard this read eventDelayedFlag--; - if ( eventDelayedFlag == 0 ) { + if (eventDelayedFlag == 0) { eventLength = -1; // reset event when we are past it insertedBases = null; eventStart = -1; @@ -193,34 +214,35 @@ public CigarOperator stepForwardOnGenome() { } } - boolean done = false; switch (curElement.getOperator()) { - case H : // ignore hard clips - case P : // ignore pads + case H: // ignore hard clips + case P: // ignore pads cigarElementCounter = curElement.getLength(); break; - case I : // insertion w.r.t. the reference - if ( generateExtendedEvents ) { + case I: // insertion w.r.t. the reference + if (generateExtendedEvents) { // we see insertions only once, when we step right onto them; the position on the read is scrolled // past the insertion right after that - if ( eventDelayedFlag > 1 ) throw new UserException.MalformedBAM(read, "Adjacent I/D events in read "+read.getReadName()); - insertedBases = Arrays.copyOfRange(read.getReadBases(),readOffset+1,readOffset+1+curElement.getLength()); - eventLength = curElement.getLength() ; + if (eventDelayedFlag > 1) + throw new UserException.MalformedBAM(read, "Adjacent I/D events in read " + read.getReadName()); + insertedBases = Arrays.copyOfRange(read.getReadBases(), readOffset + 1, readOffset + 1 + curElement.getLength()); + eventLength = curElement.getLength(); eventStart = readOffset; eventDelayedFlag = 2; // insertion causes re-entry into stepForwardOnGenome, so we set the delay to 2 // System.out.println("Inserted "+(new String (insertedBases)) +" after "+readOffset); } // continue onto the 'S' case ! - case S : // soft clip + case S: // soft clip cigarElementCounter = curElement.getLength(); readOffset += curElement.getLength(); break; - case D : // deletion w.r.t. the reference - if ( generateExtendedEvents ) { - if ( cigarElementCounter == 1) { + case D: // deletion w.r.t. the reference + if (generateExtendedEvents) { + if (cigarElementCounter == 1) { // generate an extended event only if we just stepped into the deletion (i.e. don't // generate the event at every deleted position on the ref, that's what cigarElementCounter==1 is for!) - if ( eventDelayedFlag > 1 ) throw new UserException.MalformedBAM(read, "Adjacent I/D events in read "+read.getReadName()); + if (eventDelayedFlag > 1) + throw new UserException.MalformedBAM(read, "Adjacent I/D events in read " + read.getReadName()); eventLength = curElement.getLength(); eventDelayedFlag = 2; // deletion on the ref causes an immediate return, so we have to delay by 1 only eventStart = readOffset; @@ -232,26 +254,27 @@ public CigarOperator stepForwardOnGenome() { genomeOffset++; done = true; break; - case N : // reference skip (looks and gets processed just like a "deletion", just different logical meaning) + case N: // reference skip (looks and gets processed just like a "deletion", just different logical meaning) genomeOffset++; done = true; break; - case M : + case M: readOffset++; genomeOffset++; done = true; break; - default : throw new IllegalStateException("Case statement didn't deal with cigar op: " + curElement.getOperator()); + default: + throw new IllegalStateException("Case statement didn't deal with cigar op: " + curElement.getOperator()); } - if ( generateExtendedEvents ) { - if ( eventDelayedFlag > 0 && done ) { - // if we did make a successful step on the ref, decrement delayed flag. If, upon the decrementthe, + if (generateExtendedEvents) { + if (eventDelayedFlag > 0 && done) { + // if we did make a successful step on the ref, decrement delayed flag. If, upon the decrementing the, // the flag is 1, we are standing on the reference base right after the indel (so we have to keep it). // Otherwise, we are away from the previous indel and have to clear our memories... eventDelayedFlag--; // when we notice an indel, we set delayed flag to 2, so now - // if eventDelayedFlag == 1, an indel occured right before the current base - if ( eventDelayedFlag == 0 ) { + // if eventDelayedFlag == 1, an indel occured right before the current base + if (eventDelayedFlag == 0) { eventLength = -1; // reset event when we are past it insertedBases = null; eventStart = -1; @@ -274,15 +297,15 @@ public CigarOperator stepForwardOnGenome() { // // ----------------------------------------------------------------------------------------------------------------- - public LocusIteratorByState(final Iterator samIterator, ReadProperties readInformation, GenomeLocParser genomeLocParser, Collection samples ) { + public LocusIteratorByState(final Iterator samIterator, ReadProperties readInformation, GenomeLocParser genomeLocParser, Collection samples) { this.readInfo = readInformation; this.genomeLocParser = genomeLocParser; this.samples = new ArrayList(samples); - this.readStates = new ReadStateManager(samIterator,readInformation.getDownsamplingMethod()); + this.readStates = new ReadStateManager(samIterator, readInformation.getDownsamplingMethod()); // currently the GATK expects this LocusIteratorByState to accept empty sample lists, when // there's no read data. So we need to throw this error only when samIterator.hasNext() is true - if ( this.samples.isEmpty() && samIterator.hasNext() ) { + if (this.samples.isEmpty() && samIterator.hasNext()) { throw new IllegalArgumentException("samples list must not be empty"); } } @@ -322,7 +345,7 @@ private GenomeLoc getLocation() { // ----------------------------------------------------------------------------------------------------------------- public AlignmentContext next() { lazyLoadNextAlignmentContext(); - if(!hasNext()) + if (!hasNext()) throw new NoSuchElementException("LocusIteratorByState: out of elements."); AlignmentContext currentAlignmentContext = nextAlignmentContext; nextAlignmentContext = null; @@ -334,7 +357,7 @@ public AlignmentContext next() { * nextAlignmentContext MUST BE null in order for this method to advance to the next entry. */ private void lazyLoadNextAlignmentContext() { - while(nextAlignmentContext == null && readStates.hasNext()) { + while (nextAlignmentContext == null && readStates.hasNext()) { // this call will set hasExtendedEvents to true if it picks up a read with indel right before the current position on the ref: readStates.collectPendingReads(); @@ -350,14 +373,14 @@ private void lazyLoadNextAlignmentContext() { // In this case, the subsequent call to next() will emit the normal pileup at the current base // and shift the position. if (readInfo.generateExtendedEvents() && hasExtendedEvents) { - Map fullExtendedEventPileup = new HashMap(); + Map fullExtendedEventPileup = new HashMap(); // get current location on the reference and decrement it by 1: the indels we just stepped over // are associated with the *previous* reference base - GenomeLoc loc = genomeLocParser.incPos(getLocation(),-1); + GenomeLoc loc = genomeLocParser.incPos(getLocation(), -1); boolean hasBeenSampled = false; - for(final String sample: samples) { + for (final String sample : samples) { Iterator iterator = readStates.iterator(sample); List indelPile = new ArrayList(readStates.size(sample)); hasBeenSampled |= loc.getStart() <= readStates.getDownsamplingExtent(sample); @@ -368,103 +391,108 @@ private void lazyLoadNextAlignmentContext() { nMQ0Reads = 0; int maxDeletionLength = 0; - while(iterator.hasNext()) { - SAMRecordState state = iterator.next(); - if ( state.hadIndel() ) { + while (iterator.hasNext()) { + final SAMRecordState state = iterator.next(); + final GATKSAMRecord read = (GATKSAMRecord) state.getRead(); // the actual read + final CigarOperator op = state.getCurrentCigarOperator(); // current cigar operator + final int readOffset = state.getReadOffset(); // the base offset on this read + final int eventStartOffset = state.getReadEventStartOffset(); // this will be -1 if base is not a deletion, or if base is the first deletion in the event. Otherwise, it will give the last base before the deletion began. + final int eventLength = state.getEventLength(); + +// if (op != CigarOperator.N) // N's are never added to any pileup +// continue; +// + if (state.hadIndel()) { // this read has an indel associated with the previous position on the ref size++; - if ( state.getEventBases() == null ) { + ExtendedEventPileupElement pileupElement; + if (state.getEventBases() == null) { // Deletion event nDeletions++; - maxDeletionLength = Math.max(maxDeletionLength,state.getEventLength()); - } - else nInsertions++; - indelPile.add ( new ExtendedEventPileupElement((GATKSAMRecord) state.getRead(), state.getReadEventStartOffset(), state.getEventLength(), state.getEventBases()) ); - - } else { - // HACK: The readahead mechanism for LocusIteratorByState will effectively read past the current position - // and add in extra reads that start after this indel. Skip these reads. - // My belief at this moment after empirically looking at read->ref alignment is that, in a cigar string - // like 1I76M, the first insertion is between alignment start-1 and alignment start, so we shouldn't be - // filtering these out. - // TODO: UPDATE! Eric tells me that we *might* want reads adjacent to the pileup in the pileup. Strike this block. - //if(state.getRead().getAlignmentStart() > loc.getStart()) - // continue; - - if ( state.getCurrentCigarOperator() != CigarOperator.N ) { - // this read has no indel associated with the previous position on the ref; - // we count this read in only if it has actual bases, not N span... - if ( state.getCurrentCigarOperator() != CigarOperator.D || readInfo.includeReadsWithDeletionAtLoci() ) { - - // if cigar operator is D but the read has no extended event reported (that's why we ended - // up in this branch), it means that we are currently inside a deletion that started earlier; - // we count such reads (with a longer deletion spanning over a deletion at the previous base we are - // about to report) only if includeReadsWithDeletionAtLoci is true. - size++; - indelPile.add ( new ExtendedEventPileupElement((GATKSAMRecord) state.getRead(), state.getReadOffset()-1, -1) // length=-1 --> noevent - ); - } + maxDeletionLength = Math.max(maxDeletionLength, state.getEventLength()); + pileupElement = new ExtendedEventPileupElement(read, eventStartOffset, eventLength); + } + else { // Insertion event + nInsertions++; + pileupElement = new ExtendedEventPileupElement(read, eventStartOffset, eventLength, state.getEventBases()); } + + indelPile.add(pileupElement); } - if ( state.getRead().getMappingQuality() == 0 ) { - nMQ0Reads++; + + // this read has no indel associated with the previous position on the ref. Criteria to include in the pileup are: + // we only add reads that are not N's + // we only include deletions to the pileup if the walker requests it + else if ( (op != CigarOperator.N) && (op != CigarOperator.D || readInfo.includeReadsWithDeletionAtLoci())) { + size++; + indelPile.add(new ExtendedEventPileupElement((GATKSAMRecord) state.getRead(), readOffset)); } + + + if (state.getRead().getMappingQuality() == 0) + nMQ0Reads++; + } - if( indelPile.size() != 0 ) fullExtendedEventPileup.put(sample,new ReadBackedExtendedEventPileupImpl(loc,indelPile,size,maxDeletionLength,nInsertions,nDeletions,nMQ0Reads)); + + if (indelPile.size() != 0) + fullExtendedEventPileup.put(sample, new ReadBackedExtendedEventPileupImpl(loc, indelPile, size, maxDeletionLength, nInsertions, nDeletions, nMQ0Reads)); } - hasExtendedEvents = false; // we are done with extended events prior to current ref base -// System.out.println("Indel(s) at "+loc); -// for ( ExtendedEventPileupElement pe : indelPile ) { if ( pe.isIndel() ) System.out.println(" "+pe.toString()); } + hasExtendedEvents = false; // we are done with extended events prior to current ref base nextAlignmentContext = new AlignmentContext(loc, new ReadBackedExtendedEventPileupImpl(loc, fullExtendedEventPileup), hasBeenSampled); - } else { + } + else { // this is a regular event pileup (not extended) GenomeLoc location = getLocation(); - Map fullPileup = new HashMap(); - + Map fullPileup = new HashMap(); boolean hasBeenSampled = false; - for(final String sample: samples) { + for (final String sample : samples) { Iterator iterator = readStates.iterator(sample); List pile = new ArrayList(readStates.size(sample)); hasBeenSampled |= location.getStart() <= readStates.getDownsamplingExtent(sample); - size = 0; - nDeletions = 0; - nMQ0Reads = 0; + size = 0; // number of elements in this sample's pileup + nDeletions = 0; // number of deletions in this sample's pileup + nMQ0Reads = 0; // number of MQ0 reads in this sample's pileup (warning: current implementation includes N bases that are MQ0) - while(iterator.hasNext()) { - SAMRecordState state = iterator.next(); - if ( state.getCurrentCigarOperator() != CigarOperator.D && state.getCurrentCigarOperator() != CigarOperator.N ) { - if ( filterBaseInRead((GATKSAMRecord) state.getRead(), location.getStart()) ) { - //discarded_bases++; - //printStatus("Adaptor bases", discarded_adaptor_bases); - continue; - } else { - //observed_bases++; - pile.add(new PileupElement((GATKSAMRecord) state.getRead(), state.getReadOffset())); - size++; - } - } else if ( readInfo.includeReadsWithDeletionAtLoci() && state.getCurrentCigarOperator() != CigarOperator.N ) { - size++; - pile.add(new PileupElement((GATKSAMRecord) state.getRead(), -1)); - nDeletions++; - } + while (iterator.hasNext()) { + final SAMRecordState state = iterator.next(); // state object with the read/offset information + final GATKSAMRecord read = (GATKSAMRecord) state.getRead(); // the actual read + final CigarOperator op = state.getCurrentCigarOperator(); // current cigar operator + final int readOffset = state.getReadOffset(); // the base offset on this read + final int eventStartOffset = state.getReadEventStartOffset(); // this will be -1 if base is not a deletion, or if base is the first deletion in the event. Otherwise, it will give the last base before the deletion began. - if ( state.getRead().getMappingQuality() == 0 ) { + if (op == CigarOperator.N) // N's are never added to any pileup + continue; + + if (read.getMappingQuality() == 0) nMQ0Reads++; + + if (op == CigarOperator.D) { + if (readInfo.includeReadsWithDeletionAtLoci()) { // only add deletions to the pileup if we are authorized to do so + int leftAlignedStart = (eventStartOffset < 0) ? readOffset : eventStartOffset; + pile.add(new PileupElement(read, leftAlignedStart, true)); + size++; + nDeletions++; + } + } else { + if (!filterBaseInRead(read, location.getStart())) { + pile.add(new PileupElement(read, readOffset, false)); + size++; + } } } - if( pile.size() != 0 ) - fullPileup.put(sample,new ReadBackedPileupImpl(location,pile,size,nDeletions,nMQ0Reads)); + if (pile.size() != 0) // if this pileup added at least one base, add it to the full pileup + fullPileup.put(sample, new ReadBackedPileupImpl(location, pile, size, nDeletions, nMQ0Reads)); } - updateReadStates(); // critical - must be called after we get the current state offsets and location - // if we got reads with non-D/N over the current position, we are done - if ( !fullPileup.isEmpty() ) nextAlignmentContext = new AlignmentContext(location, new ReadBackedPileupImpl(location,fullPileup),hasBeenSampled); + updateReadStates(); // critical - must be called after we get the current state offsets and location + if (!fullPileup.isEmpty()) // if we got reads with non-D/N over the current position, we are done + nextAlignmentContext = new AlignmentContext(location, new ReadBackedPileupImpl(location, fullPileup), hasBeenSampled); } } } // fast testing of position private boolean readIsPastCurrentPosition(SAMRecord read) { - if ( readStates.isEmpty() ) + if (readStates.isEmpty()) return false; else { SAMRecordState state = readStates.getFirst(); @@ -485,20 +513,18 @@ private static boolean filterBaseInRead(GATKSAMRecord rec, long pos) { } private void updateReadStates() { - for(final String sample: samples) { + for (final String sample : samples) { Iterator it = readStates.iterator(sample); - while ( it.hasNext() ) { + while (it.hasNext()) { SAMRecordState state = it.next(); CigarOperator op = state.stepForwardOnGenome(); - if ( state.hadIndel() && readInfo.generateExtendedEvents() ) hasExtendedEvents = true; - else { + if (state.hadIndel() && readInfo.generateExtendedEvents()) + hasExtendedEvents = true; + else if (op == null) { // we discard the read only when we are past its end AND indel at the end of the read (if any) was // already processed. Keeping the read state that retunred null upon stepForwardOnGenome() is safe // as the next call to stepForwardOnGenome() will return null again AND will clear hadIndel() flag. - if ( op == null ) { // we've stepped off the end of the object - //if (DEBUG) logger.debug(String.format(" removing read %s at %d", state.getRead().getReadName(), state.getRead().getAlignmentStart())); - it.remove(); - } + it.remove(); // we've stepped off the end of the object } } } @@ -508,20 +534,20 @@ public void remove() { throw new UnsupportedOperationException("Can not remove records from a SAM file via an iterator!"); } - private class ReadStateManager { + private class ReadStateManager { private final PeekableIterator iterator; private final DownsamplingMethod downsamplingMethod; private final SamplePartitioner samplePartitioner; - private final Map readStatesBySample = new HashMap(); + private final Map readStatesBySample = new HashMap(); private final int targetCoverage; private int totalReadStates = 0; public ReadStateManager(Iterator source, DownsamplingMethod downsamplingMethod) { this.iterator = new PeekableIterator(source); this.downsamplingMethod = downsamplingMethod.type != null ? downsamplingMethod : DownsamplingMethod.NONE; - switch(this.downsamplingMethod.type) { + switch (this.downsamplingMethod.type) { case BY_SAMPLE: - if(downsamplingMethod.toCoverage == null) + if (downsamplingMethod.toCoverage == null) throw new UserException.BadArgumentValue("dcov", "Downsampling coverage (-dcov) must be specified when downsampling by sample"); this.targetCoverage = downsamplingMethod.toCoverage; break; @@ -529,10 +555,10 @@ public ReadStateManager(Iterator source, DownsamplingMethod downsampl this.targetCoverage = Integer.MAX_VALUE; } - Map readSelectors = new HashMap(); - for(final String sample: samples) { - readStatesBySample.put(sample,new PerSampleReadStateManager()); - readSelectors.put(sample,downsamplingMethod.type == DownsampleType.BY_SAMPLE ? new NRandomReadSelector(null,targetCoverage) : new AllReadsSelector()); + Map readSelectors = new HashMap(); + for (final String sample : samples) { + readStatesBySample.put(sample, new PerSampleReadStateManager()); + readSelectors.put(sample, downsamplingMethod.type == DownsampleType.BY_SAMPLE ? new NRandomReadSelector(null, targetCoverage) : new AllReadsSelector()); } samplePartitioner = new SamplePartitioner(readSelectors); @@ -541,6 +567,7 @@ public ReadStateManager(Iterator source, DownsamplingMethod downsampl /** * Returns a iterator over all the reads associated with the given sample. Note that remove() is implemented * for this iterator; if present, total read states will be decremented. + * * @param sample The sample. * @return Iterator over the reads associated with that sample. */ @@ -569,6 +596,7 @@ public boolean isEmpty() { /** * Retrieves the total number of reads in the manager across all samples. + * * @return Total number of reads over all samples. */ public int size() { @@ -577,6 +605,7 @@ public int size() { /** * Retrieves the total number of reads in the manager in the given sample. + * * @param sample The sample. * @return Total number of reads in the given sample. */ @@ -587,6 +616,7 @@ public int size(final String sample) { /** * The extent of downsampling; basically, the furthest base out which has 'fallen * victim' to the downsampler. + * * @param sample Sample, downsampled independently. * @return Integer stop of the furthest undownsampled region. */ @@ -595,9 +625,9 @@ public int getDownsamplingExtent(final String sample) { } public SAMRecordState getFirst() { - for(final String sample: samples) { + for (final String sample : samples) { PerSampleReadStateManager reads = readStatesBySample.get(sample); - if(!reads.isEmpty()) + if (!reads.isEmpty()) return reads.peek(); } return null; @@ -608,19 +638,18 @@ public boolean hasNext() { } public void collectPendingReads() { - if(!iterator.hasNext()) + if (!iterator.hasNext()) return; - if(readStates.size() == 0) { + if (readStates.size() == 0) { int firstContigIndex = iterator.peek().getReferenceIndex(); int firstAlignmentStart = iterator.peek().getAlignmentStart(); - while(iterator.hasNext() && iterator.peek().getReferenceIndex() == firstContigIndex && iterator.peek().getAlignmentStart() == firstAlignmentStart) { + while (iterator.hasNext() && iterator.peek().getReferenceIndex() == firstContigIndex && iterator.peek().getAlignmentStart() == firstAlignmentStart) { samplePartitioner.submitRead(iterator.next()); } - } - else { + } else { // Fast fail in the case that the read is past the current position. - if(readIsPastCurrentPosition(iterator.peek())) + if (readIsPastCurrentPosition(iterator.peek())) return; while (iterator.hasNext() && !readIsPastCurrentPosition(iterator.peek())) { @@ -629,7 +658,7 @@ public void collectPendingReads() { } samplePartitioner.complete(); - for(final String sample: samples) { + for (final String sample : samples) { ReadSelector aggregator = samplePartitioner.getSelectedReads(sample); Collection newReads = new ArrayList(aggregator.getSelectedReads()); @@ -638,21 +667,20 @@ public void collectPendingReads() { int numReads = statesBySample.size(); int downsamplingExtent = aggregator.getDownsamplingExtent(); - if(numReads+newReads.size()<=targetCoverage || downsamplingMethod.type==DownsampleType.NONE) { + if (numReads + newReads.size() <= targetCoverage || downsamplingMethod.type == DownsampleType.NONE) { long readLimit = aggregator.getNumReadsSeen(); - addReadsToSample(statesBySample,newReads,readLimit); + addReadsToSample(statesBySample, newReads, readLimit); statesBySample.specifyNewDownsamplingExtent(downsamplingExtent); - } - else { + } else { int[] counts = statesBySample.getCountsPerAlignmentStart(); int[] updatedCounts = new int[counts.length]; - System.arraycopy(counts,0,updatedCounts,0,counts.length); + System.arraycopy(counts, 0, updatedCounts, 0, counts.length); boolean readPruned = true; - while(numReads+newReads.size()>targetCoverage && readPruned) { + while (numReads + newReads.size() > targetCoverage && readPruned) { readPruned = false; - for(int alignmentStart=updatedCounts.length-1;numReads+newReads.size()>targetCoverage&&alignmentStart>=0;alignmentStart--) { - if(updatedCounts[alignmentStart] > 1) { + for (int alignmentStart = updatedCounts.length - 1; numReads + newReads.size() > targetCoverage && alignmentStart >= 0; alignmentStart--) { + if (updatedCounts[alignmentStart] > 1) { updatedCounts[alignmentStart]--; numReads--; readPruned = true; @@ -660,7 +688,7 @@ public void collectPendingReads() { } } - if(numReads == targetCoverage) { + if (numReads == targetCoverage) { updatedCounts[0]--; numReads--; } @@ -668,18 +696,18 @@ public void collectPendingReads() { BitSet toPurge = new BitSet(readStates.size()); int readOffset = 0; - for(int i = 0; i < updatedCounts.length; i++) { + for (int i = 0; i < updatedCounts.length; i++) { int n = counts[i]; int k = updatedCounts[i]; - for(Integer purgedElement: MathUtils.sampleIndicesWithoutReplacement(n,n-k)) - toPurge.set(readOffset+purgedElement); + for (Integer purgedElement : MathUtils.sampleIndicesWithoutReplacement(n, n - k)) + toPurge.set(readOffset + purgedElement); readOffset += counts[i]; } - downsamplingExtent = Math.max(downsamplingExtent,statesBySample.purge(toPurge)); - - addReadsToSample(statesBySample,newReads,targetCoverage-numReads); + downsamplingExtent = Math.max(downsamplingExtent, statesBySample.purge(toPurge)); + + addReadsToSample(statesBySample, newReads, targetCoverage - numReads); statesBySample.specifyNewDownsamplingExtent(downsamplingExtent); } } @@ -688,23 +716,25 @@ public void collectPendingReads() { /** * Add reads with the given sample name to the given hanger entry. + * * @param readStates The list of read states to add this collection of reads. - * @param reads Reads to add. Selected reads will be pulled from this source. - * @param maxReads Maximum number of reads to add. + * @param reads Reads to add. Selected reads will be pulled from this source. + * @param maxReads Maximum number of reads to add. */ private void addReadsToSample(final PerSampleReadStateManager readStates, final Collection reads, final long maxReads) { - if(reads.isEmpty()) + if (reads.isEmpty()) return; Collection newReadStates = new LinkedList(); int readCount = 0; - for(SAMRecord read: reads) { - if(readCount < maxReads) { + for (SAMRecord read : reads) { + if (readCount < maxReads) { SAMRecordState state = new SAMRecordState(read, readInfo.generateExtendedEvents()); state.stepForwardOnGenome(); newReadStates.add(state); // TODO: What if we downsample the extended events away? - if (state.hadIndel()) hasExtendedEvents = true; + if (state.hadIndel()) + hasExtendedEvents = true; readCount++; } } @@ -735,7 +765,7 @@ public int size() { } public void specifyNewDownsamplingExtent(int downsamplingExtent) { - this.downsamplingExtent = Math.max(this.downsamplingExtent,downsamplingExtent); + this.downsamplingExtent = Math.max(this.downsamplingExtent, downsamplingExtent); } public int getDownsamplingExtent() { @@ -745,7 +775,7 @@ public int getDownsamplingExtent() { public int[] getCountsPerAlignmentStart() { int[] counts = new int[readStateCounter.size()]; int index = 0; - for(Counter counter: readStateCounter) + for (Counter counter : readStateCounter) counts[index++] = counter.getCount(); return counts; } @@ -766,7 +796,7 @@ public void remove() { wrappedIterator.remove(); Counter counter = readStateCounter.peek(); counter.decrement(); - if(counter.getCount() == 0) + if (counter.getCount() == 0) readStateCounter.remove(); } }; @@ -775,13 +805,14 @@ public void remove() { /** * Purge the given elements from the bitset. If an element in the bitset is true, purge * the corresponding read state. + * * @param elements bits from the set to purge. * @return the extent of the final downsampled read. */ public int purge(final BitSet elements) { int downsamplingExtent = 0; - if(elements.isEmpty() || readStates.isEmpty()) return downsamplingExtent; + if (elements.isEmpty() || readStates.isEmpty()) return downsamplingExtent; Iterator readStateIterator = readStates.iterator(); @@ -794,22 +825,22 @@ public int purge(final BitSet elements) { int toPurge = elements.nextSetBit(0); int removedCount = 0; - while(readStateIterator.hasNext() && toPurge >= 0) { + while (readStateIterator.hasNext() && toPurge >= 0) { SAMRecordState state = readStateIterator.next(); - downsamplingExtent = Math.max(downsamplingExtent,state.getRead().getAlignmentEnd()); + downsamplingExtent = Math.max(downsamplingExtent, state.getRead().getAlignmentEnd()); - if(readIndex == toPurge) { + if (readIndex == toPurge) { readStateIterator.remove(); currentCounter.decrement(); - if(currentCounter.getCount() == 0) + if (currentCounter.getCount() == 0) counterIterator.remove(); removedCount++; - toPurge = elements.nextSetBit(toPurge+1); + toPurge = elements.nextSetBit(toPurge + 1); } readIndex++; alignmentStartCounter--; - if(alignmentStartCounter == 0 && counterIterator.hasNext()) { + if (alignmentStartCounter == 0 && counterIterator.hasNext()) { currentCounter = counterIterator.next(); alignmentStartCounter = currentCounter.getCount(); } @@ -849,12 +880,14 @@ public void decrement() { interface ReadSelector { /** * All previous selectors in the chain have allowed this read. Submit it to this selector for consideration. + * * @param read the read to evaluate. */ public void submitRead(SAMRecord read); /** * A previous selector has deemed this read unfit. Notify this selector so that this selector's counts are valid. + * * @param read the read previously rejected. */ public void notifyReadRejected(SAMRecord read); @@ -866,12 +899,14 @@ interface ReadSelector { /** * Retrieve the number of reads seen by this selector so far. + * * @return number of reads seen. */ public long getNumReadsSeen(); /** * Return the number of reads accepted by this selector so far. + * * @return number of reads selected. */ public long getNumReadsSelected(); @@ -880,12 +915,14 @@ interface ReadSelector { * Gets the locus at which the last of the downsampled reads selected by this selector ends. The value returned will be the * last aligned position from this selection to which a downsampled read aligns -- in other words, if a read is thrown out at * position 3 whose cigar string is 76M, the value of this parameter will be 78. + * * @return If any read has been downsampled, this will return the last aligned base of the longest alignment. Else, 0. */ public int getDownsamplingExtent(); /** * Get the reads selected by this selector. + * * @return collection of reads selected by this selector. */ public Collection getSelectedReads(); @@ -911,7 +948,7 @@ public void submitRead(SAMRecord read) { public void notifyReadRejected(SAMRecord read) { readsSeen++; - downsamplingExtent = Math.max(downsamplingExtent,read.getAlignmentEnd()); + downsamplingExtent = Math.max(downsamplingExtent, read.getAlignmentEnd()); } public void complete() { @@ -949,18 +986,18 @@ class NRandomReadSelector implements ReadSelector { private final ReservoirDownsampler reservoir; private final ReadSelector chainedSelector; private long readsSeen = 0; - private int downsamplingExtent = 0; + private int downsamplingExtent = 0; public NRandomReadSelector(ReadSelector chainedSelector, long readLimit) { - this.reservoir = new ReservoirDownsampler((int)readLimit); + this.reservoir = new ReservoirDownsampler((int) readLimit); this.chainedSelector = chainedSelector; } public void submitRead(SAMRecord read) { SAMRecord displaced = reservoir.add(read); - if(displaced != null && chainedSelector != null) { + if (displaced != null && chainedSelector != null) { chainedSelector.notifyReadRejected(read); - downsamplingExtent = Math.max(downsamplingExtent,read.getAlignmentEnd()); + downsamplingExtent = Math.max(downsamplingExtent, read.getAlignmentEnd()); } readsSeen++; } @@ -970,9 +1007,9 @@ public void notifyReadRejected(SAMRecord read) { } public void complete() { - for(SAMRecord read: reservoir.getDownsampledContents()) + for (SAMRecord read : reservoir.getDownsampledContents()) chainedSelector.submitRead(read); - if(chainedSelector != null) + if (chainedSelector != null) chainedSelector.complete(); } @@ -987,7 +1024,7 @@ public long getNumReadsSelected() { public int getDownsamplingExtent() { return downsamplingExtent; - } + } public Collection getSelectedReads() { return reservoir.getDownsampledContents(); @@ -996,7 +1033,7 @@ public Collection getSelectedReads() { public void reset() { reservoir.clear(); downsamplingExtent = 0; - if(chainedSelector != null) + if (chainedSelector != null) chainedSelector.reset(); } } @@ -1005,23 +1042,23 @@ public void reset() { * Note: stores reads by sample ID string, not by sample object */ class SamplePartitioner implements ReadSelector { - private final Map readsBySample; + private final Map readsBySample; private long readsSeen = 0; - public SamplePartitioner(Map readSelectors) { + public SamplePartitioner(Map readSelectors) { readsBySample = readSelectors; } public void submitRead(SAMRecord read) { - String sampleName = read.getReadGroup()!=null ? read.getReadGroup().getSample() : null; - if(readsBySample.containsKey(sampleName)) + String sampleName = read.getReadGroup() != null ? read.getReadGroup().getSample() : null; + if (readsBySample.containsKey(sampleName)) readsBySample.get(sampleName).submitRead(read); readsSeen++; } public void notifyReadRejected(SAMRecord read) { - String sampleName = read.getReadGroup()!=null ? read.getReadGroup().getSample() : null; - if(readsBySample.containsKey(sampleName)) + String sampleName = read.getReadGroup() != null ? read.getReadGroup().getSample() : null; + if (readsBySample.containsKey(sampleName)) readsBySample.get(sampleName).notifyReadRejected(read); readsSeen++; } @@ -1040,23 +1077,23 @@ public long getNumReadsSelected() { public int getDownsamplingExtent() { int downsamplingExtent = 0; - for(ReadSelector storage: readsBySample.values()) - downsamplingExtent = Math.max(downsamplingExtent,storage.getDownsamplingExtent()); + for (ReadSelector storage : readsBySample.values()) + downsamplingExtent = Math.max(downsamplingExtent, storage.getDownsamplingExtent()); return downsamplingExtent; } - + public Collection getSelectedReads() { throw new UnsupportedOperationException("Cannot directly get selected reads from a read partitioner."); } public ReadSelector getSelectedReads(String sampleName) { - if(!readsBySample.containsKey(sampleName)) + if (!readsBySample.containsKey(sampleName)) throw new NoSuchElementException("Sample name not found"); return readsBySample.get(sampleName); } public void reset() { - for(ReadSelector storage: readsBySample.values()) + for (ReadSelector storage : readsBySample.values()) storage.reset(); readsSeen = 0; } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java index 312b505ec1..507a6559c2 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java @@ -25,13 +25,13 @@ public class BaseQualityRankSumTest extends RankSumTest { protected void fillQualsFromPileup(byte ref, byte alt, ReadBackedPileup pileup, List refQuals, List altQuals) { for ( final PileupElement p : pileup ) { if( isUsableBase(p) ) { - if ( p.getBase() == ref ) { + if ( p.getBase() == ref ) refQuals.add((double)p.getQual()); - } else if ( p.getBase() == alt ) { + else if ( p.getBase() == alt ) altQuals.add((double)p.getQual()); - } } } + } protected void fillIndelQualsFromPileup(ReadBackedPileup pileup, List refQuals, List altQuals) { // equivalent is whether indel likelihoods for reads corresponding to ref allele are more likely than reads corresponding to alt allele ? @@ -57,8 +57,6 @@ protected void fillIndelQualsFromPileup(ReadBackedPileup pileup, List re refQuals.add(-10.0*refLikelihood); else if (altLikelihood > refLikelihood + INDEL_LIKELIHOOD_THRESH) altQuals.add(-10.0*altLikelihood); - - } } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java index 0dda02421c..987579ab86 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java @@ -205,7 +205,7 @@ private static int[][] getSNPContingencyTable(Map stra for ( Map.Entry sample : stratifiedContexts.entrySet() ) { for (PileupElement p : sample.getValue().getBasePileup()) { - if ( p.isDeletion() || p.isReducedRead() ) // ignore deletions and reduced reads + if ( p.isDeletion() || p.getRead().isReducedRead() ) // ignore deletions and reduced reads continue; if ( p.getRead().getMappingQuality() < 20 || p.getQual() < 20 ) @@ -258,7 +258,7 @@ else if (context.hasBasePileup()) continue; for (final PileupElement p: pileup) { - if ( p.isReducedRead() ) // ignore reduced reads + if ( p.getRead().isReducedRead() ) // ignore reduced reads continue; if ( p.getRead().getMappingQuality() < 20) continue; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java index 551f8e2cf4..40b5aa4d5a 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java @@ -24,7 +24,6 @@ package org.broadinstitute.sting.gatk.walkers.annotator; -import net.sf.samtools.SAMRecord; import org.broadinstitute.sting.gatk.contexts.AlignmentContext; import org.broadinstitute.sting.gatk.contexts.AlignmentContextUtils; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; @@ -43,6 +42,7 @@ import org.broadinstitute.sting.utils.pileup.PileupElement; import org.broadinstitute.sting.utils.pileup.ReadBackedPileup; import org.broadinstitute.sting.utils.sam.AlignmentUtils; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import org.broadinstitute.sting.utils.variantcontext.Allele; import org.broadinstitute.sting.utils.variantcontext.Genotype; import org.broadinstitute.sting.utils.variantcontext.VariantContext; @@ -62,15 +62,15 @@ public class HaplotypeScore extends InfoFieldAnnotation implements StandardAnnot private final static char REGEXP_WILDCARD = '.'; public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc) { - if (stratifiedContexts.size() == 0 ) // size 0 means that call was made by someone else and we have no data here + if (stratifiedContexts.size() == 0) // size 0 means that call was made by someone else and we have no data here return null; - if (vc.isSNP() && !vc.isBiallelic()) + if (vc.isSNP() && !vc.isBiallelic()) return null; final AlignmentContext context = AlignmentContextUtils.joinContexts(stratifiedContexts.values()); - final int contextWingSize = Math.min(((int)ref.getWindow().size() - 1)/2, MIN_CONTEXT_WING_SIZE); + final int contextWingSize = Math.min(((int) ref.getWindow().size() - 1) / 2, MIN_CONTEXT_WING_SIZE); final int contextSize = contextWingSize * 2 + 1; final int locus = ref.getLocus().getStart() + (ref.getLocus().getStop() - ref.getLocus().getStart()) / 2; @@ -84,14 +84,14 @@ else if (context.hasBasePileup()) if (pileup == null) return null; - + final List haplotypes = computeHaplotypes(pileup, contextSize, locus, vc); - final MathUtils.RunningAverage scoreRA = new MathUtils.RunningAverage(); + final MathUtils.RunningAverage scoreRA = new MathUtils.RunningAverage(); if (haplotypes != null) { - for ( final Genotype genotype : vc.getGenotypes()) { + for (final Genotype genotype : vc.getGenotypes()) { final AlignmentContext thisContext = stratifiedContexts.get(genotype.getSampleName()); - if ( thisContext != null ) { + if (thisContext != null) { final ReadBackedPileup thisPileup; if (thisContext.hasExtendedEventPileup()) thisPileup = thisContext.getExtendedEventPileup(); @@ -102,14 +102,13 @@ else if (thisContext.hasBasePileup()) if (thisPileup != null) { if (vc.isSNP()) - scoreRA.add( scoreReadsAgainstHaplotypes(haplotypes, thisPileup, contextSize, locus) ); // Taking the simple average of all sample's score since the score can be negative and the RMS doesn't make sense + scoreRA.add(scoreReadsAgainstHaplotypes(haplotypes, thisPileup, contextSize, locus)); // Taking the simple average of all sample's score since the score can be negative and the RMS doesn't make sense else if (vc.isIndel() || vc.isMixed()) { Double d = scoreIndelsAgainstHaplotypes(thisPileup); if (d == null) return null; - scoreRA.add( d ); // Taking the simple average of all sample's score since the score can be negative and the RMS doesn't make sense - } - else + scoreRA.add(d); // Taking the simple average of all sample's score since the score can be negative and the RMS doesn't make sense + } else return null; } } @@ -122,12 +121,12 @@ else if (vc.isIndel() || vc.isMixed()) { return map; } - private class HaplotypeComparator implements Comparator{ + private class HaplotypeComparator implements Comparator { public int compare(Haplotype a, Haplotype b) { if (a.getQualitySum() < b.getQualitySum()) return 1; - if (a.getQualitySum() > b.getQualitySum()){ + if (a.getQualitySum() > b.getQualitySum()) { return -1; } return 0; @@ -137,39 +136,38 @@ public int compare(Haplotype a, Haplotype b) { private List computeHaplotypes(final ReadBackedPileup pileup, final int contextSize, final int locus, final VariantContext vc) { // Compute all possible haplotypes consistent with current pileup - int haplotypesToCompute = vc.getAlternateAlleles().size()+1; + int haplotypesToCompute = vc.getAlternateAlleles().size() + 1; final PriorityQueue candidateHaplotypeQueue = new PriorityQueue(100, new HaplotypeComparator()); final PriorityQueue consensusHaplotypeQueue = new PriorityQueue(MAX_CONSENSUS_HAPLOTYPES_TO_CONSIDER, new HaplotypeComparator()); - for ( final PileupElement p : pileup ) { + for (final PileupElement p : pileup) { final Haplotype haplotypeFromRead = getHaplotypeFromRead(p, contextSize, locus); candidateHaplotypeQueue.add(haplotypeFromRead); } // Now that priority queue has been built with all reads at context, we need to merge and find possible segregating haplotypes Haplotype elem; - while ((elem = candidateHaplotypeQueue.poll()) != null) { + while ((elem = candidateHaplotypeQueue.poll()) != null) { boolean foundHaplotypeMatch = false; Haplotype lastCheckedHaplotype = null; - for ( final Haplotype haplotypeFromList : consensusHaplotypeQueue ) { + for (final Haplotype haplotypeFromList : consensusHaplotypeQueue) { final Haplotype consensusHaplotype = getConsensusHaplotype(elem, haplotypeFromList); - if (consensusHaplotype != null) { + if (consensusHaplotype != null) { foundHaplotypeMatch = true; if (consensusHaplotype.getQualitySum() > haplotypeFromList.getQualitySum()) { consensusHaplotypeQueue.remove(haplotypeFromList); consensusHaplotypeQueue.add(consensusHaplotype); } break; - } - else { + } else { lastCheckedHaplotype = haplotypeFromList; } } if (!foundHaplotypeMatch && consensusHaplotypeQueue.size() < MAX_CONSENSUS_HAPLOTYPES_TO_CONSIDER) { consensusHaplotypeQueue.add(elem); - } else if (!foundHaplotypeMatch && lastCheckedHaplotype != null && elem.getQualitySum() > lastCheckedHaplotype.getQualitySum() ) { + } else if (!foundHaplotypeMatch && lastCheckedHaplotype != null && elem.getQualitySum() > lastCheckedHaplotype.getQualitySum()) { consensusHaplotypeQueue.remove(lastCheckedHaplotype); consensusHaplotypeQueue.add(elem); } @@ -180,12 +178,14 @@ private List computeHaplotypes(final ReadBackedPileup pileup, final i // The consensus haplotypes are in a quality-ordered priority queue, so the best haplotypes are just the ones at the front of the queue final Haplotype haplotype1 = consensusHaplotypeQueue.poll(); - Listhlist = new ArrayList(); + List hlist = new ArrayList(); hlist.add(new Haplotype(haplotype1.getBases(), 60)); - for (int k=1; k < haplotypesToCompute; k++) { + for (int k = 1; k < haplotypesToCompute; k++) { Haplotype haplotype2 = consensusHaplotypeQueue.poll(); - if(haplotype2 == null ) { haplotype2 = haplotype1; } // Sometimes only the reference haplotype can be found + if (haplotype2 == null) { + haplotype2 = haplotype1; + } // Sometimes only the reference haplotype can be found hlist.add(new Haplotype(haplotype2.getBases(), 20)); } return hlist; @@ -194,36 +194,43 @@ private List computeHaplotypes(final ReadBackedPileup pileup, final i } private Haplotype getHaplotypeFromRead(final PileupElement p, final int contextSize, final int locus) { - final SAMRecord read = p.getRead(); + final GATKSAMRecord read = p.getRead(); int readOffsetFromPileup = p.getOffset(); final byte[] haplotypeBases = new byte[contextSize]; - Arrays.fill(haplotypeBases, (byte)REGEXP_WILDCARD); + Arrays.fill(haplotypeBases, (byte) REGEXP_WILDCARD); final double[] baseQualities = new double[contextSize]; Arrays.fill(baseQualities, 0.0); byte[] readBases = read.getReadBases(); - readBases = AlignmentUtils.readToAlignmentByteArray(p.getRead().getCigar(), readBases); // Adjust the read bases based on the Cigar string + readBases = AlignmentUtils.readToAlignmentByteArray(read.getCigar(), readBases); // Adjust the read bases based on the Cigar string byte[] readQuals = read.getBaseQualities(); - readQuals = AlignmentUtils.readToAlignmentByteArray(p.getRead().getCigar(), readQuals); // Shift the location of the qual scores based on the Cigar string + readQuals = AlignmentUtils.readToAlignmentByteArray(read.getCigar(), readQuals); // Shift the location of the qual scores based on the Cigar string - readOffsetFromPileup = AlignmentUtils.calcAlignmentByteArrayOffset(p.getRead().getCigar(), readOffsetFromPileup, p.getRead().getAlignmentStart(), locus); - final int baseOffsetStart = readOffsetFromPileup - (contextSize - 1)/2; + readOffsetFromPileup = AlignmentUtils.calcAlignmentByteArrayOffset(read.getCigar(), p, read.getAlignmentStart(), locus); + final int baseOffsetStart = readOffsetFromPileup - (contextSize - 1) / 2; - for (int i = 0; i < contextSize; i++ ) { + for (int i = 0; i < contextSize; i++) { final int baseOffset = i + baseOffsetStart; - if ( baseOffset < 0 ) { + if (baseOffset < 0) { continue; } - if ( baseOffset >= readBases.length ) { + if (baseOffset >= readBases.length) { break; } - if( readQuals[baseOffset] == PileupElement.DELETION_BASE) { readQuals[baseOffset] = PileupElement.DELETION_QUAL; } - if( !BaseUtils.isRegularBase(readBases[baseOffset]) ) { readBases[baseOffset] = (byte)REGEXP_WILDCARD; readQuals[baseOffset] = (byte) 0; } // N's shouldn't be treated as distinct bases - readQuals[baseOffset] = (byte)Math.min((int)readQuals[baseOffset], p.getMappingQual()); - if( ((int)readQuals[baseOffset]) < 5 ) { readQuals[baseOffset] = (byte) 0; } // quals less than 5 are used as codes and don't have actual probabilistic meaning behind them + if (readQuals[baseOffset] == PileupElement.DELETION_BASE) { + readQuals[baseOffset] = PileupElement.DELETION_QUAL; + } + if (!BaseUtils.isRegularBase(readBases[baseOffset])) { + readBases[baseOffset] = (byte) REGEXP_WILDCARD; + readQuals[baseOffset] = (byte) 0; + } // N's shouldn't be treated as distinct bases + readQuals[baseOffset] = (byte) Math.min((int) readQuals[baseOffset], p.getMappingQual()); + if (((int) readQuals[baseOffset]) < 5) { + readQuals[baseOffset] = (byte) 0; + } // quals less than 5 are used as codes and don't have actual probabilistic meaning behind them haplotypeBases[i] = readBases[baseOffset]; - baseQualities[i] = (double)readQuals[baseOffset]; + baseQualities[i] = (double) readQuals[baseOffset]; } return new Haplotype(haplotypeBases, baseQualities); @@ -238,7 +245,7 @@ private Haplotype getConsensusHaplotype(final Haplotype haplotypeA, final Haplot } byte chA, chB; - final byte wc = (byte)REGEXP_WILDCARD; + final byte wc = (byte) REGEXP_WILDCARD; final int length = a.length; final byte[] consensusChars = new byte[length]; @@ -247,7 +254,7 @@ private Haplotype getConsensusHaplotype(final Haplotype haplotypeA, final Haplot final double[] qualsA = haplotypeA.getQuals(); final double[] qualsB = haplotypeB.getQuals(); - for (int i=0; i < length; i++) { + for (int i = 0; i < length; i++) { chA = a[i]; chB = b[i]; @@ -257,17 +264,15 @@ private Haplotype getConsensusHaplotype(final Haplotype haplotypeA, final Haplot if ((chA == wc) && (chB == wc)) { consensusChars[i] = wc; consensusQuals[i] = 0.0; - } - else if ((chA == wc)) { + } else if ((chA == wc)) { consensusChars[i] = chB; consensusQuals[i] = qualsB[i]; - } - else if ((chB == wc)){ + } else if ((chB == wc)) { consensusChars[i] = chA; consensusQuals[i] = qualsA[i]; } else { consensusChars[i] = chA; - consensusQuals[i] = qualsA[i]+qualsB[i]; + consensusQuals[i] = qualsA[i] + qualsB[i]; } } @@ -276,31 +281,33 @@ else if ((chB == wc)){ // calculate the haplotype scores by walking over all reads and comparing them to the haplotypes private double scoreReadsAgainstHaplotypes(final List haplotypes, final ReadBackedPileup pileup, final int contextSize, final int locus) { - if ( DEBUG ) System.out.printf("HAP1: %s%n", haplotypes.get(0)); - if ( DEBUG ) System.out.printf("HAP2: %s%n", haplotypes.get(1)); + if (DEBUG) System.out.printf("HAP1: %s%n", haplotypes.get(0)); + if (DEBUG) System.out.printf("HAP2: %s%n", haplotypes.get(1)); final ArrayList haplotypeScores = new ArrayList(); - for ( final PileupElement p : pileup ) { + for (final PileupElement p : pileup) { // Score all the reads in the pileup, even the filtered ones final double[] scores = new double[haplotypes.size()]; - for ( int i = 0; i < haplotypes.size(); i++ ) { + for (int i = 0; i < haplotypes.size(); i++) { final Haplotype haplotype = haplotypes.get(i); final double score = scoreReadAgainstHaplotype(p, contextSize, haplotype, locus); scores[i] = score; - if ( DEBUG ) { System.out.printf(" vs. haplotype %d = %f%n", i, score); } + if (DEBUG) { + System.out.printf(" vs. haplotype %d = %f%n", i, score); + } } haplotypeScores.add(scores); } double overallScore = 0.0; - for ( final double[] readHaplotypeScores : haplotypeScores ) { + for (final double[] readHaplotypeScores : haplotypeScores) { overallScore += MathUtils.arrayMin(readHaplotypeScores); } return overallScore; } - private double scoreReadAgainstHaplotype(final PileupElement p, final int contextSize, final Haplotype haplotype, final int locus ) { + private double scoreReadAgainstHaplotype(final PileupElement p, final int contextSize, final Haplotype haplotype, final int locus) { double expected = 0.0; double mismatches = 0.0; @@ -315,33 +322,35 @@ private double scoreReadAgainstHaplotype(final PileupElement p, final int contex // the chance that it is actually a mismatch is 1 - e, since any of the other 3 options would be a mismatch. // so the probability-weighted mismatch rate is sum_i ( matched ? e_i / 3 : 1 - e_i ) for i = 1 ... n final byte[] haplotypeBases = haplotype.getBases(); - final SAMRecord read = p.getRead(); + final GATKSAMRecord read = p.getRead(); byte[] readBases = read.getReadBases(); readBases = AlignmentUtils.readToAlignmentByteArray(p.getRead().getCigar(), readBases); // Adjust the read bases based on the Cigar string byte[] readQuals = read.getBaseQualities(); readQuals = AlignmentUtils.readToAlignmentByteArray(p.getRead().getCigar(), readQuals); // Shift the location of the qual scores based on the Cigar string int readOffsetFromPileup = p.getOffset(); - readOffsetFromPileup = AlignmentUtils.calcAlignmentByteArrayOffset(p.getRead().getCigar(), readOffsetFromPileup, p.getRead().getAlignmentStart(), locus); - final int baseOffsetStart = readOffsetFromPileup - (contextSize - 1)/2; + readOffsetFromPileup = AlignmentUtils.calcAlignmentByteArrayOffset(p.getRead().getCigar(), p, read.getAlignmentStart(), locus); + final int baseOffsetStart = readOffsetFromPileup - (contextSize - 1) / 2; - for ( int i = 0; i < contextSize; i++ ) { + for (int i = 0; i < contextSize; i++) { final int baseOffset = i + baseOffsetStart; - if ( baseOffset < 0 ) { + if (baseOffset < 0) { continue; } - if ( baseOffset >= readBases.length ) { + if (baseOffset >= readBases.length) { break; } final byte haplotypeBase = haplotypeBases[i]; final byte readBase = readBases[baseOffset]; - final boolean matched = ( readBase == haplotypeBase || haplotypeBase == (byte)REGEXP_WILDCARD ); + final boolean matched = (readBase == haplotypeBase || haplotypeBase == (byte) REGEXP_WILDCARD); byte qual = readQuals[baseOffset]; - if( qual == PileupElement.DELETION_BASE ) { qual = PileupElement.DELETION_QUAL; } // calcAlignmentByteArrayOffset fills the readQuals array with DELETION_BASE at deletions - qual = (byte)Math.min((int)qual, p.getMappingQual()); - if( ((int) qual) >= 5 ) { // quals less than 5 are used as codes and don't have actual probabilistic meaning behind them + if (qual == PileupElement.DELETION_BASE) { + qual = PileupElement.DELETION_QUAL; + } // calcAlignmentByteArrayOffset fills the readQuals array with DELETION_BASE at deletions + qual = (byte) Math.min((int) qual, p.getMappingQual()); + if (((int) qual) >= 5) { // quals less than 5 are used as codes and don't have actual probabilistic meaning behind them final double e = QualityUtils.qualToErrorProb(qual); expected += e; mismatches += matched ? e : 1.0 - e / 3.0; @@ -355,26 +364,27 @@ private double scoreReadAgainstHaplotype(final PileupElement p, final int contex } - private Double scoreIndelsAgainstHaplotypes(final ReadBackedPileup pileup) { final ArrayList haplotypeScores = new ArrayList(); - final HashMap> indelLikelihoodMap = IndelGenotypeLikelihoodsCalculationModel.getIndelLikelihoodMap(); + final HashMap> indelLikelihoodMap = IndelGenotypeLikelihoodsCalculationModel.getIndelLikelihoodMap(); - if (indelLikelihoodMap== null) + if (indelLikelihoodMap == null) return null; - for (final PileupElement p: pileup) { + for (final PileupElement p : pileup) { if (indelLikelihoodMap.containsKey(p)) { // retrieve likelihood information corresponding to this read - LinkedHashMap el = indelLikelihoodMap.get(p); + LinkedHashMap el = indelLikelihoodMap.get(p); // Score all the reads in the pileup, even the filtered ones final double[] scores = new double[el.size()]; int i = 0; - for (Allele a: el.keySet() ) { + for (Allele a : el.keySet()) { scores[i++] = -el.get(a); - if ( DEBUG ) { System.out.printf(" vs. haplotype %d = %f%n", i-1, scores[i-1]); } + if (DEBUG) { + System.out.printf(" vs. haplotype %d = %f%n", i - 1, scores[i - 1]); + } } haplotypeScores.add(scores); @@ -383,7 +393,7 @@ private Double scoreIndelsAgainstHaplotypes(final ReadBackedPileup pileup) { // indel likelihoods are stric log-probs, not phred scored double overallScore = 0.0; - for ( final double[] readHaplotypeScores : haplotypeScores ) { + for (final double[] readHaplotypeScores : haplotypeScores) { overallScore += MathUtils.arrayMin(readHaplotypeScores); } @@ -392,6 +402,11 @@ private Double scoreIndelsAgainstHaplotypes(final ReadBackedPileup pileup) { } - public List getKeyNames() { return Arrays.asList("HaplotypeScore"); } - public List getDescriptions() { return Arrays.asList(new VCFInfoHeaderLine("HaplotypeScore", 1, VCFHeaderLineType.Float, "Consistency of the site with at most two segregating haplotypes")); } + public List getKeyNames() { + return Arrays.asList("HaplotypeScore"); + } + + public List getDescriptions() { + return Arrays.asList(new VCFInfoHeaderLine("HaplotypeScore", 1, VCFHeaderLineType.Float, "Consistency of the site with at most two segregating haplotypes")); + } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java index c5a2df1fd5..e0e62cdb0f 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java @@ -30,11 +30,11 @@ public abstract class RankSumTest extends InfoFieldAnnotation implements Standar static final boolean DEBUG = false; public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc) { - if ( stratifiedContexts.size() == 0 ) + if (stratifiedContexts.size() == 0) return null; - + final GenotypesContext genotypes = vc.getGenotypes(); - if ( genotypes == null || genotypes.size() == 0 ) + if (genotypes == null || genotypes.size() == 0) return null; @@ -43,19 +43,18 @@ public Map annotate(RefMetaDataTracker tracker, AnnotatorCompati if (vc.isSNP() && vc.isBiallelic()) { // todo - no current support for multiallelic snps - for ( final Genotype genotype : genotypes.iterateInSampleNameOrder() ) { + for (final Genotype genotype : genotypes.iterateInSampleNameOrder()) { final AlignmentContext context = stratifiedContexts.get(genotype.getSampleName()); - if ( context == null ) { + if (context == null) { continue; } fillQualsFromPileup(ref.getBase(), vc.getAlternateAllele(0).getBases()[0], context.getBasePileup(), refQuals, altQuals); } - } - else if (vc.isIndel() || vc.isMixed()) { + } else if (vc.isIndel() || vc.isMixed()) { - for ( final Genotype genotype : genotypes.iterateInSampleNameOrder() ) { + for (final Genotype genotype : genotypes.iterateInSampleNameOrder()) { final AlignmentContext context = stratifiedContexts.get(genotype.getSampleName()); - if ( context == null ) { + if (context == null) { continue; } @@ -74,46 +73,47 @@ else if (context.hasBasePileup()) fillIndelQualsFromPileup(pileup, refQuals, altQuals); } - } - else + } else return null; final MannWhitneyU mannWhitneyU = new MannWhitneyU(); - for ( final Double qual : altQuals ) { + for (final Double qual : altQuals) { mannWhitneyU.add(qual, MannWhitneyU.USet.SET1); } - for ( final Double qual : refQuals ) { + for (final Double qual : refQuals) { mannWhitneyU.add(qual, MannWhitneyU.USet.SET2); } if (DEBUG) { - System.out.format("%s, REF QUALS:",this.getClass().getName()); - for ( final Double qual : refQuals ) - System.out.format("%4.1f ",qual); + System.out.format("%s, REF QUALS:", this.getClass().getName()); + for (final Double qual : refQuals) + System.out.format("%4.1f ", qual); System.out.println(); - System.out.format("%s, ALT QUALS:",this.getClass().getName()); - for ( final Double qual : altQuals ) - System.out.format("%4.1f ",qual); + System.out.format("%s, ALT QUALS:", this.getClass().getName()); + for (final Double qual : altQuals) + System.out.format("%4.1f ", qual); System.out.println(); } // we are testing that set1 (the alt bases) have lower quality scores than set2 (the ref bases) - final Pair testResults = mannWhitneyU.runOneSidedTest( MannWhitneyU.USet.SET1 ); + final Pair testResults = mannWhitneyU.runOneSidedTest(MannWhitneyU.USet.SET1); final Map map = new HashMap(); - if ( ! Double.isNaN(testResults.first) ) + if (!Double.isNaN(testResults.first)) map.put(getKeyNames().get(0), String.format("%.3f", testResults.first)); return map; } protected abstract void fillQualsFromPileup(byte ref, byte alt, ReadBackedPileup pileup, List refQuals, List altQuals); + protected abstract void fillIndelQualsFromPileup(ReadBackedPileup pileup, List refQuals, List altQuals); - protected static boolean isUsableBase( final PileupElement p ) { - return !( p.isDeletion() || - p.getMappingQual() == 0 || - p.getMappingQual() == QualityUtils.MAPPING_QUALITY_UNAVAILABLE || - ((int)p.getQual()) < QualityUtils.MIN_USABLE_Q_SCORE ); // need the unBAQed quality score here + protected static boolean isUsableBase(final PileupElement p) { + return !(p.isInsertionAtBeginningOfRead() || + p.isDeletion() || + p.getMappingQual() == 0 || + p.getMappingQual() == QualityUtils.MAPPING_QUALITY_UNAVAILABLE || + ((int) p.getQual()) < QualityUtils.MIN_USABLE_Q_SCORE); // need the unBAQed quality score here } } \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java index d762af4284..b0039d1a00 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java @@ -24,27 +24,32 @@ */ public class ReadPosRankSumTest extends RankSumTest { - public List getKeyNames() { return Arrays.asList("ReadPosRankSum"); } + public List getKeyNames() { + return Arrays.asList("ReadPosRankSum"); + } - public List getDescriptions() { return Arrays.asList(new VCFInfoHeaderLine("ReadPosRankSum", 1, VCFHeaderLineType.Float, "Z-score from Wilcoxon rank sum test of Alt vs. Ref read position bias")); } + public List getDescriptions() { + return Arrays.asList(new VCFInfoHeaderLine("ReadPosRankSum", 1, VCFHeaderLineType.Float, "Z-score from Wilcoxon rank sum test of Alt vs. Ref read position bias")); + } protected void fillQualsFromPileup(byte ref, byte alt, ReadBackedPileup pileup, List refQuals, List altQuals) { - for ( final PileupElement p : pileup ) { - if( isUsableBase(p) ) { - int readPos = AlignmentUtils.calcAlignmentByteArrayOffset(p.getRead().getCigar(), p.getOffset(), 0, 0); + for (final PileupElement p : pileup) { + if (isUsableBase(p)) { + int readPos = AlignmentUtils.calcAlignmentByteArrayOffset(p.getRead().getCigar(), p, 0, 0); final int numAlignedBases = AlignmentUtils.getNumAlignedBases(p.getRead()); - if( readPos > numAlignedBases / 2 ) { - readPos = numAlignedBases - ( readPos + 1 ); - } + if (readPos > numAlignedBases / 2) + readPos = numAlignedBases - (readPos + 1); + + + if (p.getBase() == ref) + refQuals.add((double) readPos); + else if (p.getBase() == alt) + altQuals.add((double) readPos); - if ( p.getBase() == ref ) { - refQuals.add( (double)readPos ); - } else if ( p.getBase() == alt ) { - altQuals.add( (double)readPos ); - } } } } + protected void fillIndelQualsFromPileup(ReadBackedPileup pileup, List refQuals, List altQuals) { // equivalent is whether indel likelihoods for reads corresponding to ref allele are more likely than reads corresponding to alt allele // to classify a pileup element as ref or alt, we look at the likelihood associated with the allele associated to this element. @@ -52,18 +57,15 @@ protected void fillIndelQualsFromPileup(ReadBackedPileup pileup, List re // To classify a pileup element as Ref or Alt, we look at the likelihood of corresponding alleles. // If likelihood of ref allele > highest likelihood of all alt alleles + epsilon, then this pielup element is "ref" // otherwise if highest alt allele likelihood is > ref likelihood + epsilon, then this pileup element it "alt" - final HashMap> indelLikelihoodMap = IndelGenotypeLikelihoodsCalculationModel.getIndelLikelihoodMap(); - for (final PileupElement p: pileup) { + final HashMap> indelLikelihoodMap = IndelGenotypeLikelihoodsCalculationModel.getIndelLikelihoodMap(); + for (final PileupElement p : pileup) { if (indelLikelihoodMap.containsKey(p)) { - // retrieve likelihood information corresponding to this read - LinkedHashMap el = indelLikelihoodMap.get(p); - // by design, first element in LinkedHashMap was ref allele - double refLikelihood=0.0, altLikelihood=Double.NEGATIVE_INFINITY; + LinkedHashMap el = indelLikelihoodMap.get(p); // retrieve likelihood information corresponding to this read + double refLikelihood = 0.0, altLikelihood = Double.NEGATIVE_INFINITY; // by design, first element in LinkedHashMap was ref allele for (Allele a : el.keySet()) { - if (a.isReference()) - refLikelihood =el.get(a); + refLikelihood = el.get(a); else { double like = el.get(a); if (like >= altLikelihood) @@ -75,23 +77,22 @@ protected void fillIndelQualsFromPileup(ReadBackedPileup pileup, List re final int numAlignedBases = getNumAlignedBases(p.getRead()); int rp = readPos; - if( readPos > numAlignedBases / 2 ) { - readPos = numAlignedBases - ( readPos + 1 ); + if (readPos > numAlignedBases / 2) { + readPos = numAlignedBases - (readPos + 1); } - //if (DEBUG) System.out.format("R:%s start:%d C:%s offset:%d rp:%d readPos:%d alignedB:%d\n",p.getRead().getReadName(),p.getRead().getAlignmentStart(),p.getRead().getCigarString(),p.getOffset(), rp, readPos, numAlignedBases); + //if (DEBUG) System.out.format("R:%s start:%d C:%s offset:%d rp:%d readPos:%d alignedB:%d\n",p.getRead().getReadName(),p.getRead().getAlignmentStart(),p.getRead().getCigarString(),p.getOffset(), rp, readPos, numAlignedBases); // if event is beyond span of read just return and don't consider this element. This can happen, for example, with reads // where soft clipping still left strings of low quality bases but these are later removed by indel-specific clipping. - // if (readPos < -1) + // if (readPos < -1) // return; - if (refLikelihood > (altLikelihood + INDEL_LIKELIHOOD_THRESH)) { - refQuals.add((double)readPos); + if (refLikelihood > (altLikelihood + INDEL_LIKELIHOOD_THRESH)) { + refQuals.add((double) readPos); //if (DEBUG) System.out.format("REF like: %4.1f, pos: %d\n",refLikelihood,readPos); - } - else if (altLikelihood > (refLikelihood + INDEL_LIKELIHOOD_THRESH)) { - altQuals.add((double)readPos); - //if (DEBUG) System.out.format("ALT like: %4.1f, pos: %d\n",refLikelihood,readPos); + } else if (altLikelihood > (refLikelihood + INDEL_LIKELIHOOD_THRESH)) { + altQuals.add((double) readPos); + //if (DEBUG) System.out.format("ALT like: %4.1f, pos: %d\n",refLikelihood,readPos); } @@ -115,7 +116,7 @@ int getNumClippedBasesAtStart(SAMRecord read) { // Do a stricter base clipping than provided by CIGAR string, since this one may be too conservative, // and may leave a string of Q2 bases still hanging off the reads. - for (int i=numStartClippedBases; i < unclippedReadBases.length; i++) { + for (int i = numStartClippedBases; i < unclippedReadBases.length; i++) { if (unclippedReadQuals[i] < PairHMMIndelErrorModel.BASE_QUAL_THRESHOLD) numStartClippedBases++; else @@ -134,7 +135,7 @@ int getNumClippedBasesAtEnd(SAMRecord read) { // compute total number of clipped bases (soft or hard clipped) // check for hard clips (never consider these bases): final Cigar c = read.getCigar(); - CigarElement last = c.getCigarElement(c.numCigarElements()-1); + CigarElement last = c.getCigarElement(c.numCigarElements() - 1); int numEndClippedBases = 0; if (last.getOperator() == CigarOperator.H) { @@ -145,7 +146,7 @@ int getNumClippedBasesAtEnd(SAMRecord read) { // Do a stricter base clipping than provided by CIGAR string, since this one may be too conservative, // and may leave a string of Q2 bases still hanging off the reads. - for (int i=unclippedReadBases.length-numEndClippedBases-1; i >= 0; i-- ){ + for (int i = unclippedReadBases.length - numEndClippedBases - 1; i >= 0; i--) { if (unclippedReadQuals[i] < PairHMMIndelErrorModel.BASE_QUAL_THRESHOLD) numEndClippedBases++; else @@ -157,8 +158,6 @@ int getNumClippedBasesAtEnd(SAMRecord read) { } int getOffsetFromClippedReadStart(SAMRecord read, int offset) { - - - return offset - getNumClippedBasesAtStart(read); + return offset - getNumClippedBasesAtStart(read); } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidSNPGenotypeLikelihoods.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidSNPGenotypeLikelihoods.java index ae70772305..7143606aeb 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidSNPGenotypeLikelihoods.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidSNPGenotypeLikelihoods.java @@ -278,7 +278,7 @@ public int add(PileupElement elt, boolean ignoreBadBases, boolean capBaseQualsAt if ( qual == 0 ) return 0; - if ( elt.isReducedRead() ) { + if ( elt.getRead().isReducedRead() ) { // reduced read representation if ( BaseUtils.isRegularBase( obsBase )) { int representativeCount = elt.getRepresentativeCount(); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java index 0756caf03a..9126c04956 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java @@ -60,14 +60,14 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood private final int maxAlternateAlleles; private PairHMMIndelErrorModel pairModel; - private static ThreadLocal>> indelLikelihoodMap = - new ThreadLocal>>() { - protected synchronized HashMap> initialValue() { - return new HashMap>(); + private static ThreadLocal>> indelLikelihoodMap = + new ThreadLocal>>() { + protected synchronized HashMap> initialValue() { + return new HashMap>(); } }; - private LinkedHashMap haplotypeMap; + private LinkedHashMap haplotypeMap; // gdebug removeme // todo -cleanup @@ -75,13 +75,13 @@ protected synchronized HashMap> initi private ArrayList alleleList; static { - indelLikelihoodMap.set(new HashMap>()); + indelLikelihoodMap.set(new HashMap>()); } protected IndelGenotypeLikelihoodsCalculationModel(UnifiedArgumentCollection UAC, Logger logger) { super(UAC, logger); - pairModel = new PairHMMIndelErrorModel(UAC.INDEL_GAP_OPEN_PENALTY,UAC.INDEL_GAP_CONTINUATION_PENALTY, + pairModel = new PairHMMIndelErrorModel(UAC.INDEL_GAP_OPEN_PENALTY, UAC.INDEL_GAP_CONTINUATION_PENALTY, UAC.OUTPUT_DEBUG_INDEL_INFO, !UAC.DONT_DO_BANDED_INDEL_COMPUTATION); alleleList = new ArrayList(); getAlleleListFromVCF = UAC.GenotypingMode == GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES; @@ -91,7 +91,7 @@ protected IndelGenotypeLikelihoodsCalculationModel(UnifiedArgumentCollection UAC maxAlternateAlleles = UAC.MAX_ALTERNATE_ALLELES; doMultiAllelicCalls = UAC.MULTI_ALLELIC; - haplotypeMap = new LinkedHashMap(); + haplotypeMap = new LinkedHashMap(); ignoreSNPAllelesWhenGenotypingIndels = UAC.IGNORE_SNP_ALLELES; } @@ -99,15 +99,15 @@ protected IndelGenotypeLikelihoodsCalculationModel(UnifiedArgumentCollection UAC private ArrayList computeConsensusAlleles(ReferenceContext ref, Map contexts, AlignmentContextUtils.ReadOrientation contextType, GenomeLocParser locParser) { - Allele refAllele=null, altAllele=null; + Allele refAllele = null, altAllele = null; GenomeLoc loc = ref.getLocus(); ArrayList aList = new ArrayList(); - HashMap consensusIndelStrings = new HashMap(); + HashMap consensusIndelStrings = new HashMap(); int insCount = 0, delCount = 0; // quick check of total number of indels in pileup - for ( Map.Entry sample : contexts.entrySet() ) { + for (Map.Entry sample : contexts.entrySet()) { AlignmentContext context = AlignmentContextUtils.stratify(sample.getValue(), contextType); final ReadBackedExtendedEventPileup indelPileup = context.getExtendedEventPileup(); @@ -118,21 +118,19 @@ private ArrayList computeConsensusAlleles(ReferenceContext ref, if (insCount < minIndelCountForGenotyping && delCount < minIndelCountForGenotyping) return aList; - for ( Map.Entry sample : contexts.entrySet() ) { + for (Map.Entry sample : contexts.entrySet()) { // todo -- warning, can be duplicating expensive partition here AlignmentContext context = AlignmentContextUtils.stratify(sample.getValue(), contextType); final ReadBackedExtendedEventPileup indelPileup = context.getExtendedEventPileup(); - - - for ( ExtendedEventPileupElement p : indelPileup.toExtendedIterable() ) { + for (ExtendedEventPileupElement p : indelPileup.toExtendedIterable()) { //SAMRecord read = p.getRead(); GATKSAMRecord read = ReadClipper.hardClipAdaptorSequence(p.getRead()); if (read == null) continue; - if(ReadUtils.is454Read(read)) { + if (ReadUtils.is454Read(read)) { continue; } @@ -151,62 +149,57 @@ private ArrayList computeConsensusAlleles(ReferenceContext ref, // In this case, the read could have any of the inserted bases and we need to build a consensus for (String s : consensusIndelStrings.keySet()) { int cnt = consensusIndelStrings.get(s); - if (s.startsWith(indelString)){ + if (s.startsWith(indelString)) { // case 1: current insertion is prefix of indel in hash map - consensusIndelStrings.put(s,cnt+1); + consensusIndelStrings.put(s, cnt + 1); foundKey = true; break; - } - else if (indelString.startsWith(s)) { + } else if (indelString.startsWith(s)) { // case 2: indel stored in hash table is prefix of current insertion // In this case, new bases are new key. consensusIndelStrings.remove(s); - consensusIndelStrings.put(indelString,cnt+1); + consensusIndelStrings.put(indelString, cnt + 1); foundKey = true; break; } } if (!foundKey) // none of the above: event bases not supported by previous table, so add new key - consensusIndelStrings.put(indelString,1); + consensusIndelStrings.put(indelString, 1); - } - else if (read.getAlignmentStart() == loc.getStart()+1) { + } else if (read.getAlignmentStart() == loc.getStart() + 1) { // opposite corner condition: read will start at current locus with an insertion for (String s : consensusIndelStrings.keySet()) { int cnt = consensusIndelStrings.get(s); - if (s.endsWith(indelString)){ + if (s.endsWith(indelString)) { // case 1: current insertion is suffix of indel in hash map - consensusIndelStrings.put(s,cnt+1); + consensusIndelStrings.put(s, cnt + 1); foundKey = true; break; - } - else if (indelString.endsWith(s)) { + } else if (indelString.endsWith(s)) { // case 2: indel stored in hash table is suffix of current insertion // In this case, new bases are new key. consensusIndelStrings.remove(s); - consensusIndelStrings.put(indelString,cnt+1); + consensusIndelStrings.put(indelString, cnt + 1); foundKey = true; break; } } if (!foundKey) // none of the above: event bases not supported by previous table, so add new key - consensusIndelStrings.put(indelString,1); + consensusIndelStrings.put(indelString, 1); - } - else { + } else { // normal case: insertion somewhere in the middle of a read: add count to hash map - int cnt = consensusIndelStrings.containsKey(indelString)? consensusIndelStrings.get(indelString):0; - consensusIndelStrings.put(indelString,cnt+1); + int cnt = consensusIndelStrings.containsKey(indelString) ? consensusIndelStrings.get(indelString) : 0; + consensusIndelStrings.put(indelString, cnt + 1); } - } - else if (p.isDeletion()) { - indelString = String.format("D%d",p.getEventLength()); - int cnt = consensusIndelStrings.containsKey(indelString)? consensusIndelStrings.get(indelString):0; - consensusIndelStrings.put(indelString,cnt+1); + } else if (p.isDeletion()) { + indelString = String.format("D%d", p.getEventLength()); + int cnt = consensusIndelStrings.containsKey(indelString) ? consensusIndelStrings.get(indelString) : 0; + consensusIndelStrings.put(indelString, cnt + 1); } } @@ -227,18 +220,17 @@ else if (p.isDeletion()) { // get deletion length int dLen = Integer.valueOf(s.substring(1)); // get ref bases of accurate deletion - int startIdxInReference = 1+loc.getStart()-ref.getWindow().getStart(); + int startIdxInReference = 1 + loc.getStart() - ref.getWindow().getStart(); stop = loc.getStart() + dLen; - byte[] refBases = Arrays.copyOfRange(ref.getBases(),startIdxInReference,startIdxInReference+dLen); + byte[] refBases = Arrays.copyOfRange(ref.getBases(), startIdxInReference, startIdxInReference + dLen); if (Allele.acceptableAlleleBases(refBases)) { - refAllele = Allele.create(refBases,true); + refAllele = Allele.create(refBases, true); altAllele = Allele.create(Allele.NULL_ALLELE_STRING, false); } - } - else { + } else { // insertion case - if (Allele.acceptableAlleleBases(s)) { + if (Allele.acceptableAlleleBases(s)) { refAllele = Allele.create(Allele.NULL_ALLELE_STRING, true); altAllele = Allele.create(s, false); stop = loc.getStart(); @@ -288,7 +280,7 @@ public VariantContext getLikelihoods(RefMetaDataTracker tracker, Allele alternateAlleleToUse, boolean useBAQedPileup, GenomeLocParser locParser) { - if ( tracker == null ) + if (tracker == null) return null; GenomeLoc loc = ref.getLocus(); @@ -299,12 +291,12 @@ public VariantContext getLikelihoods(RefMetaDataTracker tracker, // starting a new site: clear allele list alleleList.clear(); lastSiteVisited = ref.getLocus(); - indelLikelihoodMap.set(new HashMap>()); + indelLikelihoodMap.set(new HashMap>()); haplotypeMap.clear(); if (getAlleleListFromVCF) { - for( final VariantContext vc_input : tracker.getValues(UAC.alleles, loc) ) { - if( vc_input != null && + for (final VariantContext vc_input : tracker.getValues(UAC.alleles, loc)) { + if (vc_input != null && allowableTypes.contains(vc_input.getType()) && ref.getLocus().getStart() == vc_input.getStart()) { vc = vc_input; @@ -312,7 +304,7 @@ public VariantContext getLikelihoods(RefMetaDataTracker tracker, } } // ignore places where we don't have a variant - if ( vc == null ) + if (vc == null) return null; alleleList.clear(); @@ -324,15 +316,13 @@ public VariantContext getLikelihoods(RefMetaDataTracker tracker, else alleleList.add(a); - } - else { + } else { for (Allele a : vc.getAlleles()) alleleList.add(a); } - } - else { - alleleList = computeConsensusAlleles(ref,contexts, contextType, locParser); + } else { + alleleList = computeConsensusAlleles(ref, contexts, contextType, locParser); if (alleleList.isEmpty()) return null; } @@ -342,9 +332,9 @@ public VariantContext getLikelihoods(RefMetaDataTracker tracker, return null; // check if there is enough reference window to create haplotypes (can be an issue at end of contigs) - if (ref.getWindow().getStop() < loc.getStop()+HAPLOTYPE_SIZE) + if (ref.getWindow().getStop() < loc.getStop() + HAPLOTYPE_SIZE) return null; - if ( !(priors instanceof DiploidIndelGenotypePriors) ) + if (!(priors instanceof DiploidIndelGenotypePriors)) throw new StingException("Only diploid-based Indel priors are supported in the DINDEL GL model"); if (alleleList.isEmpty()) @@ -355,8 +345,8 @@ public VariantContext getLikelihoods(RefMetaDataTracker tracker, // look for alt allele that has biggest length distance to ref allele int maxLenDiff = 0; - for (Allele a: alleleList) { - if(a.isNonReference()) { + for (Allele a : alleleList) { + if (a.isNonReference()) { int lenDiff = Math.abs(a.getBaseString().length() - refAllele.getBaseString().length()); if (lenDiff > maxLenDiff) { maxLenDiff = lenDiff; @@ -366,11 +356,11 @@ public VariantContext getLikelihoods(RefMetaDataTracker tracker, } final int eventLength = altAllele.getBaseString().length() - refAllele.getBaseString().length(); - final int hsize = (int)ref.getWindow().size()-Math.abs(eventLength)-1; - final int numPrefBases= ref.getLocus().getStart()-ref.getWindow().getStart()+1; + final int hsize = (int) ref.getWindow().size() - Math.abs(eventLength) - 1; + final int numPrefBases = ref.getLocus().getStart() - ref.getWindow().getStart() + 1; - if (hsize <=0) { - logger.warn(String.format("Warning: event at location %s can't be genotyped, skipping",loc.toString())); + if (hsize <= 0) { + logger.warn(String.format("Warning: event at location %s can't be genotyped, skipping", loc.toString())); return null; } haplotypeMap = Haplotype.makeHaplotypeListFromAlleles(alleleList, loc.getStart(), @@ -388,7 +378,7 @@ public VariantContext getLikelihoods(RefMetaDataTracker tracker, // For each sample, get genotype likelihoods based on pileup // compute prior likelihoods on haplotypes, and initialize haplotype likelihood matrix with them. - for ( Map.Entry sample : contexts.entrySet() ) { + for (Map.Entry sample : contexts.entrySet()) { AlignmentContext context = AlignmentContextUtils.stratify(sample.getValue(), contextType); ReadBackedPileup pileup = null; @@ -397,8 +387,8 @@ public VariantContext getLikelihoods(RefMetaDataTracker tracker, else if (context.hasBasePileup()) pileup = context.getBasePileup(); - if (pileup != null ) { - final double[] genotypeLikelihoods = pairModel.computeReadHaplotypeLikelihoods( pileup, haplotypeMap, ref, eventLength, getIndelLikelihoodMap()); + if (pileup != null) { + final double[] genotypeLikelihoods = pairModel.computeReadHaplotypeLikelihoods(pileup, haplotypeMap, ref, eventLength, getIndelLikelihoodMap()); GenotypeLikelihoods likelihoods = GenotypeLikelihoods.fromLog10Likelihoods(genotypeLikelihoods); HashMap attributes = new HashMap(); @@ -407,9 +397,9 @@ else if (context.hasBasePileup()) genotypes.add(new Genotype(sample.getKey(), noCall, Genotype.NO_LOG10_PERROR, null, attributes, false)); if (DEBUG) { - System.out.format("Sample:%s Alleles:%s GL:",sample.getKey(), alleleList.toString()); - for (int k=0; k < genotypeLikelihoods.length; k++) - System.out.format("%1.4f ",genotypeLikelihoods[k]); + System.out.format("Sample:%s Alleles:%s GL:", sample.getKey(), alleleList.toString()); + for (int k = 0; k < genotypeLikelihoods.length; k++) + System.out.format("%1.4f ", genotypeLikelihoods[k]); System.out.println(); } } @@ -421,21 +411,21 @@ else if (context.hasBasePileup()) private int calculateEndPos(Collection alleles, Allele refAllele, GenomeLoc loc) { // for indels, stop location is one more than ref allele length boolean hasNullAltAllele = false; - for ( Allele a : alleles ) { - if ( a.isNull() ) { + for (Allele a : alleles) { + if (a.isNull()) { hasNullAltAllele = true; break; } } int endLoc = loc.getStart() + refAllele.length(); - if( !hasNullAltAllele ) + if (!hasNullAltAllele) endLoc--; return endLoc; } - public static HashMap> getIndelLikelihoodMap() { + public static HashMap> getIndelLikelihoodMap() { return indelLikelihoodMap.get(); } @@ -443,8 +433,8 @@ public static HashMap> getIndelLikeli // so that per-sample DP will include deletions covering the event. protected int getFilteredDepth(ReadBackedPileup pileup) { int count = 0; - for ( PileupElement p : pileup ) { - if (p.isDeletion() || BaseUtils.isRegularBase(p.getBase()) ) + for (PileupElement p : pileup) { + if (p.isDeletion() || p.isInsertionAtBeginningOfRead() || BaseUtils.isRegularBase(p.getBase())) count++; } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java index 81c766e4de..d9ee2ba1b0 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java @@ -212,7 +212,7 @@ public ReadBackedPileup createBAQedPileup( final ReadBackedPileup pileup ) { public class BAQedPileupElement extends PileupElement { public BAQedPileupElement( final PileupElement PE ) { - super(PE.getRead(), PE.getOffset()); + super(PE.getRead(), PE.getOffset(), PE.isDeletion()); } @Override diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java b/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java index 586b86490b..1fa7101ca3 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java @@ -40,7 +40,7 @@ * @author mhanna * @version 0.1 */ -public abstract class AbstractReadBackedPileup,PE extends PileupElement> implements ReadBackedPileup { +public abstract class AbstractReadBackedPileup, PE extends PileupElement> implements ReadBackedPileup { protected final GenomeLoc loc; protected final PileupElementTracker pileupElementTracker; @@ -55,23 +55,18 @@ public abstract class AbstractReadBackedPileup reads, List offsets ) { + public AbstractReadBackedPileup(GenomeLoc loc, List reads, List offsets) { this.loc = loc; - this.pileupElementTracker = readsOffsets2Pileup(reads,offsets); + this.pileupElementTracker = readsOffsets2Pileup(reads, offsets); } - public AbstractReadBackedPileup(GenomeLoc loc, List reads, int offset ) { - this.loc = loc; - this.pileupElementTracker = readsOffsets2Pileup(reads,offset); - } /** * Create a new version of a read backed pileup at loc without any aligned reads - * */ public AbstractReadBackedPileup(GenomeLoc loc) { this(loc, new UnifiedPileupElementTracker()); @@ -81,11 +76,10 @@ public AbstractReadBackedPileup(GenomeLoc loc) { * Create a new version of a read backed pileup at loc, using the reads and their corresponding * offsets. This lower level constructure assumes pileup is well-formed and merely keeps a * pointer to pileup. Don't go changing the data in pileup. - * */ public AbstractReadBackedPileup(GenomeLoc loc, List pileup) { - if ( loc == null ) throw new ReviewedStingException("Illegal null genomeloc in ReadBackedPileup"); - if ( pileup == null ) throw new ReviewedStingException("Illegal null pileup in ReadBackedPileup"); + if (loc == null) throw new ReviewedStingException("Illegal null genomeloc in ReadBackedPileup"); + if (pileup == null) throw new ReviewedStingException("Illegal null pileup in ReadBackedPileup"); this.loc = loc; this.pileupElementTracker = new UnifiedPileupElementTracker(pileup); @@ -94,12 +88,13 @@ public AbstractReadBackedPileup(GenomeLoc loc, List pileup) { /** * Optimization of above constructor where all of the cached data is provided + * * @param loc * @param pileup */ public AbstractReadBackedPileup(GenomeLoc loc, List pileup, int size, int nDeletions, int nMQ0Reads) { - if ( loc == null ) throw new ReviewedStingException("Illegal null genomeloc in UnifiedReadBackedPileup"); - if ( pileup == null ) throw new ReviewedStingException("Illegal null pileup in UnifiedReadBackedPileup"); + if (loc == null) throw new ReviewedStingException("Illegal null genomeloc in UnifiedReadBackedPileup"); + if (pileup == null) throw new ReviewedStingException("Illegal null pileup in UnifiedReadBackedPileup"); this.loc = loc; this.pileupElementTracker = new UnifiedPileupElementTracker(pileup); @@ -115,16 +110,21 @@ protected AbstractReadBackedPileup(GenomeLoc loc, PileupElementTracker track calculateCachedData(); } - protected AbstractReadBackedPileup(GenomeLoc loc, Map> pileupsBySample) { + protected AbstractReadBackedPileup(GenomeLoc loc, Map> pileupsBySample) { this.loc = loc; PerSamplePileupElementTracker tracker = new PerSamplePileupElementTracker(); - for(Map.Entry> pileupEntry: pileupsBySample.entrySet()) { - tracker.addElements(pileupEntry.getKey(),pileupEntry.getValue().pileupElementTracker); + for (Map.Entry> pileupEntry : pileupsBySample.entrySet()) { + tracker.addElements(pileupEntry.getKey(), pileupEntry.getValue().pileupElementTracker); addPileupToCumulativeStats(pileupEntry.getValue()); } this.pileupElementTracker = tracker; } + public AbstractReadBackedPileup(GenomeLoc loc, List reads, int offset) { + this.loc = loc; + this.pileupElementTracker = readsOffsets2Pileup(reads, offset); + } + /** * Calculate cached sizes, nDeletion, and base counts for the pileup. This calculation is done upfront, * so you pay the cost at the start, but it's more efficient to do this rather than pay the cost of calling @@ -135,12 +135,12 @@ protected void calculateCachedData() { nDeletions = 0; nMQ0Reads = 0; - for ( PileupElement p : pileupElementTracker ) { + for (PileupElement p : pileupElementTracker) { size++; - if ( p.isDeletion() ) { + if (p.isDeletion()) { nDeletions++; } - if ( p.getRead().getMappingQuality() == 0 ) { + if (p.getRead().getMappingQuality() == 0) { nMQ0Reads++; } } @@ -148,12 +148,12 @@ protected void calculateCachedData() { protected void calculateAbstractSize() { abstractSize = 0; - for ( PileupElement p : pileupElementTracker ) { + for (PileupElement p : pileupElementTracker) { abstractSize += p.getRepresentativeCount(); } } - protected void addPileupToCumulativeStats(AbstractReadBackedPileup pileup) { + protected void addPileupToCumulativeStats(AbstractReadBackedPileup pileup) { size += pileup.getNumberOfElements(); abstractSize += pileup.depthOfCoverage(); nDeletions += pileup.getNumberOfDeletions(); @@ -167,14 +167,17 @@ protected void addPileupToCumulativeStats(AbstractReadBackedPileup pileu * @param offsets * @return */ - private PileupElementTracker readsOffsets2Pileup(List reads, List offsets ) { - if ( reads == null ) throw new ReviewedStingException("Illegal null read list in UnifiedReadBackedPileup"); - if ( offsets == null ) throw new ReviewedStingException("Illegal null offsets list in UnifiedReadBackedPileup"); - if ( reads.size() != offsets.size() ) throw new ReviewedStingException("Reads and offset lists have different sizes!"); + private PileupElementTracker readsOffsets2Pileup(List reads, List offsets) { + if (reads == null) throw new ReviewedStingException("Illegal null read list in UnifiedReadBackedPileup"); + if (offsets == null) throw new ReviewedStingException("Illegal null offsets list in UnifiedReadBackedPileup"); + if (reads.size() != offsets.size()) + throw new ReviewedStingException("Reads and offset lists have different sizes!"); UnifiedPileupElementTracker pileup = new UnifiedPileupElementTracker(); - for ( int i = 0; i < reads.size(); i++ ) { - pileup.add(createNewPileupElement(reads.get(i),offsets.get(i))); + for (int i = 0; i < reads.size(); i++) { + GATKSAMRecord read = reads.get(i); + int offset = offsets.get(i); + pileup.add(createNewPileupElement(read, offset, BaseUtils.simpleBaseToBaseIndex(read.getReadBases()[offset]) == BaseUtils.D)); } return pileup; @@ -187,20 +190,21 @@ private PileupElementTracker readsOffsets2Pileup(List reads, * @param offset * @return */ - private PileupElementTracker readsOffsets2Pileup(List reads, int offset ) { - if ( reads == null ) throw new ReviewedStingException("Illegal null read list in UnifiedReadBackedPileup"); - if ( offset < 0 ) throw new ReviewedStingException("Illegal offset < 0 UnifiedReadBackedPileup"); + private PileupElementTracker readsOffsets2Pileup(List reads, int offset) { + if (reads == null) throw new ReviewedStingException("Illegal null read list in UnifiedReadBackedPileup"); + if (offset < 0) throw new ReviewedStingException("Illegal offset < 0 UnifiedReadBackedPileup"); UnifiedPileupElementTracker pileup = new UnifiedPileupElementTracker(); - for ( int i = 0; i < reads.size(); i++ ) { - pileup.add(createNewPileupElement( reads.get(i), offset )); + for (GATKSAMRecord read : reads) { + pileup.add(createNewPileupElement(read, offset, BaseUtils.simpleBaseToBaseIndex(read.getReadBases()[offset]) == BaseUtils.D)); } return pileup; } - protected abstract AbstractReadBackedPileup createNewPileup(GenomeLoc loc, PileupElementTracker pileupElementTracker); - protected abstract PE createNewPileupElement(GATKSAMRecord read, int offset); + protected abstract AbstractReadBackedPileup createNewPileup(GenomeLoc loc, PileupElementTracker pileupElementTracker); + + protected abstract PE createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion); // -------------------------------------------------------- // @@ -217,32 +221,31 @@ private PileupElementTracker readsOffsets2Pileup(List reads, */ @Override public RBP getPileupWithoutDeletions() { - if ( getNumberOfDeletions() > 0 ) { - if(pileupElementTracker instanceof PerSamplePileupElementTracker) { - PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker)pileupElementTracker; + if (getNumberOfDeletions() > 0) { + if (pileupElementTracker instanceof PerSamplePileupElementTracker) { + PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker) pileupElementTracker; PerSamplePileupElementTracker filteredTracker = new PerSamplePileupElementTracker(); - for(final String sample: tracker.getSamples()) { + for (final String sample : tracker.getSamples()) { PileupElementTracker perSampleElements = tracker.getElements(sample); - AbstractReadBackedPileup pileup = createNewPileup(loc,perSampleElements).getPileupWithoutDeletions(); - filteredTracker.addElements(sample,pileup.pileupElementTracker); + AbstractReadBackedPileup pileup = createNewPileup(loc, perSampleElements).getPileupWithoutDeletions(); + filteredTracker.addElements(sample, pileup.pileupElementTracker); } - return (RBP)createNewPileup(loc,filteredTracker); + return (RBP) createNewPileup(loc, filteredTracker); - } - else { - UnifiedPileupElementTracker tracker = (UnifiedPileupElementTracker)pileupElementTracker; + } else { + UnifiedPileupElementTracker tracker = (UnifiedPileupElementTracker) pileupElementTracker; UnifiedPileupElementTracker filteredTracker = new UnifiedPileupElementTracker(); - for ( PE p : tracker ) { - if ( !p.isDeletion() ) { + for (PE p : tracker) { + if (!p.isDeletion()) { filteredTracker.add(p); } } - return (RBP)createNewPileup(loc, filteredTracker); + return (RBP) createNewPileup(loc, filteredTracker); } } else { - return (RBP)this; + return (RBP) this; } } @@ -256,21 +259,20 @@ public RBP getPileupWithoutDeletions() { */ @Override public RBP getOverlappingFragmentFilteredPileup() { - if(pileupElementTracker instanceof PerSamplePileupElementTracker) { - PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker)pileupElementTracker; + if (pileupElementTracker instanceof PerSamplePileupElementTracker) { + PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker) pileupElementTracker; PerSamplePileupElementTracker filteredTracker = new PerSamplePileupElementTracker(); - for(final String sample: tracker.getSamples()) { + for (final String sample : tracker.getSamples()) { PileupElementTracker perSampleElements = tracker.getElements(sample); - AbstractReadBackedPileup pileup = createNewPileup(loc,perSampleElements).getOverlappingFragmentFilteredPileup(); - filteredTracker.addElements(sample,pileup.pileupElementTracker); + AbstractReadBackedPileup pileup = createNewPileup(loc, perSampleElements).getOverlappingFragmentFilteredPileup(); + filteredTracker.addElements(sample, pileup.pileupElementTracker); } - return (RBP)createNewPileup(loc,filteredTracker); - } - else { - Map filteredPileup = new HashMap(); + return (RBP) createNewPileup(loc, filteredTracker); + } else { + Map filteredPileup = new HashMap(); - for ( PE p : pileupElementTracker ) { + for (PE p : pileupElementTracker) { String readName = p.getRead().getReadName(); // if we've never seen this read before, life is good @@ -292,10 +294,10 @@ public RBP getOverlappingFragmentFilteredPileup() { } UnifiedPileupElementTracker filteredTracker = new UnifiedPileupElementTracker(); - for(PE filteredElement: filteredPileup.values()) + for (PE filteredElement : filteredPileup.values()) filteredTracker.add(filteredElement); - return (RBP)createNewPileup(loc,filteredTracker); + return (RBP) createNewPileup(loc, filteredTracker); } } @@ -309,300 +311,299 @@ public RBP getOverlappingFragmentFilteredPileup() { */ @Override public RBP getPileupWithoutMappingQualityZeroReads() { - if ( getNumberOfMappingQualityZeroReads() > 0 ) { - if(pileupElementTracker instanceof PerSamplePileupElementTracker) { - PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker)pileupElementTracker; + if (getNumberOfMappingQualityZeroReads() > 0) { + if (pileupElementTracker instanceof PerSamplePileupElementTracker) { + PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker) pileupElementTracker; PerSamplePileupElementTracker filteredTracker = new PerSamplePileupElementTracker(); - for(final String sample: tracker.getSamples()) { + for (final String sample : tracker.getSamples()) { PileupElementTracker perSampleElements = tracker.getElements(sample); - AbstractReadBackedPileup pileup = createNewPileup(loc,perSampleElements).getPileupWithoutMappingQualityZeroReads(); - filteredTracker.addElements(sample,pileup.pileupElementTracker); + AbstractReadBackedPileup pileup = createNewPileup(loc, perSampleElements).getPileupWithoutMappingQualityZeroReads(); + filteredTracker.addElements(sample, pileup.pileupElementTracker); } - return (RBP)createNewPileup(loc,filteredTracker); + return (RBP) createNewPileup(loc, filteredTracker); - } - else { - UnifiedPileupElementTracker tracker = (UnifiedPileupElementTracker)pileupElementTracker; + } else { + UnifiedPileupElementTracker tracker = (UnifiedPileupElementTracker) pileupElementTracker; UnifiedPileupElementTracker filteredTracker = new UnifiedPileupElementTracker(); - for ( PE p : tracker ) { - if ( p.getRead().getMappingQuality() > 0 ) { + for (PE p : tracker) { + if (p.getRead().getMappingQuality() > 0) { filteredTracker.add(p); } } - return (RBP)createNewPileup(loc, filteredTracker); + return (RBP) createNewPileup(loc, filteredTracker); } } else { - return (RBP)this; + return (RBP) this; } } public RBP getPositiveStrandPileup() { - if(pileupElementTracker instanceof PerSamplePileupElementTracker) { - PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker)pileupElementTracker; + if (pileupElementTracker instanceof PerSamplePileupElementTracker) { + PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker) pileupElementTracker; PerSamplePileupElementTracker filteredTracker = new PerSamplePileupElementTracker(); - for(final String sample: tracker.getSamples()) { + for (final String sample : tracker.getSamples()) { PileupElementTracker perSampleElements = tracker.getElements(sample); - AbstractReadBackedPileup pileup = createNewPileup(loc,perSampleElements).getPositiveStrandPileup(); - filteredTracker.addElements(sample,pileup.pileupElementTracker); + AbstractReadBackedPileup pileup = createNewPileup(loc, perSampleElements).getPositiveStrandPileup(); + filteredTracker.addElements(sample, pileup.pileupElementTracker); } - return (RBP)createNewPileup(loc,filteredTracker); - } - else { - UnifiedPileupElementTracker tracker = (UnifiedPileupElementTracker)pileupElementTracker; + return (RBP) createNewPileup(loc, filteredTracker); + } else { + UnifiedPileupElementTracker tracker = (UnifiedPileupElementTracker) pileupElementTracker; UnifiedPileupElementTracker filteredTracker = new UnifiedPileupElementTracker(); - for ( PE p : tracker ) { - if ( !p.getRead().getReadNegativeStrandFlag() ) { + for (PE p : tracker) { + if (!p.getRead().getReadNegativeStrandFlag()) { filteredTracker.add(p); } } - return (RBP)createNewPileup(loc, filteredTracker); + return (RBP) createNewPileup(loc, filteredTracker); } } /** * Gets the pileup consisting of only reads on the negative strand. + * * @return A read-backed pileup consisting only of reads on the negative strand. */ public RBP getNegativeStrandPileup() { - if(pileupElementTracker instanceof PerSamplePileupElementTracker) { - PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker)pileupElementTracker; + if (pileupElementTracker instanceof PerSamplePileupElementTracker) { + PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker) pileupElementTracker; PerSamplePileupElementTracker filteredTracker = new PerSamplePileupElementTracker(); - for(final String sample: tracker.getSamples()) { + for (final String sample : tracker.getSamples()) { PileupElementTracker perSampleElements = tracker.getElements(sample); - AbstractReadBackedPileup pileup = createNewPileup(loc,perSampleElements).getNegativeStrandPileup(); - filteredTracker.addElements(sample,pileup.pileupElementTracker); + AbstractReadBackedPileup pileup = createNewPileup(loc, perSampleElements).getNegativeStrandPileup(); + filteredTracker.addElements(sample, pileup.pileupElementTracker); } - return (RBP)createNewPileup(loc,filteredTracker); - } - else { - UnifiedPileupElementTracker tracker = (UnifiedPileupElementTracker)pileupElementTracker; + return (RBP) createNewPileup(loc, filteredTracker); + } else { + UnifiedPileupElementTracker tracker = (UnifiedPileupElementTracker) pileupElementTracker; UnifiedPileupElementTracker filteredTracker = new UnifiedPileupElementTracker(); - for ( PE p : tracker ) { - if ( p.getRead().getReadNegativeStrandFlag() ) { + for (PE p : tracker) { + if (p.getRead().getReadNegativeStrandFlag()) { filteredTracker.add(p); } } - return (RBP)createNewPileup(loc, filteredTracker); + return (RBP) createNewPileup(loc, filteredTracker); } } /** * Gets a pileup consisting of all those elements passed by a given filter. + * * @param filter Filter to use when testing for elements. * @return a pileup without the given filtered elements. */ public RBP getFilteredPileup(PileupElementFilter filter) { - if(pileupElementTracker instanceof PerSamplePileupElementTracker) { - PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker)pileupElementTracker; + if (pileupElementTracker instanceof PerSamplePileupElementTracker) { + PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker) pileupElementTracker; PerSamplePileupElementTracker filteredTracker = new PerSamplePileupElementTracker(); - for(final String sample: tracker.getSamples()) { + for (final String sample : tracker.getSamples()) { PileupElementTracker perSampleElements = tracker.getElements(sample); - AbstractReadBackedPileup pileup = createNewPileup(loc,perSampleElements).getFilteredPileup(filter); - filteredTracker.addElements(sample,pileup.pileupElementTracker); + AbstractReadBackedPileup pileup = createNewPileup(loc, perSampleElements).getFilteredPileup(filter); + filteredTracker.addElements(sample, pileup.pileupElementTracker); } - return (RBP)createNewPileup(loc,filteredTracker); - } - else { + return (RBP) createNewPileup(loc, filteredTracker); + } else { UnifiedPileupElementTracker filteredTracker = new UnifiedPileupElementTracker(); - for ( PE p : pileupElementTracker ) { - if( filter.allow(p) ) + for (PE p : pileupElementTracker) { + if (filter.allow(p)) filteredTracker.add(p); } - return (RBP)createNewPileup(loc, filteredTracker); + return (RBP) createNewPileup(loc, filteredTracker); } } - /** Returns subset of this pileup that contains only bases with quality >= minBaseQ, coming from + /** + * Returns subset of this pileup that contains only bases with quality >= minBaseQ, coming from * reads with mapping qualities >= minMapQ. This method allocates and returns a new instance of ReadBackedPileup. + * * @param minBaseQ * @param minMapQ * @return */ @Override - public RBP getBaseAndMappingFilteredPileup( int minBaseQ, int minMapQ ) { - if(pileupElementTracker instanceof PerSamplePileupElementTracker) { - PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker)pileupElementTracker; + public RBP getBaseAndMappingFilteredPileup(int minBaseQ, int minMapQ) { + if (pileupElementTracker instanceof PerSamplePileupElementTracker) { + PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker) pileupElementTracker; PerSamplePileupElementTracker filteredTracker = new PerSamplePileupElementTracker(); - for(final String sample: tracker.getSamples()) { + for (final String sample : tracker.getSamples()) { PileupElementTracker perSampleElements = tracker.getElements(sample); - AbstractReadBackedPileup pileup = createNewPileup(loc,perSampleElements).getBaseAndMappingFilteredPileup(minBaseQ,minMapQ); - filteredTracker.addElements(sample,pileup.pileupElementTracker); + AbstractReadBackedPileup pileup = createNewPileup(loc, perSampleElements).getBaseAndMappingFilteredPileup(minBaseQ, minMapQ); + filteredTracker.addElements(sample, pileup.pileupElementTracker); } - return (RBP)createNewPileup(loc,filteredTracker); - } - else { + return (RBP) createNewPileup(loc, filteredTracker); + } else { UnifiedPileupElementTracker filteredTracker = new UnifiedPileupElementTracker(); - for ( PE p : pileupElementTracker ) { - if ( p.getRead().getMappingQuality() >= minMapQ && + for (PE p : pileupElementTracker) { + if (p.getRead().getMappingQuality() >= minMapQ && (p.isDeletion() || - ((p instanceof ExtendedEventPileupElement) && ((ExtendedEventPileupElement)p).getType() == ExtendedEventPileupElement.Type.NOEVENT) || - p.getQual() >= minBaseQ) ) { + ((p instanceof ExtendedEventPileupElement) && ((ExtendedEventPileupElement) p).getType() == ExtendedEventPileupElement.Type.NOEVENT) || + p.getQual() >= minBaseQ)) { filteredTracker.add(p); } } - return (RBP)createNewPileup(loc, filteredTracker); + return (RBP) createNewPileup(loc, filteredTracker); } } - /** Returns subset of this pileup that contains only bases with quality >= minBaseQ. + /** + * Returns subset of this pileup that contains only bases with quality >= minBaseQ. * This method allocates and returns a new instance of ReadBackedPileup. + * * @param minBaseQ * @return */ @Override - public RBP getBaseFilteredPileup( int minBaseQ ) { + public RBP getBaseFilteredPileup(int minBaseQ) { return getBaseAndMappingFilteredPileup(minBaseQ, -1); } - /** Returns subset of this pileup that contains only bases coming from reads with mapping quality >= minMapQ. + /** + * Returns subset of this pileup that contains only bases coming from reads with mapping quality >= minMapQ. * This method allocates and returns a new instance of ReadBackedPileup. + * * @param minMapQ * @return */ @Override - public RBP getMappingFilteredPileup( int minMapQ ) { + public RBP getMappingFilteredPileup(int minMapQ) { return getBaseAndMappingFilteredPileup(-1, minMapQ); } /** * Gets a list of the read groups represented in this pileup. + * * @return */ @Override public Collection getReadGroups() { Set readGroups = new HashSet(); - for(PileupElement pileupElement: this) + for (PileupElement pileupElement : this) readGroups.add(pileupElement.getRead().getReadGroup().getReadGroupId()); return readGroups; } /** * Gets the pileup for a given read group. Horrendously inefficient at this point. + * * @param targetReadGroupId Identifier for the read group. * @return A read-backed pileup containing only the reads in the given read group. */ @Override public RBP getPileupForReadGroup(String targetReadGroupId) { - if(pileupElementTracker instanceof PerSamplePileupElementTracker) { - PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker)pileupElementTracker; + if (pileupElementTracker instanceof PerSamplePileupElementTracker) { + PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker) pileupElementTracker; PerSamplePileupElementTracker filteredTracker = new PerSamplePileupElementTracker(); - for(final String sample: tracker.getSamples()) { + for (final String sample : tracker.getSamples()) { PileupElementTracker perSampleElements = tracker.getElements(sample); - AbstractReadBackedPileup pileup = createNewPileup(loc,perSampleElements).getPileupForReadGroup(targetReadGroupId); - if(pileup != null) - filteredTracker.addElements(sample,pileup.pileupElementTracker); + AbstractReadBackedPileup pileup = createNewPileup(loc, perSampleElements).getPileupForReadGroup(targetReadGroupId); + if (pileup != null) + filteredTracker.addElements(sample, pileup.pileupElementTracker); } - return filteredTracker.size()>0 ? (RBP)createNewPileup(loc,filteredTracker) : null; - } - else { + return filteredTracker.size() > 0 ? (RBP) createNewPileup(loc, filteredTracker) : null; + } else { UnifiedPileupElementTracker filteredTracker = new UnifiedPileupElementTracker(); - for(PE p: pileupElementTracker) { + for (PE p : pileupElementTracker) { GATKSAMRecord read = p.getRead(); - if(targetReadGroupId != null) { - if(read.getReadGroup() != null && targetReadGroupId.equals(read.getReadGroup().getReadGroupId())) + if (targetReadGroupId != null) { + if (read.getReadGroup() != null && targetReadGroupId.equals(read.getReadGroup().getReadGroupId())) filteredTracker.add(p); - } - else { - if(read.getReadGroup() == null || read.getReadGroup().getReadGroupId() == null) + } else { + if (read.getReadGroup() == null || read.getReadGroup().getReadGroupId() == null) filteredTracker.add(p); } } - return filteredTracker.size()>0 ? (RBP)createNewPileup(loc,filteredTracker) : null; + return filteredTracker.size() > 0 ? (RBP) createNewPileup(loc, filteredTracker) : null; } } /** * Gets the pileup for a set of read groups. Horrendously inefficient at this point. + * * @param rgSet List of identifiers for the read groups. * @return A read-backed pileup containing only the reads in the given read groups. */ @Override public RBP getPileupForReadGroups(final HashSet rgSet) { - if(pileupElementTracker instanceof PerSamplePileupElementTracker) { - PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker)pileupElementTracker; + if (pileupElementTracker instanceof PerSamplePileupElementTracker) { + PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker) pileupElementTracker; PerSamplePileupElementTracker filteredTracker = new PerSamplePileupElementTracker(); - for(final String sample: tracker.getSamples()) { + for (final String sample : tracker.getSamples()) { PileupElementTracker perSampleElements = tracker.getElements(sample); - AbstractReadBackedPileup pileup = createNewPileup(loc,perSampleElements).getPileupForReadGroups(rgSet); - if(pileup != null) - filteredTracker.addElements(sample,pileup.pileupElementTracker); + AbstractReadBackedPileup pileup = createNewPileup(loc, perSampleElements).getPileupForReadGroups(rgSet); + if (pileup != null) + filteredTracker.addElements(sample, pileup.pileupElementTracker); } - return filteredTracker.size()>0 ? (RBP)createNewPileup(loc,filteredTracker) : null; - } - else { + return filteredTracker.size() > 0 ? (RBP) createNewPileup(loc, filteredTracker) : null; + } else { UnifiedPileupElementTracker filteredTracker = new UnifiedPileupElementTracker(); - for(PE p: pileupElementTracker) { + for (PE p : pileupElementTracker) { GATKSAMRecord read = p.getRead(); - if(rgSet != null && !rgSet.isEmpty()) { - if(read.getReadGroup() != null && rgSet.contains(read.getReadGroup().getReadGroupId())) + if (rgSet != null && !rgSet.isEmpty()) { + if (read.getReadGroup() != null && rgSet.contains(read.getReadGroup().getReadGroupId())) filteredTracker.add(p); - } - else { - if(read.getReadGroup() == null || read.getReadGroup().getReadGroupId() == null) + } else { + if (read.getReadGroup() == null || read.getReadGroup().getReadGroupId() == null) filteredTracker.add(p); } } - return filteredTracker.size()>0 ? (RBP)createNewPileup(loc,filteredTracker) : null; + return filteredTracker.size() > 0 ? (RBP) createNewPileup(loc, filteredTracker) : null; } } @Override public RBP getPileupForLane(String laneID) { - if(pileupElementTracker instanceof PerSamplePileupElementTracker) { - PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker)pileupElementTracker; + if (pileupElementTracker instanceof PerSamplePileupElementTracker) { + PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker) pileupElementTracker; PerSamplePileupElementTracker filteredTracker = new PerSamplePileupElementTracker(); - for(final String sample: tracker.getSamples()) { + for (final String sample : tracker.getSamples()) { PileupElementTracker perSampleElements = tracker.getElements(sample); - AbstractReadBackedPileup pileup = createNewPileup(loc,perSampleElements).getPileupForLane(laneID); - if(pileup != null) - filteredTracker.addElements(sample,pileup.pileupElementTracker); + AbstractReadBackedPileup pileup = createNewPileup(loc, perSampleElements).getPileupForLane(laneID); + if (pileup != null) + filteredTracker.addElements(sample, pileup.pileupElementTracker); } - return filteredTracker.size()>0 ? (RBP)createNewPileup(loc,filteredTracker) : null; - } - else { + return filteredTracker.size() > 0 ? (RBP) createNewPileup(loc, filteredTracker) : null; + } else { UnifiedPileupElementTracker filteredTracker = new UnifiedPileupElementTracker(); - for(PE p: pileupElementTracker) { + for (PE p : pileupElementTracker) { GATKSAMRecord read = p.getRead(); - if(laneID != null) { - if(read.getReadGroup() != null && - (read.getReadGroup().getReadGroupId().startsWith(laneID + ".")) || // lane is the same, but sample identifier is different - (read.getReadGroup().getReadGroupId().equals(laneID))) // in case there is no sample identifier, they have to be exactly the same + if (laneID != null) { + if (read.getReadGroup() != null && + (read.getReadGroup().getReadGroupId().startsWith(laneID + ".")) || // lane is the same, but sample identifier is different + (read.getReadGroup().getReadGroupId().equals(laneID))) // in case there is no sample identifier, they have to be exactly the same filteredTracker.add(p); - } - else { - if(read.getReadGroup() == null || read.getReadGroup().getReadGroupId() == null) + } else { + if (read.getReadGroup() == null || read.getReadGroup().getReadGroupId() == null) filteredTracker.add(p); } } - return filteredTracker.size()>0 ? (RBP)createNewPileup(loc,filteredTracker) : null; + return filteredTracker.size() > 0 ? (RBP) createNewPileup(loc, filteredTracker) : null; } } public Collection getSamples() { - if(pileupElementTracker instanceof PerSamplePileupElementTracker) { - PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker)pileupElementTracker; + if (pileupElementTracker instanceof PerSamplePileupElementTracker) { + PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker) pileupElementTracker; return new HashSet(tracker.getSamples()); - } - else { + } else { Collection sampleNames = new HashSet(); - for(PileupElement p: this) { + for (PileupElement p : this) { GATKSAMRecord read = p.getRead(); String sampleName = read.getReadGroup() != null ? read.getReadGroup().getSample() : null; sampleNames.add(sampleName); @@ -619,103 +620,98 @@ public Collection getSamples() { */ @Override public RBP getDownsampledPileup(int desiredCoverage) { - if ( getNumberOfElements() <= desiredCoverage ) - return (RBP)this; + if (getNumberOfElements() <= desiredCoverage) + return (RBP) this; // randomly choose numbers corresponding to positions in the reads list TreeSet positions = new TreeSet(); - for ( int i = 0; i < desiredCoverage; /* no update */ ) { - if ( positions.add(GenomeAnalysisEngine.getRandomGenerator().nextInt(size)) ) + for (int i = 0; i < desiredCoverage; /* no update */) { + if (positions.add(GenomeAnalysisEngine.getRandomGenerator().nextInt(size))) i++; } - if(pileupElementTracker instanceof PerSamplePileupElementTracker) { - PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker)pileupElementTracker; + if (pileupElementTracker instanceof PerSamplePileupElementTracker) { + PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker) pileupElementTracker; PerSamplePileupElementTracker filteredTracker = new PerSamplePileupElementTracker(); int current = 0; - for(final String sample: tracker.getSamples()) { + for (final String sample : tracker.getSamples()) { PileupElementTracker perSampleElements = tracker.getElements(sample); List filteredPileup = new ArrayList(); - for(PileupElement p: perSampleElements) { - if(positions.contains(current)) + for (PileupElement p : perSampleElements) { + if (positions.contains(current)) filteredPileup.add(p); } - if(!filteredPileup.isEmpty()) { - AbstractReadBackedPileup pileup = createNewPileup(loc,perSampleElements); - filteredTracker.addElements(sample,pileup.pileupElementTracker); + if (!filteredPileup.isEmpty()) { + AbstractReadBackedPileup pileup = createNewPileup(loc, perSampleElements); + filteredTracker.addElements(sample, pileup.pileupElementTracker); } current++; } - return (RBP)createNewPileup(loc,filteredTracker); - } - else { - UnifiedPileupElementTracker tracker = (UnifiedPileupElementTracker)pileupElementTracker; + return (RBP) createNewPileup(loc, filteredTracker); + } else { + UnifiedPileupElementTracker tracker = (UnifiedPileupElementTracker) pileupElementTracker; UnifiedPileupElementTracker filteredTracker = new UnifiedPileupElementTracker(); Iterator positionIter = positions.iterator(); - while ( positionIter.hasNext() ) { - int nextReadToKeep = (Integer)positionIter.next(); + while (positionIter.hasNext()) { + int nextReadToKeep = (Integer) positionIter.next(); filteredTracker.add(tracker.get(nextReadToKeep)); } - return (RBP)createNewPileup(getLocation(), filteredTracker); + return (RBP) createNewPileup(getLocation(), filteredTracker); } } @Override public RBP getPileupForSamples(Collection sampleNames) { - if(pileupElementTracker instanceof PerSamplePileupElementTracker) { - PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker)pileupElementTracker; + if (pileupElementTracker instanceof PerSamplePileupElementTracker) { + PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker) pileupElementTracker; PileupElementTracker filteredElements = tracker.getElements(sampleNames); - return filteredElements != null ? (RBP)createNewPileup(loc,filteredElements) : null; - } - else { + return filteredElements != null ? (RBP) createNewPileup(loc, filteredElements) : null; + } else { HashSet hashSampleNames = new HashSet(sampleNames); // to speed up the "contains" access in the for loop UnifiedPileupElementTracker filteredTracker = new UnifiedPileupElementTracker(); - for(PE p: pileupElementTracker) { + for (PE p : pileupElementTracker) { GATKSAMRecord read = p.getRead(); - if(sampleNames != null) { // still checking on sampleNames because hashSampleNames will never be null. And empty means something else. - if(read.getReadGroup() != null && hashSampleNames.contains(read.getReadGroup().getSample())) + if (sampleNames != null) { // still checking on sampleNames because hashSampleNames will never be null. And empty means something else. + if (read.getReadGroup() != null && hashSampleNames.contains(read.getReadGroup().getSample())) filteredTracker.add(p); - } - else { - if(read.getReadGroup() == null || read.getReadGroup().getSample() == null) + } else { + if (read.getReadGroup() == null || read.getReadGroup().getSample() == null) filteredTracker.add(p); } } - return filteredTracker.size()>0 ? (RBP)createNewPileup(loc,filteredTracker) : null; + return filteredTracker.size() > 0 ? (RBP) createNewPileup(loc, filteredTracker) : null; } } @Override public RBP getPileupForSample(String sampleName) { - if(pileupElementTracker instanceof PerSamplePileupElementTracker) { - PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker)pileupElementTracker; + if (pileupElementTracker instanceof PerSamplePileupElementTracker) { + PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker) pileupElementTracker; PileupElementTracker filteredElements = tracker.getElements(sampleName); - return filteredElements != null ? (RBP)createNewPileup(loc,filteredElements) : null; - } - else { + return filteredElements != null ? (RBP) createNewPileup(loc, filteredElements) : null; + } else { UnifiedPileupElementTracker filteredTracker = new UnifiedPileupElementTracker(); - for(PE p: pileupElementTracker) { + for (PE p : pileupElementTracker) { GATKSAMRecord read = p.getRead(); - if(sampleName != null) { - if(read.getReadGroup() != null && sampleName.equals(read.getReadGroup().getSample())) + if (sampleName != null) { + if (read.getReadGroup() != null && sampleName.equals(read.getReadGroup().getSample())) filteredTracker.add(p); - } - else { - if(read.getReadGroup() == null || read.getReadGroup().getSample() == null) + } else { + if (read.getReadGroup() == null || read.getReadGroup().getSample() == null) filteredTracker.add(p); } } - return filteredTracker.size()>0 ? (RBP)createNewPileup(loc,filteredTracker) : null; + return filteredTracker.size() > 0 ? (RBP) createNewPileup(loc, filteredTracker) : null; } } @@ -727,9 +723,9 @@ public RBP getPileupForSample(String sampleName) { /** * The best way to access PileupElements where you only care about the bases and quals in the pileup. - * + *

* for (PileupElement p : this) { doSomething(p); } - * + *

* Provides efficient iteration of the data. * * @return @@ -739,9 +735,17 @@ public Iterator iterator() { return new Iterator() { private final Iterator wrappedIterator = pileupElementTracker.iterator(); - public boolean hasNext() { return wrappedIterator.hasNext(); } - public PileupElement next() { return wrappedIterator.next(); } - public void remove() { throw new UnsupportedOperationException("Cannot remove from a pileup element iterator"); } + public boolean hasNext() { + return wrappedIterator.hasNext(); + } + + public PileupElement next() { + return wrappedIterator.next(); + } + + public void remove() { + throw new UnsupportedOperationException("Cannot remove from a pileup element iterator"); + } }; } @@ -784,7 +788,7 @@ public int getNumberOfElements() { */ @Override public int depthOfCoverage() { - if ( abstractSize == -1 ) + if (abstractSize == -1) calculateAbstractSize(); return abstractSize; } @@ -794,7 +798,7 @@ public int depthOfCoverage() { */ @Override public boolean isEmpty() { - return size==0; + return size == 0; } @@ -816,19 +820,18 @@ public GenomeLoc getLocation() { public int[] getBaseCounts() { int[] counts = new int[4]; - if(pileupElementTracker instanceof PerSamplePileupElementTracker) { - PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker)pileupElementTracker; - for(final String sample: tracker.getSamples()) { - int[] countsBySample = createNewPileup(loc,tracker.getElements(sample)).getBaseCounts(); - for(int i = 0; i < counts.length; i++) + if (pileupElementTracker instanceof PerSamplePileupElementTracker) { + PerSamplePileupElementTracker tracker = (PerSamplePileupElementTracker) pileupElementTracker; + for (final String sample : tracker.getSamples()) { + int[] countsBySample = createNewPileup(loc, tracker.getElements(sample)).getBaseCounts(); + for (int i = 0; i < counts.length; i++) counts[i] += countsBySample[i]; } - } - else { - for ( PileupElement pile : this ) { + } else { + for (PileupElement pile : this) { // skip deletion sites - if ( ! pile.isDeletion() ) { - int index = BaseUtils.simpleBaseToBaseIndex((char)pile.getBase()); + if (!pile.isDeletion()) { + int index = BaseUtils.simpleBaseToBaseIndex((char) pile.getBase()); if (index != -1) counts[index]++; } @@ -857,65 +860,80 @@ public String getPileupString(Character ref) { /** * Returns a list of the reads in this pileup. Note this call costs O(n) and allocates fresh lists each time + * * @return */ @Override public List getReads() { List reads = new ArrayList(getNumberOfElements()); - for ( PileupElement pile : this ) { reads.add(pile.getRead()); } + for (PileupElement pile : this) { + reads.add(pile.getRead()); + } return reads; } /** * Returns a list of the offsets in this pileup. Note this call costs O(n) and allocates fresh lists each time + * * @return */ @Override public List getOffsets() { List offsets = new ArrayList(getNumberOfElements()); - for ( PileupElement pile : this ) { offsets.add(pile.getOffset()); } + for (PileupElement pile : this) { + offsets.add(pile.getOffset()); + } return offsets; } /** * Returns an array of the bases in this pileup. Note this call costs O(n) and allocates fresh array each time + * * @return */ @Override public byte[] getBases() { byte[] v = new byte[getNumberOfElements()]; int pos = 0; - for ( PileupElement pile : pileupElementTracker ) { v[pos++] = pile.getBase(); } + for (PileupElement pile : pileupElementTracker) { + v[pos++] = pile.getBase(); + } return v; } /** * Returns an array of the quals in this pileup. Note this call costs O(n) and allocates fresh array each time + * * @return */ @Override public byte[] getQuals() { byte[] v = new byte[getNumberOfElements()]; int pos = 0; - for ( PileupElement pile : pileupElementTracker ) { v[pos++] = pile.getQual(); } + for (PileupElement pile : pileupElementTracker) { + v[pos++] = pile.getQual(); + } return v; } /** * Get an array of the mapping qualities + * * @return */ @Override public byte[] getMappingQuals() { byte[] v = new byte[getNumberOfElements()]; int pos = 0; - for ( PileupElement pile : pileupElementTracker ) { v[pos++] = (byte)pile.getRead().getMappingQuality(); } + for (PileupElement pile : pileupElementTracker) { + v[pos++] = (byte) pile.getRead().getMappingQuality(); + } return v; } - static String quals2String( byte[] quals ) { + static String quals2String(byte[] quals) { StringBuilder qualStr = new StringBuilder(); - for ( int qual : quals ) { + for (int qual : quals) { qual = Math.min(qual, 63); // todo: fixme, this isn't a good idea char qualChar = (char) (33 + qual); // todo: warning, this is illegal for qual > 63 qualStr.append(qualChar); diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/ExtendedEventPileupElement.java b/public/java/src/org/broadinstitute/sting/utils/pileup/ExtendedEventPileupElement.java index 1e5e4d4e5a..1d7e6f636b 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/ExtendedEventPileupElement.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/ExtendedEventPileupElement.java @@ -12,7 +12,7 @@ * are seen on the base-by-base basis (i.e. the pileup does keep the information about the current reference base being deleted * in some reads), but the information about the extended event (deletion length, string of all deleted bases) is not kept. * The insertions that may be present in some reads are not seen at all in such strict reference traversal mode. - * + *

* By convention, any extended event (indel) is mapped onto the reference at the last base prior to the event (i.e. * last base before the insertion or deletion). If the special "extended" traversal mode is turned on and there is * an indel in at least one read that maps onto the reference position Z, the walker's map function will be called twice: @@ -22,9 +22,9 @@ * (covered) reference position. Note that if the extended event at Z was a deletion, the "standard" base pileup at * Z+1 and following bases may still contain deleted bases. However the fully extended event call will be performed * only once, at the position where the indel maps (starts). - * + *

* This class wraps an "extended" event (indel) so that in can be added to a pileup of events at a given location. - * + *

* Created by IntelliJ IDEA. * User: asivache * Date: Dec 21, 2009 @@ -39,40 +39,52 @@ public enum Type { private Type type = null; private int eventLength = -1; private String eventBases = null; // if it is a deletion, we do not have information about the actual deleted bases - // in the read itself, so we fill the string with D's; for insertions we keep actual inserted bases + // in the read itself, so we fill the string with D's; for insertions we keep actual inserted bases private SAMRecord read; private int offset; // position in the read immediately BEFORE the event // This is broken! offset is always zero because these member variables are shadowed by base class - /** Constructor for extended pileup element (indel). + + public ExtendedEventPileupElement(GATKSAMRecord read, int offset, int eventLength, String eventBases, Type type) { + super(read, offset, type == Type.DELETION); + this.read = read; + this.offset = offset; + this.eventLength = eventLength; + this.eventBases = eventBases; + this.type = type; + } + + /** + * Quick constructor for insertions. * - * @param read the read, in which the indel is observed - * @param offset position in the read immediately before the indel (can be -1 if read starts with an insertion) - * @param length length of the indel (number of inserted or deleted bases); length <=0 indicates that the read has no indel (NOEVENT) + * @param read the read, in which the indel is observed + * @param offset position in the read immediately before the indel (can be -1 if read starts with an insertion) + * @param length length of the indel (number of inserted or deleted bases); length <=0 indicates that the read has no indel (NOEVENT) * @param eventBases inserted bases. null indicates that the event is a deletion; ignored if length<=0 (noevent) */ - public ExtendedEventPileupElement( GATKSAMRecord read, int offset, int length, byte[] eventBases ) { - super(read, offset); - this.eventLength = length; - if ( length <= 0 ) type = Type.NOEVENT; - else { - if ( eventBases != null ) { - this.eventBases = new String(eventBases).toUpperCase(); - type = Type.INSERTION; - } else { - type = Type.DELETION; - } - } + public ExtendedEventPileupElement(GATKSAMRecord read, int offset, int length, byte[] eventBases) { + this(read, offset, length, new String(eventBases).toUpperCase(), Type.INSERTION); } - /** Constructor for deletion or noevent calls - does not take event bases as an argument (as those should - * be null or are ignored in these cases anyway) - * @param read - * @param offset - * @param length + /** + * Quick constructor for non indels (matches) + * + * @param read the read + * @param offset where in the read the match is */ - public ExtendedEventPileupElement( GATKSAMRecord read, int offset, int length ) { - this(read,offset, length, null); + public ExtendedEventPileupElement(GATKSAMRecord read, int offset) { + this(read, offset, -1, null, Type.NOEVENT); + } + + /** + * Quick constructor for deletions + * + * @param read the read + * @param offset the last base before the deletion starts (left aligned deletion) + * @param length length of this deletion + */ + public ExtendedEventPileupElement(GATKSAMRecord read, int offset, int length) { + this(read, offset, length, null, Type.DELETION); } public boolean isDeletion() { @@ -87,46 +99,54 @@ public boolean isIndel() { return isDeletion() || isInsertion(); } - public Type getType() { return type; } + public Type getType() { + return type; + } // The offset can be negative with insertions at the start of the read, but a valid base does exist at this position with // a valid base quality. The following code attempts to compensate for that.' @Override public byte getBase() { - return getBase(offset >= 0 ? offset : offset+eventLength); + return getBase(offset >= 0 ? offset : offset + eventLength); } @Override public int getBaseIndex() { - return getBaseIndex(offset >= 0 ? offset : offset+eventLength); + return getBaseIndex(offset >= 0 ? offset : offset + eventLength); } @Override public byte getQual() { - return getQual(offset >= 0 ? offset : offset+eventLength); + return getQual(offset >= 0 ? offset : offset + eventLength); } - /** Returns length of the event (number of inserted or deleted bases */ - public int getEventLength() { return eventLength; } + /** + * Returns length of the event (number of inserted or deleted bases + */ + public int getEventLength() { + return eventLength; + } - /** Returns actual sequence of inserted bases, or a null if the event is a deletion or if there is no event in the associated read. - * */ - public String getEventBases() { return eventBases; } + /** + * Returns actual sequence of inserted bases, or a null if the event is a deletion or if there is no event in the associated read. + */ + public String getEventBases() { + return eventBases; + } @Override public String toString() { char c = '.'; - String fillStr = null ; - if ( isDeletion() ) { + String fillStr = null; + if (isDeletion()) { c = '-'; - char [] filler = new char[eventLength]; + char[] filler = new char[eventLength]; Arrays.fill(filler, 'D'); fillStr = new String(filler); - } - else if ( isInsertion() ) c = '+'; - return String.format("%s @ %d = %c%s MQ%d", getRead().getReadName(), getOffset(), c, isIndel()? - (isInsertion() ? eventBases : fillStr ): "", getMappingQual()); + } else if (isInsertion()) c = '+'; + return String.format("%s @ %d = %c%s MQ%d", getRead().getReadName(), getOffset(), c, isIndel() ? + (isInsertion() ? eventBases : fillStr) : "", getMappingQual()); } } diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java b/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java index 2d13d6e59d..73f010d404 100755 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java @@ -3,6 +3,7 @@ import com.google.java.contract.Ensures; import com.google.java.contract.Requires; import org.broadinstitute.sting.utils.BaseUtils; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; /** @@ -21,25 +22,61 @@ public class PileupElement implements Comparable { protected final GATKSAMRecord read; protected final int offset; + protected final boolean isDeletion; + + /** + * Creates a new pileup element. + * + * @param read the read we are adding to the pileup + * @param offset the position in the read for this base. All deletions must be left aligned! (-1 is only allowed for reads starting with insertions) + * @param isDeletion whether or not this base is a deletion + */ @Requires({ "read != null", "offset >= -1", "offset <= read.getReadLength()"}) - public PileupElement( GATKSAMRecord read, int offset ) { + public PileupElement(GATKSAMRecord read, int offset, boolean isDeletion) { + if (offset < 0 && isDeletion) + throw new ReviewedStingException("Pileup Element cannot create a deletion with a negative offset"); + this.read = read; this.offset = offset; + this.isDeletion = isDeletion; } + // /** +// * Creates a NON DELETION pileup element. +// * +// * use this constructor only for insertions and matches/mismatches. +// * @param read the read we are adding to the pileup +// * @param offset the position in the read for this base. All deletions must be left aligned! (-1 is only allowed for reads starting with insertions) +// */ +// @Requires({ +// "read != null", +// "offset >= -1", +// "offset <= read.getReadLength()"}) +// public PileupElement( GATKSAMRecord read, int offset ) { +// this(read, offset, false); +// } +// public boolean isDeletion() { + return isDeletion; + } + + public boolean isInsertionAtBeginningOfRead() { return offset == -1; } @Ensures("result != null") - public GATKSAMRecord getRead() { return read; } + public GATKSAMRecord getRead() { + return read; + } @Ensures("result == offset") - public int getOffset() { return offset; } + public int getOffset() { + return offset; + } public byte getBase() { return getBase(offset); @@ -59,30 +96,30 @@ public int getMappingQual() { @Ensures("result != null") public String toString() { - return String.format("%s @ %d = %c Q%d", getRead().getReadName(), getOffset(), (char)getBase(), getQual()); + return String.format("%s @ %d = %c Q%d", getRead().getReadName(), getOffset(), (char) getBase(), getQual()); } protected byte getBase(final int offset) { - return isDeletion() ? DELETION_BASE : read.getReadBases()[offset]; + return (isDeletion() || isInsertionAtBeginningOfRead()) ? DELETION_BASE : read.getReadBases()[offset]; } protected int getBaseIndex(final int offset) { - return BaseUtils.simpleBaseToBaseIndex(isDeletion() ? DELETION_BASE : read.getReadBases()[offset]); + return BaseUtils.simpleBaseToBaseIndex((isDeletion() || isInsertionAtBeginningOfRead()) ? DELETION_BASE : read.getReadBases()[offset]); } protected byte getQual(final int offset) { - return isDeletion() ? DELETION_QUAL : read.getBaseQualities()[offset]; + return (isDeletion() || isInsertionAtBeginningOfRead()) ? DELETION_QUAL : read.getBaseQualities()[offset]; } @Override public int compareTo(final PileupElement pileupElement) { - if ( offset < pileupElement.offset ) + if (offset < pileupElement.offset) return -1; - else if ( offset > pileupElement.offset ) + else if (offset > pileupElement.offset) return 1; - else if ( read.getAlignmentStart() < pileupElement.read.getAlignmentStart() ) + else if (read.getAlignmentStart() < pileupElement.read.getAlignmentStart()) return -1; - else if ( read.getAlignmentStart() > pileupElement.read.getAlignmentStart() ) + else if (read.getAlignmentStart() > pileupElement.read.getAlignmentStart()) return 1; else return 0; @@ -94,13 +131,26 @@ else if ( read.getAlignmentStart() > pileupElement.read.getAlignmentStart() ) // // -------------------------------------------------------------------------- - public boolean isReducedRead() { - return read.isReducedRead(); - } - +// public boolean isReducedRead() { +// return read.isReducedRead(); +// } + + /** + * Returns the number of elements in the pileup element. + *

+ * Unless this is a reduced read, the number of elements in a pileup element is one. In the event of + * this being a reduced read and a deletion, we return the average number of elements between the left + * and right elements to the deletion. We assume the deletion to be left aligned. + * + * @return + */ public int getRepresentativeCount() { - // TODO -- if we ever decide to reduce the representation of deletions then this will need to be fixed - return (!isDeletion() && isReducedRead()) ? read.getReducedCount(offset) : 1; + int representativeCount = 1; + + if (read.isReducedRead() && !isInsertionAtBeginningOfRead()) + representativeCount = (isDeletion()) ? Math.round((read.getReducedCount(offset) + read.getReducedCount(offset + 1)) / 2) : read.getReducedCount(offset); + + return representativeCount; } } \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java index 43ad063523..bf67d1a706 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java @@ -30,33 +30,34 @@ import java.util.*; -public class ReadBackedExtendedEventPileupImpl extends AbstractReadBackedPileup implements ReadBackedExtendedEventPileup { +public class ReadBackedExtendedEventPileupImpl extends AbstractReadBackedPileup implements ReadBackedExtendedEventPileup { private int nInsertions; private int maxDeletionLength; // cached value of the length of the longest deletion observed at the site public ReadBackedExtendedEventPileupImpl(GenomeLoc loc, List pileupElements) { - super(loc,pileupElements); + super(loc, pileupElements); } public ReadBackedExtendedEventPileupImpl(GenomeLoc loc, PileupElementTracker tracker) { - super(loc,tracker); + super(loc, tracker); } /** * Optimization of above constructor where all of the cached data is provided + * * @param loc * @param pileup */ public ReadBackedExtendedEventPileupImpl(GenomeLoc loc, List pileup, int size, - int maxDeletionLength, int nInsertions, int nDeletions, int nMQ0Reads) { - super(loc,pileup,size,nDeletions,nMQ0Reads); + int maxDeletionLength, int nInsertions, int nDeletions, int nMQ0Reads) { + super(loc, pileup, size, nDeletions, nMQ0Reads); this.maxDeletionLength = maxDeletionLength; this.nInsertions = nInsertions; } // this is the good new one - public ReadBackedExtendedEventPileupImpl(GenomeLoc loc, Map pileupElementsBySample) { - super(loc,pileupElementsBySample); + public ReadBackedExtendedEventPileupImpl(GenomeLoc loc, Map pileupElementsBySample) { + super(loc, pileupElementsBySample); } /** @@ -71,31 +72,31 @@ protected void calculateCachedData() { nInsertions = 0; nMQ0Reads = 0; - for ( ExtendedEventPileupElement p : this.toExtendedIterable() ) { + for (ExtendedEventPileupElement p : this.toExtendedIterable()) { - if ( p.isDeletion() ) { + if (p.isDeletion()) { maxDeletionLength = Math.max(maxDeletionLength, p.getEventLength()); } else { - if ( p.isInsertion() ) nInsertions++; + if (p.isInsertion()) nInsertions++; } } } @Override - protected void addPileupToCumulativeStats(AbstractReadBackedPileup pileup) { + protected void addPileupToCumulativeStats(AbstractReadBackedPileup pileup) { super.addPileupToCumulativeStats(pileup); - ReadBackedExtendedEventPileup extendedEventPileup = ((ReadBackedExtendedEventPileup)pileup); + ReadBackedExtendedEventPileup extendedEventPileup = ((ReadBackedExtendedEventPileup) pileup); this.nInsertions += extendedEventPileup.getNumberOfInsertions(); this.maxDeletionLength += extendedEventPileup.getMaxDeletionLength(); } @Override protected ReadBackedExtendedEventPileupImpl createNewPileup(GenomeLoc loc, PileupElementTracker tracker) { - return new ReadBackedExtendedEventPileupImpl(loc,tracker); + return new ReadBackedExtendedEventPileupImpl(loc, tracker); } @Override - protected ExtendedEventPileupElement createNewPileupElement(GATKSAMRecord read, int offset) { + protected ExtendedEventPileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion) { throw new UnsupportedOperationException("Not enough information provided to create a new pileup element"); } @@ -110,10 +111,12 @@ public int getNumberOfInsertions() { return nInsertions; } - /** Returns the length of the longest deletion observed at the site this + /** + * Returns the length of the longest deletion observed at the site this * pileup is associated with (NOTE: by convention, both insertions and deletions * are associated with genomic location immediately before the actual event). If * there are no deletions at the site, returns 0. + * * @return */ @Override @@ -123,36 +126,47 @@ public int getMaxDeletionLength() { public Iterable toExtendedIterable() { return new Iterable() { - public Iterator iterator() { return pileupElementTracker.iterator(); } + public Iterator iterator() { + return pileupElementTracker.iterator(); + } }; } /** * Returns an array of the events in this pileup ('I', 'D', or '.'). Note this call costs O(n) and allocates fresh array each time + * * @return */ @Override public byte[] getEvents() { byte[] v = new byte[getNumberOfElements()]; int i = 0; - for ( ExtendedEventPileupElement e : this.toExtendedIterable() ) { - switch ( e.getType() ) { - case INSERTION: v[i] = 'I'; break; - case DELETION: v[i] = 'D'; break; - case NOEVENT: v[i] = '.'; break; - default: throw new ReviewedStingException("Unknown event type encountered: "+e.getType()); + for (ExtendedEventPileupElement e : this.toExtendedIterable()) { + switch (e.getType()) { + case INSERTION: + v[i] = 'I'; + break; + case DELETION: + v[i] = 'D'; + break; + case NOEVENT: + v[i] = '.'; + break; + default: + throw new ReviewedStingException("Unknown event type encountered: " + e.getType()); } i++; } return v; - } + } - /** A shortcut for getEventStringsWithCounts(null); + /** + * A shortcut for getEventStringsWithCounts(null); * * @return */ @Override - public List> getEventStringsWithCounts() { + public List> getEventStringsWithCounts() { return getEventStringsWithCounts(null); } @@ -163,44 +177,48 @@ public String getShortPileupString() { // insertion, deletion or no-event, respectively. return String.format("%s %s E %s", getLocation().getContig(), getLocation().getStart(), // chromosome name and coordinate - new String(getEvents()) ); + new String(getEvents())); } - /** Returns String representation of all distinct extended events (indels) at the site along with + /** + * Returns String representation of all distinct extended events (indels) at the site along with * observation counts (numbers of reads) for each distinct event. If refBases is null, a simple string representation for * deletions will be generated as "D" (i.e. "5D"); if the reference bases are provided, the actual * deleted sequence will be used in the string representation (e.g. "-AAC"). - * @param refBases reference bases, starting with the current locus (i.e. the one immediately before the indel), and - * extending far enough to accomodate the longest deletion (i.e. size of refBases must be at least 1+) + * + * @param refBases reference bases, starting with the current locus (i.e. the one immediately before the indel), and + * extending far enough to accomodate the longest deletion (i.e. size of refBases must be at least 1+) * @return list of distinct events; first element of a pair is a string representation of the event, second element - * gives the number of reads, in which that event was observed + * gives the number of reads, in which that event was observed */ @Override - public List> getEventStringsWithCounts(byte[] refBases) { - Map events = new HashMap(); + public List> getEventStringsWithCounts(byte[] refBases) { + Map events = new HashMap(); - for ( ExtendedEventPileupElement e : this.toExtendedIterable() ) { + for (ExtendedEventPileupElement e : this.toExtendedIterable()) { Integer cnt; String indel = null; - switch ( e.getType() ) { + switch (e.getType()) { case INSERTION: - indel = "+"+e.getEventBases(); + indel = "+" + e.getEventBases(); break; case DELETION: - indel = getDeletionString(e.getEventLength(),refBases); + indel = getDeletionString(e.getEventLength(), refBases); break; - case NOEVENT: continue; - default: throw new ReviewedStingException("Unknown event type encountered: "+e.getType()); + case NOEVENT: + continue; + default: + throw new ReviewedStingException("Unknown event type encountered: " + e.getType()); } cnt = events.get(indel); - if ( cnt == null ) events.put(indel,1); - else events.put(indel,cnt.intValue()+1); + if (cnt == null) events.put(indel, 1); + else events.put(indel, cnt.intValue() + 1); } - List> eventList = new ArrayList>(events.size()); - for ( Map.Entry m : events.entrySet() ) { - eventList.add( new Pair(m.getKey(),m.getValue())); + List> eventList = new ArrayList>(events.size()); + for (Map.Entry m : events.entrySet()) { + eventList.add(new Pair(m.getKey(), m.getValue())); } return eventList; } @@ -208,18 +226,19 @@ public List> getEventStringsWithCounts(byte[] refBases) { /** * Builds string representation of the deletion event. If refBases is null, the representation will be * "D" (e.g. "5D"); if the reference bases are provided, a verbose representation (e.g. "-AAC") - * will be generated. NOTE: refBases must start with the base prior to the actual deletion (i.e. deleted + * will be generated. NOTE: refBases must start with the base prior to the actual deletion (i.e. deleted * base(s) are refBase[1], refBase[2], ...), and the length of the passed array must be sufficient to accomodate the * deletion length (i.e. size of refBase must be at least length+1). + * * @param length * @param refBases * @return */ private String getDeletionString(int length, byte[] refBases) { - if ( refBases == null ) { - return Integer.toString(length)+"D"; // if we do not have reference bases, we can only report something like "5D" + if (refBases == null) { + return Integer.toString(length) + "D"; // if we do not have reference bases, we can only report something like "5D" } else { - return "-"+new String(refBases,1,length).toUpperCase(); + return "-" + new String(refBases, 1, length).toUpperCase(); } } } \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java index b7445be8dd..66ddbe95d6 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java @@ -29,48 +29,49 @@ import java.util.List; import java.util.Map; -public class ReadBackedPileupImpl extends AbstractReadBackedPileup implements ReadBackedPileup { +public class ReadBackedPileupImpl extends AbstractReadBackedPileup implements ReadBackedPileup { public ReadBackedPileupImpl(GenomeLoc loc) { super(loc); } - public ReadBackedPileupImpl(GenomeLoc loc, List reads, List offsets ) { - super(loc,reads,offsets); + public ReadBackedPileupImpl(GenomeLoc loc, List reads, List offsets) { + super(loc, reads, offsets); } - public ReadBackedPileupImpl(GenomeLoc loc, List reads, int offset ) { - super(loc,reads,offset); + public ReadBackedPileupImpl(GenomeLoc loc, List reads, int offset) { + super(loc, reads, offset); } public ReadBackedPileupImpl(GenomeLoc loc, List pileupElements) { - super(loc,pileupElements); + super(loc, pileupElements); } - public ReadBackedPileupImpl(GenomeLoc loc, Map pileupElementsBySample) { - super(loc,pileupElementsBySample); + public ReadBackedPileupImpl(GenomeLoc loc, Map pileupElementsBySample) { + super(loc, pileupElementsBySample); } /** * Optimization of above constructor where all of the cached data is provided + * * @param loc * @param pileup */ public ReadBackedPileupImpl(GenomeLoc loc, List pileup, int size, int nDeletions, int nMQ0Reads) { - super(loc,pileup,size,nDeletions,nMQ0Reads); + super(loc, pileup, size, nDeletions, nMQ0Reads); } protected ReadBackedPileupImpl(GenomeLoc loc, PileupElementTracker tracker) { - super(loc,tracker); + super(loc, tracker); } @Override protected ReadBackedPileupImpl createNewPileup(GenomeLoc loc, PileupElementTracker tracker) { - return new ReadBackedPileupImpl(loc,tracker); + return new ReadBackedPileupImpl(loc, tracker); } @Override - protected PileupElement createNewPileupElement(GATKSAMRecord read, int offset) { - return new PileupElement(read,offset); + protected PileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion) { + return new PileupElement(read, offset, isDeletion); } } diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/AlignmentUtils.java b/public/java/src/org/broadinstitute/sting/utils/sam/AlignmentUtils.java index b8e8921014..3b27364182 100644 --- a/public/java/src/org/broadinstitute/sting/utils/sam/AlignmentUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/AlignmentUtils.java @@ -1,745 +1,774 @@ -/* - * Copyright (c) 2010 The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR - * THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.utils.sam; - -import net.sf.samtools.Cigar; -import net.sf.samtools.CigarElement; -import net.sf.samtools.CigarOperator; -import net.sf.samtools.SAMRecord; -import org.broadinstitute.sting.gatk.contexts.ReferenceContext; -import org.broadinstitute.sting.utils.BaseUtils; -import org.broadinstitute.sting.utils.Utils; -import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import org.broadinstitute.sting.utils.pileup.PileupElement; -import org.broadinstitute.sting.utils.pileup.ReadBackedPileup; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.BitSet; - - -public class AlignmentUtils { - - public static class MismatchCount { - public int numMismatches = 0; - public long mismatchQualities = 0; - } - - public static long mismatchingQualities(SAMRecord r, byte[] refSeq, int refIndex) { - return getMismatchCount(r, refSeq, refIndex).mismatchQualities; - } - - public static MismatchCount getMismatchCount(SAMRecord r, byte[] refSeq, int refIndex) { - return getMismatchCount(r,refSeq,refIndex,0,r.getReadLength()); - } - - // todo -- this code and mismatchesInRefWindow should be combined and optimized into a single - // todo -- high performance implementation. We can do a lot better than this right now - public static MismatchCount getMismatchCount(SAMRecord r, byte[] refSeq, int refIndex, int startOnRead, int nReadBases) { - MismatchCount mc = new MismatchCount(); - - int readIdx = 0; - int endOnRead = startOnRead + nReadBases - 1; // index of the last base on read we want to count - byte[] readSeq = r.getReadBases(); - Cigar c = r.getCigar(); - for (int i = 0 ; i < c.numCigarElements() ; i++) { - - if ( readIdx > endOnRead ) break; - - CigarElement ce = c.getCigarElement(i); - switch ( ce.getOperator() ) { - case M: - for (int j = 0 ; j < ce.getLength() ; j++, refIndex++, readIdx++ ) { - if ( refIndex >= refSeq.length ) - continue; - if ( readIdx < startOnRead ) continue; - if ( readIdx > endOnRead ) break; - byte refChr = refSeq[refIndex]; - byte readChr = readSeq[readIdx]; - // Note: we need to count X/N's as mismatches because that's what SAM requires - //if ( BaseUtils.simpleBaseToBaseIndex(readChr) == -1 || - // BaseUtils.simpleBaseToBaseIndex(refChr) == -1 ) - // continue; // do not count Ns/Xs/etc ? - if ( readChr != refChr ) { - mc.numMismatches++; - mc.mismatchQualities += r.getBaseQualities()[readIdx]; - } - } - break; - case I: - case S: - readIdx += ce.getLength(); - break; - case D: - case N: - refIndex += ce.getLength(); - break; - case H: - case P: - break; - default: throw new ReviewedStingException("The " + ce.getOperator() + " cigar element is not currently supported"); - } - - } - return mc; - } - - /** Returns the number of mismatches in the pileup within the given reference context. - * - * @param pileup the pileup with reads - * @param ref the reference context - * @param ignoreTargetSite if true, ignore mismatches at the target locus (i.e. the center of the window) - * @return the number of mismatches - */ - public static int mismatchesInRefWindow(ReadBackedPileup pileup, ReferenceContext ref, boolean ignoreTargetSite) { - int mismatches = 0; - for ( PileupElement p : pileup ) - mismatches += mismatchesInRefWindow(p, ref, ignoreTargetSite); - return mismatches; - } - - /** Returns the number of mismatches in the pileup element within the given reference context. - * - * @param p the pileup element - * @param ref the reference context - * @param ignoreTargetSite if true, ignore mismatches at the target locus (i.e. the center of the window) - * @return the number of mismatches - */ - public static int mismatchesInRefWindow(PileupElement p, ReferenceContext ref, boolean ignoreTargetSite) { - return mismatchesInRefWindow(p, ref, ignoreTargetSite, false); - } - - /** Returns the number of mismatches in the pileup element within the given reference context. - * - * @param p the pileup element - * @param ref the reference context - * @param ignoreTargetSite if true, ignore mismatches at the target locus (i.e. the center of the window) - * @param qualitySumInsteadOfMismatchCount if true, return the quality score sum of the mismatches rather than the count - * @return the number of mismatches - */ - public static int mismatchesInRefWindow(PileupElement p, ReferenceContext ref, boolean ignoreTargetSite, boolean qualitySumInsteadOfMismatchCount) { - int sum = 0; - - int windowStart = ref.getWindow().getStart(); - int windowStop = ref.getWindow().getStop(); - byte[] refBases = ref.getBases(); - byte[] readBases = p.getRead().getReadBases(); - byte[] readQualities = p.getRead().getBaseQualities(); - Cigar c = p.getRead().getCigar(); - - int readIndex = 0; - int currentPos = p.getRead().getAlignmentStart(); - int refIndex = Math.max(0, currentPos - windowStart); - - for (int i = 0 ; i < c.numCigarElements() ; i++) { - CigarElement ce = c.getCigarElement(i); - int cigarElementLength = ce.getLength(); - switch ( ce.getOperator() ) { - case M: - for (int j = 0; j < cigarElementLength; j++, readIndex++, currentPos++) { - // are we past the ref window? - if ( currentPos > windowStop ) - break; - - // are we before the ref window? - if ( currentPos < windowStart ) - continue; - - byte refChr = refBases[refIndex++]; - - // do we need to skip the target site? - if ( ignoreTargetSite && ref.getLocus().getStart() == currentPos ) - continue; - - byte readChr = readBases[readIndex]; - if ( readChr != refChr ) - sum += (qualitySumInsteadOfMismatchCount) ? readQualities[readIndex] : 1; - } - break; - case I: - case S: - readIndex += cigarElementLength; - break; - case D: - case N: - currentPos += cigarElementLength; - if ( currentPos > windowStart ) - refIndex += Math.min(cigarElementLength, currentPos - windowStart); - break; - case H: - case P: - break; - } - } - - return sum; - } - - /** Returns the number of mismatches in the pileup element within the given reference context. - * - * @param read the SAMRecord - * @param ref the reference context - * @param maxMismatches the maximum number of surrounding mismatches we tolerate to consider a base good - * @param windowSize window size (on each side) to test - * @return a bitset representing which bases are good - */ - public static BitSet mismatchesInRefWindow(SAMRecord read, ReferenceContext ref, int maxMismatches, int windowSize) { - // first determine the positions with mismatches - int readLength = read.getReadLength(); - BitSet mismatches = new BitSet(readLength); - - // it's possible we aren't starting at the beginning of a read, - // and we don't need to look at any of the previous context outside our window - // (although we do need future context) - int readStartPos = Math.max(read.getAlignmentStart(), ref.getLocus().getStart() - windowSize); - int currentReadPos = read.getAlignmentStart(); - - byte[] refBases = ref.getBases(); - int refIndex = readStartPos - ref.getWindow().getStart(); - if ( refIndex < 0 ) { - throw new IllegalStateException("When calculating mismatches, we somehow don't have enough previous reference context for read " + read.getReadName() + " at position " + ref.getLocus()); - } - - byte[] readBases = read.getReadBases(); - int readIndex = 0; - - Cigar c = read.getCigar(); - - for (int i = 0 ; i < c.numCigarElements() ; i++) { - CigarElement ce = c.getCigarElement(i); - int cigarElementLength = ce.getLength(); - switch ( ce.getOperator() ) { - case M: - for (int j = 0; j < cigarElementLength; j++, readIndex++) { - // skip over unwanted bases - if ( currentReadPos++ < readStartPos ) - continue; - - // this is possible if reads extend beyond the contig end - if ( refIndex >= refBases.length ) - break; - - byte refChr = refBases[refIndex]; - byte readChr = readBases[readIndex]; - if ( readChr != refChr ) - mismatches.set(readIndex); - - refIndex++; - } - break; - case I: - case S: - readIndex += cigarElementLength; - break; - case D: - case N: - if ( currentReadPos >= readStartPos ) - refIndex += cigarElementLength; - currentReadPos += cigarElementLength; - break; - case H: - case P: - break; - } - } - - // all bits are set to false by default - BitSet result = new BitSet(readLength); - - int currentPos = 0, leftPos = 0, rightPos; - int mismatchCount = 0; - - // calculate how many mismatches exist in the windows to the left/right - for ( rightPos = 1; rightPos <= windowSize && rightPos < readLength; rightPos++) { - if ( mismatches.get(rightPos) ) - mismatchCount++; - } - if ( mismatchCount <= maxMismatches ) - result.set(currentPos); - - // now, traverse over the read positions - while ( currentPos < readLength ) { - // add a new rightmost position - if ( rightPos < readLength && mismatches.get(rightPos++) ) - mismatchCount++; - // re-penalize the previous position - if ( mismatches.get(currentPos++) ) - mismatchCount++; - // don't penalize the current position - if ( mismatches.get(currentPos) ) - mismatchCount--; - // subtract the leftmost position - if ( leftPos < currentPos - windowSize && mismatches.get(leftPos++) ) - mismatchCount--; - - if ( mismatchCount <= maxMismatches ) - result.set(currentPos); - } - - return result; - } - /** Returns number of alignment blocks (continuous stretches of aligned bases) in the specified alignment. - * This method follows closely the SAMRecord::getAlignmentBlocks() implemented in samtools library, but - * it only counts blocks without actually allocating and filling the list of blocks themselves. Hence, this method is - * a much more efficient alternative to r.getAlignmentBlocks.size() in the situations when this number is all that is needed. - * Formally, this method simply returns the number of M elements in the cigar. - * @param r alignment - * @return number of continuous alignment blocks (i.e. 'M' elements of the cigar; all indel and clipping elements are ignored). - */ - public static int getNumAlignmentBlocks(final SAMRecord r) { - int n = 0; - final Cigar cigar = r.getCigar(); - if (cigar == null) return 0; - - for (final CigarElement e : cigar.getCigarElements()) { - if (e.getOperator() == CigarOperator.M ) n++; - } - - return n; - } - - public static int getNumAlignedBases(final SAMRecord r) { - int n = 0; - final Cigar cigar = r.getCigar(); - if (cigar == null) return 0; - - for (final CigarElement e : cigar.getCigarElements()) { - if (e.getOperator() == CigarOperator.M ) { n += e.getLength(); } - } - - return n; - } - - public static byte[] alignmentToByteArray( final Cigar cigar, final byte[] read, final byte[] ref ) { - - final byte[] alignment = new byte[read.length]; - int refPos = 0; - int alignPos = 0; - - for ( int iii = 0 ; iii < cigar.numCigarElements() ; iii++ ) { - - final CigarElement ce = cigar.getCigarElement(iii); - final int elementLength = ce.getLength(); - - switch( ce.getOperator() ) { - case I: - case S: - for ( int jjj = 0; jjj < elementLength; jjj++ ) { - alignment[alignPos++] = '+'; - } - break; - case D: - case N: - refPos += elementLength; - break; - case M: - for ( int jjj = 0; jjj < elementLength; jjj++ ) { - alignment[alignPos] = ref[refPos]; - alignPos++; - refPos++; - } - break; - case H: - case P: - break; - default: - throw new ReviewedStingException( "Unsupported cigar operator: " + ce.getOperator() ); - } - } - return alignment; - } - - public static int calcAlignmentByteArrayOffset( final Cigar cigar, int pileupOffset, final int alignmentStart, final int refLocus ) { - - boolean atDeletion = false; - if(pileupOffset == -1) { - atDeletion = true; - pileupOffset = refLocus - alignmentStart; - final CigarElement ce = cigar.getCigarElement(0); - if( ce.getOperator() == CigarOperator.S ) { - pileupOffset += ce.getLength(); - } - } - int pos = 0; - int alignmentPos = 0; - for ( int iii = 0 ; iii < cigar.numCigarElements() ; iii++ ) { - - final CigarElement ce = cigar.getCigarElement(iii); - final int elementLength = ce.getLength(); - - switch( ce.getOperator() ) { - case I: - case S: - pos += elementLength; - if( pos >= pileupOffset ) { - return alignmentPos; - } - break; - case D: - case N: - if(!atDeletion) { - alignmentPos += elementLength; - } else { - if( pos + elementLength - 1 >= pileupOffset ) { - return alignmentPos + (pileupOffset - pos); - } else { - pos += elementLength; - alignmentPos += elementLength; - } - } - break; - case M: - if( pos + elementLength - 1 >= pileupOffset ) { - return alignmentPos + (pileupOffset - pos); - } else { - pos += elementLength; - alignmentPos += elementLength; - } - break; - case H: - case P: - break; - default: - throw new ReviewedStingException( "Unsupported cigar operator: " + ce.getOperator() ); - } - } - return alignmentPos; - } - - public static byte[] readToAlignmentByteArray( final Cigar cigar, final byte[] read ) { - - int alignmentLength = 0; - for ( int iii = 0 ; iii < cigar.numCigarElements() ; iii++ ) { - - final CigarElement ce = cigar.getCigarElement(iii); - final int elementLength = ce.getLength(); - - switch( ce.getOperator() ) { - case I: - case S: - break; - case D: - case N: - alignmentLength += elementLength; - break; - case M: - alignmentLength += elementLength; - break; - case H: - case P: - break; - default: - throw new ReviewedStingException( "Unsupported cigar operator: " + ce.getOperator() ); - } - } - - final byte[] alignment = new byte[alignmentLength]; - int alignPos = 0; - int readPos = 0; - for ( int iii = 0 ; iii < cigar.numCigarElements() ; iii++ ) { - - final CigarElement ce = cigar.getCigarElement(iii); - final int elementLength = ce.getLength(); - - switch( ce.getOperator() ) { - case I: - if( alignPos > 0 ) { - if( alignment[alignPos-1] == BaseUtils.A ) { alignment[alignPos-1] = PileupElement.A_FOLLOWED_BY_INSERTION_BASE; } - else if( alignment[alignPos-1] == BaseUtils.C ) { alignment[alignPos-1] = PileupElement.C_FOLLOWED_BY_INSERTION_BASE; } - else if( alignment[alignPos-1] == BaseUtils.T ) { alignment[alignPos-1] = PileupElement.T_FOLLOWED_BY_INSERTION_BASE; } - else if( alignment[alignPos-1] == BaseUtils.G ) { alignment[alignPos-1] = PileupElement.G_FOLLOWED_BY_INSERTION_BASE; } - } - case S: - for ( int jjj = 0; jjj < elementLength; jjj++ ) { - readPos++; - } - break; - case D: - case N: - for ( int jjj = 0; jjj < elementLength; jjj++ ) { - alignment[alignPos] = PileupElement.DELETION_BASE; - alignPos++; - } - break; - case M: - for ( int jjj = 0; jjj < elementLength; jjj++ ) { - alignment[alignPos] = read[readPos]; - alignPos++; - readPos++; - } - break; - case H: - case P: - break; - default: - throw new ReviewedStingException( "Unsupported cigar operator: " + ce.getOperator() ); - } - } - return alignment; - } - - /** - * Due to (unfortunate) multiple ways to indicate that read is unmapped allowed by SAM format - * specification, one may need this convenience shortcut. Checks both 'read unmapped' flag and - * alignment reference index/start. - * @param r record - * @return true if read is unmapped - */ - public static boolean isReadUnmapped(final SAMRecord r) { - if ( r.getReadUnmappedFlag() ) return true; - - // our life would be so much easier if all sam files followed the specs. In reality, - // sam files (including those generated by maq or bwa) miss headers altogether. When - // reading such a SAM file, reference name is set, but since there is no sequence dictionary, - // null is always returned for referenceIndex. Let's be paranoid here, and make sure that - // we do not call the read "unmapped" when it has only reference name set with ref. index missing - // or vice versa. - if ( ( r.getReferenceIndex() != null && r.getReferenceIndex() != SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX - || r.getReferenceName() != null && !r.getReferenceName().equals(SAMRecord.NO_ALIGNMENT_REFERENCE_NAME) ) - && r.getAlignmentStart() != SAMRecord.NO_ALIGNMENT_START ) return false ; - return true; - } - - /** - * Due to (unfortunate) multiple ways to indicate that read/mate is unmapped allowed by SAM format - * specification, one may need this convenience shortcut. Checks both 'mate unmapped' flag and - * alignment reference index/start of the mate. - * @param r sam record for the read - * @return true if read's mate is unmapped - */ - public static boolean isMateUnmapped(final SAMRecord r) { - if ( r.getMateUnmappedFlag() ) return true; - - // our life would be so much easier if all sam files followed the specs. In reality, - // sam files (including those generated by maq or bwa) miss headers altogether. When - // reading such a SAM file, reference name is set, but since there is no sequence dictionary, - // null is always returned for referenceIndex. Let's be paranoid here, and make sure that - // we do not call the read "unmapped" when it has only reference name set with ref. index missing - // or vice versa. - if ( ( r.getMateReferenceIndex() != null && r.getMateReferenceIndex() != SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX - || r.getMateReferenceName() != null && !r.getMateReferenceName().equals(SAMRecord.NO_ALIGNMENT_REFERENCE_NAME) ) - && r.getMateAlignmentStart() != SAMRecord.NO_ALIGNMENT_START ) return false ; - return true; - } - - /** Returns true is read is mapped and mapped uniquely (Q>0). - * - * @param read - * @return - */ - public static boolean isReadUniquelyMapped(SAMRecord read) { - return ( ! AlignmentUtils.isReadUnmapped(read) ) && read.getMappingQuality() > 0; - } - - /** Returns the array of base qualitites in the order the bases were read on the machine (i.e. always starting from - * cycle 1). In other words, if the read is unmapped or aligned in the forward direction, the read's own base - * qualities are returned as stored in the SAM record; if the read is aligned in the reverse direction, the array - * of read's base qualitites is inverted (in this case new array is allocated and returned). - * @param read - * @return - */ - public static byte [] getQualsInCycleOrder(SAMRecord read) { - if ( isReadUnmapped(read) || ! read.getReadNegativeStrandFlag() ) return read.getBaseQualities(); - - return Utils.reverse(read.getBaseQualities()); - } - - /** Returns the array of original base qualitites (before recalibration) in the order the bases were read on the machine (i.e. always starting from - * cycle 1). In other words, if the read is unmapped or aligned in the forward direction, the read's own base - * qualities are returned as stored in the SAM record; if the read is aligned in the reverse direction, the array - * of read's base qualitites is inverted (in this case new array is allocated and returned). If no original base qualities - * are available this method will throw a runtime exception. - * @param read - * @return - */ - public static byte [] getOriginalQualsInCycleOrder(SAMRecord read) { - if ( isReadUnmapped(read) || ! read.getReadNegativeStrandFlag() ) return read.getOriginalBaseQualities(); - - return Utils.reverse(read.getOriginalBaseQualities()); - } - - /** Takes the alignment of the read sequence readSeq to the reference sequence refSeq - * starting at 0-based position refIndex on the refSeq and specified by its cigar. - * The last argument readIndex specifies 0-based position on the read where the alignment described by the - * cigar starts. Usually cigars specify alignments of the whole read to the ref, so that readIndex is normally 0. - * Use non-zero readIndex only when the alignment cigar represents alignment of a part of the read. The refIndex in this case - * should be the position where the alignment of that part of the read starts at. In other words, both refIndex and readIndex are - * always the positions where the cigar starts on the ref and on the read, respectively. - * - * If the alignment has an indel, then this method attempts moving this indel left across a stretch of repetitive bases. For instance, if the original cigar - * specifies that (any) one AT is deleted from a repeat sequence TATATATA, the output cigar will always mark the leftmost AT - * as deleted. If there is no indel in the original cigar, or the indel position is determined unambiguously (i.e. inserted/deleted sequence - * is not repeated), the original cigar is returned. - * @param cigar structure of the original alignment - * @param refSeq reference sequence the read is aligned to - * @param readSeq read sequence - * @param refIndex 0-based alignment start position on ref - * @param readIndex 0-based alignment start position on read - * @return a cigar, in which indel is guaranteed to be placed at the leftmost possible position across a repeat (if any) - */ - public static Cigar leftAlignIndel(Cigar cigar, final byte[] refSeq, final byte[] readSeq, final int refIndex, final int readIndex) { - - int indexOfIndel = -1; - for ( int i = 0; i < cigar.numCigarElements(); i++ ) { - CigarElement ce = cigar.getCigarElement(i); - if ( ce.getOperator() == CigarOperator.D || ce.getOperator() == CigarOperator.I ) { - // if there is more than 1 indel, don't left align - if ( indexOfIndel != -1 ) - return cigar; - indexOfIndel = i; - } - } - - // if there is no indel or if the alignment starts with an insertion (so that there - // is no place on the read to move that insertion further left), we are done - if ( indexOfIndel < 1 ) return cigar; - - final int indelLength = cigar.getCigarElement(indexOfIndel).getLength(); - - byte[] altString = createIndelString(cigar, indexOfIndel, refSeq, readSeq, refIndex, readIndex); - if ( altString == null ) - return cigar; - - Cigar newCigar = cigar; - for ( int i = 0; i < indelLength; i++ ) { - newCigar = moveCigarLeft(newCigar, indexOfIndel); - byte[] newAltString = createIndelString(newCigar, indexOfIndel, refSeq, readSeq, refIndex, readIndex); - - // check to make sure we haven't run off the end of the read - boolean reachedEndOfRead = cigarHasZeroSizeElement(newCigar); - - if ( Arrays.equals(altString, newAltString) ) { - cigar = newCigar; - i = -1; - if ( reachedEndOfRead ) - cigar = cleanUpCigar(cigar); - } - - if ( reachedEndOfRead ) - break; - } - - return cigar; - } - - private static boolean cigarHasZeroSizeElement(Cigar c) { - for ( CigarElement ce : c.getCigarElements() ) { - if ( ce.getLength() == 0 ) - return true; - } - return false; - } - - private static Cigar cleanUpCigar(Cigar c) { - ArrayList elements = new ArrayList(c.numCigarElements()-1); - for ( CigarElement ce : c.getCigarElements() ) { - if ( ce.getLength() != 0 && - (elements.size() != 0 || ce.getOperator() != CigarOperator.D) ) { - elements.add(ce); - } - } - return new Cigar(elements); - } - - private static Cigar moveCigarLeft(Cigar cigar, int indexOfIndel) { - // get the first few elements - ArrayList elements = new ArrayList(cigar.numCigarElements()); - for ( int i = 0; i < indexOfIndel - 1; i++) - elements.add(cigar.getCigarElement(i)); - - // get the indel element and move it left one base - CigarElement ce = cigar.getCigarElement(indexOfIndel-1); - elements.add(new CigarElement(ce.getLength()-1, ce.getOperator())); - elements.add(cigar.getCigarElement(indexOfIndel)); - if ( indexOfIndel+1 < cigar.numCigarElements() ) { - ce = cigar.getCigarElement(indexOfIndel+1); - elements.add(new CigarElement(ce.getLength()+1, ce.getOperator())); - } else { - elements.add(new CigarElement(1, CigarOperator.M)); - } - - // get the last few elements - for ( int i = indexOfIndel + 2; i < cigar.numCigarElements(); i++) - elements.add(cigar.getCigarElement(i)); - return new Cigar(elements); - } - - private static byte[] createIndelString(final Cigar cigar, final int indexOfIndel, final byte[] refSeq, final byte[] readSeq, int refIndex, int readIndex) { - CigarElement indel = cigar.getCigarElement(indexOfIndel); - int indelLength = indel.getLength(); - - int totalRefBases = 0; - for ( int i = 0; i < indexOfIndel; i++ ) { - CigarElement ce = cigar.getCigarElement(i); - int length = ce.getLength(); - - switch( ce.getOperator() ) { - case M: - readIndex += length; - refIndex += length; - totalRefBases += length; - break; - case S: - readIndex += length; - break; - case N: - refIndex += length; - totalRefBases += length; - break; - default: - break; - } - } - - // sometimes, when there are very large known indels, we won't have enough reference sequence to cover them - if ( totalRefBases + indelLength > refSeq.length ) - indelLength -= (totalRefBases + indelLength - refSeq.length); - - // the indel-based reference string - byte[] alt = new byte[refSeq.length + (indelLength * (indel.getOperator() == CigarOperator.D ? -1 : 1))]; - - // add the bases before the indel, making sure it's not aligned off the end of the reference - if ( refIndex > alt.length || refIndex > refSeq.length ) - return null; - System.arraycopy(refSeq, 0, alt, 0, refIndex); - int currentPos = refIndex; - - // take care of the indel - if ( indel.getOperator() == CigarOperator.D ) { - refIndex += indelLength; - } else { - System.arraycopy(readSeq, readIndex, alt, currentPos, indelLength); - currentPos += indelLength; - } - - // add the bases after the indel, making sure it's not aligned off the end of the reference - if ( refSeq.length - refIndex > alt.length - currentPos ) - return null; - System.arraycopy(refSeq, refIndex, alt, currentPos, refSeq.length - refIndex); - - return alt; - } -} +/* + * Copyright (c) 2010 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR + * THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.sam; + +import net.sf.samtools.Cigar; +import net.sf.samtools.CigarElement; +import net.sf.samtools.CigarOperator; +import net.sf.samtools.SAMRecord; +import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.utils.BaseUtils; +import org.broadinstitute.sting.utils.Utils; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import org.broadinstitute.sting.utils.pileup.PileupElement; +import org.broadinstitute.sting.utils.pileup.ReadBackedPileup; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; + + +public class AlignmentUtils { + + public static class MismatchCount { + public int numMismatches = 0; + public long mismatchQualities = 0; + } + + public static long mismatchingQualities(SAMRecord r, byte[] refSeq, int refIndex) { + return getMismatchCount(r, refSeq, refIndex).mismatchQualities; + } + + public static MismatchCount getMismatchCount(SAMRecord r, byte[] refSeq, int refIndex) { + return getMismatchCount(r, refSeq, refIndex, 0, r.getReadLength()); + } + + // todo -- this code and mismatchesInRefWindow should be combined and optimized into a single + // todo -- high performance implementation. We can do a lot better than this right now + public static MismatchCount getMismatchCount(SAMRecord r, byte[] refSeq, int refIndex, int startOnRead, int nReadBases) { + MismatchCount mc = new MismatchCount(); + + int readIdx = 0; + int endOnRead = startOnRead + nReadBases - 1; // index of the last base on read we want to count + byte[] readSeq = r.getReadBases(); + Cigar c = r.getCigar(); + for (int i = 0; i < c.numCigarElements(); i++) { + + if (readIdx > endOnRead) break; + + CigarElement ce = c.getCigarElement(i); + switch (ce.getOperator()) { + case M: + for (int j = 0; j < ce.getLength(); j++, refIndex++, readIdx++) { + if (refIndex >= refSeq.length) + continue; + if (readIdx < startOnRead) continue; + if (readIdx > endOnRead) break; + byte refChr = refSeq[refIndex]; + byte readChr = readSeq[readIdx]; + // Note: we need to count X/N's as mismatches because that's what SAM requires + //if ( BaseUtils.simpleBaseToBaseIndex(readChr) == -1 || + // BaseUtils.simpleBaseToBaseIndex(refChr) == -1 ) + // continue; // do not count Ns/Xs/etc ? + if (readChr != refChr) { + mc.numMismatches++; + mc.mismatchQualities += r.getBaseQualities()[readIdx]; + } + } + break; + case I: + case S: + readIdx += ce.getLength(); + break; + case D: + case N: + refIndex += ce.getLength(); + break; + case H: + case P: + break; + default: + throw new ReviewedStingException("The " + ce.getOperator() + " cigar element is not currently supported"); + } + + } + return mc; + } + + /** + * Returns the number of mismatches in the pileup within the given reference context. + * + * @param pileup the pileup with reads + * @param ref the reference context + * @param ignoreTargetSite if true, ignore mismatches at the target locus (i.e. the center of the window) + * @return the number of mismatches + */ + public static int mismatchesInRefWindow(ReadBackedPileup pileup, ReferenceContext ref, boolean ignoreTargetSite) { + int mismatches = 0; + for (PileupElement p : pileup) + mismatches += mismatchesInRefWindow(p, ref, ignoreTargetSite); + return mismatches; + } + + /** + * Returns the number of mismatches in the pileup element within the given reference context. + * + * @param p the pileup element + * @param ref the reference context + * @param ignoreTargetSite if true, ignore mismatches at the target locus (i.e. the center of the window) + * @return the number of mismatches + */ + public static int mismatchesInRefWindow(PileupElement p, ReferenceContext ref, boolean ignoreTargetSite) { + return mismatchesInRefWindow(p, ref, ignoreTargetSite, false); + } + + /** + * Returns the number of mismatches in the pileup element within the given reference context. + * + * @param p the pileup element + * @param ref the reference context + * @param ignoreTargetSite if true, ignore mismatches at the target locus (i.e. the center of the window) + * @param qualitySumInsteadOfMismatchCount + * if true, return the quality score sum of the mismatches rather than the count + * @return the number of mismatches + */ + public static int mismatchesInRefWindow(PileupElement p, ReferenceContext ref, boolean ignoreTargetSite, boolean qualitySumInsteadOfMismatchCount) { + int sum = 0; + + int windowStart = ref.getWindow().getStart(); + int windowStop = ref.getWindow().getStop(); + byte[] refBases = ref.getBases(); + byte[] readBases = p.getRead().getReadBases(); + byte[] readQualities = p.getRead().getBaseQualities(); + Cigar c = p.getRead().getCigar(); + + int readIndex = 0; + int currentPos = p.getRead().getAlignmentStart(); + int refIndex = Math.max(0, currentPos - windowStart); + + for (int i = 0; i < c.numCigarElements(); i++) { + CigarElement ce = c.getCigarElement(i); + int cigarElementLength = ce.getLength(); + switch (ce.getOperator()) { + case M: + for (int j = 0; j < cigarElementLength; j++, readIndex++, currentPos++) { + // are we past the ref window? + if (currentPos > windowStop) + break; + + // are we before the ref window? + if (currentPos < windowStart) + continue; + + byte refChr = refBases[refIndex++]; + + // do we need to skip the target site? + if (ignoreTargetSite && ref.getLocus().getStart() == currentPos) + continue; + + byte readChr = readBases[readIndex]; + if (readChr != refChr) + sum += (qualitySumInsteadOfMismatchCount) ? readQualities[readIndex] : 1; + } + break; + case I: + case S: + readIndex += cigarElementLength; + break; + case D: + case N: + currentPos += cigarElementLength; + if (currentPos > windowStart) + refIndex += Math.min(cigarElementLength, currentPos - windowStart); + break; + case H: + case P: + break; + } + } + + return sum; + } + + /** + * Returns the number of mismatches in the pileup element within the given reference context. + * + * @param read the SAMRecord + * @param ref the reference context + * @param maxMismatches the maximum number of surrounding mismatches we tolerate to consider a base good + * @param windowSize window size (on each side) to test + * @return a bitset representing which bases are good + */ + public static BitSet mismatchesInRefWindow(SAMRecord read, ReferenceContext ref, int maxMismatches, int windowSize) { + // first determine the positions with mismatches + int readLength = read.getReadLength(); + BitSet mismatches = new BitSet(readLength); + + // it's possible we aren't starting at the beginning of a read, + // and we don't need to look at any of the previous context outside our window + // (although we do need future context) + int readStartPos = Math.max(read.getAlignmentStart(), ref.getLocus().getStart() - windowSize); + int currentReadPos = read.getAlignmentStart(); + + byte[] refBases = ref.getBases(); + int refIndex = readStartPos - ref.getWindow().getStart(); + if (refIndex < 0) { + throw new IllegalStateException("When calculating mismatches, we somehow don't have enough previous reference context for read " + read.getReadName() + " at position " + ref.getLocus()); + } + + byte[] readBases = read.getReadBases(); + int readIndex = 0; + + Cigar c = read.getCigar(); + + for (int i = 0; i < c.numCigarElements(); i++) { + CigarElement ce = c.getCigarElement(i); + int cigarElementLength = ce.getLength(); + switch (ce.getOperator()) { + case M: + for (int j = 0; j < cigarElementLength; j++, readIndex++) { + // skip over unwanted bases + if (currentReadPos++ < readStartPos) + continue; + + // this is possible if reads extend beyond the contig end + if (refIndex >= refBases.length) + break; + + byte refChr = refBases[refIndex]; + byte readChr = readBases[readIndex]; + if (readChr != refChr) + mismatches.set(readIndex); + + refIndex++; + } + break; + case I: + case S: + readIndex += cigarElementLength; + break; + case D: + case N: + if (currentReadPos >= readStartPos) + refIndex += cigarElementLength; + currentReadPos += cigarElementLength; + break; + case H: + case P: + break; + } + } + + // all bits are set to false by default + BitSet result = new BitSet(readLength); + + int currentPos = 0, leftPos = 0, rightPos; + int mismatchCount = 0; + + // calculate how many mismatches exist in the windows to the left/right + for (rightPos = 1; rightPos <= windowSize && rightPos < readLength; rightPos++) { + if (mismatches.get(rightPos)) + mismatchCount++; + } + if (mismatchCount <= maxMismatches) + result.set(currentPos); + + // now, traverse over the read positions + while (currentPos < readLength) { + // add a new rightmost position + if (rightPos < readLength && mismatches.get(rightPos++)) + mismatchCount++; + // re-penalize the previous position + if (mismatches.get(currentPos++)) + mismatchCount++; + // don't penalize the current position + if (mismatches.get(currentPos)) + mismatchCount--; + // subtract the leftmost position + if (leftPos < currentPos - windowSize && mismatches.get(leftPos++)) + mismatchCount--; + + if (mismatchCount <= maxMismatches) + result.set(currentPos); + } + + return result; + } + + /** + * Returns number of alignment blocks (continuous stretches of aligned bases) in the specified alignment. + * This method follows closely the SAMRecord::getAlignmentBlocks() implemented in samtools library, but + * it only counts blocks without actually allocating and filling the list of blocks themselves. Hence, this method is + * a much more efficient alternative to r.getAlignmentBlocks.size() in the situations when this number is all that is needed. + * Formally, this method simply returns the number of M elements in the cigar. + * + * @param r alignment + * @return number of continuous alignment blocks (i.e. 'M' elements of the cigar; all indel and clipping elements are ignored). + */ + public static int getNumAlignmentBlocks(final SAMRecord r) { + int n = 0; + final Cigar cigar = r.getCigar(); + if (cigar == null) return 0; + + for (final CigarElement e : cigar.getCigarElements()) { + if (e.getOperator() == CigarOperator.M) n++; + } + + return n; + } + + public static int getNumAlignedBases(final SAMRecord r) { + int n = 0; + final Cigar cigar = r.getCigar(); + if (cigar == null) return 0; + + for (final CigarElement e : cigar.getCigarElements()) + if (e.getOperator() == CigarOperator.M) + n += e.getLength(); + + return n; + } + + public static byte[] alignmentToByteArray(final Cigar cigar, final byte[] read, final byte[] ref) { + + final byte[] alignment = new byte[read.length]; + int refPos = 0; + int alignPos = 0; + + for (int iii = 0; iii < cigar.numCigarElements(); iii++) { + + final CigarElement ce = cigar.getCigarElement(iii); + final int elementLength = ce.getLength(); + + switch (ce.getOperator()) { + case I: + case S: + for (int jjj = 0; jjj < elementLength; jjj++) { + alignment[alignPos++] = '+'; + } + break; + case D: + case N: + refPos += elementLength; + break; + case M: + for (int jjj = 0; jjj < elementLength; jjj++) { + alignment[alignPos] = ref[refPos]; + alignPos++; + refPos++; + } + break; + case H: + case P: + break; + default: + throw new ReviewedStingException("Unsupported cigar operator: " + ce.getOperator()); + } + } + return alignment; + } + + public static int calcAlignmentByteArrayOffset(final Cigar cigar, PileupElement pileup, final int alignmentStart, final int refLocus) { + int pileupOffset = pileup.getOffset(); + + // Special case for reads starting with insertion + if (pileup.isInsertionAtBeginningOfRead()) + return 0; + + // Reassign the offset if we are in the middle of a deletion because of the modified representation of the read bases + if (pileup.isDeletion()) { + pileupOffset = refLocus - alignmentStart; + final CigarElement ce = cigar.getCigarElement(0); + if (ce.getOperator() == CigarOperator.S) { + pileupOffset += ce.getLength(); + } + } + + int pos = 0; + int alignmentPos = 0; + + for (int iii = 0; iii < cigar.numCigarElements(); iii++) { + final CigarElement ce = cigar.getCigarElement(iii); + final int elementLength = ce.getLength(); + + switch (ce.getOperator()) { + case I: + case S: + pos += elementLength; + if (pos >= pileupOffset) { + return alignmentPos; + } + break; + case D: + case N: + if (!pileup.isDeletion()) { + alignmentPos += elementLength; + } else { + if (pos + elementLength - 1 >= pileupOffset) { + return alignmentPos + (pileupOffset - pos); + } else { + pos += elementLength; + alignmentPos += elementLength; + } + } + break; + case M: + if (pos + elementLength - 1 >= pileupOffset) { + return alignmentPos + (pileupOffset - pos); + } else { + pos += elementLength; + alignmentPos += elementLength; + } + break; + case H: + case P: + break; + default: + throw new ReviewedStingException("Unsupported cigar operator: " + ce.getOperator()); + } + } + + return alignmentPos; + } + + public static byte[] readToAlignmentByteArray(final Cigar cigar, final byte[] read) { + + int alignmentLength = 0; + for (int iii = 0; iii < cigar.numCigarElements(); iii++) { + + final CigarElement ce = cigar.getCigarElement(iii); + final int elementLength = ce.getLength(); + + switch (ce.getOperator()) { + case I: + case S: + break; + case D: + case N: + alignmentLength += elementLength; + break; + case M: + alignmentLength += elementLength; + break; + case H: + case P: + break; + default: + throw new ReviewedStingException("Unsupported cigar operator: " + ce.getOperator()); + } + } + + final byte[] alignment = new byte[alignmentLength]; + int alignPos = 0; + int readPos = 0; + for (int iii = 0; iii < cigar.numCigarElements(); iii++) { + + final CigarElement ce = cigar.getCigarElement(iii); + final int elementLength = ce.getLength(); + + switch (ce.getOperator()) { + case I: + if (alignPos > 0) { + if (alignment[alignPos - 1] == BaseUtils.A) { + alignment[alignPos - 1] = PileupElement.A_FOLLOWED_BY_INSERTION_BASE; + } else if (alignment[alignPos - 1] == BaseUtils.C) { + alignment[alignPos - 1] = PileupElement.C_FOLLOWED_BY_INSERTION_BASE; + } else if (alignment[alignPos - 1] == BaseUtils.T) { + alignment[alignPos - 1] = PileupElement.T_FOLLOWED_BY_INSERTION_BASE; + } else if (alignment[alignPos - 1] == BaseUtils.G) { + alignment[alignPos - 1] = PileupElement.G_FOLLOWED_BY_INSERTION_BASE; + } + } + case S: + for (int jjj = 0; jjj < elementLength; jjj++) { + readPos++; + } + break; + case D: + case N: + for (int jjj = 0; jjj < elementLength; jjj++) { + alignment[alignPos] = PileupElement.DELETION_BASE; + alignPos++; + } + break; + case M: + for (int jjj = 0; jjj < elementLength; jjj++) { + alignment[alignPos] = read[readPos]; + alignPos++; + readPos++; + } + break; + case H: + case P: + break; + default: + throw new ReviewedStingException("Unsupported cigar operator: " + ce.getOperator()); + } + } + return alignment; + } + + /** + * Due to (unfortunate) multiple ways to indicate that read is unmapped allowed by SAM format + * specification, one may need this convenience shortcut. Checks both 'read unmapped' flag and + * alignment reference index/start. + * + * @param r record + * @return true if read is unmapped + */ + public static boolean isReadUnmapped(final SAMRecord r) { + if (r.getReadUnmappedFlag()) return true; + + // our life would be so much easier if all sam files followed the specs. In reality, + // sam files (including those generated by maq or bwa) miss headers altogether. When + // reading such a SAM file, reference name is set, but since there is no sequence dictionary, + // null is always returned for referenceIndex. Let's be paranoid here, and make sure that + // we do not call the read "unmapped" when it has only reference name set with ref. index missing + // or vice versa. + if ((r.getReferenceIndex() != null && r.getReferenceIndex() != SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX + || r.getReferenceName() != null && !r.getReferenceName().equals(SAMRecord.NO_ALIGNMENT_REFERENCE_NAME)) + && r.getAlignmentStart() != SAMRecord.NO_ALIGNMENT_START) return false; + return true; + } + + /** + * Due to (unfortunate) multiple ways to indicate that read/mate is unmapped allowed by SAM format + * specification, one may need this convenience shortcut. Checks both 'mate unmapped' flag and + * alignment reference index/start of the mate. + * + * @param r sam record for the read + * @return true if read's mate is unmapped + */ + public static boolean isMateUnmapped(final SAMRecord r) { + if (r.getMateUnmappedFlag()) return true; + + // our life would be so much easier if all sam files followed the specs. In reality, + // sam files (including those generated by maq or bwa) miss headers altogether. When + // reading such a SAM file, reference name is set, but since there is no sequence dictionary, + // null is always returned for referenceIndex. Let's be paranoid here, and make sure that + // we do not call the read "unmapped" when it has only reference name set with ref. index missing + // or vice versa. + if ((r.getMateReferenceIndex() != null && r.getMateReferenceIndex() != SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX + || r.getMateReferenceName() != null && !r.getMateReferenceName().equals(SAMRecord.NO_ALIGNMENT_REFERENCE_NAME)) + && r.getMateAlignmentStart() != SAMRecord.NO_ALIGNMENT_START) return false; + return true; + } + + /** + * Returns true is read is mapped and mapped uniquely (Q>0). + * + * @param read + * @return + */ + public static boolean isReadUniquelyMapped(SAMRecord read) { + return (!AlignmentUtils.isReadUnmapped(read)) && read.getMappingQuality() > 0; + } + + /** + * Returns the array of base qualitites in the order the bases were read on the machine (i.e. always starting from + * cycle 1). In other words, if the read is unmapped or aligned in the forward direction, the read's own base + * qualities are returned as stored in the SAM record; if the read is aligned in the reverse direction, the array + * of read's base qualitites is inverted (in this case new array is allocated and returned). + * + * @param read + * @return + */ + public static byte[] getQualsInCycleOrder(SAMRecord read) { + if (isReadUnmapped(read) || !read.getReadNegativeStrandFlag()) return read.getBaseQualities(); + + return Utils.reverse(read.getBaseQualities()); + } + + /** + * Returns the array of original base qualitites (before recalibration) in the order the bases were read on the machine (i.e. always starting from + * cycle 1). In other words, if the read is unmapped or aligned in the forward direction, the read's own base + * qualities are returned as stored in the SAM record; if the read is aligned in the reverse direction, the array + * of read's base qualitites is inverted (in this case new array is allocated and returned). If no original base qualities + * are available this method will throw a runtime exception. + * + * @param read + * @return + */ + public static byte[] getOriginalQualsInCycleOrder(SAMRecord read) { + if (isReadUnmapped(read) || !read.getReadNegativeStrandFlag()) return read.getOriginalBaseQualities(); + + return Utils.reverse(read.getOriginalBaseQualities()); + } + + /** + * Takes the alignment of the read sequence readSeq to the reference sequence refSeq + * starting at 0-based position refIndex on the refSeq and specified by its cigar. + * The last argument readIndex specifies 0-based position on the read where the alignment described by the + * cigar starts. Usually cigars specify alignments of the whole read to the ref, so that readIndex is normally 0. + * Use non-zero readIndex only when the alignment cigar represents alignment of a part of the read. The refIndex in this case + * should be the position where the alignment of that part of the read starts at. In other words, both refIndex and readIndex are + * always the positions where the cigar starts on the ref and on the read, respectively. + *

+ * If the alignment has an indel, then this method attempts moving this indel left across a stretch of repetitive bases. For instance, if the original cigar + * specifies that (any) one AT is deleted from a repeat sequence TATATATA, the output cigar will always mark the leftmost AT + * as deleted. If there is no indel in the original cigar, or the indel position is determined unambiguously (i.e. inserted/deleted sequence + * is not repeated), the original cigar is returned. + * + * @param cigar structure of the original alignment + * @param refSeq reference sequence the read is aligned to + * @param readSeq read sequence + * @param refIndex 0-based alignment start position on ref + * @param readIndex 0-based alignment start position on read + * @return a cigar, in which indel is guaranteed to be placed at the leftmost possible position across a repeat (if any) + */ + public static Cigar leftAlignIndel(Cigar cigar, final byte[] refSeq, final byte[] readSeq, final int refIndex, final int readIndex) { + + int indexOfIndel = -1; + for (int i = 0; i < cigar.numCigarElements(); i++) { + CigarElement ce = cigar.getCigarElement(i); + if (ce.getOperator() == CigarOperator.D || ce.getOperator() == CigarOperator.I) { + // if there is more than 1 indel, don't left align + if (indexOfIndel != -1) + return cigar; + indexOfIndel = i; + } + } + + // if there is no indel or if the alignment starts with an insertion (so that there + // is no place on the read to move that insertion further left), we are done + if (indexOfIndel < 1) return cigar; + + final int indelLength = cigar.getCigarElement(indexOfIndel).getLength(); + + byte[] altString = createIndelString(cigar, indexOfIndel, refSeq, readSeq, refIndex, readIndex); + if (altString == null) + return cigar; + + Cigar newCigar = cigar; + for (int i = 0; i < indelLength; i++) { + newCigar = moveCigarLeft(newCigar, indexOfIndel); + byte[] newAltString = createIndelString(newCigar, indexOfIndel, refSeq, readSeq, refIndex, readIndex); + + // check to make sure we haven't run off the end of the read + boolean reachedEndOfRead = cigarHasZeroSizeElement(newCigar); + + if (Arrays.equals(altString, newAltString)) { + cigar = newCigar; + i = -1; + if (reachedEndOfRead) + cigar = cleanUpCigar(cigar); + } + + if (reachedEndOfRead) + break; + } + + return cigar; + } + + private static boolean cigarHasZeroSizeElement(Cigar c) { + for (CigarElement ce : c.getCigarElements()) { + if (ce.getLength() == 0) + return true; + } + return false; + } + + private static Cigar cleanUpCigar(Cigar c) { + ArrayList elements = new ArrayList(c.numCigarElements() - 1); + for (CigarElement ce : c.getCigarElements()) { + if (ce.getLength() != 0 && + (elements.size() != 0 || ce.getOperator() != CigarOperator.D)) { + elements.add(ce); + } + } + return new Cigar(elements); + } + + private static Cigar moveCigarLeft(Cigar cigar, int indexOfIndel) { + // get the first few elements + ArrayList elements = new ArrayList(cigar.numCigarElements()); + for (int i = 0; i < indexOfIndel - 1; i++) + elements.add(cigar.getCigarElement(i)); + + // get the indel element and move it left one base + CigarElement ce = cigar.getCigarElement(indexOfIndel - 1); + elements.add(new CigarElement(ce.getLength() - 1, ce.getOperator())); + elements.add(cigar.getCigarElement(indexOfIndel)); + if (indexOfIndel + 1 < cigar.numCigarElements()) { + ce = cigar.getCigarElement(indexOfIndel + 1); + elements.add(new CigarElement(ce.getLength() + 1, ce.getOperator())); + } else { + elements.add(new CigarElement(1, CigarOperator.M)); + } + + // get the last few elements + for (int i = indexOfIndel + 2; i < cigar.numCigarElements(); i++) + elements.add(cigar.getCigarElement(i)); + return new Cigar(elements); + } + + private static byte[] createIndelString(final Cigar cigar, final int indexOfIndel, final byte[] refSeq, final byte[] readSeq, int refIndex, int readIndex) { + CigarElement indel = cigar.getCigarElement(indexOfIndel); + int indelLength = indel.getLength(); + + int totalRefBases = 0; + for (int i = 0; i < indexOfIndel; i++) { + CigarElement ce = cigar.getCigarElement(i); + int length = ce.getLength(); + + switch (ce.getOperator()) { + case M: + readIndex += length; + refIndex += length; + totalRefBases += length; + break; + case S: + readIndex += length; + break; + case N: + refIndex += length; + totalRefBases += length; + break; + default: + break; + } + } + + // sometimes, when there are very large known indels, we won't have enough reference sequence to cover them + if (totalRefBases + indelLength > refSeq.length) + indelLength -= (totalRefBases + indelLength - refSeq.length); + + // the indel-based reference string + byte[] alt = new byte[refSeq.length + (indelLength * (indel.getOperator() == CigarOperator.D ? -1 : 1))]; + + // add the bases before the indel, making sure it's not aligned off the end of the reference + if (refIndex > alt.length || refIndex > refSeq.length) + return null; + System.arraycopy(refSeq, 0, alt, 0, refIndex); + int currentPos = refIndex; + + // take care of the indel + if (indel.getOperator() == CigarOperator.D) { + refIndex += indelLength; + } else { + System.arraycopy(readSeq, readIndex, alt, currentPos, indelLength); + currentPos += indelLength; + } + + // add the bases after the indel, making sure it's not aligned off the end of the reference + if (refSeq.length - refIndex > alt.length - currentPos) + return null; + System.arraycopy(refSeq, refIndex, alt, currentPos, refSeq.length - refIndex); + + return alt; + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/ArtificialSAMUtils.java b/public/java/src/org/broadinstitute/sting/utils/sam/ArtificialSAMUtils.java index 542adea775..8661d5ad06 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/ArtificialSAMUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/ArtificialSAMUtils.java @@ -27,7 +27,7 @@ public class ArtificialSAMUtils { * @param chromosomeSize how large each chromosome is * @param readsPerChomosome how many reads to make in each chromosome. They'll be aligned from position 1 to x (which is the number of reads) */ - public static void createArtificialBamFile( String filename, int numberOfChromosomes, int startingChromosome, int chromosomeSize, int readsPerChomosome ) { + public static void createArtificialBamFile(String filename, int numberOfChromosomes, int startingChromosome, int chromosomeSize, int readsPerChomosome) { SAMFileHeader header = createArtificialSamHeader(numberOfChromosomes, startingChromosome, chromosomeSize); File outFile = new File(filename); @@ -51,7 +51,7 @@ public static void createArtificialBamFile( String filename, int numberOfChromos * @param chromosomeSize how large each chromosome is * @param readsPerChomosome how many reads to make in each chromosome. They'll be aligned from position 1 to x (which is the number of reads) */ - public static void createArtificialSamFile( String filename, int numberOfChromosomes, int startingChromosome, int chromosomeSize, int readsPerChomosome ) { + public static void createArtificialSamFile(String filename, int numberOfChromosomes, int startingChromosome, int chromosomeSize, int readsPerChomosome) { SAMFileHeader header = createArtificialSamHeader(numberOfChromosomes, startingChromosome, chromosomeSize); File outFile = new File(filename); @@ -72,16 +72,15 @@ public static void createArtificialSamFile( String filename, int numberOfChromos * @param numberOfChromosomes the number of chromosomes to create * @param startingChromosome the starting number for the chromosome (most likely set to 1) * @param chromosomeSize the length of each chromosome - * * @return */ - public static SAMFileHeader createArtificialSamHeader( int numberOfChromosomes, int startingChromosome, int chromosomeSize ) { + public static SAMFileHeader createArtificialSamHeader(int numberOfChromosomes, int startingChromosome, int chromosomeSize) { SAMFileHeader header = new SAMFileHeader(); header.setSortOrder(net.sf.samtools.SAMFileHeader.SortOrder.coordinate); SAMSequenceDictionary dict = new SAMSequenceDictionary(); // make up some sequence records for (int x = startingChromosome; x < startingChromosome + numberOfChromosomes; x++) { - SAMSequenceRecord rec = new SAMSequenceRecord("chr" + ( x ), chromosomeSize /* size */); + SAMSequenceRecord rec = new SAMSequenceRecord("chr" + (x), chromosomeSize /* size */); rec.setSequenceLength(chromosomeSize); dict.addSequence(rec); } @@ -95,10 +94,9 @@ public static SAMFileHeader createArtificialSamHeader( int numberOfChromosomes, * @param header the header to set * @param readGroupID the read group ID tag * @param sampleName the sample name - * * @return the adjusted SAMFileHeader */ - public static SAMFileHeader createDefaultReadGroup( SAMFileHeader header, String readGroupID, String sampleName ) { + public static SAMFileHeader createDefaultReadGroup(SAMFileHeader header, String readGroupID, String sampleName) { SAMReadGroupRecord rec = new SAMReadGroupRecord(readGroupID); rec.setSample(sampleName); List readGroups = new ArrayList(); @@ -113,10 +111,9 @@ public static SAMFileHeader createDefaultReadGroup( SAMFileHeader header, String * @param header the header to set * @param readGroupIDs the read group ID tags * @param sampleNames the sample names - * * @return the adjusted SAMFileHeader */ - public static SAMFileHeader createEnumeratedReadGroups( SAMFileHeader header, List readGroupIDs, List sampleNames ) { + public static SAMFileHeader createEnumeratedReadGroups(SAMFileHeader header, List readGroupIDs, List sampleNames) { if (readGroupIDs.size() != sampleNames.size()) { throw new ReviewedStingException("read group count and sample name count must be the same"); } @@ -137,18 +134,16 @@ public static SAMFileHeader createEnumeratedReadGroups( SAMFileHeader header, Li /** * Create an artificial read based on the parameters. The cigar string will be *M, where * is the length of the read * - * * @param header the SAM header to associate the read with * @param name the name of the read * @param refIndex the reference index, i.e. what chromosome to associate it with * @param alignmentStart where to start the alignment * @param length the length of the read - * * @return the artificial read */ public static GATKSAMRecord createArtificialRead(SAMFileHeader header, String name, int refIndex, int alignmentStart, int length) { - if( (refIndex == SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX && alignmentStart != SAMRecord.NO_ALIGNMENT_START) || - (refIndex != SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX && alignmentStart == SAMRecord.NO_ALIGNMENT_START) ) + if ((refIndex == SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX && alignmentStart != SAMRecord.NO_ALIGNMENT_START) || + (refIndex != SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX && alignmentStart == SAMRecord.NO_ALIGNMENT_START)) throw new ReviewedStingException("Invalid alignment start for artificial read, start = " + alignmentStart); GATKSAMRecord record = new GATKSAMRecord(header); record.setReadName(name); @@ -183,10 +178,9 @@ public static GATKSAMRecord createArtificialRead(SAMFileHeader header, String na * @param alignmentStart where to start the alignment * @param bases the sequence of the read * @param qual the qualities of the read - * * @return the artificial read */ - public static GATKSAMRecord createArtificialRead( SAMFileHeader header, String name, int refIndex, int alignmentStart, byte[] bases, byte[] qual ) { + public static GATKSAMRecord createArtificialRead(SAMFileHeader header, String name, int refIndex, int alignmentStart, byte[] bases, byte[] qual) { if (bases.length != qual.length) { throw new ReviewedStingException("Passed in read string is different length then the quality array"); } @@ -210,10 +204,9 @@ public static GATKSAMRecord createArtificialRead( SAMFileHeader header, String n * @param bases the sequence of the read * @param qual the qualities of the read * @param cigar the cigar string of the read - * * @return the artificial read */ - public static GATKSAMRecord createArtificialRead( SAMFileHeader header, String name, int refIndex, int alignmentStart, byte[] bases, byte[] qual, String cigar ) { + public static GATKSAMRecord createArtificialRead(SAMFileHeader header, String name, int refIndex, int alignmentStart, byte[] bases, byte[] qual, String cigar) { GATKSAMRecord rec = createArtificialRead(header, name, refIndex, alignmentStart, bases, qual); rec.setCigarString(cigar); return rec; @@ -221,22 +214,21 @@ public static GATKSAMRecord createArtificialRead( SAMFileHeader header, String n /** * Create an artificial read with the following default parameters : - * header: - * numberOfChromosomes = 1 - * startingChromosome = 1 - * chromosomeSize = 1000000 - * read: - * name = "default_read" - * refIndex = 0 - * alignmentStart = 1 - * - * @param bases the sequence of the read - * @param qual the qualities of the read - * @param cigar the cigar string of the read + * header: + * numberOfChromosomes = 1 + * startingChromosome = 1 + * chromosomeSize = 1000000 + * read: + * name = "default_read" + * refIndex = 0 + * alignmentStart = 1 * + * @param bases the sequence of the read + * @param qual the qualities of the read + * @param cigar the cigar string of the read * @return the artificial read */ - public static GATKSAMRecord createArtificialRead( byte[] bases, byte[] qual, String cigar ) { + public static GATKSAMRecord createArtificialRead(byte[] bases, byte[] qual, String cigar) { SAMFileHeader header = ArtificialSAMUtils.createArtificialSamHeader(1, 1, 1000000); return ArtificialSAMUtils.createArtificialRead(header, "default_read", 0, 10000, bases, qual, cigar); } @@ -253,7 +245,7 @@ public final static List createPair(SAMFileHeader header, String right.setProperPairFlag(true); left.setFirstOfPairFlag(leftIsFirst); - right.setFirstOfPairFlag(! leftIsFirst); + right.setFirstOfPairFlag(!leftIsFirst); left.setReadNegativeStrandFlag(leftIsNegative); left.setMateNegativeStrandFlag(!leftIsNegative); @@ -279,11 +271,10 @@ public final static List createPair(SAMFileHeader header, String * @param startingChr the chromosome (reference ID) to start from * @param endingChr the id to end with * @param readCount the number of reads per chromosome - * * @return StingSAMIterator representing the specified amount of fake data */ - public static StingSAMIterator mappedReadIterator( int startingChr, int endingChr, int readCount ) { - SAMFileHeader header = createArtificialSamHeader(( endingChr - startingChr ) + 1, startingChr, readCount + DEFAULT_READ_LENGTH); + public static StingSAMIterator mappedReadIterator(int startingChr, int endingChr, int readCount) { + SAMFileHeader header = createArtificialSamHeader((endingChr - startingChr) + 1, startingChr, readCount + DEFAULT_READ_LENGTH); return new ArtificialSAMQueryIterator(startingChr, endingChr, readCount, 0, header); } @@ -295,11 +286,10 @@ public static StingSAMIterator mappedReadIterator( int startingChr, int endingCh * @param endingChr the id to end with * @param readCount the number of reads per chromosome * @param unmappedReadCount the count of unmapped reads to place at the end of the iterator, like in a sorted bam file - * * @return StingSAMIterator representing the specified amount of fake data */ - public static StingSAMIterator mappedAndUnmappedReadIterator( int startingChr, int endingChr, int readCount, int unmappedReadCount ) { - SAMFileHeader header = createArtificialSamHeader(( endingChr - startingChr ) + 1, startingChr, readCount + DEFAULT_READ_LENGTH); + public static StingSAMIterator mappedAndUnmappedReadIterator(int startingChr, int endingChr, int readCount, int unmappedReadCount) { + SAMFileHeader header = createArtificialSamHeader((endingChr - startingChr) + 1, startingChr, readCount + DEFAULT_READ_LENGTH); return new ArtificialSAMQueryIterator(startingChr, endingChr, readCount, unmappedReadCount, header); } @@ -310,11 +300,10 @@ public static StingSAMIterator mappedAndUnmappedReadIterator( int startingChr, i * @param startingChr the chromosome (reference ID) to start from * @param endingChr the id to end with * @param readCount the number of reads per chromosome - * * @return StingSAMIterator representing the specified amount of fake data */ - public static ArtificialSAMQueryIterator queryReadIterator( int startingChr, int endingChr, int readCount ) { - SAMFileHeader header = createArtificialSamHeader(( endingChr - startingChr ) + 1, startingChr, readCount + DEFAULT_READ_LENGTH); + public static ArtificialSAMQueryIterator queryReadIterator(int startingChr, int endingChr, int readCount) { + SAMFileHeader header = createArtificialSamHeader((endingChr - startingChr) + 1, startingChr, readCount + DEFAULT_READ_LENGTH); return new ArtificialSAMQueryIterator(startingChr, endingChr, readCount, 0, header); } @@ -326,11 +315,10 @@ public static ArtificialSAMQueryIterator queryReadIterator( int startingChr, int * @param endingChr the id to end with * @param readCount the number of reads per chromosome * @param unmappedReadCount the count of unmapped reads to place at the end of the iterator, like in a sorted bam file - * * @return StingSAMIterator representing the specified amount of fake data */ - public static StingSAMIterator queryReadIterator( int startingChr, int endingChr, int readCount, int unmappedReadCount ) { - SAMFileHeader header = createArtificialSamHeader(( endingChr - startingChr ) + 1, startingChr, readCount + DEFAULT_READ_LENGTH); + public static StingSAMIterator queryReadIterator(int startingChr, int endingChr, int readCount, int unmappedReadCount) { + SAMFileHeader header = createArtificialSamHeader((endingChr - startingChr) + 1, startingChr, readCount + DEFAULT_READ_LENGTH); return new ArtificialSAMQueryIterator(startingChr, endingChr, readCount, unmappedReadCount, header); } @@ -345,6 +333,7 @@ private final static int ranIntInclusive(Random ran, int start, int stop) { * reads created that have readLen bases. Pairs are sampled from a gaussian distribution with mean insert * size of insertSize and variation of insertSize / 10. The first read will be in the pileup, and the second * may be, depending on where this sampled insertSize puts it. + * * @param header * @param loc * @param readLen @@ -360,22 +349,22 @@ public static ReadBackedPileup createReadBackedPileup(final SAMFileHeader header final int pos = loc.getStart(); final List pileupElements = new ArrayList(); - for ( int i = 0; i < pileupSize / 2; i++ ) { + for (int i = 0; i < pileupSize / 2; i++) { final String readName = "read" + i; final int leftStart = ranIntInclusive(ran, 1, pos); - final int fragmentSize = (int)(ran.nextGaussian() * insertSizeVariation + insertSize); + final int fragmentSize = (int) (ran.nextGaussian() * insertSizeVariation + insertSize); final int rightStart = leftStart + fragmentSize - readLen; - if ( rightStart <= 0 ) continue; + if (rightStart <= 0) continue; List pair = createPair(header, readName, readLen, leftStart, rightStart, leftIsFirst, leftIsNegative); final GATKSAMRecord left = pair.get(0); final GATKSAMRecord right = pair.get(1); - pileupElements.add(new PileupElement(left, pos - leftStart)); + pileupElements.add(new PileupElement(left, pos - leftStart, false)); - if ( pos >= right.getAlignmentStart() && pos <= right.getAlignmentEnd() ) { - pileupElements.add(new PileupElement(right, pos - rightStart)); + if (pos >= right.getAlignmentStart() && pos <= right.getAlignmentEnd()) { + pileupElements.add(new PileupElement(right, pos - rightStart, false)); } } diff --git a/public/java/test/org/broadinstitute/sting/BaseTest.java b/public/java/test/org/broadinstitute/sting/BaseTest.java index 61829dcfc2..626b91cbfe 100755 --- a/public/java/test/org/broadinstitute/sting/BaseTest.java +++ b/public/java/test/org/broadinstitute/sting/BaseTest.java @@ -1,13 +1,20 @@ package org.broadinstitute.sting; -import org.apache.log4j.*; +import org.apache.log4j.AppenderSkeleton; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.PatternLayout; import org.apache.log4j.spi.LoggingEvent; import org.broadinstitute.sting.commandline.CommandLineUtils; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.io.IOUtils; -import java.io.*; -import java.util.*; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java index 5cdf12f1bd..e9b4fc2113 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java @@ -28,7 +28,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest { public void testMultiSamplePilot1() { WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec( baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000", 1, - Arrays.asList("d61c7055bd09024abb8902bde6bd3960")); + Arrays.asList("653172b43b19003d9f7df6dab21f4b09")); executeTest("test MultiSample Pilot1", spec); } @@ -227,7 +227,7 @@ public void testSimpleIndels() { " -o %s" + " -L 1:10,000,000-10,500,000", 1, - Arrays.asList("b11df6587e4e16cb819d76a900446946")); + Arrays.asList("bd9d3d50a1f49605d7cd592a0f446899")); executeTest(String.format("test indel caller in SLX"), spec); } @@ -255,7 +255,7 @@ public void testMultiTechnologyIndels() { " -o %s" + " -L 1:10,000,000-10,500,000", 1, - Arrays.asList("59068bc8888ad5f08790946066d76602")); + Arrays.asList("91cd6d2e3972b0b8e4064bb35a33241f")); executeTest(String.format("test indel calling, multiple technologies"), spec); } @@ -294,7 +294,7 @@ public void testWithIndelAllelesPassedIn4() { WalkerTest.WalkerTestSpec spec4 = new WalkerTest.WalkerTestSpec( baseCommandIndelsb37 + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "ALL.wgs.union_v2_chr20_100_110K.20101123.indels.sites.vcf -I " + validationDataLocation + "phase1_GBR_realigned.chr20.100K-110K.bam -o %s -L 20:100,000-110,000", 1, - Arrays.asList("fcd590a55f5fec2a9b7e628187d6b8a8")); + Arrays.asList("877de5b0cc61dc54636062df6399b978")); executeTest("test MultiSample Phase1 indels with complicated records", spec4); } diff --git a/public/java/test/org/broadinstitute/sting/utils/sam/ReadUtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/sam/ReadUtilsUnitTest.java index 367f6294df..1a8086a1b4 100755 --- a/public/java/test/org/broadinstitute/sting/utils/sam/ReadUtilsUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/sam/ReadUtilsUnitTest.java @@ -42,12 +42,12 @@ public void testReducedReads() { @Test public void testReducedReadPileupElement() { - PileupElement readp = new PileupElement(read, 0); - PileupElement reducedreadp = new PileupElement(reducedRead, 0); + PileupElement readp = new PileupElement(read, 0, false); + PileupElement reducedreadp = new PileupElement(reducedRead, 0, false); - Assert.assertFalse(readp.isReducedRead()); + Assert.assertFalse(readp.getRead().isReducedRead()); - Assert.assertTrue(reducedreadp.isReducedRead()); + Assert.assertTrue(reducedreadp.getRead().isReducedRead()); Assert.assertEquals(reducedreadp.getRepresentativeCount(), REDUCED_READ_COUNTS[0]); Assert.assertEquals(reducedreadp.getQual(), readp.getQual()); } From 97499529c73b9cb8b262bbada2945c14004992fd Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Tue, 24 Jan 2012 16:13:53 -0500 Subject: [PATCH 128/356] another small bug with the file extension. --- .../sting/queue/qscripts/PacbioProcessingPipeline.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala index d5f7512e4c..c64eef7f76 100755 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/PacbioProcessingPipeline.scala @@ -62,12 +62,12 @@ class PacbioProcessingPipeline extends QScript { var USE_BWA: Boolean = false var resetQuals: Boolean = true - if (file.endsWith(".fasta") || file.endsWith(".fq")) { + if (file.endsWith(".fasta") || file.endsWith(".fq") || file.endsWith(".fastq")) { if (bwaPath == null) { throw new UserException("You provided a fasta/fastq file but didn't provide the path for BWA"); } USE_BWA = true - if (file.endsWith(".fq")) + if (file.endsWith(".fq") || file.endsWith(".fastq")) resetQuals = false } From 8f7d9bff0ab47a1529fc9f5fc96f2d7b4c62c780 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Wed, 25 Jan 2012 00:12:19 -0500 Subject: [PATCH 129/356] Fix MultiplyLikelihood logic, minor alterations to scala scripts. From 9818c69df67fe5c930df4b62d895b5b9b15fac07 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Wed, 25 Jan 2012 09:32:52 -0500 Subject: [PATCH 130/356] Can now specify active regions to process at the command line, mainly for debugging purposes --- .../traversals/TraverseActiveRegions.java | 4 +-- .../gatk/walkers/ActiveRegionWalker.java | 32 +++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java index ebfcc0c29b..cf15cc92b1 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java +++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java @@ -92,7 +92,7 @@ public T traverse( final ActiveRegionWalker walker, // Call the walkers isActive function for this locus and add them to the list to be integrated later if( initialIntervals.overlaps(location) ) { - final boolean isActive = walker.isActive( tracker, refContext, locus ); + final boolean isActive = ( walker.presetActiveRegions == null ? walker.isActive( tracker, refContext, locus ) : walker.presetActiveRegions.overlaps(location) ); isActiveList.add( new ActiveRegion(location, isActive, engine.getGenomeLocParser(), activeRegionExtension ) ); } @@ -109,7 +109,7 @@ public T traverse( final ActiveRegionWalker walker, if( !locusView.hasNext() ) { // Call the walkers isActive function for this locus and add them to the list to be integrated later if( initialIntervals.overlaps(location) ) { - final boolean isActive = walker.isActive( tracker, refContext, locus ); + final boolean isActive = ( walker.presetActiveRegions == null ? walker.isActive( tracker, refContext, locus ) : walker.presetActiveRegions.overlaps(location) ); isActiveList.add( new ActiveRegion(location, isActive, engine.getGenomeLocParser(), activeRegionExtension ) ); } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java index d7e170d739..508aebb5cd 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java @@ -1,6 +1,11 @@ package org.broadinstitute.sting.gatk.walkers; import net.sf.picard.reference.IndexedFastaSequenceFile; +import org.broad.tribble.Feature; +import org.broadinstitute.sting.commandline.Input; +import org.broadinstitute.sting.commandline.IntervalBinding; +import org.broadinstitute.sting.commandline.Output; +import org.broadinstitute.sting.commandline.RodBinding; import org.broadinstitute.sting.gatk.contexts.AlignmentContext; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; import org.broadinstitute.sting.gatk.filters.DuplicateReadFilter; @@ -14,8 +19,10 @@ import org.broadinstitute.sting.utils.GenomeLocSortedSet; import org.broadinstitute.sting.utils.activeregion.ActiveRegion; import org.broadinstitute.sting.utils.interval.IntervalMergingRule; +import org.broadinstitute.sting.utils.interval.IntervalSetRule; import org.broadinstitute.sting.utils.interval.IntervalUtils; +import java.io.PrintStream; import java.util.ArrayList; import java.util.List; @@ -32,6 +39,31 @@ @ReadFilters({UnmappedReadFilter.class, NotPrimaryAlignmentFilter.class, DuplicateReadFilter.class, FailsVendorQualityCheckFilter.class}) public abstract class ActiveRegionWalker extends Walker { + @Output(fullName="activeRegionOut", shortName="ARO", doc="Output the active region to this interval list file", required = false) + protected PrintStream activeRegionOutStream = null; + + @Input(fullName="activeRegionIn", shortName="AR", doc="Use this interval list file as the active regions to process", required = false) + protected List> activeRegionBindings = null; + + public GenomeLocSortedSet presetActiveRegions = null; + + @Override + public void initialize() { + if( activeRegionBindings == null ) { return; } + List allIntervals = new ArrayList(0); + for ( IntervalBinding intervalBinding : activeRegionBindings ) { + List intervals = intervalBinding.getIntervals(this.getToolkit()); + + if ( intervals.isEmpty() ) { + logger.warn("The interval file " + intervalBinding.getSource() + " contains no intervals that could be parsed."); + } + + allIntervals = IntervalUtils.mergeListsBySetOperator(intervals, allIntervals, IntervalSetRule.UNION); + } + + presetActiveRegions = IntervalUtils.sortAndMergeIntervals(this.getToolkit().getGenomeLocParser(), allIntervals, IntervalMergingRule.ALL); + } + // Do we actually want to operate on the context? public boolean filter(final RefMetaDataTracker tracker, final ReferenceContext ref, final AlignmentContext context) { return true; // We are keeping all the reads From bbefe4a272f440262c717eef51750b4f6a138c37 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Wed, 25 Jan 2012 09:47:06 -0500 Subject: [PATCH 131/356] Added option to be able to write out the active regions to an interval list file --- .../sting/gatk/traversals/TraverseActiveRegions.java | 11 ++++++++++- .../sting/gatk/walkers/ActiveRegionWalker.java | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java index cf15cc92b1..f5e936a092 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java +++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java @@ -128,7 +128,16 @@ public T traverse( final ActiveRegionWalker walker, // add these blocks of work to the work queue final ArrayList activeRegions = integrateActiveList( isActiveList ); logger.debug("Integrated " + isActiveList.size() + " isActive calls into " + activeRegions.size() + " regions." ); - workQueue.addAll( activeRegions ); + if( walker.activeRegionOutStream == null ) { + workQueue.addAll( activeRegions ); + } else { // Just want to output the active regions to a file, not actually process them + for( final ActiveRegion activeRegion : activeRegions ) { + if( activeRegion.isActive ) { + walker.activeRegionOutStream.println( activeRegion.getLocation() ); + } + } + } + // Since we've sufficiently past this point (or this contig!) in the workQueue we can unload those regions and process them if( !workQueue.isEmpty() ) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java index 508aebb5cd..98308ee111 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java @@ -40,7 +40,7 @@ public abstract class ActiveRegionWalker extends Walker { @Output(fullName="activeRegionOut", shortName="ARO", doc="Output the active region to this interval list file", required = false) - protected PrintStream activeRegionOutStream = null; + public PrintStream activeRegionOutStream = null; @Input(fullName="activeRegionIn", shortName="AR", doc="Use this interval list file as the active regions to process", required = false) protected List> activeRegionBindings = null; From 7a26fcb86f8e61c72e3e55d1f1a1c8d768f8fbc2 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Wed, 25 Jan 2012 09:51:13 -0500 Subject: [PATCH 132/356] Setting the max alternate alleles for the exact model in the HaplotypeCaller's copy of the UG engine. From ea3d4d60f2f99e1211ecccbfafc25c4d43ef12b7 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Wed, 25 Jan 2012 11:35:13 -0500 Subject: [PATCH 133/356] This annotation requires rods and should be annotated as such --- .../sting/gatk/walkers/annotator/MVLikelihoodRatio.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MVLikelihoodRatio.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MVLikelihoodRatio.java index b9e6a5b2bc..889cc634c3 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MVLikelihoodRatio.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MVLikelihoodRatio.java @@ -8,6 +8,7 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatibleWalker; import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ExperimentalAnnotation; import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation; +import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.RodRequiringAnnotation; import org.broadinstitute.sting.utils.MendelianViolation; import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType; import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine; @@ -23,7 +24,7 @@ * Time: 12:24 PM * To change this template use File | Settings | File Templates. */ -public class MVLikelihoodRatio extends InfoFieldAnnotation implements ExperimentalAnnotation { +public class MVLikelihoodRatio extends InfoFieldAnnotation implements ExperimentalAnnotation, RodRequiringAnnotation { private MendelianViolation mendelianViolation = null; private String motherId; From e349b4b14b2f66efd56119dc75014f7a5190ee90 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Wed, 25 Jan 2012 11:35:54 -0500 Subject: [PATCH 134/356] Allow appending with the dbSNP ID even if a (different) ID is already present for the variant rod. --- .../walkers/annotator/VariantAnnotator.java | 9 +++++++-- .../annotator/VariantAnnotatorEngine.java | 17 +++++++++++++---- .../interfaces/AnnotatorCompatibleWalker.java | 2 +- .../walkers/genotyper/UnifiedGenotyper.java | 2 +- .../VariantAnnotatorIntegrationTest.java | 8 ++++++++ 5 files changed, 30 insertions(+), 8 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java index 69560c7cb1..5312c41367 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java @@ -32,7 +32,6 @@ import org.broadinstitute.sting.gatk.contexts.AlignmentContextUtils; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; -import org.broadinstitute.sting.gatk.samples.SampleDB; import org.broadinstitute.sting.gatk.walkers.*; import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.*; import org.broadinstitute.sting.utils.BaseUtils; @@ -84,7 +83,6 @@ public class VariantAnnotator extends RodWalker implements Ann @ArgumentCollection protected StandardVariantContextInputArgumentCollection variantCollection = new StandardVariantContextInputArgumentCollection(); - public RodBinding getVariantRodBinding() { return variantCollection.variants; } /** * The INFO field will be annotated with information on the most biologically-significant effect @@ -163,6 +161,13 @@ public class VariantAnnotator extends RodWalker implements Ann @Argument(fullName="list", shortName="ls", doc="List the available annotations and exit") protected Boolean LIST = false; + /** + * By default, the dbSNP ID is added only when the ID field in the variant VCF is empty. + */ + @Argument(fullName="alwaysAppendDbsnpId", shortName="alwaysAppendDbsnpId", doc="In conjunction with the dbSNP binding, append the dbSNP ID even when the variant VCF already has the ID field populated") + protected Boolean ALWAYS_APPEND_DBSNP_ID = false; + public boolean alwaysAppendDbsnpId() { return ALWAYS_APPEND_DBSNP_ID; } + @Hidden @Argument(fullName="vcfContainsOnlyIndels", shortName="dels",doc="Use if you are annotating an indel vcf, currently VERY experimental", required = false) protected boolean indelsOnly = false; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java index 98d2fe17b1..90d0ad7402 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java @@ -195,11 +195,20 @@ public VariantContext annotateContext(RefMetaDataTracker tracker, ReferenceConte private VariantContext annotateDBs(RefMetaDataTracker tracker, ReferenceContext ref, VariantContext vc, Map infoAnnotations) { for ( Map.Entry, String> dbSet : dbAnnotations.entrySet() ) { if ( dbSet.getValue().equals(VCFConstants.DBSNP_KEY) ) { - String rsID = VCFUtils.rsIDOfFirstRealVariant(tracker.getValues(dbSet.getKey(), ref.getLocus()), vc.getType()); + final String rsID = VCFUtils.rsIDOfFirstRealVariant(tracker.getValues(dbSet.getKey(), ref.getLocus()), vc.getType()); + + // put the DB key into the INFO field infoAnnotations.put(VCFConstants.DBSNP_KEY, rsID != null); - // annotate dbsnp id if available and not already there - if ( rsID != null && vc.emptyID() ) - vc = new VariantContextBuilder(vc).id(rsID).make(); + + // add the ID if appropriate + if ( rsID != null ) { + if ( vc.emptyID() ) { + vc = new VariantContextBuilder(vc).id(rsID).make(); + } else if ( walker.alwaysAppendDbsnpId() && vc.getID().indexOf(rsID) == -1 ) { + final String newRsID = vc.getID() + VCFConstants.ID_FIELD_SEPARATOR + rsID; + vc = new VariantContextBuilder(vc).id(newRsID).make(); + } + } } else { boolean overlapsComp = false; for ( VariantContext comp : tracker.getValues(dbSet.getKey(), ref.getLocus()) ) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/AnnotatorCompatibleWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/AnnotatorCompatibleWalker.java index 7200f841bc..1331ad5df1 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/AnnotatorCompatibleWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/AnnotatorCompatibleWalker.java @@ -8,9 +8,9 @@ public interface AnnotatorCompatibleWalker { // getter methods for various used bindings - public abstract RodBinding getVariantRodBinding(); public abstract RodBinding getSnpEffRodBinding(); public abstract RodBinding getDbsnpRodBinding(); public abstract List> getCompRodBindings(); public abstract List> getResourceRodBindings(); + public abstract boolean alwaysAppendDbsnpId(); } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java index 369c2d0c68..5a269087ca 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java @@ -126,10 +126,10 @@ public class UnifiedGenotyper extends LocusWalker getDbsnpRodBinding() { return dbsnp.dbsnp; } - public RodBinding getVariantRodBinding() { return null; } public RodBinding getSnpEffRodBinding() { return null; } public List> getCompRodBindings() { return Collections.emptyList(); } public List> getResourceRodBindings() { return Collections.emptyList(); } + public boolean alwaysAppendDbsnpId() { return false; } /** * A raw, unfiltered, highly specific callset in VCF format. diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java index 14f7457b82..0d9d9bcd89 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java @@ -110,6 +110,14 @@ public void testDBTagWithDbsnp() { executeTest("getting DB tag with dbSNP", spec); } + @Test + public void testMultipleIdsWithDbsnp() { + WalkerTestSpec spec = new WalkerTestSpec( + baseTestString() + " --alwaysAppendDbsnpId --dbsnp " + b36dbSNP129 + " -G Standard --variant " + validationDataLocation + "vcfexample3withIDs.vcf -L " + validationDataLocation + "vcfexample3withIDs.vcf", 1, + Arrays.asList("cd7e3d43b8f5579c461b3e588a295fa8")); + executeTest("adding multiple IDs with dbSNP", spec); + } + @Test public void testDBTagWithHapMap() { WalkerTestSpec spec = new WalkerTestSpec( From fb863dc6a70ca1b23d8e5c07ab1ad237c840e63a Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Wed, 25 Jan 2012 11:50:12 -0500 Subject: [PATCH 135/356] Warn user when trying to run with EMIT_ALL_SITES with indels; better docs for that option. --- .../sting/gatk/walkers/genotyper/UnifiedGenotyper.java | 6 ++++++ .../gatk/walkers/genotyper/UnifiedGenotyperEngine.java | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java index 5a269087ca..5f84f62ec8 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java @@ -205,6 +205,12 @@ public static class UGStatistics { * **/ public void initialize() { + // warn the user for misusing EMIT_ALL_SITES + if ( UAC.OutputMode == UnifiedGenotyperEngine.OUTPUT_MODE.EMIT_ALL_SITES && + UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.DISCOVERY && + UAC.GLmodel != GenotypeLikelihoodsCalculationModel.Model.SNP ) + logger.warn("Note that the EMIT_ALL_SITES option is intended only for point mutations (SNPs) in DISCOVERY mode or generally when running in GENOTYPE_GIVEN_ALLELES mode; it will by no means produce a comprehensive set of indels in DISCOVERY mode"); + // get all of the unique sample names Set samples = SampleUtils.getSAMFileSamples(getToolkit().getSAMFileHeader()); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java index ee5aed3e59..ba4b224453 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java @@ -54,8 +54,9 @@ public enum OUTPUT_MODE { EMIT_VARIANTS_ONLY, /** produces calls at variant sites and confident reference sites */ EMIT_ALL_CONFIDENT_SITES, - /** produces calls at any callable site regardless of confidence; this argument is intended for point - * mutations (SNPs) only and while some indel calls may be produced they are by no means comprehensive */ + /** produces calls at any callable site regardless of confidence; this argument is intended only for point + * mutations (SNPs) in DISCOVERY mode or generally when running in GENOTYPE_GIVEN_ALLELES mode; it will by + * no means produce a comprehensive set of indels in DISCOVERY mode */ EMIT_ALL_SITES } From 96b62daff39a21d478db8d2e443c13e27b863792 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Wed, 25 Jan 2012 11:55:33 -0500 Subject: [PATCH 136/356] Minor tweak to the warning message. --- .../sting/gatk/walkers/genotyper/UnifiedGenotyper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java index 5f84f62ec8..b1495ac7d0 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java @@ -209,7 +209,7 @@ public void initialize() { if ( UAC.OutputMode == UnifiedGenotyperEngine.OUTPUT_MODE.EMIT_ALL_SITES && UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.DISCOVERY && UAC.GLmodel != GenotypeLikelihoodsCalculationModel.Model.SNP ) - logger.warn("Note that the EMIT_ALL_SITES option is intended only for point mutations (SNPs) in DISCOVERY mode or generally when running in GENOTYPE_GIVEN_ALLELES mode; it will by no means produce a comprehensive set of indels in DISCOVERY mode"); + logger.warn("WARNING: note that the EMIT_ALL_SITES option is intended only for point mutations (SNPs) in DISCOVERY mode or generally when running in GENOTYPE_GIVEN_ALLELES mode; it will by no means produce a comprehensive set of indels in DISCOVERY mode"); // get all of the unique sample names Set samples = SampleUtils.getSAMFileSamples(getToolkit().getSAMFileHeader()); From 2799a1b686763543b95e7815809aa898cc2de101 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Wed, 25 Jan 2012 12:15:51 -0500 Subject: [PATCH 137/356] Catch exception for bad type and throw as a TribbleException --- .../sting/utils/codecs/vcf/VCFCompoundHeaderLine.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFCompoundHeaderLine.java b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFCompoundHeaderLine.java index bb822f2edf..97166833b1 100755 --- a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFCompoundHeaderLine.java +++ b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFCompoundHeaderLine.java @@ -24,6 +24,7 @@ package org.broadinstitute.sting.utils.codecs.vcf; +import org.broad.tribble.TribbleException; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import java.util.Arrays; @@ -149,7 +150,11 @@ protected VCFCompoundHeaderLine(String line, VCFHeaderVersion version, Supported count = Integer.valueOf(numberStr); } - type = VCFHeaderLineType.valueOf(mapping.get("Type")); + try { + type = VCFHeaderLineType.valueOf(mapping.get("Type")); + } catch (Exception e) { + throw new TribbleException(mapping.get("Type") + " is not a valid type in the VCF specification (note that types are case-sensitive)"); + } if (type == VCFHeaderLineType.Flag && !allowFlagValues()) throw new IllegalArgumentException("Flag is an unsupported type for this kind of field"); From 05816955aa7945bc0422c8d3d117691b9a43bb52 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Wed, 25 Jan 2012 14:28:21 -0500 Subject: [PATCH 138/356] It was possible that we'd clean up a matrix column too early when a dependent column aborted early (with not enough probability mass) because we weren't being smart about the order in which we created dependencies. Fixed. --- .../genotyper/ExactAFCalculationModel.java | 79 +++++++++++-------- 1 file changed, 45 insertions(+), 34 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java index 1594c92cb0..24d7696b52 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java @@ -27,7 +27,6 @@ import org.apache.log4j.Logger; import org.broadinstitute.sting.utils.MathUtils; -import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.variantcontext.*; import java.io.PrintStream; @@ -177,11 +176,11 @@ public static void linearExactMultiAllelic(final GenotypesContext GLs, ACqueue.add(zeroSet); indexesToACset.put(zeroSet.ACcounts, zeroSet); - // optimization: create the temporary storage for computing L(j,k) just once - final int maxPossibleDependencies = numAlternateAlleles + (numAlternateAlleles * (numAlternateAlleles + 1) / 2) + 1; - final double[][] tempLog10ConformationLikelihoods = new double[numSamples+1][maxPossibleDependencies]; - for ( int i = 0; i < maxPossibleDependencies; i++ ) - tempLog10ConformationLikelihoods[0][i] = Double.NEGATIVE_INFINITY; + // optimization: create the temporary storage for computing L(j,k) just once + final int maxPossibleDependencies = numAlternateAlleles + (numAlternateAlleles * (numAlternateAlleles + 1) / 2) + 1; + final double[][] tempLog10ConformationLikelihoods = new double[numSamples+1][maxPossibleDependencies]; + for ( int i = 0; i < maxPossibleDependencies; i++ ) + tempLog10ConformationLikelihoods[0][i] = Double.NEGATIVE_INFINITY; // keep processing while we have AC conformations that need to be calculated double maxLog10L = Double.NEGATIVE_INFINITY; @@ -204,7 +203,7 @@ private static double calculateAlleleCountConformation(final ExactACset set, final HashMap indexesToACset, final double[][] log10AlleleFrequencyPriors, final AlleleFrequencyCalculationResult result, - final double[][] tempLog10ConformationLikelihoods) { + final double[][] tempLog10ConformationLikelihoods) { //if ( DEBUG ) // System.out.printf(" *** computing LofK for set=%s%n", set.ACcounts); @@ -256,14 +255,24 @@ private static double calculateAlleleCountConformation(final ExactACset set, // add conformations for the k+2 case if it makes sense; note that the 2 new alleles may be the same or different if ( ACwiggle > 1 ) { - for ( int allele_i = 0; allele_i < numAltAlleles; allele_i++ ) { - for ( int allele_j = allele_i; allele_j < numAltAlleles; allele_j++ ) { + // IMPORTANT: we must first add the cases where the 2 new alleles are different so that the queue maintains its ordering + for ( int allele_i = 0; allele_i < numAltAlleles - 1; allele_i++ ) { + for ( int allele_j = allele_i + 1; allele_j < numAltAlleles; allele_j++ ) { + if ( allele_i == allele_j ) + continue; final int[] ACcountsClone = set.ACcounts.getCounts().clone(); ACcountsClone[allele_i]++; ACcountsClone[allele_j]++; lastSet = updateACset(ACcountsClone, numChr, set, ++PLindex , ACqueue, indexesToACset); } } + + // now we can deal with the case where the 2 new alleles are the same + for ( int allele_i = 0; allele_i < numAltAlleles; allele_i++ ) { + final int[] ACcountsClone = set.ACcounts.getCounts().clone(); + ACcountsClone[allele_i] += 2; + lastSet = updateACset(ACcountsClone, numChr, set, ++PLindex , ACqueue, indexesToACset); + } } // if the last dependent set was not at the back of the queue (i.e. not just added), then we need to iterate @@ -298,6 +307,8 @@ private static ExactACset updateACset(final int[] ACcounts, } // add the given dependency to the set + //if ( DEBUG ) + // System.out.println(" *** adding dependency from " + index + " to " + callingSet.ACcounts); final ExactACset set = indexesToACset.get(index); set.ACsetIndexToPLIndex.put(callingSet.ACcounts, PLsetIndex); return wasInQueue ? null : set; @@ -317,7 +328,7 @@ private static void computeLofK(final ExactACset set, final HashMap indexesToACset, final double[][] log10AlleleFrequencyPriors, final AlleleFrequencyCalculationResult result, - final double[][] tempLog10ConformationLikelihoods) { + final double[][] tempLog10ConformationLikelihoods) { set.log10Likelihoods[0] = 0.0; // the zero case final int totalK = set.getACsum(); @@ -329,40 +340,40 @@ private static void computeLofK(final ExactACset set, } // k > 0 for at least one k else { - // deal with the non-AA possible conformations - int conformationIndex = 1; - for ( Map.Entry mapping : set.ACsetIndexToPLIndex.entrySet() ) { - //if ( DEBUG ) - // System.out.printf(" *** evaluating set=%s which depends on set=%s%n", set.ACcounts, mapping.getKey()); + // deal with the non-AA possible conformations + int conformationIndex = 1; + for ( Map.Entry mapping : set.ACsetIndexToPLIndex.entrySet() ) { + //if ( DEBUG ) + // System.out.printf(" *** evaluating set=%s which depends on set=%s%n", set.ACcounts, mapping.getKey()); - ExactACset dependent = indexesToACset.get(mapping.getKey()); + ExactACset dependent = indexesToACset.get(mapping.getKey()); - for ( int j = 1; j < set.log10Likelihoods.length; j++ ) { + for ( int j = 1; j < set.log10Likelihoods.length; j++ ) { - if ( totalK <= 2*j ) { // skip impossible conformations - final double[] gl = genotypeLikelihoods.get(j); - tempLog10ConformationLikelihoods[j][conformationIndex] = - determineCoefficient(mapping.getValue(), j, set.ACcounts.getCounts(), totalK) + dependent.log10Likelihoods[j-1] + gl[mapping.getValue()]; + if ( totalK <= 2*j ) { // skip impossible conformations + final double[] gl = genotypeLikelihoods.get(j); + tempLog10ConformationLikelihoods[j][conformationIndex] = + determineCoefficient(mapping.getValue(), j, set.ACcounts.getCounts(), totalK) + dependent.log10Likelihoods[j-1] + gl[mapping.getValue()]; } else { - tempLog10ConformationLikelihoods[j][conformationIndex] = Double.NEGATIVE_INFINITY; - } + tempLog10ConformationLikelihoods[j][conformationIndex] = Double.NEGATIVE_INFINITY; + } } - conformationIndex++; - } + conformationIndex++; + } - // finally, deal with the AA case (which depends on previous cells in this column) and then update the L(j,k) value + // finally, deal with the AA case (which depends on previous cells in this column) and then update the L(j,k) value final int numPaths = set.ACsetIndexToPLIndex.size() + 1; - for ( int j = 1; j < set.log10Likelihoods.length; j++ ) { + for ( int j = 1; j < set.log10Likelihoods.length; j++ ) { - if ( totalK < 2*j-1 ) { - final double[] gl = genotypeLikelihoods.get(j); - tempLog10ConformationLikelihoods[j][0] = MathUtils.log10Cache[2*j-totalK] + MathUtils.log10Cache[2*j-totalK-1] + set.log10Likelihoods[j-1] + gl[HOM_REF_INDEX]; - } else { - tempLog10ConformationLikelihoods[j][0] = Double.NEGATIVE_INFINITY; - } + if ( totalK < 2*j-1 ) { + final double[] gl = genotypeLikelihoods.get(j); + tempLog10ConformationLikelihoods[j][0] = MathUtils.log10Cache[2*j-totalK] + MathUtils.log10Cache[2*j-totalK-1] + set.log10Likelihoods[j-1] + gl[HOM_REF_INDEX]; + } else { + tempLog10ConformationLikelihoods[j][0] = Double.NEGATIVE_INFINITY; + } - final double logDenominator = MathUtils.log10Cache[2*j] + MathUtils.log10Cache[2*j-1]; + final double logDenominator = MathUtils.log10Cache[2*j] + MathUtils.log10Cache[2*j-1]; final double log10Max = MathUtils.approximateLog10SumLog10(tempLog10ConformationLikelihoods[j], numPaths); set.log10Likelihoods[j] = log10Max - logDenominator; } From 8e2d372ab0649a003745ffbd319e64e6f5df2e25 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Wed, 25 Jan 2012 14:41:34 -0500 Subject: [PATCH 139/356] Use remove instead of setting the value to null --- .../sting/gatk/walkers/genotyper/ExactAFCalculationModel.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java index 24d7696b52..aee0030898 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java @@ -214,7 +214,7 @@ private static double calculateAlleleCountConformation(final ExactACset set, // clean up memory if ( !preserveData ) { for ( ExactACcounts index : set.dependentACsetsToDelete ) { - indexesToACset.put(index, null); + indexesToACset.remove(index); //if ( DEBUG ) // System.out.printf(" *** removing used set=%s after seeing final dependent set=%s%n", index, set.ACcounts); } @@ -229,7 +229,7 @@ private static double calculateAlleleCountConformation(final ExactACset set, // no reason to keep this data around because nothing depends on it if ( !preserveData ) - indexesToACset.put(set.ACcounts, null); + indexesToACset.remove(set.ACcounts); return log10LofK; } From ef335a5812e6f6e18ea8227b95361bde9c3826df Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Wed, 25 Jan 2012 15:15:42 -0500 Subject: [PATCH 140/356] Better implementation of the fix; PL index is now traversed in order. --- .../genotyper/ExactAFCalculationModel.java | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java index aee0030898..d75be23bed 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java @@ -27,6 +27,7 @@ import org.apache.log4j.Logger; import org.broadinstitute.sting.utils.MathUtils; +import org.broadinstitute.sting.utils.collections.Pair; import org.broadinstitute.sting.utils.variantcontext.*; import java.io.PrintStream; @@ -194,6 +195,16 @@ public static void linearExactMultiAllelic(final GenotypesContext GLs, } } + private static final class DependentSet { + public final int[] ACcounts; + public final int PLindex; + + public DependentSet(final int[] ACcounts, final int PLindex) { + this.ACcounts = ACcounts; + this.PLindex = PLindex; + } + } + private static double calculateAlleleCountConformation(final ExactACset set, final ArrayList genotypeLikelihoods, final double maxLog10L, @@ -255,24 +266,27 @@ private static double calculateAlleleCountConformation(final ExactACset set, // add conformations for the k+2 case if it makes sense; note that the 2 new alleles may be the same or different if ( ACwiggle > 1 ) { - // IMPORTANT: we must first add the cases where the 2 new alleles are different so that the queue maintains its ordering - for ( int allele_i = 0; allele_i < numAltAlleles - 1; allele_i++ ) { - for ( int allele_j = allele_i + 1; allele_j < numAltAlleles; allele_j++ ) { - if ( allele_i == allele_j ) - continue; + final ArrayList differentAlleles = new ArrayList(numAltAlleles * numAltAlleles); + final ArrayList sameAlleles = new ArrayList(numAltAlleles); + + for ( int allele_i = 0; allele_i < numAltAlleles; allele_i++ ) { + for ( int allele_j = allele_i; allele_j < numAltAlleles; allele_j++ ) { final int[] ACcountsClone = set.ACcounts.getCounts().clone(); ACcountsClone[allele_i]++; ACcountsClone[allele_j]++; - lastSet = updateACset(ACcountsClone, numChr, set, ++PLindex , ACqueue, indexesToACset); + + if ( allele_i == allele_j ) + sameAlleles.add(new DependentSet(ACcountsClone, ++PLindex)); + else + differentAlleles.add(new DependentSet(ACcountsClone, ++PLindex)); } } - // now we can deal with the case where the 2 new alleles are the same - for ( int allele_i = 0; allele_i < numAltAlleles; allele_i++ ) { - final int[] ACcountsClone = set.ACcounts.getCounts().clone(); - ACcountsClone[allele_i] += 2; - lastSet = updateACset(ACcountsClone, numChr, set, ++PLindex , ACqueue, indexesToACset); - } + // IMPORTANT: we must first add the cases where the 2 new alleles are different so that the queue maintains its ordering + for ( DependentSet dependent : differentAlleles ) + lastSet = updateACset(dependent.ACcounts, numChr, set, dependent.PLindex, ACqueue, indexesToACset); + for ( DependentSet dependent : sameAlleles ) + lastSet = updateACset(dependent.ACcounts, numChr, set, dependent.PLindex, ACqueue, indexesToACset); } // if the last dependent set was not at the back of the queue (i.e. not just added), then we need to iterate From 66772d0ebf003c5a1fea28f4752495e10526593f Mon Sep 17 00:00:00 2001 From: Guillermo del Angel Date: Wed, 25 Jan 2012 15:41:08 -0500 Subject: [PATCH 141/356] Next iteration in the pool caller: more bug fixes, start of big refactoring to clean up interfaces, moved up a lot of attributes that really belonged to a site up from the Pool class, added by default option to filter out a call if there's no reference depth (instead of just skipping the call which makes it hard to figure out what happened afterwards). From 4337dcd7e429ee690956ca5d56ab809842da5661 Mon Sep 17 00:00:00 2001 From: Guillermo del Angel Date: Wed, 25 Jan 2012 15:53:03 -0500 Subject: [PATCH 142/356] More pool caller bug fixes: the QUAL field was actually multiplied by 10 (accounting for a lot of singletons that shouldn't have been there), and correct AD output From db645a94ca0e5f533402284367b5ce042de61ad7 Mon Sep 17 00:00:00 2001 From: Menachem Fromer Date: Wed, 25 Jan 2012 16:10:59 -0500 Subject: [PATCH 143/356] Added options to make the batch-merger more all-inclusive: keep all indels, SNPs (even filtered ones) but maintain their annotations. Also, VariantContextUtils.simpleMerge can now merge variants of all types using the Hidden non-default enum MultipleAllelesMergeType=MIX_TYPES --- .../genotyper/ExactAFCalculationModel.java | 5 +- .../walkers/genotyper/UGCalcLikelihoods.java | 114 ------------- .../walkers/genotyper/UGCallVariants.java | 152 ------------------ .../walkers/variantutils/CombineVariants.java | 29 +++- .../variantcontext/VariantContextUtils.java | 13 ++ 5 files changed, 38 insertions(+), 275 deletions(-) delete mode 100755 public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UGCalcLikelihoods.java delete mode 100755 public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UGCallVariants.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java index 1594c92cb0..a91928bc34 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java @@ -27,7 +27,6 @@ import org.apache.log4j.Logger; import org.broadinstitute.sting.utils.MathUtils; -import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.variantcontext.*; import java.io.PrintStream; @@ -39,6 +38,8 @@ public class ExactAFCalculationModel extends AlleleFrequencyCalculationModel { private final static double MAX_LOG10_ERROR_TO_STOP_EARLY = 6; // we want the calculation to be accurate to 1 / 10^6 + // TODO: PERMITS WALKER USED TO HAVE A TEMPORARY FIX to prevent NullPointerException caused by bug: + public static boolean PRESERVE_AC_DATA = false; protected ExactAFCalculationModel(UnifiedArgumentCollection UAC, int N, Logger logger, PrintStream verboseWriter) { super(UAC, N, logger, verboseWriter); @@ -51,7 +52,7 @@ public void getLog10PNonRef(final GenotypesContext GLs, final int numAlleles = alleles.size(); //linearExact(GLs, log10AlleleFrequencyPriors[0], log10AlleleFrequencyLikelihoods, log10AlleleFrequencyPosteriors); - linearExactMultiAllelic(GLs, numAlleles - 1, log10AlleleFrequencyPriors, result, false); + linearExactMultiAllelic(GLs, numAlleles - 1, log10AlleleFrequencyPriors, result, PRESERVE_AC_DATA); } private static final ArrayList getGLs(GenotypesContext GLs) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UGCalcLikelihoods.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UGCalcLikelihoods.java deleted file mode 100755 index c7e5773937..0000000000 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UGCalcLikelihoods.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright (c) 2010 The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR - * THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.gatk.walkers.genotyper; - -import org.broadinstitute.sting.commandline.ArgumentCollection; -import org.broadinstitute.sting.commandline.Output; -import org.broadinstitute.sting.gatk.DownsampleType; -import org.broadinstitute.sting.gatk.contexts.AlignmentContext; -import org.broadinstitute.sting.gatk.contexts.ReferenceContext; -import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; -import org.broadinstitute.sting.gatk.walkers.*; -import org.broadinstitute.sting.utils.SampleUtils; -import org.broadinstitute.sting.utils.baq.BAQ; -import org.broadinstitute.sting.utils.codecs.vcf.*; -import org.broadinstitute.sting.utils.variantcontext.VariantContext; - -import java.util.HashSet; -import java.util.Set; - - -/** - * Uses the UG engine to determine per-sample genotype likelihoods and emits them as a VCF (using PLs). - * Absolutely not supported or recommended for public use. - * Run this as you would the UnifiedGenotyper, except that you must additionally pass in a VCF bound to - * the name 'allele' so we know which alternate allele to use at each site. - */ -@BAQMode(QualityMode = BAQ.QualityMode.ADD_TAG, ApplicationTime = BAQ.ApplicationTime.ON_INPUT) -@Reference(window=@Window(start=-200,stop=200)) -@By(DataSource.READS) -@Downsample(by=DownsampleType.BY_SAMPLE, toCoverage=250) -public class UGCalcLikelihoods extends LocusWalker implements TreeReducible { - - @ArgumentCollection private UnifiedArgumentCollection UAC = new UnifiedArgumentCollection(); - - // control the output - @Output(doc="File to which variants should be written",required=true) - protected VCFWriter writer = null; - - // the calculation arguments - private UnifiedGenotyperEngine UG_engine = null; - - // enable deletions in the pileup - public boolean includeReadsWithDeletionAtLoci() { return true; } - - // enable extended events for indels - public boolean generateExtendedEvents() { return UAC.GLmodel != GenotypeLikelihoodsCalculationModel.Model.SNP; } - - public void initialize() { - // get all of the unique sample names - Set samples = SampleUtils.getSAMFileSamples(getToolkit().getSAMFileHeader()); - - UG_engine = new UnifiedGenotyperEngine(getToolkit(), UAC, logger, null, null, samples); - - // initialize the header - Set headerInfo = new HashSet(); - headerInfo.add(new VCFInfoHeaderLine(VCFConstants.DOWNSAMPLED_KEY, 0, VCFHeaderLineType.Flag, "Were any of the samples downsampled?")); - headerInfo.add(new VCFFormatHeaderLine(VCFConstants.GENOTYPE_KEY, 1, VCFHeaderLineType.String, "Genotype")); - headerInfo.add(new VCFFormatHeaderLine(VCFConstants.DEPTH_KEY, 1, VCFHeaderLineType.Integer, "Read Depth (only filtered reads used for calling)")); - headerInfo.add(new VCFFormatHeaderLine(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY, 3, VCFHeaderLineType.Float, "Normalized, Phred-scaled likelihoods for AA,AB,BB genotypes where A=ref and B=alt; not applicable if site is not biallelic")); - - writer.writeHeader(new VCFHeader(headerInfo, samples)) ; - } - - public VariantCallContext map(RefMetaDataTracker tracker, ReferenceContext refContext, AlignmentContext rawContext) { - VariantContext call = UG_engine.calculateLikelihoods(tracker, refContext, rawContext); - return call == null ? null : new VariantCallContext(call, true); - } - - public Integer reduceInit() { return 0; } - - public Integer treeReduce(Integer lhs, Integer rhs) { - return lhs + rhs; - } - - public Integer reduce(VariantCallContext value, Integer sum) { - if ( value == null ) - return sum; - - try { - writer.add(value); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException(e.getMessage() + "; this is often caused by using the --assume_single_sample_reads argument with the wrong sample name"); - } - - return sum + 1; - } - - public void onTraversalDone(Integer sum) { - logger.info(String.format("Visited bases: %d", sum)); - } -} diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UGCallVariants.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UGCallVariants.java deleted file mode 100755 index 97f7b21eb7..0000000000 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UGCallVariants.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright (c) 2010, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.gatk.walkers.genotyper; - -import org.broadinstitute.sting.commandline.ArgumentCollection; -import org.broadinstitute.sting.commandline.Input; -import org.broadinstitute.sting.commandline.Output; -import org.broadinstitute.sting.commandline.RodBinding; -import org.broadinstitute.sting.gatk.contexts.AlignmentContext; -import org.broadinstitute.sting.gatk.contexts.ReferenceContext; -import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; -import org.broadinstitute.sting.gatk.walkers.RodWalker; -import org.broadinstitute.sting.utils.SampleUtils; -import org.broadinstitute.sting.utils.codecs.vcf.*; -import org.broadinstitute.sting.utils.exceptions.UserException; -import org.broadinstitute.sting.utils.variantcontext.*; - -import java.util.*; - -/** - * Uses the UG engine to call variants based off of VCFs annotated with GLs (or PLs). - * Absolutely not supported or recommended for public use. - * Run this as you would the UnifiedGenotyper, except that instead of '-I reads' it expects any number - * of GL/PL-annotated VCFs bound to a name starting with 'variant'. - */ -public class UGCallVariants extends RodWalker { - - @ArgumentCollection - private UnifiedArgumentCollection UAC = new UnifiedArgumentCollection(); - - @Input(fullName="variant", shortName = "V", doc="Input VCF file", required=true) - public List> variants; - - // control the output - @Output(doc="File to which variants should be written",required=true) - protected VCFWriter writer = null; - - // the calculation arguments - private UnifiedGenotyperEngine UG_engine = null; - - // variant track names - private Set trackNames = new HashSet(); - - public void initialize() { - - for ( RodBinding rb : variants ) - trackNames.add(rb.getName()); - Set samples = SampleUtils.getSampleListWithVCFHeader(getToolkit(), trackNames); - - UG_engine = new UnifiedGenotyperEngine(getToolkit(), UAC, logger, null, null, samples); - - Set headerInfo = new HashSet(); - headerInfo.add(new VCFInfoHeaderLine(VCFConstants.ALLELE_FREQUENCY_KEY, -1, VCFHeaderLineType.Float, "Allele Frequency, for each ALT allele, in the same order as listed")); - headerInfo.add(new VCFInfoHeaderLine(VCFConstants.ALLELE_COUNT_KEY, -1, VCFHeaderLineType.Integer, "Allele count in genotypes, for each ALT allele, in the same order as listed")); - headerInfo.add(new VCFInfoHeaderLine(VCFConstants.ALLELE_NUMBER_KEY, 1, VCFHeaderLineType.Integer, "Total number of alleles in called genotypes")); - headerInfo.add(new VCFFormatHeaderLine(VCFConstants.GENOTYPE_KEY, 1, VCFHeaderLineType.String, "Genotype")); - headerInfo.add(new VCFFormatHeaderLine(VCFConstants.GENOTYPE_QUALITY_KEY, 1, VCFHeaderLineType.Float, "Genotype Quality")); - headerInfo.add(new VCFFormatHeaderLine(VCFConstants.DEPTH_KEY, 1, VCFHeaderLineType.Integer, "Read Depth (only filtered reads used for calling)")); - headerInfo.add(new VCFFormatHeaderLine(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY, 3, VCFHeaderLineType.Float, "Normalized, Phred-scaled likelihoods for AA,AB,BB genotypes where A=ref and B=alt; not applicable if site is not biallelic")); - if ( UAC.STANDARD_CONFIDENCE_FOR_EMITTING < UAC.STANDARD_CONFIDENCE_FOR_CALLING ) - headerInfo.add(new VCFFilterHeaderLine(UnifiedGenotyperEngine.LOW_QUAL_FILTER_NAME, "Low quality")); - - // initialize the header - writer.writeHeader(new VCFHeader(headerInfo, samples)); - } - - public VariantCallContext map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) { - if ( tracker == null ) - return null; - - List VCs = tracker.getValues(variants, context.getLocation()); - - VariantContext mergedVC = mergeVCsWithGLs(VCs); - if ( mergedVC == null ) - return null; - - return UG_engine.calculateGenotypes(tracker, ref, context, mergedVC); - } - - public Integer reduceInit() { return 0; } - - public Integer reduce(VariantCallContext value, Integer sum) { - if ( value == null ) - return sum; - - try { - VariantContextBuilder builder = new VariantContextBuilder(value); - VariantContextUtils.calculateChromosomeCounts(builder, true); - writer.add(builder.make()); - } catch (IllegalArgumentException e) { - throw new IllegalArgumentException(e.getMessage() + "; this is often caused by using the --assume_single_sample_reads argument with the wrong sample name"); - } - - return sum + 1; - } - - public void onTraversalDone(Integer result) { - logger.info(String.format("Visited sites: %d", result)); - } - - private static VariantContext mergeVCsWithGLs(List VCs) { - // we can't use the VCUtils classes because our VCs can all be no-calls - if ( VCs.size() == 0 ) - return null; - - VariantContext variantVC = null; - GenotypesContext genotypes = GenotypesContext.create(); - for ( VariantContext vc : VCs ) { - if ( variantVC == null && vc.isVariant() ) - variantVC = vc; - genotypes.addAll(getGenotypesWithGLs(vc.getGenotypes())); - } - - if ( variantVC == null ) { - VariantContext vc = VCs.get(0); - throw new UserException("There is no ALT allele in any of the VCF records passed in at " + vc.getChr() + ":" + vc.getStart()); - } - - return new VariantContextBuilder(variantVC).source("VCwithGLs").genotypes(genotypes).make(); - } - - private static GenotypesContext getGenotypesWithGLs(GenotypesContext genotypes) { - GenotypesContext genotypesWithGLs = GenotypesContext.create(genotypes.size()); - for ( final Genotype g : genotypes ) { - if ( g.hasLikelihoods() && g.getLikelihoods().getAsVector() != null ) - genotypesWithGLs.add(g); - } - return genotypesWithGLs; - } -} \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/CombineVariants.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/CombineVariants.java index af05c0dc4e..684b9102a7 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/CombineVariants.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/CombineVariants.java @@ -120,6 +120,10 @@ public class CombineVariants extends RodWalker { @Argument(shortName="filteredRecordsMergeType", doc="Determines how we should handle records seen at the same site in the VCF, but with different FILTER fields", required=false) public VariantContextUtils.FilteredRecordMergeType filteredRecordsMergeType = VariantContextUtils.FilteredRecordMergeType.KEEP_IF_ANY_UNFILTERED; + @Hidden + @Argument(shortName="multipleAllelesMergeType", doc="Determines how we should handle records seen at the same site in the VCF, but with different allele types (for example, SNP vs. indel)", required=false) + public VariantContextUtils.MultipleAllelesMergeType multipleAllelesMergeType = VariantContextUtils.MultipleAllelesMergeType.BY_TYPE; + /** * Used when taking the union of variants that contain genotypes. A complete priority list MUST be provided. */ @@ -236,13 +240,24 @@ public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentCo return 0; List mergedVCs = new ArrayList(); - Map> VCsByType = VariantContextUtils.separateVariantContextsByType(vcs); - // iterate over the types so that it's deterministic - for ( VariantContext.Type type : VariantContext.Type.values() ) { - if ( VCsByType.containsKey(type) ) - mergedVCs.add(VariantContextUtils.simpleMerge(getToolkit().getGenomeLocParser(), VCsByType.get(type), - priority, filteredRecordsMergeType, genotypeMergeOption, true, printComplexMerges, - SET_KEY, filteredAreUncalled, MERGE_INFO_WITH_MAX_AC)); + + if (multipleAllelesMergeType == VariantContextUtils.MultipleAllelesMergeType.BY_TYPE) { + Map> VCsByType = VariantContextUtils.separateVariantContextsByType(vcs); + // iterate over the types so that it's deterministic + for (VariantContext.Type type : VariantContext.Type.values()) { + if (VCsByType.containsKey(type)) + mergedVCs.add(VariantContextUtils.simpleMerge(getToolkit().getGenomeLocParser(), VCsByType.get(type), + priority, filteredRecordsMergeType, genotypeMergeOption, true, printComplexMerges, + SET_KEY, filteredAreUncalled, MERGE_INFO_WITH_MAX_AC)); + } + } + else if (multipleAllelesMergeType == VariantContextUtils.MultipleAllelesMergeType.MIX_TYPES) { + mergedVCs.add(VariantContextUtils.simpleMerge(getToolkit().getGenomeLocParser(), vcs, + priority, filteredRecordsMergeType, genotypeMergeOption, true, printComplexMerges, + SET_KEY, filteredAreUncalled, MERGE_INFO_WITH_MAX_AC)); + } + else { + logger.warn("Ignoring all records at site " + ref.getLocus()); } for ( VariantContext mergedVC : mergedVCs ) { diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java index 39045ea212..179c91660a 100755 --- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java @@ -29,6 +29,7 @@ import org.apache.commons.jexl2.JexlEngine; import org.apache.log4j.Logger; import org.broad.tribble.util.popgen.HardyWeinbergCalculation; +import org.broadinstitute.sting.commandline.Hidden; import org.broadinstitute.sting.utils.BaseUtils; import org.broadinstitute.sting.utils.GenomeLoc; import org.broadinstitute.sting.utils.GenomeLocParser; @@ -471,6 +472,18 @@ public enum FilteredRecordMergeType { KEEP_UNCONDITIONAL } + @Hidden + public enum MultipleAllelesMergeType { + /** + * Combine only alleles of the same type (SNP, indel, etc.) into a single VCF record. + */ + BY_TYPE, + /** + * Merge all allele types at the same start position into the same VCF record. + */ + MIX_TYPES + } + /** * Merges VariantContexts into a single hybrid. Takes genotypes for common samples in priority order, if provided. * If uniqifySamples is true, the priority order is ignored and names are created by concatenating the VC name with From add6918f32d1322debba7bd5dcfc5d22a27c6c2f Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Wed, 25 Jan 2012 16:21:10 -0500 Subject: [PATCH 144/356] Cleaner, more efficient way of determining the last dependent set in the queue. --- .../genotyper/ExactAFCalculationModel.java | 56 +++++++++---------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java index d75be23bed..363b74ceb2 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java @@ -27,7 +27,7 @@ import org.apache.log4j.Logger; import org.broadinstitute.sting.utils.MathUtils; -import org.broadinstitute.sting.utils.collections.Pair; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.variantcontext.*; import java.io.PrintStream; @@ -166,7 +166,7 @@ public static void linearExactMultiAllelic(final GenotypesContext GLs, final int numChr = 2*numSamples; // queue of AC conformations to process - final Queue ACqueue = new LinkedList(); + final LinkedList ACqueue = new LinkedList(); // mapping of ExactACset indexes to the objects final HashMap indexesToACset = new HashMap(numChr+1); @@ -210,7 +210,7 @@ private static double calculateAlleleCountConformation(final ExactACset set, final double maxLog10L, final int numChr, final boolean preserveData, - final Queue ACqueue, + final LinkedList ACqueue, final HashMap indexesToACset, final double[][] log10AlleleFrequencyPriors, final AlleleFrequencyCalculationResult result, @@ -250,7 +250,6 @@ private static double calculateAlleleCountConformation(final ExactACset set, if ( ACwiggle == 0 ) // all alternate alleles already sum to 2N so we cannot possibly go to higher frequencies return log10LofK; - ExactACset lastSet = null; // keep track of the last set placed in the queue so that we can tell it to clean us up when done processing final int numAltAlleles = set.ACcounts.getCounts().length; // genotype likelihoods are a linear vector that can be thought of as a row-wise upper triangular matrix of log10Likelihoods. @@ -261,7 +260,7 @@ private static double calculateAlleleCountConformation(final ExactACset set, for ( int allele = 0; allele < numAltAlleles; allele++ ) { final int[] ACcountsClone = set.ACcounts.getCounts().clone(); ACcountsClone[allele]++; - lastSet = updateACset(ACcountsClone, numChr, set, ++PLindex, ACqueue, indexesToACset); + updateACset(ACcountsClone, numChr, set, ++PLindex, ACqueue, indexesToACset); } // add conformations for the k+2 case if it makes sense; note that the 2 new alleles may be the same or different @@ -284,20 +283,17 @@ private static double calculateAlleleCountConformation(final ExactACset set, // IMPORTANT: we must first add the cases where the 2 new alleles are different so that the queue maintains its ordering for ( DependentSet dependent : differentAlleles ) - lastSet = updateACset(dependent.ACcounts, numChr, set, dependent.PLindex, ACqueue, indexesToACset); + updateACset(dependent.ACcounts, numChr, set, dependent.PLindex, ACqueue, indexesToACset); for ( DependentSet dependent : sameAlleles ) - lastSet = updateACset(dependent.ACcounts, numChr, set, dependent.PLindex, ACqueue, indexesToACset); + updateACset(dependent.ACcounts, numChr, set, dependent.PLindex, ACqueue, indexesToACset); } - // if the last dependent set was not at the back of the queue (i.e. not just added), then we need to iterate - // over all the dependent sets to find the last one in the queue (otherwise it will be cleaned up too early) - if ( !preserveData && lastSet == null ) { - //if ( DEBUG ) - // System.out.printf(" *** iterating over dependent sets for set=%s%n", set.ACcounts); - lastSet = determineLastDependentSetInQueue(set.ACcounts, ACqueue); + // determine which is the last dependent set in the queue (not necessarily the last one added above) so we can know when it is safe to clean up this column + if ( !preserveData ) { + final ExactACset lastSet = determineLastDependentSetInQueue(set.ACcounts, ACqueue); + if ( lastSet != null ) + lastSet.dependentACsetsToDelete.add(set.ACcounts); } - if ( lastSet != null ) - lastSet.dependentACsetsToDelete.add(set.ACcounts); return log10LofK; } @@ -305,19 +301,17 @@ private static double calculateAlleleCountConformation(final ExactACset set, // adds the ExactACset represented by the ACcounts to the ACqueue if not already there (creating it if needed) and // also adds it as a dependency to the given callingSetIndex. // returns the ExactACset if that set was not already in the queue and null otherwise. - private static ExactACset updateACset(final int[] ACcounts, - final int numChr, - final ExactACset callingSet, - final int PLsetIndex, - final Queue ACqueue, - final HashMap indexesToACset) { + private static void updateACset(final int[] ACcounts, + final int numChr, + final ExactACset callingSet, + final int PLsetIndex, + final Queue ACqueue, + final HashMap indexesToACset) { final ExactACcounts index = new ExactACcounts(ACcounts); - boolean wasInQueue = true; if ( !indexesToACset.containsKey(index) ) { ExactACset set = new ExactACset(numChr/2 +1, index); indexesToACset.put(index, set); ACqueue.add(set); - wasInQueue = false; } // add the given dependency to the set @@ -325,16 +319,18 @@ private static ExactACset updateACset(final int[] ACcounts, // System.out.println(" *** adding dependency from " + index + " to " + callingSet.ACcounts); final ExactACset set = indexesToACset.get(index); set.ACsetIndexToPLIndex.put(callingSet.ACcounts, PLsetIndex); - return wasInQueue ? null : set; } - private static ExactACset determineLastDependentSetInQueue(final ExactACcounts callingSetIndex, final Queue ACqueue) { - ExactACset set = null; - for ( ExactACset queued : ACqueue ) { - if ( queued.dependentACsetsToDelete.contains(callingSetIndex) ) - set = queued; + private static ExactACset determineLastDependentSetInQueue(final ExactACcounts callingSetIndex, final LinkedList ACqueue) { + Iterator reverseIterator = ACqueue.descendingIterator(); + while ( reverseIterator.hasNext() ) { + final ExactACset queued = reverseIterator.next(); + if ( queued.ACsetIndexToPLIndex.containsKey(callingSetIndex) ) + return queued; } - return set; + + // shouldn't get here + throw new ReviewedStingException("Error: no sets in the queue currently hold " + callingSetIndex + " as a dependent!"); } private static void computeLofK(final ExactACset set, From ddaf51a50ffea902068ad554367eceadcf37c86b Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Wed, 25 Jan 2012 19:18:51 -0500 Subject: [PATCH 145/356] Updated one integration test for indels --- .../gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java index e9b4fc2113..7285b0fb8a 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java @@ -294,7 +294,7 @@ public void testWithIndelAllelesPassedIn4() { WalkerTest.WalkerTestSpec spec4 = new WalkerTest.WalkerTestSpec( baseCommandIndelsb37 + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "ALL.wgs.union_v2_chr20_100_110K.20101123.indels.sites.vcf -I " + validationDataLocation + "phase1_GBR_realigned.chr20.100K-110K.bam -o %s -L 20:100,000-110,000", 1, - Arrays.asList("877de5b0cc61dc54636062df6399b978")); + Arrays.asList("1d1956fd7b0f0d30935674b2f5019860")); executeTest("test MultiSample Phase1 indels with complicated records", spec4); } From 9a60887567d677dacc43fb4aced8a93ed308e54e Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Wed, 25 Jan 2012 19:41:41 -0500 Subject: [PATCH 146/356] Lost an import in the merge --- .../gatk/walkers/genotyper/ExactAFCalculationModel.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java index 560ade2d8d..d604e8d62c 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java @@ -27,6 +27,7 @@ import org.apache.log4j.Logger; import org.broadinstitute.sting.utils.MathUtils; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.variantcontext.*; import java.io.PrintStream; @@ -38,9 +39,6 @@ public class ExactAFCalculationModel extends AlleleFrequencyCalculationModel { private final static double MAX_LOG10_ERROR_TO_STOP_EARLY = 6; // we want the calculation to be accurate to 1 / 10^6 - // TODO: PERMITS WALKER USED TO HAVE A TEMPORARY FIX to prevent NullPointerException caused by bug: - public static boolean PRESERVE_AC_DATA = false; - protected ExactAFCalculationModel(UnifiedArgumentCollection UAC, int N, Logger logger, PrintStream verboseWriter) { super(UAC, N, logger, verboseWriter); } @@ -52,7 +50,7 @@ public void getLog10PNonRef(final GenotypesContext GLs, final int numAlleles = alleles.size(); //linearExact(GLs, log10AlleleFrequencyPriors[0], log10AlleleFrequencyLikelihoods, log10AlleleFrequencyPosteriors); - linearExactMultiAllelic(GLs, numAlleles - 1, log10AlleleFrequencyPriors, result, PRESERVE_AC_DATA); + linearExactMultiAllelic(GLs, numAlleles - 1, log10AlleleFrequencyPriors, result, false); } private static final ArrayList getGLs(GenotypesContext GLs) { From 702a2d768fc437411f55b2a95e37d4845ad3c24f Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Wed, 25 Jan 2012 19:42:55 -0500 Subject: [PATCH 147/356] Initial version of multi-allelic summary module in VariantEval --- .../evaluators/MultiallelicSummary.java | 181 ++++++++++++++++++ .../variantcontext/VariantContextUtils.java | 8 + 2 files changed, 189 insertions(+) create mode 100644 public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java new file mode 100644 index 0000000000..835f6ca8c0 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.walkers.varianteval.evaluators; + +import org.apache.log4j.Logger; +import org.broadinstitute.sting.gatk.contexts.AlignmentContext; +import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; +import org.broadinstitute.sting.gatk.walkers.varianteval.VariantEvalWalker; +import org.broadinstitute.sting.gatk.walkers.varianteval.util.Analysis; +import org.broadinstitute.sting.gatk.walkers.varianteval.util.DataPoint; +import org.broadinstitute.sting.utils.GenomeLoc; +import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants; +import org.broadinstitute.sting.utils.exceptions.UserException; +import org.broadinstitute.sting.utils.interval.IntervalUtils; +import org.broadinstitute.sting.utils.variantcontext.*; + +import java.util.*; + +@Analysis(description = "Evaluation summary for multi-allelic variants") +public class MultiallelicSummary extends VariantEvaluator implements StandardEval { + final protected static Logger logger = Logger.getLogger(MultiallelicSummary.class); + + public enum Type { + SNP, INDEL + } + + // basic counts on various rates found + @DataPoint(description = "Number of processed loci") + public long nProcessedLoci = 0; + + @DataPoint(description = "Number of SNPs") + public int nSNPs = 0; + @DataPoint(description = "Number of multi-allelic SNPs") + public int nMultiSNPs = 0; + @DataPoint(description = "% processed sites that are multi-allelic SNPs", format = "%.5f") + public double processedMultiSnpRatio = 0; + @DataPoint(description = "% SNP sites that are multi-allelic", format = "%.3f") + public double variantMultiSnpRatio = 0; + + @DataPoint(description = "Number of Indels") + public int nIndels = 0; + @DataPoint(description = "Number of multi-allelic Indels") + public int nMultiIndels = 0; + @DataPoint(description = "% processed sites that are multi-allelic Indels", format = "%.5f") + public double processedMultiIndelRatio = 0; + @DataPoint(description = "% Indel sites that are multi-allelic", format = "%.3f") + public double variantMultiIndelRatio = 0; + + @DataPoint(description = "Number of Transitions") + public int nTi = 0; + @DataPoint(description = "Number of Transversions") + public int nTv = 0; + @DataPoint(description = "Overall TiTv ratio", format = "%.2f") + public double TiTvRatio = 0; + + @DataPoint(description = "Multi-allelic SNPs partially known") + public int knownSNPsPartial = 0; + @DataPoint(description = "Multi-allelic SNPs completely known") + public int knownSNPsComplete = 0; + @DataPoint(description = "Multi-allelic SNP Novelty Rate") + public String SNPNoveltyRate = "NA"; + + @DataPoint(description = "Multi-allelic Indels partially known") + public int knownIndelsPartial = 0; + @DataPoint(description = "Multi-allelic Indels completely known") + public int knownIndelsComplete = 0; + @DataPoint(description = "Multi-allelic Indel Novelty Rate") + public String indelNoveltyRate = "NA"; + + // TODO -- Also, AF distributions (pairwise like TiTv) + + public void initialize(VariantEvalWalker walker) {} + + @Override public boolean enabled() { return true; } + + public int getComparisonOrder() { + return 2; + } + + public void update0(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) { + nProcessedLoci += context.getSkippedBases() + (ref == null ? 0 : 1); + } + + + public String update2(VariantContext eval, VariantContext comp, RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) { + if ( eval == null || eval.isMonomorphicInSamples() ) + return null; + + // update counts + switch ( eval.getType() ) { + case SNP: + nSNPs++; + if ( !eval.isBiallelic() ) { + nMultiSNPs++; + calculatePairwiseTiTv(eval); + calculateSNPPairwiseNovelty(eval, comp); + } + break; + case INDEL: + nIndels++; + if ( !eval.isBiallelic() ) { + nMultiIndels++; + calculateIndelPairwiseNovelty(eval, comp); + } + break; + default: + throw new UserException.BadInput("Unexpected variant context type: " + eval); + } + + return null; // we don't capture any interesting sites + } + + private void calculatePairwiseTiTv(VariantContext vc) { + for ( Allele alt : vc.getAlternateAlleles() ) { + if ( VariantContextUtils.isTransition(vc.getReference(), alt) ) + nTi++; + else + nTv++; + } + } + + private void calculateSNPPairwiseNovelty(VariantContext eval, VariantContext comp) { + if ( comp == null ) + return; + + int knownAlleles = 0; + for ( Allele alt : eval.getAlternateAlleles() ) { + if ( comp.getAlternateAlleles().contains(alt) ) + knownAlleles++; + } + + if ( knownAlleles == eval.getAlternateAlleles().size() ) + knownSNPsComplete++; + else if ( knownAlleles > 0 ) + knownSNPsPartial++; + } + + private void calculateIndelPairwiseNovelty(VariantContext eval, VariantContext comp) { + } + + private final String noveltyRate(final int all, final int known) { + final int novel = all - known; + final double rate = (novel / (1.0 * all)); + return all == 0 ? "NA" : String.format("%.2f", rate); + } + + public void finalizeEvaluation() { + processedMultiSnpRatio = (double)nMultiSNPs / (double)nProcessedLoci; + variantMultiSnpRatio = (double)nMultiSNPs / (double)nSNPs; + processedMultiIndelRatio = (double)nMultiIndels / (double)nProcessedLoci; + variantMultiIndelRatio = (double)nMultiIndels / (double)nIndels; + + TiTvRatio = (double)nTi / (double)nTv; + + SNPNoveltyRate = noveltyRate(nMultiSNPs, knownSNPsPartial + knownSNPsComplete); + indelNoveltyRate = noveltyRate(nMultiSNPs, knownIndelsPartial + knownIndelsComplete); + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java index 179c91660a..c79bbaace7 100755 --- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java @@ -1073,6 +1073,14 @@ public static boolean isTransversion(VariantContext context) { return getSNPSubstitutionType(context) == BaseUtils.BaseSubstitutionType.TRANSVERSION; } + public static boolean isTransition(Allele ref, Allele alt) { + return BaseUtils.SNPSubstitutionType(ref.getBases()[0], alt.getBases()[0]) == BaseUtils.BaseSubstitutionType.TRANSITION; + } + + public static boolean isTransversion(Allele ref, Allele alt) { + return BaseUtils.SNPSubstitutionType(ref.getBases()[0], alt.getBases()[0]) == BaseUtils.BaseSubstitutionType.TRANSVERSION; + } + /** * create a genome location, given a variant context * @param genomeLocParser parser From 5b9c8ab01b4f2c8bf42a7209fd72293c495c60e3 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Wed, 25 Jan 2012 21:53:20 -0500 Subject: [PATCH 148/356] Another quick update missed in the merge From 774e5400426fb1f2ace3b4b2cc64b69a35345f70 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Thu, 26 Jan 2012 00:31:41 -0500 Subject: [PATCH 149/356] Fixing broken test From c5e81be9781881cc9a79f99e696b47cb578b8ba2 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Thu, 26 Jan 2012 00:37:06 -0500 Subject: [PATCH 150/356] Adding pairwise AF table. Not polished at all, but usable none-the-less. --- .../evaluators/MultiallelicSummary.java | 142 ++++++++++++------ 1 file changed, 97 insertions(+), 45 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java index 835f6ca8c0..6094385e65 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java @@ -31,10 +31,10 @@ import org.broadinstitute.sting.gatk.walkers.varianteval.VariantEvalWalker; import org.broadinstitute.sting.gatk.walkers.varianteval.util.Analysis; import org.broadinstitute.sting.gatk.walkers.varianteval.util.DataPoint; -import org.broadinstitute.sting.utils.GenomeLoc; +import org.broadinstitute.sting.gatk.walkers.varianteval.util.TableType; +import org.broadinstitute.sting.utils.MathUtils; import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants; import org.broadinstitute.sting.utils.exceptions.UserException; -import org.broadinstitute.sting.utils.interval.IntervalUtils; import org.broadinstitute.sting.utils.variantcontext.*; import java.util.*; @@ -90,7 +90,59 @@ public enum Type { @DataPoint(description = "Multi-allelic Indel Novelty Rate") public String indelNoveltyRate = "NA"; - // TODO -- Also, AF distributions (pairwise like TiTv) + @DataPoint(description="Histogram of allele frequencies") + AFHistogram AFhistogram = new AFHistogram(); + + /* + * AF histogram table object + */ + static class AFHistogram implements TableType { + private Object[] colKeys, rowKeys = {"pairwise_AF"}; + private int[] AFhistogram; + + private static final double AFincrement = 0.01; + private static final int numBins = (int)(1.00 / AFincrement); + + public AFHistogram() { + colKeys = initColKeys(); + AFhistogram = new int[colKeys.length]; + } + + public Object[] getColumnKeys() { + return colKeys; + } + + public Object[] getRowKeys() { + return rowKeys; + } + + public Object getCell(int row, int col) { + return AFhistogram[col]; + } + + private static Object[] initColKeys() { + ArrayList keyList = new ArrayList(numBins + 1); + for ( double a = 0.00; a <= 1.01; a += AFincrement ) { + keyList.add(String.format("%.2f", a)); + } + return keyList.toArray(); + } + + public String getName() { return "AFHistTable"; } + + public void update(VariantContext vc) { + final Object obj = vc.getAttribute(VCFConstants.ALLELE_FREQUENCY_KEY, null); + if ( obj == null || !(obj instanceof List) ) + return; + + List list = (List)obj; + for ( String str : list ) { + final double AF = Double.valueOf(str); + final int bin = (int)(numBins * MathUtils.round(AF, 2)); + AFhistogram[bin]++; + } + } + } public void initialize(VariantEvalWalker walker) {} @@ -104,58 +156,58 @@ public void update0(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentC nProcessedLoci += context.getSkippedBases() + (ref == null ? 0 : 1); } - public String update2(VariantContext eval, VariantContext comp, RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) { if ( eval == null || eval.isMonomorphicInSamples() ) - return null; + return null; // update counts switch ( eval.getType() ) { - case SNP: - nSNPs++; - if ( !eval.isBiallelic() ) { - nMultiSNPs++; - calculatePairwiseTiTv(eval); - calculateSNPPairwiseNovelty(eval, comp); - } - break; + case SNP: + nSNPs++; + if ( !eval.isBiallelic() ) { + nMultiSNPs++; + calculatePairwiseTiTv(eval); + calculateSNPPairwiseNovelty(eval, comp); + } + break; case INDEL: - nIndels++; - if ( !eval.isBiallelic() ) { - nMultiIndels++; - calculateIndelPairwiseNovelty(eval, comp); - } - break; + nIndels++; + if ( !eval.isBiallelic() ) { + nMultiIndels++; + calculateIndelPairwiseNovelty(eval, comp); + } + break; default: throw new UserException.BadInput("Unexpected variant context type: " + eval); } - + AFhistogram.update(eval); + return null; // we don't capture any interesting sites } private void calculatePairwiseTiTv(VariantContext vc) { - for ( Allele alt : vc.getAlternateAlleles() ) { - if ( VariantContextUtils.isTransition(vc.getReference(), alt) ) - nTi++; - else - nTv++; - } + for ( Allele alt : vc.getAlternateAlleles() ) { + if ( VariantContextUtils.isTransition(vc.getReference(), alt) ) + nTi++; + else + nTv++; + } } private void calculateSNPPairwiseNovelty(VariantContext eval, VariantContext comp) { - if ( comp == null ) - return; - - int knownAlleles = 0; - for ( Allele alt : eval.getAlternateAlleles() ) { - if ( comp.getAlternateAlleles().contains(alt) ) - knownAlleles++; - } - - if ( knownAlleles == eval.getAlternateAlleles().size() ) - knownSNPsComplete++; - else if ( knownAlleles > 0 ) - knownSNPsPartial++; + if ( comp == null ) + return; + + int knownAlleles = 0; + for ( Allele alt : eval.getAlternateAlleles() ) { + if ( comp.getAlternateAlleles().contains(alt) ) + knownAlleles++; + } + + if ( knownAlleles == eval.getAlternateAlleles().size() ) + knownSNPsComplete++; + else if ( knownAlleles > 0 ) + knownSNPsPartial++; } private void calculateIndelPairwiseNovelty(VariantContext eval, VariantContext comp) { @@ -168,14 +220,14 @@ private final String noveltyRate(final int all, final int known) { } public void finalizeEvaluation() { - processedMultiSnpRatio = (double)nMultiSNPs / (double)nProcessedLoci; - variantMultiSnpRatio = (double)nMultiSNPs / (double)nSNPs; - processedMultiIndelRatio = (double)nMultiIndels / (double)nProcessedLoci; - variantMultiIndelRatio = (double)nMultiIndels / (double)nIndels; + processedMultiSnpRatio = (double)nMultiSNPs / (double)nProcessedLoci; + variantMultiSnpRatio = (double)nMultiSNPs / (double)nSNPs; + processedMultiIndelRatio = (double)nMultiIndels / (double)nProcessedLoci; + variantMultiIndelRatio = (double)nMultiIndels / (double)nIndels; TiTvRatio = (double)nTi / (double)nTv; - SNPNoveltyRate = noveltyRate(nMultiSNPs, knownSNPsPartial + knownSNPsComplete); - indelNoveltyRate = noveltyRate(nMultiSNPs, knownIndelsPartial + knownIndelsComplete); + SNPNoveltyRate = noveltyRate(nMultiSNPs, knownSNPsPartial + knownSNPsComplete); + indelNoveltyRate = noveltyRate(nMultiSNPs, knownIndelsPartial + knownIndelsComplete); } } From 859dd882c90ababb92f165479d6171b8b8ec9ce7 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Thu, 26 Jan 2012 00:38:16 -0500 Subject: [PATCH 151/356] Don't make it standard for now --- .../walkers/varianteval/evaluators/MultiallelicSummary.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java index 6094385e65..9113e75382 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java @@ -40,7 +40,7 @@ import java.util.*; @Analysis(description = "Evaluation summary for multi-allelic variants") -public class MultiallelicSummary extends VariantEvaluator implements StandardEval { +public class MultiallelicSummary extends VariantEvaluator { // implements StandardEval { final protected static Logger logger = Logger.getLogger(MultiallelicSummary.class); public enum Type { From 390d493049f66d01b0ef925f572358a862b767e0 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Thu, 26 Jan 2012 11:37:08 -0500 Subject: [PATCH 152/356] Updating ActiveRegionWalker interface to output a probability of active status instead of a boolean. Integrator runs a band-pass filter over this probability to produce actual active regions. First version of HaplotypeCaller which decides for itself where to trigger and assembles those regions. --- .../traversals/TraverseActiveRegions.java | 90 ++++++++++--------- .../gatk/walkers/ActiveRegionWalker.java | 4 +- .../walkers/genotyper/UnifiedGenotyper.java | 2 + 3 files changed, 52 insertions(+), 44 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java index f5e936a092..83daa4d80e 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java +++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java @@ -10,14 +10,12 @@ import org.broadinstitute.sting.gatk.walkers.*; import org.broadinstitute.sting.utils.GenomeLoc; import org.broadinstitute.sting.utils.GenomeLocSortedSet; +import org.broadinstitute.sting.utils.MathUtils; import org.broadinstitute.sting.utils.activeregion.ActiveRegion; import org.broadinstitute.sting.utils.pileup.PileupElement; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; -import java.util.ArrayList; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.Queue; +import java.util.*; /** * Created by IntelliJ IDEA. @@ -54,7 +52,8 @@ public T traverse( final ActiveRegionWalker walker, if ( locusView.hasNext() ) { // trivial optimization to avoid unnecessary processing when there's nothing here at all int minStart = Integer.MAX_VALUE; - final ArrayList isActiveList = new ArrayList(); + final ArrayList isActiveList = new ArrayList(); + GenomeLoc firstIsActiveStart = null; //ReferenceOrderedView referenceOrderedDataView = new ReferenceOrderedView( dataProvider ); ReferenceOrderedView referenceOrderedDataView = null; @@ -91,11 +90,15 @@ public T traverse( final ActiveRegionWalker walker, final RefMetaDataTracker tracker = referenceOrderedDataView.getReferenceOrderedDataAtLocus(locus.getLocation(), refContext); // Call the walkers isActive function for this locus and add them to the list to be integrated later - if( initialIntervals.overlaps(location) ) { - final boolean isActive = ( walker.presetActiveRegions == null ? walker.isActive( tracker, refContext, locus ) : walker.presetActiveRegions.overlaps(location) ); - isActiveList.add( new ActiveRegion(location, isActive, engine.getGenomeLocParser(), activeRegionExtension ) ); + if( initialIntervals.overlaps( location ) ) { + final double isActiveProb = ( walker.presetActiveRegions == null ? walker.isActive( tracker, refContext, locus ) + : ( walker.presetActiveRegions.overlaps(location) ? 1.0 : 0.0 ) ); + isActiveList.add( isActiveProb ); + if( firstIsActiveStart == null ) { + firstIsActiveStart = locus.getLocation(); + } } - + // Grab all the previously unseen reads from this pileup and add them to the massive read list for( final PileupElement p : locus.getBasePileup() ) { final SAMRecord read = p.getRead(); @@ -104,15 +107,9 @@ public T traverse( final ActiveRegionWalker walker, } } - // If this is the last pileup for this shard then need to first do a special walker.isActive() call - // and then calculate the minimum alignment start so that we know which active regions in the work queue are now safe to process + // If this is the last pileup for this shard calculate the minimum alignment start so that we know + // which active regions in the work queue are now safe to process if( !locusView.hasNext() ) { - // Call the walkers isActive function for this locus and add them to the list to be integrated later - if( initialIntervals.overlaps(location) ) { - final boolean isActive = ( walker.presetActiveRegions == null ? walker.isActive( tracker, refContext, locus ) : walker.presetActiveRegions.overlaps(location) ); - isActiveList.add( new ActiveRegion(location, isActive, engine.getGenomeLocParser(), activeRegionExtension ) ); - } - for( final PileupElement p : locus.getBasePileup() ) { final SAMRecord read = p.getRead(); if( !myReads.contains(read) ) { @@ -121,12 +118,12 @@ public T traverse( final ActiveRegionWalker walker, if( read.getAlignmentStart() < minStart ) { minStart = read.getAlignmentStart(); } } } - printProgress(dataProvider.getShard(),locus.getLocation()); + printProgress(dataProvider.getShard(), locus.getLocation()); } // Take the individual isActive calls and integrate them into contiguous active regions and // add these blocks of work to the work queue - final ArrayList activeRegions = integrateActiveList( isActiveList ); + final ArrayList activeRegions = integrateActiveList( isActiveList, firstIsActiveStart, activeRegionExtension ); logger.debug("Integrated " + isActiveList.size() + " isActive calls into " + activeRegions.size() + " regions." ); if( walker.activeRegionOutStream == null ) { workQueue.addAll( activeRegions ); @@ -137,14 +134,11 @@ public T traverse( final ActiveRegionWalker walker, } } } - // Since we've sufficiently past this point (or this contig!) in the workQueue we can unload those regions and process them - if( !workQueue.isEmpty() ) { - while( workQueue.peek().getExtendedLoc().getStop() < minStart || !workQueue.peek().getExtendedLoc().getContig().equals(dataProvider.getLocus().getContig()) ) { - final ActiveRegion activeRegion = workQueue.remove(); - sum = processActiveRegion( activeRegion, myReads, workQueue, sum, walker ); - } + while( workQueue.peek() != null && (workQueue.peek().getExtendedLoc().getStop() < minStart || !workQueue.peek().getExtendedLoc().getContig().equals(dataProvider.getLocus().getContig())) ) { + final ActiveRegion activeRegion = workQueue.remove(); + sum = processActiveRegion( activeRegion, myReads, workQueue, sum, walker ); } } @@ -184,7 +178,7 @@ private T processActiveRegion( final ActiveRegion activeRegion, final LinkedHash } for( final ActiveRegion otherRegionToTest : workQueue ) { if( !bestRegion.equals(otherRegionToTest) && otherRegionToTest.getExtendedLoc().overlapsP( readLoc ) ) { - activeRegion.add( (GATKSAMRecord) read ); + otherRegionToTest.add( (GATKSAMRecord) read ); } } } @@ -218,31 +212,43 @@ else if( dataSource == DataSource.REFERENCE_ORDERED_DATA ) throw new UnsupportedOperationException("Unsupported traversal type: " + dataSource); } - // integrate active regions into contiguous chunks with identical active status - private ArrayList integrateActiveList( final ArrayList activeList ) { + // band-pass filter the list of isActive probabilities and turn into active regions + private ArrayList integrateActiveList( final ArrayList activeList, final GenomeLoc firstIsActiveStart, final int activeRegionExtension ) { + + final double ACTIVE_PROB_THRESHOLD = 0.2; final ArrayList returnList = new ArrayList(); if( activeList.size() == 0 ) { return returnList; } else if( activeList.size() == 1 ) { - returnList.add( new ActiveRegion( engine.getGenomeLocParser().createGenomeLoc(activeList.get(0).getLocation().getContig(), activeList.get(0).getLocation().getStart(), activeList.get(0).getLocation().getStart()), - activeList.get(0).isActive, engine.getGenomeLocParser(), activeList.get(0).getExtension() ) ); + returnList.add( new ActiveRegion( engine.getGenomeLocParser().createGenomeLoc(firstIsActiveStart.getContig(), firstIsActiveStart.getStart(), firstIsActiveStart.getStart()), + activeList.get(0) > ACTIVE_PROB_THRESHOLD, engine.getGenomeLocParser(), activeRegionExtension ) ); return returnList; } else { - ActiveRegion prevLocus = activeList.get(0); - ActiveRegion startLocus = prevLocus; - for( final ActiveRegion thisLocus : activeList ) { - if( prevLocus.isActive != thisLocus.isActive || !prevLocus.getLocation().contiguousP( thisLocus.getLocation() ) ) { - returnList.add( new ActiveRegion( engine.getGenomeLocParser().createGenomeLoc(startLocus.getLocation().getContig(), startLocus.getLocation().getStart(), prevLocus.getLocation().getStart()), - prevLocus.isActive, engine.getGenomeLocParser(), startLocus.getExtension() ) ); - startLocus = thisLocus; + final Double[] activeProbArray = activeList.toArray(new Double[activeList.size()]); + final double[] filteredProbArray = new double[activeProbArray.length]; + final int FILTER_SIZE = 10; + final int MAX_ACTIVE_REGION = 200; + for( int iii = 0; iii < activeProbArray.length; iii++ ) { + double maxVal = 0; + for( int jjj = Math.max( 0, iii-FILTER_SIZE); jjj < Math.min( activeList.size(), iii+FILTER_SIZE); jjj++ ) { + if( activeProbArray[jjj] > maxVal ) { maxVal = activeProbArray[jjj]; } } - prevLocus = thisLocus; + filteredProbArray[iii] = maxVal; } - // output the last region if necessary - if( startLocus != prevLocus ) { - returnList.add( new ActiveRegion( engine.getGenomeLocParser().createGenomeLoc(startLocus.getLocation().getContig(), startLocus.getLocation().getStart(), prevLocus.getLocation().getStart()), - prevLocus.isActive, engine.getGenomeLocParser(), startLocus.getExtension() ) ); + + boolean curStatus = filteredProbArray[0] > ACTIVE_PROB_THRESHOLD; + int curStart = 0; + for(int iii = 1; iii < filteredProbArray.length; iii++ ) { + final boolean thisStatus = filteredProbArray[iii] > ACTIVE_PROB_THRESHOLD; + if( curStatus != thisStatus || (iii-curStart) > MAX_ACTIVE_REGION ) { + returnList.add( new ActiveRegion( engine.getGenomeLocParser().createGenomeLoc(firstIsActiveStart.getContig(), firstIsActiveStart.getStart() + curStart, firstIsActiveStart.getStart() + (iii-1)), + curStatus, engine.getGenomeLocParser(), activeRegionExtension ) ); + curStatus = thisStatus; + curStart = iii; + } } + returnList.add( new ActiveRegion( engine.getGenomeLocParser().createGenomeLoc(firstIsActiveStart.getContig(), firstIsActiveStart.getStart() + curStart, firstIsActiveStart.getStart() + (filteredProbArray.length-1)), + curStatus, engine.getGenomeLocParser(), activeRegionExtension ) ); return returnList; } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java index 98308ee111..244870c78a 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java @@ -73,8 +73,8 @@ public boolean wantsNonPrimaryReads() { return false; } - // Determine active status over the AlignmentContext - public abstract boolean isActive(final RefMetaDataTracker tracker, final ReferenceContext ref, final AlignmentContext context); + // Determine probability of active status over the AlignmentContext + public abstract double isActive(final RefMetaDataTracker tracker, final ReferenceContext ref, final AlignmentContext context); // Map over the ActiveRegion public abstract MapType map(final ActiveRegion activeRegion, final ReadMetaDataTracker metaDataTracker); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java index 369c2d0c68..35295284cb 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java @@ -169,9 +169,11 @@ public class UnifiedGenotyper extends LocusWalker Date: Thu, 26 Jan 2012 12:43:52 -0500 Subject: [PATCH 153/356] Allow segments of genome to be excluded in generating a reference panel. Occasionally targets would contain no variation (typically, in the middle of the centromere), which beagle doesn't particularly like, and errors out rather than producing empty output files. The best way to deal with these is to just exclude the regions on a second-pass, and the remaining bits will be gathered with no additional work. AllelePair is being mean and not telling me what genotype it sees when it finds a non-diploid genotype, but i suspect it's a no-call (".") rather than a no call ("./."). --- .../broadinstitute/sting/gatk/walkers/phasing/AllelePair.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/AllelePair.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/AllelePair.java index cb123c8683..c629bd313b 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/AllelePair.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/AllelePair.java @@ -36,7 +36,7 @@ public class AllelePair { public AllelePair(Genotype gt) { if (gt.getPloidy() != 2) - throw new ReviewedStingException("AllelePair must have ploidy of 2!"); + throw new ReviewedStingException("AllelePair must have ploidy of 2! incoming gt was"+gt.toBriefString()); this.top = gt.getAllele(0); this.bottom = gt.getAllele(1); From 67c89cadad872fd3bde68634fc7887e9d38713ef Mon Sep 17 00:00:00 2001 From: Guillermo del Angel Date: Thu, 26 Jan 2012 12:52:00 -0500 Subject: [PATCH 154/356] Fixes for pool caller to match UG outputs at certain sites: implement min base qual/min mapping qual read filter so those reads are filtered from pileups, and implemented filter for sites that have a too large a fraction of deletions From 9c6fda7e1597e272a988ef42f99c6ffd8ea489f8 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 26 Jan 2012 12:54:11 -0500 Subject: [PATCH 155/356] Yup. I was right. --- .../broadinstitute/sting/gatk/walkers/phasing/AllelePair.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/AllelePair.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/AllelePair.java index c629bd313b..aca9e21af3 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/AllelePair.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/AllelePair.java @@ -35,6 +35,10 @@ public class AllelePair { private Allele bottom; public AllelePair(Genotype gt) { + if ( gt.isNoCall() ) { + // do nothing + return; + } if (gt.getPloidy() != 2) throw new ReviewedStingException("AllelePair must have ploidy of 2! incoming gt was"+gt.toBriefString()); From 673ceadd114a5a54e4d19aad1d844d14797b2377 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 26 Jan 2012 13:06:36 -0500 Subject: [PATCH 156/356] While this fix worked for the evaluator module, it could potentially have bad effects in the phasing walkers. Special-case nocalls in the PhasingEvaluator and return AllelePair to previous state. --- .../broadinstitute/sting/gatk/walkers/phasing/AllelePair.java | 4 ---- .../varianteval/evaluators/GenotypePhasingEvaluator.java | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/AllelePair.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/AllelePair.java index aca9e21af3..c629bd313b 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/AllelePair.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/AllelePair.java @@ -35,10 +35,6 @@ public class AllelePair { private Allele bottom; public AllelePair(Genotype gt) { - if ( gt.isNoCall() ) { - // do nothing - return; - } if (gt.getPloidy() != 2) throw new ReviewedStingException("AllelePair must have ploidy of 2! incoming gt was"+gt.toBriefString()); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/GenotypePhasingEvaluator.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/GenotypePhasingEvaluator.java index 07cd95997f..f4369401b7 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/GenotypePhasingEvaluator.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/GenotypePhasingEvaluator.java @@ -119,7 +119,7 @@ public String update2(VariantContext eval, VariantContext comp, RefMetaDataTrack if (evalSampGenotypes != null) evalSampGt = evalSampGenotypes.get(samp); - if (compSampGt == null || evalSampGt == null) { // Since either comp or eval (or both) are missing the site, the best we can do is hope to preserve phase [if the non-missing one preserves phase] + if (compSampGt == null || evalSampGt == null || compSampGt.isNoCall() || evalSampGt.isNoCall()) { // Since either comp or eval (or both) are missing the site, the best we can do is hope to preserve phase [if the non-missing one preserves phase] // Having an unphased site breaks the phasing for the sample [does NOT permit "transitive phasing"] - hence, must reset phasing knowledge for both comp and eval [put a null CompEvalGenotypes]: if (isNonNullButUnphased(compSampGt) || isNonNullButUnphased(evalSampGt)) samplePrevGenotypes.put(samp, null); @@ -209,7 +209,7 @@ public static boolean isRelevantToPhasing(VariantContext vc) { } public boolean isNonNullButUnphased(Genotype gt) { - return (gt != null && !genotypesArePhasedAboveThreshold(gt)); + return (gt != null && !gt.isNoCall() && !genotypesArePhasedAboveThreshold(gt)); } public boolean genotypesArePhasedAboveThreshold(Genotype gt) { From dbe9eb70fe301a6d43426c93bb290daffb337ffa Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Thu, 26 Jan 2012 13:25:22 -0500 Subject: [PATCH 157/356] Updating HC integration tests after merge From cdff23269d619ff8515d37c86110a468eff37de3 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Thu, 26 Jan 2012 15:56:33 -0500 Subject: [PATCH 158/356] HaplotypeCaller now uses insertions and softclipped bases as possible triggers. LocusIteratorByState tags pileup elements with the required info to make this calculation efficient. The days of the extended event pileup are coming to a close. --- .../gatk/iterators/LocusIteratorByState.java | 9 ++++- ...NPGenotypeLikelihoodsCalculationModel.java | 2 +- .../pileup/AbstractReadBackedPileup.java | 6 +-- .../pileup/ExtendedEventPileupElement.java | 4 +- .../sting/utils/pileup/PileupElement.java | 37 +++++++++---------- .../ReadBackedExtendedEventPileupImpl.java | 2 +- .../utils/pileup/ReadBackedPileupImpl.java | 4 +- .../sting/utils/sam/ArtificialSAMUtils.java | 4 +- .../sting/utils/sam/ReadUtilsUnitTest.java | 4 +- 9 files changed, 39 insertions(+), 33 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java b/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java index f1ffa121b1..2257cc139b 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java +++ b/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java @@ -176,6 +176,10 @@ public String toString() { return String.format("%s ro=%d go=%d co=%d cec=%d %s", read.getReadName(), readOffset, genomeOffset, cigarOffset, cigarElementCounter, curElement); } + public CigarOperator peekForwardOnGenome() { + return ( cigarElementCounter + 1 > curElement.getLength() && cigarOffset + 1 < nCigarElements ? cigar.getCigarElement(cigarOffset + 1) : curElement ).getOperator(); + } + public CigarOperator stepForwardOnGenome() { // we enter this method with readOffset = index of the last processed base on the read // (-1 if we did not process a single base yet); this can be last matching base, or last base of an insertion @@ -455,6 +459,7 @@ else if ( (op != CigarOperator.N) && (op != CigarOperator.D || readInfo.includeR final SAMRecordState state = iterator.next(); // state object with the read/offset information final GATKSAMRecord read = (GATKSAMRecord) state.getRead(); // the actual read final CigarOperator op = state.getCurrentCigarOperator(); // current cigar operator + final CigarOperator nextOp = state.peekForwardOnGenome(); // next cigar operator final int readOffset = state.getReadOffset(); // the base offset on this read final int eventStartOffset = state.getReadEventStartOffset(); // this will be -1 if base is not a deletion, or if base is the first deletion in the event. Otherwise, it will give the last base before the deletion began. @@ -467,13 +472,13 @@ else if ( (op != CigarOperator.N) && (op != CigarOperator.D || readInfo.includeR if (op == CigarOperator.D) { if (readInfo.includeReadsWithDeletionAtLoci()) { // only add deletions to the pileup if we are authorized to do so int leftAlignedStart = (eventStartOffset < 0) ? readOffset : eventStartOffset; - pile.add(new PileupElement(read, leftAlignedStart, true)); + pile.add(new PileupElement(read, leftAlignedStart, true, nextOp == CigarOperator.I, false)); size++; nDeletions++; } } else { if (!filterBaseInRead(read, location.getStart())) { - pile.add(new PileupElement(read, readOffset, false)); + pile.add(new PileupElement(read, readOffset, false, nextOp == CigarOperator.I, op == CigarOperator.S)); size++; } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java index d9ee2ba1b0..5980ff356a 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java @@ -212,7 +212,7 @@ public ReadBackedPileup createBAQedPileup( final ReadBackedPileup pileup ) { public class BAQedPileupElement extends PileupElement { public BAQedPileupElement( final PileupElement PE ) { - super(PE.getRead(), PE.getOffset(), PE.isDeletion()); + super(PE.getRead(), PE.getOffset(), PE.isDeletion(), PE.isBeforeInsertion(), PE.isSoftClipped()); } @Override diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java b/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java index 1fa7101ca3..f4fa9e9416 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java @@ -177,7 +177,7 @@ private PileupElementTracker readsOffsets2Pileup(List reads, for (int i = 0; i < reads.size(); i++) { GATKSAMRecord read = reads.get(i); int offset = offsets.get(i); - pileup.add(createNewPileupElement(read, offset, BaseUtils.simpleBaseToBaseIndex(read.getReadBases()[offset]) == BaseUtils.D)); + pileup.add(createNewPileupElement(read, offset, false, false, false)); // only used to create fake pileups for testing so ancillary information is not important } return pileup; @@ -196,7 +196,7 @@ private PileupElementTracker readsOffsets2Pileup(List reads, UnifiedPileupElementTracker pileup = new UnifiedPileupElementTracker(); for (GATKSAMRecord read : reads) { - pileup.add(createNewPileupElement(read, offset, BaseUtils.simpleBaseToBaseIndex(read.getReadBases()[offset]) == BaseUtils.D)); + pileup.add(createNewPileupElement(read, offset, false, false, false)); // only used to create fake pileups for testing so ancillary information is not important } return pileup; @@ -204,7 +204,7 @@ private PileupElementTracker readsOffsets2Pileup(List reads, protected abstract AbstractReadBackedPileup createNewPileup(GenomeLoc loc, PileupElementTracker pileupElementTracker); - protected abstract PE createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion); + protected abstract PE createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeInsertion, boolean isSoftClipped); // -------------------------------------------------------- // diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/ExtendedEventPileupElement.java b/public/java/src/org/broadinstitute/sting/utils/pileup/ExtendedEventPileupElement.java index 1d7e6f636b..921da2a1f1 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/ExtendedEventPileupElement.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/ExtendedEventPileupElement.java @@ -31,6 +31,8 @@ * Time: 2:57:55 PM * To change this template use File | Settings | File Templates. */ + +// Extended events are slated for removal public class ExtendedEventPileupElement extends PileupElement { public enum Type { NOEVENT, DELETION, INSERTION @@ -46,7 +48,7 @@ public enum Type { public ExtendedEventPileupElement(GATKSAMRecord read, int offset, int eventLength, String eventBases, Type type) { - super(read, offset, type == Type.DELETION); + super(read, offset, type == Type.DELETION, false, false); // extended events are slated for removal this.read = read; this.offset = offset; this.eventLength = eventLength; diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java b/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java index 73f010d404..87aa31c476 100755 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java @@ -23,47 +23,46 @@ public class PileupElement implements Comparable { protected final GATKSAMRecord read; protected final int offset; protected final boolean isDeletion; + protected final boolean isBeforeInsertion; + protected final boolean isSoftClipped; /** * Creates a new pileup element. * - * @param read the read we are adding to the pileup - * @param offset the position in the read for this base. All deletions must be left aligned! (-1 is only allowed for reads starting with insertions) - * @param isDeletion whether or not this base is a deletion + * @param read the read we are adding to the pileup + * @param offset the position in the read for this base. All deletions must be left aligned! (-1 is only allowed for reads starting with insertions) + * @param isDeletion whether or not this base is a deletion + * @param isBeforeInsertion whether or not this base is before an insertion + * @param isSoftClipped whether or not this base was softclipped */ @Requires({ "read != null", "offset >= -1", "offset <= read.getReadLength()"}) - public PileupElement(GATKSAMRecord read, int offset, boolean isDeletion) { + public PileupElement(final GATKSAMRecord read, final int offset, final boolean isDeletion, final boolean isBeforeInsertion, final boolean isSoftClipped) { if (offset < 0 && isDeletion) throw new ReviewedStingException("Pileup Element cannot create a deletion with a negative offset"); this.read = read; this.offset = offset; this.isDeletion = isDeletion; + this.isBeforeInsertion = isBeforeInsertion; + this.isSoftClipped = isSoftClipped; } - // /** -// * Creates a NON DELETION pileup element. -// * -// * use this constructor only for insertions and matches/mismatches. -// * @param read the read we are adding to the pileup -// * @param offset the position in the read for this base. All deletions must be left aligned! (-1 is only allowed for reads starting with insertions) -// */ -// @Requires({ -// "read != null", -// "offset >= -1", -// "offset <= read.getReadLength()"}) -// public PileupElement( GATKSAMRecord read, int offset ) { -// this(read, offset, false); -// } -// public boolean isDeletion() { return isDeletion; } + public boolean isBeforeInsertion() { + return isBeforeInsertion; + } + + public boolean isSoftClipped() { + return isSoftClipped; + } + public boolean isInsertionAtBeginningOfRead() { return offset == -1; } diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java index bf67d1a706..641c63f6c7 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java @@ -96,7 +96,7 @@ protected ReadBackedExtendedEventPileupImpl createNewPileup(GenomeLoc loc, Pileu } @Override - protected ExtendedEventPileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion) { + protected ExtendedEventPileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeInsertion, boolean isSoftClipped) { throw new UnsupportedOperationException("Not enough information provided to create a new pileup element"); } diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java index 66ddbe95d6..965e74e8bb 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java @@ -71,7 +71,7 @@ protected ReadBackedPileupImpl createNewPileup(GenomeLoc loc, PileupElementTrack } @Override - protected PileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion) { - return new PileupElement(read, offset, isDeletion); + protected PileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeInsertion, boolean isSoftClipped) { + return new PileupElement(read, offset, isDeletion, isBeforeInsertion, isSoftClipped); } } diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/ArtificialSAMUtils.java b/public/java/src/org/broadinstitute/sting/utils/sam/ArtificialSAMUtils.java index 8661d5ad06..1175a038f0 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/ArtificialSAMUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/ArtificialSAMUtils.java @@ -361,10 +361,10 @@ public static ReadBackedPileup createReadBackedPileup(final SAMFileHeader header final GATKSAMRecord left = pair.get(0); final GATKSAMRecord right = pair.get(1); - pileupElements.add(new PileupElement(left, pos - leftStart, false)); + pileupElements.add(new PileupElement(left, pos - leftStart, false, false, false)); if (pos >= right.getAlignmentStart() && pos <= right.getAlignmentEnd()) { - pileupElements.add(new PileupElement(right, pos - rightStart, false)); + pileupElements.add(new PileupElement(right, pos - rightStart, false, false, false)); } } diff --git a/public/java/test/org/broadinstitute/sting/utils/sam/ReadUtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/sam/ReadUtilsUnitTest.java index 1a8086a1b4..b7a22ca1a1 100755 --- a/public/java/test/org/broadinstitute/sting/utils/sam/ReadUtilsUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/sam/ReadUtilsUnitTest.java @@ -42,8 +42,8 @@ public void testReducedReads() { @Test public void testReducedReadPileupElement() { - PileupElement readp = new PileupElement(read, 0, false); - PileupElement reducedreadp = new PileupElement(reducedRead, 0, false); + PileupElement readp = new PileupElement(read, 0, false, false, false); + PileupElement reducedreadp = new PileupElement(reducedRead, 0, false, false, false); Assert.assertFalse(readp.getRead().isReducedRead()); From 07f72516ae299a61014b3a1fe8a1ba6c34140687 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Thu, 26 Jan 2012 16:14:25 -0500 Subject: [PATCH 159/356] Unsupported platform should be a user error --- .../sting/gatk/walkers/recalibration/CycleCovariate.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java index 6b4fec04e8..b0819ee691 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java @@ -159,7 +159,7 @@ else if( FLOW_CYCLE_PLATFORMS.contains(ngsPlatform) ) { } } else { - throw new IllegalStateException("This method hasn't been implemented yet for " + read.getReadGroup().getPlatform()); + throw new UserException("The platform (" + read.getReadGroup().getPlatform() + ") associated with read group " + read.getReadGroup() + " is not a recognized platform. Implemented options are e.g. illumina, 454, and solid"); } } From 0d4027104f2d511aacf8bc01dea00d0c5f0fb2ec Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Thu, 26 Jan 2012 16:07:29 -0500 Subject: [PATCH 160/356] Reduced reads are now aware of their original alignments * Added annotations for reads that had been soft clipped prior to being reduced so that we can later recuperate their original alignments (start and end). * Tags keep the alignment shifts, not real alignment, for better compression * Tags are defined in the GATKSAMRecord * GATKSAMRecord has new functionality to retrieve original alignment start of all reads (trimmed or not) -- getOriginalAlignmentStart() and getOriginalAligmentEnd() * Updated ReduceReads MD5s accordingly --- .../sting/utils/sam/GATKSAMRecord.java | 52 ++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java index 913548ecc3..f17772f405 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java @@ -43,7 +43,10 @@ * */ public class GATKSAMRecord extends BAMRecord { - public static final String REDUCED_READ_CONSENSUS_TAG = "RR"; + // ReduceReads specific attribute tags + public static final String REDUCED_READ_CONSENSUS_TAG = "RR"; // marks a synthetic read produced by the ReduceReads tool + public static final String REDUCED_READ_ORIGINAL_ALIGNMENT_START_SHIFT = "OS"; // reads that are clipped may use this attribute to keep track of their original alignment start + public static final String REDUCED_READ_ORIGINAL_ALIGNMENT_END_SHIFT = "OE"; // reads that are clipped may use this attribute to keep track of their original alignment end // the SAMRecord data we're caching private String mReadString = null; @@ -321,6 +324,36 @@ public int getSoftEnd() { return (lastOperator == CigarOperator.HARD_CLIP) ? stop-1 : stop+shift-1 ; } + /** + * Determines the original alignment start of a previously clipped read. + * + * This is useful for reads that have been trimmed to a variant region and lost the information of it's original alignment end + * + * @return the alignment start of a read before it was clipped + */ + public int getOriginalAlignmentStart() { + int originalAlignmentStart = getUnclippedStart(); + Integer alignmentShift = (Integer) getAttribute(REDUCED_READ_ORIGINAL_ALIGNMENT_START_SHIFT); + if (alignmentShift != null) + originalAlignmentStart += alignmentShift; + return originalAlignmentStart; + } + + /** + * Determines the original alignment end of a previously clipped read. + * + * This is useful for reads that have been trimmed to a variant region and lost the information of it's original alignment end + * + * @return the alignment end of a read before it was clipped + */ + public int getOriginalAlignmentEnd() { + int originalAlignmentEnd = getUnclippedEnd(); + Integer alignmentShift = (Integer) getAttribute(REDUCED_READ_ORIGINAL_ALIGNMENT_END_SHIFT); + if (alignmentShift != null) + originalAlignmentEnd -= alignmentShift; + return originalAlignmentEnd; + } + /** * Creates an empty GATKSAMRecord with the read's header, read group and mate * information, but empty (not-null) fields: @@ -363,4 +396,21 @@ public static GATKSAMRecord emptyRead(GATKSAMRecord read) { return emptyRead; } + /** + * Shallow copy of everything, except for the attribute list and the temporary attributes. + * A new list of the attributes is created for both, but the attributes themselves are copied by reference. + * This should be safe because callers should never modify a mutable value returned by any of the get() methods anyway. + * + * @return a shallow copy of the GATKSAMRecord + * @throws CloneNotSupportedException + */ + @Override + public Object clone() throws CloneNotSupportedException { + final GATKSAMRecord clone = (GATKSAMRecord) super.clone(); + if (temporaryAttributes != null) { + for (Object attribute : temporaryAttributes.keySet()) + clone.setTemporaryAttribute(attribute, temporaryAttributes.get(attribute)); + } + return clone; + } } From 246e085ec9773320843f50da47066cd8313f659c Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Thu, 26 Jan 2012 16:59:08 -0500 Subject: [PATCH 161/356] Unit tests for GATKSAMRecord class * new unit tests for the alignment shift properties of reduce reads * moved unit tests from ReadUtils that were actually testing GATKSAMRecord, not any of the ReadUtils to it. * cleaned up ReadUtilsUnitTest --- .../utils/sam/GATKSAMRecordUnitTest.java | 83 +++++++++++++++++++ .../sting/utils/sam/ReadUtilsUnitTest.java | 46 ---------- 2 files changed, 83 insertions(+), 46 deletions(-) create mode 100755 public/java/test/org/broadinstitute/sting/utils/sam/GATKSAMRecordUnitTest.java diff --git a/public/java/test/org/broadinstitute/sting/utils/sam/GATKSAMRecordUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/sam/GATKSAMRecordUnitTest.java new file mode 100755 index 0000000000..317b320d3e --- /dev/null +++ b/public/java/test/org/broadinstitute/sting/utils/sam/GATKSAMRecordUnitTest.java @@ -0,0 +1,83 @@ +package org.broadinstitute.sting.utils.sam; + +import net.sf.samtools.SAMFileHeader; +import org.broadinstitute.sting.BaseTest; +import org.broadinstitute.sting.utils.pileup.PileupElement; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + + +public class GATKSAMRecordUnitTest extends BaseTest { + GATKSAMRecord read, reducedRead; + final static String BASES = "ACTG"; + final static String QUALS = "!+5?"; + final private static byte[] REDUCED_READ_COUNTS = new byte[]{10, 20, 30, 40, 1}; + final private static byte[] REDUCED_READ_COUNTS_TAG = new byte[]{10, 10, 20, 30, -9}; // just the offsets + + @BeforeClass + public void init() { + SAMFileHeader header = ArtificialSAMUtils.createArtificialSamHeader(1, 1, 1000); + read = ArtificialSAMUtils.createArtificialRead(header, "read1", 0, 1, BASES.length()); + read.setReadUnmappedFlag(true); + read.setReadBases(new String(BASES).getBytes()); + read.setBaseQualityString(new String(QUALS)); + + reducedRead = ArtificialSAMUtils.createArtificialRead(header, "reducedRead", 0, 1, BASES.length()); + reducedRead.setReadBases(BASES.getBytes()); + reducedRead.setBaseQualityString(QUALS); + reducedRead.setAttribute(GATKSAMRecord.REDUCED_READ_CONSENSUS_TAG, REDUCED_READ_COUNTS_TAG); + } + + @Test + public void testReducedReads() { + Assert.assertFalse(read.isReducedRead(), "isReducedRead is false for normal read"); + Assert.assertEquals(read.getReducedReadCounts(), null, "No reduced read tag in normal read"); + + Assert.assertTrue(reducedRead.isReducedRead(), "isReducedRead is true for reduced read"); + for (int i = 0; i < reducedRead.getReadLength(); i++) { + Assert.assertEquals(reducedRead.getReducedCount(i), REDUCED_READ_COUNTS[i], "Reduced read count not set to the expected value at " + i); + } + } + + @Test + public void testReducedReadPileupElement() { + PileupElement readp = new PileupElement(read, 0, false); + PileupElement reducedreadp = new PileupElement(reducedRead, 0, false); + + Assert.assertFalse(readp.getRead().isReducedRead()); + + Assert.assertTrue(reducedreadp.getRead().isReducedRead()); + Assert.assertEquals(reducedreadp.getRepresentativeCount(), REDUCED_READ_COUNTS[0]); + Assert.assertEquals(reducedreadp.getQual(), readp.getQual()); + } + + @Test + public void testGetOriginalAlignments() { + final byte [] bases = {'A', 'A', 'A', 'A', 'A', 'A', 'A', 'A'}; + final byte [] quals = {20 , 20 , 20 , 20 , 20 , 20 , 20 , 20 }; + GATKSAMRecord read = ArtificialSAMUtils.createArtificialRead(bases, quals, "6M"); + + // A regular read with all matches + Assert.assertEquals(read.getAlignmentStart(), read.getOriginalAlignmentStart()); + Assert.assertEquals(read.getAlignmentEnd(), read.getOriginalAlignmentEnd()); + + // Alignment start shifted + int alignmentShift = 2; + read.setAttribute(GATKSAMRecord.REDUCED_READ_ORIGINAL_ALIGNMENT_START_SHIFT, alignmentShift); + Assert.assertEquals(read.getAlignmentStart() + alignmentShift, read.getOriginalAlignmentStart()); + Assert.assertEquals(read.getAlignmentEnd(), read.getOriginalAlignmentEnd()); + + // Both alignments shifted + read.setAttribute(GATKSAMRecord.REDUCED_READ_ORIGINAL_ALIGNMENT_END_SHIFT, alignmentShift); + Assert.assertEquals(read.getAlignmentStart() + alignmentShift, read.getOriginalAlignmentStart()); + Assert.assertEquals(read.getAlignmentEnd() - alignmentShift, read.getOriginalAlignmentEnd()); + + // Alignment end shifted + read.setAttribute(GATKSAMRecord.REDUCED_READ_ORIGINAL_ALIGNMENT_START_SHIFT, null); + Assert.assertEquals(read.getAlignmentStart(), read.getOriginalAlignmentStart()); + Assert.assertEquals(read.getAlignmentEnd() - alignmentShift, read.getOriginalAlignmentEnd()); + + } + +} diff --git a/public/java/test/org/broadinstitute/sting/utils/sam/ReadUtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/sam/ReadUtilsUnitTest.java index b7a22ca1a1..7598f62a6e 100755 --- a/public/java/test/org/broadinstitute/sting/utils/sam/ReadUtilsUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/sam/ReadUtilsUnitTest.java @@ -1,57 +1,11 @@ package org.broadinstitute.sting.utils.sam; -import net.sf.samtools.SAMFileHeader; import org.broadinstitute.sting.BaseTest; -import org.broadinstitute.sting.utils.pileup.PileupElement; import org.testng.Assert; -import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; public class ReadUtilsUnitTest extends BaseTest { - GATKSAMRecord read, reducedRead; - final static String BASES = "ACTG"; - final static String QUALS = "!+5?"; - final private static byte[] REDUCED_READ_COUNTS = new byte[]{10, 20, 30, 40, 1}; - final private static byte[] REDUCED_READ_COUNTS_TAG = new byte[]{10, 10, 20, 30, -9}; // just the offsets - - @BeforeTest - public void init() { - SAMFileHeader header = ArtificialSAMUtils.createArtificialSamHeader(1, 1, 1000); - read = ArtificialSAMUtils.createArtificialRead(header, "read1", 0, 1, BASES.length()); - read.setReadUnmappedFlag(true); - read.setReadBases(new String(BASES).getBytes()); - read.setBaseQualityString(new String(QUALS)); - - reducedRead = ArtificialSAMUtils.createArtificialRead(header, "reducedRead", 0, 1, BASES.length()); - reducedRead.setReadBases(BASES.getBytes()); - reducedRead.setBaseQualityString(QUALS); - reducedRead.setAttribute(GATKSAMRecord.REDUCED_READ_CONSENSUS_TAG, REDUCED_READ_COUNTS_TAG); - } - - @Test - public void testReducedReads() { - Assert.assertFalse(read.isReducedRead(), "isReducedRead is false for normal read"); - Assert.assertEquals(read.getReducedReadCounts(), null, "No reduced read tag in normal read"); - - Assert.assertTrue(reducedRead.isReducedRead(), "isReducedRead is true for reduced read"); - for (int i = 0; i < reducedRead.getReadLength(); i++) { - Assert.assertEquals(reducedRead.getReducedCount(i), REDUCED_READ_COUNTS[i], "Reduced read count not set to the expected value at " + i); - } - } - - @Test - public void testReducedReadPileupElement() { - PileupElement readp = new PileupElement(read, 0, false, false, false); - PileupElement reducedreadp = new PileupElement(reducedRead, 0, false, false, false); - - Assert.assertFalse(readp.getRead().isReducedRead()); - - Assert.assertTrue(reducedreadp.getRead().isReducedRead()); - Assert.assertEquals(reducedreadp.getRepresentativeCount(), REDUCED_READ_COUNTS[0]); - Assert.assertEquals(reducedreadp.getQual(), readp.getQual()); - } - @Test public void testGetAdaptorBoundary() { final byte[] bases = {'A', 'C', 'G', 'T', 'A', 'C', 'G', 'T'}; From d54e2376714c55bb78dc71a87c40d206456dab31 Mon Sep 17 00:00:00 2001 From: Menachem Fromer Date: Thu, 26 Jan 2012 18:25:30 -0500 Subject: [PATCH 162/356] Take advantage of Eric's fix for multiAllelic AC calculation, and also add fix to have the original allele's INFO field be passed through for batch merging From 2a565ebf90ea53e0a252fe097c778365aff3714a Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Thu, 26 Jan 2012 19:58:42 -0500 Subject: [PATCH 163/356] embarrassing fix-up, thanks Khalid. --- .../broadinstitute/sting/utils/sam/GATKSAMRecordUnitTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/java/test/org/broadinstitute/sting/utils/sam/GATKSAMRecordUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/sam/GATKSAMRecordUnitTest.java index 317b320d3e..729503f843 100755 --- a/public/java/test/org/broadinstitute/sting/utils/sam/GATKSAMRecordUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/sam/GATKSAMRecordUnitTest.java @@ -42,8 +42,8 @@ public void testReducedReads() { @Test public void testReducedReadPileupElement() { - PileupElement readp = new PileupElement(read, 0, false); - PileupElement reducedreadp = new PileupElement(reducedRead, 0, false); + PileupElement readp = new PileupElement(read, 0, false, false, false); + PileupElement reducedreadp = new PileupElement(reducedRead, 0, false, false, false); Assert.assertFalse(readp.getRead().isReducedRead()); From cb04c0bf1136b351c703e9f3486527301f086067 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 27 Jan 2012 08:20:45 -0500 Subject: [PATCH 164/356] Removing javaassist 3.7, lucene library dependancies --- ivy.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ivy.xml b/ivy.xml index f5ff15c30c..f7c64aec6f 100644 --- a/ivy.xml +++ b/ivy.xml @@ -41,7 +41,7 @@ - + @@ -66,7 +66,7 @@ - + From 13d1626f51878d3973d43dd27a9a78e10119878d Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 27 Jan 2012 08:24:17 -0500 Subject: [PATCH 165/356] Minor improvements in ref QC walker. Unfortunately this doesn't actually catch Chris's error --- .../sting/gatk/walkers/qc/QCRefWalker.java | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/QCRefWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/QCRefWalker.java index bddf27d84b..ab5324e39e 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/QCRefWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/QCRefWalker.java @@ -83,7 +83,7 @@ public void initialize() { } private final void throwError(ReferenceContext ref, String message) { - throw new StingException(String.format("Site %s failed: %s", ref, message)); + throw new StingException(String.format("Site %s failed: %s", ref.getLocus(), message)); } public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) { @@ -92,13 +92,13 @@ public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentCo contigName = locusContigName; ReferenceSequence refSeq = uncachedRef.getSequence(contigName); contigStart = 1; - contigEnd = contigStart + refSeq.length(); + contigEnd = contigStart + refSeq.length() - 1; uncachedBases = uncachedRef.getSubsequenceAt(contigName, contigStart, contigEnd).getBases(); - logger.warn(String.format("Loading contig %s (%d-%d)", contigName, contigStart, contigEnd)); + logger.info(String.format("Loading contig %s (%d-%d)", contigName, contigStart, contigEnd)); } final byte refBase = ref.getBase(); - if (! ( BaseUtils.isRegularBase(refBase) || BaseUtils.isNBase(refBase) ) ) + if (! ( BaseUtils.isRegularBase(refBase) || isExtendFastaBase(refBase) ) ) throwError(ref, String.format("Refbase isn't a regular base (%d %c)", refBase, (char)refBase)); // check bases are equal @@ -114,6 +114,28 @@ public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentCo return 1; } + private static final boolean isExtendFastaBase(final byte b) { + switch ( b ) { + case 'U': + case 'R': + case 'Y': + case 'K': + case 'M': + case 'S': + case 'W': + case 'B': + case 'D': + case 'H': + case 'V': + case 'N': + case 'X': + case '-': + return true; + default: + return false; + } + } + public Integer reduceInit() { return 0; } From ec9920b04f7a95918ed3b76a4e971143ed8084b0 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Fri, 27 Jan 2012 08:51:39 -0500 Subject: [PATCH 166/356] Updating the SAM TAG for Original Alignment Start to "OP" per Mark's recommendation to reuse the Indel Realigner tag that made it to the SAM spec. The Alignment end tag is still "OE" as there is no official tag to reuse. --- .../src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java index f17772f405..03b794ae35 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java @@ -45,7 +45,7 @@ public class GATKSAMRecord extends BAMRecord { // ReduceReads specific attribute tags public static final String REDUCED_READ_CONSENSUS_TAG = "RR"; // marks a synthetic read produced by the ReduceReads tool - public static final String REDUCED_READ_ORIGINAL_ALIGNMENT_START_SHIFT = "OS"; // reads that are clipped may use this attribute to keep track of their original alignment start + public static final String REDUCED_READ_ORIGINAL_ALIGNMENT_START_SHIFT = "OP"; // reads that are clipped may use this attribute to keep track of their original alignment start public static final String REDUCED_READ_ORIGINAL_ALIGNMENT_END_SHIFT = "OE"; // reads that are clipped may use this attribute to keep track of their original alignment end // the SAMRecord data we're caching From f8f2152f9c2f90e4d151436ca8ddedbb27c89333 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Fri, 27 Jan 2012 10:53:24 -0500 Subject: [PATCH 167/356] fixing ReduceReads MD5s now that we're using OP instead of OS. From 052a4bdb9cfd5f182e0a4e779f357ea63ea4449f Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Fri, 27 Jan 2012 11:13:30 -0500 Subject: [PATCH 168/356] Turning off PHONE HOME option in the MDCP * MDCP is for internal use and there is no need to report to the Amazon cloud. * Reporting to ASW_S3 is not allowing jobs to finish, this is probably a bug. --- .../queue/qscripts/MethodsDevelopmentCallingPipeline.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala index 2f0715ae90..b860358cac 100755 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala @@ -25,9 +25,6 @@ class MethodsDevelopmentCallingPipeline extends QScript { @Argument(shortName="noIndels", doc="do not call indels with the Unified Genotyper", required=false) var noIndels: Boolean = false - @Argument(shortName="LOCAL_ET", doc="Doesn't use the AWS S3 storage for ET option", required=false) - var LOCAL_ET: Boolean = false - @Argument(shortName="mbq", doc="The minimum Phred-Scaled quality score threshold to be considered a good base.", required=false) var minimumBaseQuality: Int = -1 @@ -203,7 +200,7 @@ class MethodsDevelopmentCallingPipeline extends QScript { trait UNIVERSAL_GATK_ARGS extends CommandLineGATK { logging_level = "INFO"; memoryLimit = 4; - phone_home = if ( LOCAL_ET ) GATKRunReport.PhoneHomeOption.STANDARD else GATKRunReport.PhoneHomeOption.AWS_S3 + phone_home = GATKRunReport.PhoneHomeOption.NO_ET } def bai(bam: File) = new File(bam + ".bai") From fc08235ff374e3eba8d326c67e9698f9c9280523 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Fri, 27 Jan 2012 15:12:37 -0500 Subject: [PATCH 169/356] Bug fix in active region traversal, locusView.getNext() skips over pileups with zero coverage but still need to count them in the active probability integrator --- .../traversals/TraverseActiveRegions.java | 43 +++++++++++-------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java index 83daa4d80e..562a6d1d09 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java +++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java @@ -10,7 +10,6 @@ import org.broadinstitute.sting.gatk.walkers.*; import org.broadinstitute.sting.utils.GenomeLoc; import org.broadinstitute.sting.utils.GenomeLocSortedSet; -import org.broadinstitute.sting.utils.MathUtils; import org.broadinstitute.sting.utils.activeregion.ActiveRegion; import org.broadinstitute.sting.utils.pileup.PileupElement; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; @@ -63,25 +62,26 @@ public T traverse( final ActiveRegionWalker walker, referenceOrderedDataView = (RodLocusView)locusView; // We keep processing while the next reference location is within the interval + GenomeLoc prevLoc = null; while( locusView.hasNext() ) { final AlignmentContext locus = locusView.next(); GenomeLoc location = locus.getLocation(); + if(prevLoc != null) { + for(int iii = prevLoc.getStart() + 1; iii < location.getStart(); iii++ ) { + final GenomeLoc fakeLoc = engine.getGenomeLocParser().createGenomeLoc(prevLoc.getContig(), iii, iii); + if( initialIntervals.overlaps( fakeLoc ) ) { + final double isActiveProb = ( walker.presetActiveRegions == null ? walker.isActive( null, null, null ) + : ( walker.presetActiveRegions.overlaps(fakeLoc) ? 1.0 : 0.0 ) ); + isActiveList.add( isActiveProb ); + if( firstIsActiveStart == null ) { + firstIsActiveStart = fakeLoc; + } + } + } + } dataProvider.getShard().getReadMetrics().incrementNumIterations(); - if ( locus.hasExtendedEventPileup() ) { - // if the alignment context we received holds an "extended" pileup (i.e. pileup of insertions/deletions - // associated with the current site), we need to update the location. The updated location still starts - // at the current genomic position, but it has to span the length of the longest deletion (if any). - location = engine.getGenomeLocParser().setStop(location,location.getStop()+locus.getExtendedEventPileup().getMaxDeletionLength()); - - // it is possible that the new expanded location spans the current shard boundary; the next method ensures - // that when it is the case, the reference sequence held by the ReferenceView will be reloaded so that - // the view has all the bases we are gonna need. If the location fits within the current view bounds, - // the next call will not do anything to the view: - referenceView.expandBoundsToAccomodateLoc(location); - } - // create reference context. Note that if we have a pileup of "extended events", the context will // hold the (longest) stretch of deleted reference bases (if deletions are present in the pileup). final ReferenceContext refContext = referenceView.getReferenceContext(location); @@ -95,7 +95,7 @@ public T traverse( final ActiveRegionWalker walker, : ( walker.presetActiveRegions.overlaps(location) ? 1.0 : 0.0 ) ); isActiveList.add( isActiveProb ); if( firstIsActiveStart == null ) { - firstIsActiveStart = locus.getLocation(); + firstIsActiveStart = location; } } @@ -118,6 +118,7 @@ public T traverse( final ActiveRegionWalker walker, if( read.getAlignmentStart() < minStart ) { minStart = read.getAlignmentStart(); } } } + prevLoc = location; printProgress(dataProvider.getShard(), locus.getLocation()); } @@ -230,7 +231,7 @@ private ArrayList integrateActiveList( final ArrayList act final int MAX_ACTIVE_REGION = 200; for( int iii = 0; iii < activeProbArray.length; iii++ ) { double maxVal = 0; - for( int jjj = Math.max( 0, iii-FILTER_SIZE); jjj < Math.min( activeList.size(), iii+FILTER_SIZE); jjj++ ) { + for( int jjj = Math.max(0, iii-FILTER_SIZE); jjj < Math.min(activeList.size(), iii+FILTER_SIZE); jjj++ ) { if( activeProbArray[jjj] > maxVal ) { maxVal = activeProbArray[jjj]; } } filteredProbArray[iii] = maxVal; @@ -241,14 +242,18 @@ private ArrayList integrateActiveList( final ArrayList act for(int iii = 1; iii < filteredProbArray.length; iii++ ) { final boolean thisStatus = filteredProbArray[iii] > ACTIVE_PROB_THRESHOLD; if( curStatus != thisStatus || (iii-curStart) > MAX_ACTIVE_REGION ) { - returnList.add( new ActiveRegion( engine.getGenomeLocParser().createGenomeLoc(firstIsActiveStart.getContig(), firstIsActiveStart.getStart() + curStart, firstIsActiveStart.getStart() + (iii-1)), + returnList.add( new ActiveRegion( + engine.getGenomeLocParser().createGenomeLoc(firstIsActiveStart.getContig(), firstIsActiveStart.getStart() + curStart, firstIsActiveStart.getStart() + (iii-1)), curStatus, engine.getGenomeLocParser(), activeRegionExtension ) ); curStatus = thisStatus; curStart = iii; } } - returnList.add( new ActiveRegion( engine.getGenomeLocParser().createGenomeLoc(firstIsActiveStart.getContig(), firstIsActiveStart.getStart() + curStart, firstIsActiveStart.getStart() + (filteredProbArray.length-1)), - curStatus, engine.getGenomeLocParser(), activeRegionExtension ) ); + if( curStart != filteredProbArray.length-1 ) { + returnList.add( new ActiveRegion( + engine.getGenomeLocParser().createGenomeLoc(firstIsActiveStart.getContig(), firstIsActiveStart.getStart() + curStart, firstIsActiveStart.getStart() + (filteredProbArray.length-1)), + curStatus, engine.getGenomeLocParser(), activeRegionExtension ) ); + } return returnList; } } From 3164c8dee57cb84a3c60c38f67e196a7fc25038e Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sun, 29 Jan 2012 15:14:58 -0500 Subject: [PATCH 170/356] S3 upload now directly creates the XML report in memory and puts that in S3 -- This is a partial fix for the problem with uploading S3 logs reported by Mauricio. There the problem is that the java.io.tmpdir is not accessible (network just hangs). Because of that the s3 upload fails because the underlying system uses tmpdir for caching, etc. As far as I can tell there's no way around this bug -- you cannot overload the java.io.tmpdir programmatically and even if I could what value would we use? The only solution seems to me is to detect that tmpdir is hanging (how?!) and fail with a meaningful error. --- ivy.xml | 2 +- .../sting/gatk/phonehome/GATKRunReport.java | 111 +++++++++--------- 2 files changed, 54 insertions(+), 59 deletions(-) diff --git a/ivy.xml b/ivy.xml index f7c64aec6f..06296c6b4a 100644 --- a/ivy.xml +++ b/ivy.xml @@ -72,7 +72,7 @@ - + diff --git a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java index f098655376..e8627ef4c9 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java +++ b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java @@ -264,22 +264,8 @@ private void postReportToStream(OutputStream stream) { } } - /** - * Opens the destination file and writes a gzipped version of the XML report there. - * - * @param destination - * @throws IOException - */ - private void postReportToFile(File destination) throws IOException { - BufferedOutputStream out = - new BufferedOutputStream( - new GZIPOutputStream( - new FileOutputStream(destination))); - try { - postReportToStream(out); - } finally { - out.close(); - } + private final String getKey() { + return getID() + ".report.xml.gz"; } /** @@ -288,16 +274,21 @@ private void postReportToFile(File destination) throws IOException { * That is, postReport() is guarenteed not to fail for any reason. */ private File postReportToLocalDisk(File rootDir) { - String filename = getID() + ".report.xml.gz"; - File file = new File(rootDir, filename); + final String filename = getKey(); + final File destination = new File(rootDir, filename); + try { - postReportToFile(file); - logger.debug("Wrote report to " + file); - return file; + final BufferedOutputStream out = new BufferedOutputStream( + new GZIPOutputStream( + new FileOutputStream(destination))); + postReportToStream(out); + out.close(); + logger.debug("Wrote report to " + destination); + return destination; } catch ( Exception e ) { // we catch everything, and no matter what eat the error exceptDuringRunReport("Couldn't read report file", e); - file.delete(); + destination.delete(); return null; } } @@ -305,42 +296,46 @@ private File postReportToLocalDisk(File rootDir) { private void postReportToAWSS3() { // modifying example code from http://jets3t.s3.amazonaws.com/toolkit/code-samples.html this.hostName = Utils.resolveHostname(); // we want to fill in the host name - File localFile = postReportToLocalDisk(new File("./")); - logger.debug("Generating GATK report to AWS S3 based on local file " + localFile); - if ( localFile != null ) { // we succeeded in creating the local file - localFile.deleteOnExit(); - try { - // stop us from printing the annoying, and meaningless, mime types warning - Logger mimeTypeLogger = Logger.getLogger(org.jets3t.service.utils.Mimetypes.class); - mimeTypeLogger.setLevel(Level.FATAL); - - // Your Amazon Web Services (AWS) login credentials are required to manage S3 accounts. These credentials - // are stored in an AWSCredentials object: - - // IAM GATK user credentials -- only right is to PutObject into GATK_Run_Report bucket - String awsAccessKey = "AKIAJXU7VIHBPDW4TDSQ"; // GATK AWS user - String awsSecretKey = "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"; // GATK AWS user - AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey); - - // To communicate with S3, create a class that implements an S3Service. We will use the REST/HTTP - // implementation based on HttpClient, as this is the most robust implementation provided with JetS3t. - S3Service s3Service = new RestS3Service(awsCredentials); - - // Create an S3Object based on a file, with Content-Length set automatically and - // Content-Type set based on the file's extension (using the Mimetypes utility class) - S3Object fileObject = new S3Object(localFile); - //logger.info("Created S3Object" + fileObject); - //logger.info("Uploading " + localFile + " to AWS bucket"); - S3Object s3Object = s3Service.putObject(REPORT_BUCKET_NAME, fileObject); - logger.debug("Uploaded to AWS: " + s3Object); - logger.info("Uploaded run statistics report to AWS S3"); - } catch ( S3ServiceException e ) { - exceptDuringRunReport("S3 exception occurred", e); - } catch ( NoSuchAlgorithmException e ) { - exceptDuringRunReport("Couldn't calculate MD5", e); - } catch ( IOException e ) { - exceptDuringRunReport("Couldn't read report file", e); - } + final String key = getKey(); + logger.debug("Generating GATK report to AWS S3 with key " + key); + try { + // create an byte output stream so we can capture the output as a byte[] + final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(8096); + final OutputStream outputStream = new GZIPOutputStream(byteStream); + postReportToStream(outputStream); + outputStream.close(); + final byte[] report = byteStream.toByteArray(); + + // stop us from printing the annoying, and meaningless, mime types warning + Logger mimeTypeLogger = Logger.getLogger(org.jets3t.service.utils.Mimetypes.class); + mimeTypeLogger.setLevel(Level.FATAL); + + // Your Amazon Web Services (AWS) login credentials are required to manage S3 accounts. These credentials + // are stored in an AWSCredentials object: + + // IAM GATK user credentials -- only right is to PutObject into GATK_Run_Report bucket + String awsAccessKey = "AKIAJXU7VIHBPDW4TDSQ"; // GATK AWS user + String awsSecretKey = "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"; // GATK AWS user + AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey); + + // To communicate with S3, create a class that implements an S3Service. We will use the REST/HTTP + // implementation based on HttpClient, as this is the most robust implementation provided with JetS3t. + S3Service s3Service = new RestS3Service(awsCredentials); + + // Create an S3Object based on a file, with Content-Length set automatically and + // Content-Type set based on the file's extension (using the Mimetypes utility class) + S3Object fileObject = new S3Object(key, report); + //logger.info("Created S3Object" + fileObject); + //logger.info("Uploading " + localFile + " to AWS bucket"); + S3Object s3Object = s3Service.putObject(REPORT_BUCKET_NAME, fileObject); + logger.debug("Uploaded to AWS: " + s3Object); + logger.info("Uploaded run statistics report to AWS S3"); + } catch ( S3ServiceException e ) { + exceptDuringRunReport("S3 exception occurred", e); + } catch ( NoSuchAlgorithmException e ) { + exceptDuringRunReport("Couldn't calculate MD5", e); + } catch ( IOException e ) { + exceptDuringRunReport("Couldn't read report file", e); } } From 3186f0f1b0f786b51849c6d8cd51fafad1187d0f Mon Sep 17 00:00:00 2001 From: Menachem Fromer Date: Sun, 29 Jan 2012 23:38:32 -0500 Subject: [PATCH 171/356] Try more memory and fewer ALT alleles so that we don't run out of memory From f1e07f169e7f04d51ae1c1113854437f72d1304e Mon Sep 17 00:00:00 2001 From: Menachem Fromer Date: Mon, 30 Jan 2012 02:22:13 -0500 Subject: [PATCH 172/356] Only apply filters if there are filters to apply From d5d4fa8a88ee599ea716ccdcecc0e6892b93e6eb Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Mon, 30 Jan 2012 09:50:14 -0500 Subject: [PATCH 173/356] Fixed discordance bug reported by Brad Chapman discordance now reports discordance between genotypes as well (just like concordance) --- .../walkers/variantutils/SelectVariants.java | 18 ++++++++---------- .../SelectVariantsIntegrationTest.java | 2 +- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java index 6d94ffe6da..5eef7fb66c 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java @@ -25,22 +25,20 @@ package org.broadinstitute.sting.gatk.walkers.variantutils; import org.broadinstitute.sting.commandline.*; +import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.gatk.arguments.StandardVariantContextInputArgumentCollection; +import org.broadinstitute.sting.gatk.contexts.AlignmentContext; +import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; import org.broadinstitute.sting.gatk.samples.Sample; +import org.broadinstitute.sting.gatk.walkers.RodWalker; import org.broadinstitute.sting.gatk.walkers.TreeReducible; +import org.broadinstitute.sting.utils.MendelianViolation; +import org.broadinstitute.sting.utils.SampleUtils; import org.broadinstitute.sting.utils.codecs.vcf.*; import org.broadinstitute.sting.utils.exceptions.UserException; -import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; -import org.broadinstitute.sting.utils.MendelianViolation; import org.broadinstitute.sting.utils.text.XReadLines; import org.broadinstitute.sting.utils.variantcontext.*; -import org.broadinstitute.sting.commandline.Argument; -import org.broadinstitute.sting.commandline.Output; -import org.broadinstitute.sting.gatk.contexts.AlignmentContext; -import org.broadinstitute.sting.gatk.contexts.ReferenceContext; -import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; -import org.broadinstitute.sting.gatk.walkers.RodWalker; -import org.broadinstitute.sting.utils.SampleUtils; import java.io.File; import java.io.FileNotFoundException; @@ -557,7 +555,7 @@ private boolean isDiscordant (VariantContext vc, Collection comp // Look for this sample in the all vcs of the comp ROD track. boolean foundVariant = false; for (VariantContext compVC : compVCs) { - if (sampleHasVariant(compVC.getGenotype(g.getSampleName()))) { + if (haveSameGenotypes(g, compVC.getGenotype(g.getSampleName()))) { foundVariant = true; break; } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariantsIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariantsIntegrationTest.java index 042de2a27d..9577966b72 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariantsIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariantsIntegrationTest.java @@ -58,7 +58,7 @@ public void testDiscordance() { WalkerTestSpec spec = new WalkerTestSpec( "-T SelectVariants -R " + hg19Reference + " -sn NA12878 -L 20:1012700-1020000 --variant " + b37hapmapGenotypes + " -disc " + testFile + " -o %s -NO_HEADER", 1, - Arrays.asList("78e6842325f1f1bc9ab30d5e7737ee6e") + Arrays.asList("929bbb96381541c162dc7e5462e26ea2") ); executeTest("testDiscordance--" + testFile, spec); From abb91cf26b50b246896cc82d1b84d3135027fc5b Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Mon, 30 Jan 2012 15:36:12 -0500 Subject: [PATCH 174/356] Increasing the size of the active regions that are produced by the active probability integrator, more context is needed to call more complex events --- .../sting/gatk/traversals/TraverseActiveRegions.java | 6 +++--- .../genotyper/IndelGenotypeLikelihoodsCalculationModel.java | 2 +- .../varianteval/evaluators/IndelLengthHistogram.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java index 562a6d1d09..769bec720e 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java +++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java @@ -216,7 +216,7 @@ else if( dataSource == DataSource.REFERENCE_ORDERED_DATA ) // band-pass filter the list of isActive probabilities and turn into active regions private ArrayList integrateActiveList( final ArrayList activeList, final GenomeLoc firstIsActiveStart, final int activeRegionExtension ) { - final double ACTIVE_PROB_THRESHOLD = 0.2; + final double ACTIVE_PROB_THRESHOLD = 0.2; // BUGBUG: needs to be set-able by the walker author final ArrayList returnList = new ArrayList(); if( activeList.size() == 0 ) { return returnList; @@ -227,8 +227,8 @@ private ArrayList integrateActiveList( final ArrayList act } else { final Double[] activeProbArray = activeList.toArray(new Double[activeList.size()]); final double[] filteredProbArray = new double[activeProbArray.length]; - final int FILTER_SIZE = 10; - final int MAX_ACTIVE_REGION = 200; + final int FILTER_SIZE = 50; // BUGBUG: needs to be set-able by the walker author + final int MAX_ACTIVE_REGION = 425; // BUGBUG: needs to be set-able by the walker author for( int iii = 0; iii < activeProbArray.length; iii++ ) { double maxVal = 0; for( int jjj = Math.max(0, iii-FILTER_SIZE); jjj < Math.min(activeList.size(), iii+FILTER_SIZE); jjj++ ) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java index 9126c04956..0422fbf035 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java @@ -335,7 +335,7 @@ public VariantContext getLikelihoods(RefMetaDataTracker tracker, if (ref.getWindow().getStop() < loc.getStop() + HAPLOTYPE_SIZE) return null; if (!(priors instanceof DiploidIndelGenotypePriors)) - throw new StingException("Only diploid-based Indel priors are supported in the DINDEL GL model"); + throw new StingException("Only diploid-based Indel priors are supported in the INDEL GL model"); if (alleleList.isEmpty()) return null; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/IndelLengthHistogram.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/IndelLengthHistogram.java index ccec9af126..6cf8b7c2c6 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/IndelLengthHistogram.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/IndelLengthHistogram.java @@ -15,7 +15,7 @@ * @Author chartl * @Date May 26, 2010 */ -@Analysis(name = "Indel length histograms", description = "Shows the distrbution of insertion/deletion event lengths (negative for deletion, positive for insertion)") +@Analysis(name = "Indel length histograms", description = "Shows the distribution of insertion/deletion event lengths (negative for deletion, positive for insertion)") public class IndelLengthHistogram extends VariantEvaluator { private static final int SIZE_LIMIT = 100; @DataPoint(description="Histogram of indel lengths") From e7ace8efc43641d6e75ff2f55756a78c24326d6b Mon Sep 17 00:00:00 2001 From: Menachem Fromer Date: Mon, 30 Jan 2012 21:00:16 -0500 Subject: [PATCH 175/356] Fix NullPointerException caused in cases with too many ALT alleles From 17dbe9a95dd1f638d050a4dc9cf41227fdf562c1 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Mon, 30 Jan 2012 21:37:02 -0500 Subject: [PATCH 176/356] A few cleanups in the LocusIteratorByState * No more N's in the extended event pileups * Only add to the pileup MQ0 counter if the read actually goes into the pileup --- .../gatk/iterators/LocusIteratorByState.java | 71 ++++++++++--------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java b/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java index 2257cc139b..34ac17f497 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java +++ b/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java @@ -74,16 +74,16 @@ public class LocusIteratorByState extends LocusIterator { static private class SAMRecordState { SAMRecord read; - int readOffset = -1; // how far are we offset from the start of the read bases? - int genomeOffset = -1; // how far are we offset from the alignment start on the genome? + int readOffset = -1; // how far are we offset from the start of the read bases? + int genomeOffset = -1; // how far are we offset from the alignment start on the genome? Cigar cigar = null; int cigarOffset = -1; CigarElement curElement = null; int nCigarElements = 0; - // how far are we into a single cigarElement - int cigarElementCounter = -1; + + int cigarElementCounter = -1; // how far are we into a single cigarElement // The logical model for generating extended events is as follows: the "record state" implements the traversal // along the reference; thus stepForwardOnGenome() returns on every and only on actual reference bases. This @@ -93,19 +93,19 @@ static private class SAMRecordState { // stepForwardOnGenome(). The next call to stepForwardOnGenome() will clear that memory (as we remember only extended // events immediately preceding the current reference base). - boolean generateExtendedEvents = true; // should we generate an additional, special pile for indels between the ref bases? - // the only purpose of this flag is to shield away a few additional lines of code - // when extended piles are not needed, it may not be even worth it... + boolean generateExtendedEvents = true; // should we generate an additional, special pile for indels between the ref bases? + // the only purpose of this flag is to shield away a few additional lines of code + // when extended piles are not needed, it may not be even worth it... - byte[] insertedBases = null; // remember full inserted sequence if we are generating piles of extended events (indels) - int eventLength = -1; // will be set to the length of insertion/deletion if we are generating piles of extended events - byte eventDelayedFlag = 0; // will be set to non-0 if there was an event (indel) right before the - // current base on the ref. We use a counter-like variable here since clearing the indel event is - // delayed by one base, so we need to remember how long ago we have seen the actual event + byte[] insertedBases = null; // remember full inserted sequence if we are generating piles of extended events (indels) + int eventLength = -1; // will be set to the length of insertion/deletion if we are generating piles of extended events + byte eventDelayedFlag = 0; // will be set to non-0 if there was an event (indel) right before the + // current base on the ref. We use a counter-like variable here since clearing the indel event is + // delayed by one base, so we need to remember how long ago we have seen the actual event - int eventStart = -1; // where on the read the extended event starts (i.e. the last position on the read prior to the - // event, or -1 if alignment starts with an insertion); this one is easy to recompute on the fly, - // we cache it here mainly for convenience + int eventStart = -1; // where on the read the extended event starts (i.e. the last position on the read prior to the + // event, or -1 if alignment starts with an insertion); this one is easy to recompute on the fly, + // we cache it here mainly for convenience public SAMRecordState(SAMRecord read, boolean extended) { @@ -241,6 +241,8 @@ public CigarOperator stepForwardOnGenome() { readOffset += curElement.getLength(); break; case D: // deletion w.r.t. the reference + if (readOffset < 0) // we don't want reads starting with deletion, this is a malformed cigar string + throw new UserException.MalformedBAM(read, "Read starting with deletion. Cigar: " + read.getCigarString()); if (generateExtendedEvents) { if (cigarElementCounter == 1) { // generate an extended event only if we just stepped into the deletion (i.e. don't @@ -403,9 +405,9 @@ private void lazyLoadNextAlignmentContext() { final int eventStartOffset = state.getReadEventStartOffset(); // this will be -1 if base is not a deletion, or if base is the first deletion in the event. Otherwise, it will give the last base before the deletion began. final int eventLength = state.getEventLength(); -// if (op != CigarOperator.N) // N's are never added to any pileup -// continue; -// + if (op == CigarOperator.N) // N's are never added to any pileup + continue; + if (state.hadIndel()) { // this read has an indel associated with the previous position on the ref size++; ExtendedEventPileupElement pileupElement; @@ -413,27 +415,26 @@ private void lazyLoadNextAlignmentContext() { nDeletions++; maxDeletionLength = Math.max(maxDeletionLength, state.getEventLength()); pileupElement = new ExtendedEventPileupElement(read, eventStartOffset, eventLength); - } + } else { // Insertion event nInsertions++; pileupElement = new ExtendedEventPileupElement(read, eventStartOffset, eventLength, state.getEventBases()); } + if (read.getMappingQuality() == 0) + nMQ0Reads++; indelPile.add(pileupElement); } - // this read has no indel associated with the previous position on the ref. Criteria to include in the pileup are: - // we only add reads that are not N's - // we only include deletions to the pileup if the walker requests it - else if ( (op != CigarOperator.N) && (op != CigarOperator.D || readInfo.includeReadsWithDeletionAtLoci())) { + // this read has no indel so add it to the pileup as a NOEVENT: + // a deletion that didn't start here (therefore, not an extended event) + // we add (mis)matches as no events. + else if (op != CigarOperator.D || readInfo.includeReadsWithDeletionAtLoci()) { size++; indelPile.add(new ExtendedEventPileupElement((GATKSAMRecord) state.getRead(), readOffset)); + if (read.getMappingQuality() == 0) + nMQ0Reads++; } - - - if (state.getRead().getMappingQuality() == 0) - nMQ0Reads++; - } if (indelPile.size() != 0) @@ -461,25 +462,25 @@ else if ( (op != CigarOperator.N) && (op != CigarOperator.D || readInfo.includeR final CigarOperator op = state.getCurrentCigarOperator(); // current cigar operator final CigarOperator nextOp = state.peekForwardOnGenome(); // next cigar operator final int readOffset = state.getReadOffset(); // the base offset on this read - final int eventStartOffset = state.getReadEventStartOffset(); // this will be -1 if base is not a deletion, or if base is the first deletion in the event. Otherwise, it will give the last base before the deletion began. if (op == CigarOperator.N) // N's are never added to any pileup continue; - if (read.getMappingQuality() == 0) - nMQ0Reads++; - if (op == CigarOperator.D) { if (readInfo.includeReadsWithDeletionAtLoci()) { // only add deletions to the pileup if we are authorized to do so - int leftAlignedStart = (eventStartOffset < 0) ? readOffset : eventStartOffset; - pile.add(new PileupElement(read, leftAlignedStart, true, nextOp == CigarOperator.I, false)); + pile.add(new PileupElement(read, readOffset, true, nextOp == CigarOperator.I, false)); size++; nDeletions++; + if (read.getMappingQuality() == 0) + nMQ0Reads++; } - } else { + } + else { if (!filterBaseInRead(read, location.getStart())) { pile.add(new PileupElement(read, readOffset, false, nextOp == CigarOperator.I, op == CigarOperator.S)); size++; + if (read.getMappingQuality() == 0) + nMQ0Reads++; } } } From d8a4d788543dcc667f49950e11cef7ba6249b75c Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Tue, 31 Jan 2012 10:49:06 -0500 Subject: [PATCH 177/356] Bugfix for exceptions with unknown source whose error was not being shown in tableau From 2f2f039c37067f2ac7e94f26f6fae78fe5ba335a Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Tue, 31 Jan 2012 10:49:46 -0500 Subject: [PATCH 178/356] Better flow for byNegTrainingFraction From a7f5d26326af8052e0eeaa295c401b9cb58ffe05 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Tue, 31 Jan 2012 11:17:05 -0500 Subject: [PATCH 179/356] No more synthetic reads starting/ending with deletions bug reported by Kristian Cibulskis that we were generating filtered data synthetic reads with leading deletions. Added integration test. From a630db1703bd30b2258149fc9a00c7c4f4a88531 Mon Sep 17 00:00:00 2001 From: Matt Hanna Date: Tue, 31 Jan 2012 11:58:21 -0500 Subject: [PATCH 180/356] Oops...HierarchicalMicroScheduler was transforming any exception from the walker level into a ReviewedStingException. Thanks to Ryan for pointing this out. --- .../gatk/executive/HierarchicalMicroScheduler.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java index eec4408200..433c7d82fb 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java +++ b/public/java/src/org/broadinstitute/sting/gatk/executive/HierarchicalMicroScheduler.java @@ -11,6 +11,7 @@ import org.broadinstitute.sting.gatk.walkers.TreeReducible; import org.broadinstitute.sting.gatk.walkers.Walker; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import org.broadinstitute.sting.utils.exceptions.StingException; import org.broadinstitute.sting.utils.threading.ThreadPoolMonitor; import java.util.Collection; @@ -101,7 +102,7 @@ public Object execute( Walker walker, Iterable shardStrategy ) { while (isShardTraversePending() || isTreeReducePending()) { // Check for errors during execution. if(hasTraversalErrorOccurred()) - throw new ReviewedStingException("An error has occurred during the traversal.",getTraversalError()); + throw getTraversalError(); // Too many files sitting around taking up space? Merge them. if (isMergeLimitExceeded()) @@ -344,10 +345,15 @@ private synchronized boolean hasTraversalErrorOccurred() { return error != null; } - private synchronized Throwable getTraversalError() { + private synchronized StingException getTraversalError() { if(!hasTraversalErrorOccurred()) throw new ReviewedStingException("User has attempted to retrieve a traversal error when none exists"); - return error; + + // If the error is already a StingException, pass it along as is. Otherwise, wrap it. + if(error instanceof StingException) + return (StingException)error; + else + return new ReviewedStingException("An error occurred during the traversal.",error); } /** From febc63455783b0cace8089dd1db03b39368a082b Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Tue, 31 Jan 2012 16:06:14 -0500 Subject: [PATCH 181/356] Changing PileupElement's isSoftClipped to isNextToSoftClip since soft clipped bases aren't actually added to pileups, oops. Removing the intrinsic clustered variants filter from the HaplotypeCaller --- .../sting/gatk/iterators/LocusIteratorByState.java | 4 ++-- .../SNPGenotypeLikelihoodsCalculationModel.java | 2 +- .../sting/utils/pileup/AbstractReadBackedPileup.java | 2 +- .../sting/utils/pileup/PileupElement.java | 12 ++++++------ .../pileup/ReadBackedExtendedEventPileupImpl.java | 2 +- .../sting/utils/pileup/ReadBackedPileupImpl.java | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java b/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java index 2257cc139b..703308da3f 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java +++ b/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java @@ -472,13 +472,13 @@ else if ( (op != CigarOperator.N) && (op != CigarOperator.D || readInfo.includeR if (op == CigarOperator.D) { if (readInfo.includeReadsWithDeletionAtLoci()) { // only add deletions to the pileup if we are authorized to do so int leftAlignedStart = (eventStartOffset < 0) ? readOffset : eventStartOffset; - pile.add(new PileupElement(read, leftAlignedStart, true, nextOp == CigarOperator.I, false)); + pile.add(new PileupElement(read, leftAlignedStart, true, nextOp == CigarOperator.I, nextOp == CigarOperator.S || (state.getGenomeOffset() == 0 && read.getSoftStart() != read.getAlignmentStart()))); size++; nDeletions++; } } else { if (!filterBaseInRead(read, location.getStart())) { - pile.add(new PileupElement(read, readOffset, false, nextOp == CigarOperator.I, op == CigarOperator.S)); + pile.add(new PileupElement(read, readOffset, false, nextOp == CigarOperator.I, nextOp == CigarOperator.S || (state.getGenomeOffset() == 0 && read.getSoftStart() != read.getAlignmentStart()))); size++; } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java index 5980ff356a..ea53c815d1 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java @@ -212,7 +212,7 @@ public ReadBackedPileup createBAQedPileup( final ReadBackedPileup pileup ) { public class BAQedPileupElement extends PileupElement { public BAQedPileupElement( final PileupElement PE ) { - super(PE.getRead(), PE.getOffset(), PE.isDeletion(), PE.isBeforeInsertion(), PE.isSoftClipped()); + super(PE.getRead(), PE.getOffset(), PE.isDeletion(), PE.isBeforeInsertion(), PE.isNextToSoftClip()); } @Override diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java b/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java index f4fa9e9416..82e4038421 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java @@ -204,7 +204,7 @@ private PileupElementTracker readsOffsets2Pileup(List reads, protected abstract AbstractReadBackedPileup createNewPileup(GenomeLoc loc, PileupElementTracker pileupElementTracker); - protected abstract PE createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeInsertion, boolean isSoftClipped); + protected abstract PE createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeInsertion, boolean isNextToSoftClip); // -------------------------------------------------------- // diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java b/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java index 87aa31c476..d67261ba26 100755 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java @@ -24,7 +24,7 @@ public class PileupElement implements Comparable { protected final int offset; protected final boolean isDeletion; protected final boolean isBeforeInsertion; - protected final boolean isSoftClipped; + protected final boolean isNextToSoftClip; /** @@ -34,13 +34,13 @@ public class PileupElement implements Comparable { * @param offset the position in the read for this base. All deletions must be left aligned! (-1 is only allowed for reads starting with insertions) * @param isDeletion whether or not this base is a deletion * @param isBeforeInsertion whether or not this base is before an insertion - * @param isSoftClipped whether or not this base was softclipped + * @param isNextToSoftClip whether or not this base is next to a soft clipped base */ @Requires({ "read != null", "offset >= -1", "offset <= read.getReadLength()"}) - public PileupElement(final GATKSAMRecord read, final int offset, final boolean isDeletion, final boolean isBeforeInsertion, final boolean isSoftClipped) { + public PileupElement(final GATKSAMRecord read, final int offset, final boolean isDeletion, final boolean isBeforeInsertion, final boolean isNextToSoftClip) { if (offset < 0 && isDeletion) throw new ReviewedStingException("Pileup Element cannot create a deletion with a negative offset"); @@ -48,7 +48,7 @@ public PileupElement(final GATKSAMRecord read, final int offset, final boolean i this.offset = offset; this.isDeletion = isDeletion; this.isBeforeInsertion = isBeforeInsertion; - this.isSoftClipped = isSoftClipped; + this.isNextToSoftClip = isNextToSoftClip; } public boolean isDeletion() { @@ -59,8 +59,8 @@ public boolean isBeforeInsertion() { return isBeforeInsertion; } - public boolean isSoftClipped() { - return isSoftClipped; + public boolean isNextToSoftClip() { + return isNextToSoftClip; } public boolean isInsertionAtBeginningOfRead() { diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java index 641c63f6c7..df334f557f 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java @@ -96,7 +96,7 @@ protected ReadBackedExtendedEventPileupImpl createNewPileup(GenomeLoc loc, Pileu } @Override - protected ExtendedEventPileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeInsertion, boolean isSoftClipped) { + protected ExtendedEventPileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeInsertion, boolean isNextToSoftClip) { throw new UnsupportedOperationException("Not enough information provided to create a new pileup element"); } diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java index 965e74e8bb..20b1000017 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java @@ -71,7 +71,7 @@ protected ReadBackedPileupImpl createNewPileup(GenomeLoc loc, PileupElementTrack } @Override - protected PileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeInsertion, boolean isSoftClipped) { - return new PileupElement(read, offset, isDeletion, isBeforeInsertion, isSoftClipped); + protected PileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeInsertion, boolean isNextToSoftClip) { + return new PileupElement(read, offset, isDeletion, isBeforeInsertion, isNextToSoftClip); } } From 08c7c07f25039954fc06d83d1a7e0fd17391ae9d Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Tue, 31 Jan 2012 17:14:57 -0500 Subject: [PATCH 182/356] Added the option of not compressing read names to ReduceReads * When scatter/gathering, name compression cannot guarantee uniqueness. If uniqueness is important, it is recommended to turn compression off for scatter/gathering ReduceReads. From 579627568e5cc29e48263e2fe881fa081935adee Mon Sep 17 00:00:00 2001 From: Menachem Fromer Date: Tue, 31 Jan 2012 23:39:39 -0500 Subject: [PATCH 183/356] Limit to 3 ALT alleles From e8528bc5269952ad4ba99ba90d6bf7365cbdb40b Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Wed, 1 Feb 2012 09:43:19 -0500 Subject: [PATCH 184/356] updating HaplotypeCaller integration tests From 810996cfcae1983e382a5fd83443564a106b5c8d Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Wed, 1 Feb 2012 10:39:03 -0500 Subject: [PATCH 185/356] Introducing: VariantsToPed, the world's most annoying walker! And also a busted QScript to run it that I need Khalid's help debugging ( frownie face ). Note that VariantsToPed and PlinkSeq generate the same binary file (up to strand flips...thanks PlinkSeq), so I know it's working properly. Hooray! --- .../walkers/variantutils/VariantsToPed.java | 198 ++++++++++++++++++ .../sting/queue/qscripts/lib/VcfToPed.scala | 162 ++++++++++++++ 2 files changed, 360 insertions(+) create mode 100644 public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToPed.java create mode 100644 public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToPed.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToPed.java new file mode 100644 index 0000000000..32b2dd06cc --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToPed.java @@ -0,0 +1,198 @@ +package org.broadinstitute.sting.gatk.walkers.variantutils; + +import org.broadinstitute.sting.commandline.*; +import org.broadinstitute.sting.gatk.arguments.DbsnpArgumentCollection; +import org.broadinstitute.sting.gatk.arguments.StandardVariantContextInputArgumentCollection; +import org.broadinstitute.sting.gatk.contexts.AlignmentContext; +import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; +import org.broadinstitute.sting.gatk.walkers.RodWalker; +import org.broadinstitute.sting.utils.codecs.vcf.VCFHeader; +import org.broadinstitute.sting.utils.codecs.vcf.VCFUtils; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import org.broadinstitute.sting.utils.exceptions.UserException; +import org.broadinstitute.sting.utils.text.XReadLines; +import org.broadinstitute.sting.utils.variantcontext.Genotype; +import org.broadinstitute.sting.utils.variantcontext.VariantContext; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintStream; +import java.util.*; + +/** + * Yet another VCF to Ped converter. The world actually does need one that will + * work efficiently on large VCFs (or at least give a progress bar). This + * produces a binary ped file in SNP-major mode. + */ +public class VariantsToPed extends RodWalker { + @ArgumentCollection + protected StandardVariantContextInputArgumentCollection variantCollection = new StandardVariantContextInputArgumentCollection(); + + @ArgumentCollection + protected DbsnpArgumentCollection dbsnp = new DbsnpArgumentCollection(); + + @Input(shortName="m",fullName = "metaData",required=true,doc="Sample metadata file. You may specify a .fam file (in which case it will be copied to the file you provide as fam output)") + File metaDataFile; + + @Output(shortName="bed",fullName = "bed",required=true,doc="output ped file") + PrintStream outBed; + + @Output(shortName="bim",fullName="bim",required=true,doc="output map file") + PrintStream outBim; + + @Output(shortName="fam",fullName="fam",required=true,doc="output fam file") + PrintStream outFam; + + private ValidateVariants vv = new ValidateVariants(); + + private static double APPROX_CM_PER_BP = 1000000.0/750000.0; + + private static final byte HOM_REF = 0x0; + private static final byte HOM_VAR = 0x3; + private static final byte HET = 0x2; + private static final byte NO_CALL = 0x1; + + // note that HET and NO_CALL are flippd from the documentation: that's because + // plink actually reads these in backwards; and we want to use a shift operator + // to put these in the appropriate location + + public void initialize() { + vv.variantCollection = variantCollection; + vv.dbsnp = dbsnp; + vv.DO_NOT_VALIDATE_FILTERED = true; + vv.type = ValidateVariants.ValidationType.REF; + // write magic bits into the ped file + try { + outBed.write(new byte[] { (byte) 0x6c, (byte) 0x1b, 0x1 }); + } catch (IOException e) { + throw new ReviewedStingException("error writing to output file."); + } + // write to the fam file, the first six columns of the standard ped file + // first, load data from the input meta data file + Map> metaValues = new HashMap>(); + try { + if ( metaDataFile.getAbsolutePath().endsWith(".fam") ) { + for ( String line : new XReadLines(metaDataFile) ) { + outFam.printf("%s%n",line); + } + } else { + for ( String line : new XReadLines(metaDataFile) ) { + String[] split = line.split("\\t"); + String sampleID = split[0]; + String keyVals = split[1]; + HashMap values = new HashMap(); + for ( String kvp : keyVals.split(";") ) { + String[] kvp_split = kvp.split("="); + values.put(kvp_split[0],kvp_split[1]); + } + metaValues.put(sampleID,values); + } + } + } catch (FileNotFoundException e) { + throw new UserException("Meta data file not found: "+metaDataFile.getAbsolutePath(),e); + } + // family ID, individual ID, Paternal ID, Maternal ID, Sex, Phenotype + int dummyID = 0; // increments for dummy parental and family IDs used + // want to be especially careful to maintain order here + Map headers = VCFUtils.getVCFHeadersFromRods(getToolkit()); + for ( Map.Entry header : headers.entrySet() ) { + if ( ! header.getKey().equals(variantCollection.variants.getName()) && ! metaDataFile.getAbsolutePath().endsWith(".fam") ) { + continue; + } + for ( String sample : header.getValue().getGenotypeSamples() ) { + Map mVals = metaValues.get(sample); + if ( mVals == null ) { + throw new UserException("No metadata provided for sample "+sample); + } + if ( ! mVals.containsKey("phenotype") ) { + throw new UserException("No phenotype data provided for sample "+sample); + } + String fid = mVals.containsKey("fid") ? mVals.get("fid") : String.format("dummy_%d",++dummyID); + String pid = mVals.containsKey("dad") ? mVals.get("dad") : String.format("dummy_%d",++dummyID); + String mid = mVals.containsKey("mom") ? mVals.get("mom") : String.format("dummy_%d",++dummyID); + String sex = mVals.containsKey("sex") ? mVals.get("sex") : "3"; + String pheno = mVals.get("phenotype"); + outFam.printf("%s\t%s\t%s\t%s\t%s\t%s%n",fid,pid,sample,mid,sex,pheno); + } + } + } + + public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) { + if ( tracker == null || ! tracker.hasValues(variantCollection.variants) || + tracker.getFirstValue(variantCollection.variants).isFiltered() || + ! tracker.getFirstValue(variantCollection.variants).isSNP() || + ! tracker.getFirstValue(variantCollection.variants).isBiallelic()) { + return 0; + } + try { + vv.map(tracker,ref,context); + } catch (UserException e) { + throw new UserException("Input VCF file is invalid; we cannot guarantee the resulting ped file. "+ + "Please run ValidateVariants for more detailed information."); + } + + VariantContext vc = tracker.getFirstValue(variantCollection.variants); + // write an entry into the map file + outBim.printf("%s\t%s\t%.2f\t%d\t%s\t%s%n",vc.getChr(),getID(vc),APPROX_CM_PER_BP*vc.getStart(),vc.getStart(), + vc.getReference().getBaseString(),vc.getAlternateAllele(0).getBaseString()); + // write an entry into the bed file + int buf = 0; + int idx = 0; + byte out = 0x0; + byte[] toWrite = new byte[1+(vc.getNSamples()/4)]; + for (Genotype g : vc.getGenotypes() ) { + out |= getEncoding(g,buf); + if ( buf == 3 ) { + toWrite[idx] = out; + buf = 0; + out = 0x0; + idx++; + } else { + buf++; + } + } + if ( out != 0x0 ) { + toWrite[idx]=out; + } + try { + outBed.write(toWrite); + } catch (IOException e) { + throw new ReviewedStingException("Error writing to output file"); + } + + return 1; + } + + public Integer reduce(Integer m, Integer r) { + return r + m; + } + + public Integer reduceInit() { + return 0; + } + + private static byte getEncoding(Genotype g, int offset) { + byte b; + if ( g.isHomRef() ) { + b = HOM_REF; + } else if ( g.isHomVar() ) { + b = HOM_VAR; + } else if ( g.isHet() ) { + b = HET; + } else { + b = NO_CALL; + } + + return (byte) (b << (2*offset)); + } + + private static String getID(VariantContext v) { + if ( v.hasID() ) { + return v.getID(); + } else { + return String.format("SNP-%s-%d",v.getChr(),v.getStart()); + } + } +} diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala new file mode 100644 index 0000000000..04f73d562a --- /dev/null +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala @@ -0,0 +1,162 @@ +package org.broadinstitute.sting.queue.qscripts.lib + +import org.broadinstitute.sting.queue.QScript +import org.broadinstitute.sting.commandline.Input +import org.broadinstitute.sting.queue.library.ipf.vcf.VCFExtractIntervals +import org.broadinstitute.sting.utils.text.XReadLines +import collection.JavaConversions._ +import java.io._ +import org.broadinstitute.sting.queue.extensions.gatk.VariantsToPed + +/** + * Created by IntelliJ IDEA. + * User: chartl + * Date: 1/31/12 + * Time: 10:46 PM + * To change this template use File | Settings | File Templates. + */ + +class VcfToPed extends QScript { + + @Input(shortName = "V", fullName="Variants", required=true,doc="VCF to convert to ped") + var variants : File = _ + + @Output(shortName = "B", fullName="Bed",required=true,doc="Name of the ped output file (fam and bim will use the root of this file)") + var bed : File = _ + + @Input(shortName = "M", fullName="Meta",required=true,doc="The sample metadata file, can be a .fam or [NAME]\\tkey1=val1;key2=val2") + var meta : File = _ + + @Input(shortName = "Int", fullName="Intervals",required=false,doc="Intervals. If not specified script will produce them and exit.") + var intervals : File = _ + + @Argument(shortName="R",fullName="Ref",required=false,doc="Reference file") + var ref : File = new File("/humgen/1kg/references/human_g1k_v37.fasta") + + @Argument(shortName="D",fullName="dbsnp",required=false,doc="dbsnp file") + var dbsnp : File = new File("/humgen/gsa-hpprojects/GATK/data/dbsnp_129_b37.vcf") + + val tmpdir : File = System.getProperty("java.io.tmpdir") + + def script = { + if ( intervals == null ) { + val ivals : File = swapExt(variants,".vcf",".intervals.list") + val extract : VCFExtractIntervals = new VCFExtractIntervals(variants,ivals,false) + add(extract) + } else { + var iXRL = new XReadLines(intervals) + var chunk = 1; + var subListFile = swapExt(tmpdir,variants,".vcf",".chunk%d.list".format(chunk)) + var subList = new PrintStream(subListFile) + var nL = 0; + var bedOuts : List[File] = Nil; + var bimOuts : List[File] = Nil + var lastFam : File = null; + while ( iXRL.hasNext ) { + subList.printf("%s%n",iXRL.next()) + nL = nL + 1 + if ( nL > 100000 ) { + val toPed : VariantsToPed = new VariantsToPed + toPed.memoryLimit = 2 + toPed.reference_sequence = ref + toPed.intervals :+= new File(subListFile) + toPed.dbsnp = dbsnp + toPed.variant = variants + toPed.metaData = meta + lazy val base : String = bed.getName.stripSuffix(".bed")+"_%".format(chunk) + lazy val tBed = new File(tmpdir,base+".bed") + lazy val bim = new File(tmpdir,base+".bim") + lazy val fam = new File(tmpdir,base+".fam") + toPed.bed = tBed + toPed.bim = bim + toPed.fam = fam + add(toPed) + subList.close() + chunk = chunk + 1 + subListFile = swapExt(tmpdir,variants,".vcf",".chunk%d.list".format(chunk)) + subList = new PrintStream(subListFile) + bedOuts :+= tBed + bimOuts :+= bim + lastFam = fam + nL = 0; + } + } + + if ( nL > 0 ) { + val toPed : VariantsToPed = new VariantsToPed + toPed.reference_sequence = ref + toPed.intervals :+= new File(subListFile) + toPed.dbsnp = dbsnp + toPed.variant = variants + toPed.metaData = meta + lazy val base : String = bed.getName.stripSuffix(".bed")+"_%".format(chunk) + lazy val tBed = new File(tmpdir,base+".bed") + lazy val bim = new File(tmpdir,base+".bim") + lazy val fam = new File(tmpdir,base+".fam") + toPed.bed = tBed + toPed.bim = bim + toPed.fam = fam + lastFam = fam + add(toPed) + subList.close() + bedOuts :+= tBed + bimOuts :+= bim + } + + var gatherUP = new MyPedGather + gatherUP.binPed = bedOuts + gatherUP.bim = bimOuts + gatherUP.outPed = bed + gatherUP.outBim = swapExt(bed,".bed",".bim") + + add(gatherUP) + + class copyFam extends InProcessFunction { + @Input(doc="fam") var inFam = lastFam + @Output(doc="fam") var outFam = swapExt(bed,".bed",".fam") + + def run = { + var stream = new PrintStream(outFam) + asScalaIterator(new XReadLines(inFam)).foreach( u => { + stream.printf("%s%n",u) + }) + stream.close() + } + } + + add(new copyFam) + } + + } + + class MyPedGather extends InProcessFunction { + @Input(doc="Peds to be merged") var binPed: List[File] = Nil + @Input(doc="Bims to be merged") var bim : List[File] = Nil + @Output(doc="The final Ped to write to") var outPed : File = _ + @Output(doc="The final bim to write to") var outBim : File = _ + + def run : Unit = { + var stream : PrintStream = new PrintStream(outPed) + stream.write((List[Byte](0x6c.toByte,0x1b.toByte,0x1.toByte)).toArray) + binPed.map(u => new FileInputStream(u) ).foreach( u => { + u.skip(3) + var b = -1 + do { + b = u.read() + stream.write(b.toByte) + } while ( b != -1 ) + }) + stream.close() + + stream = new PrintStream(outBim) + bim.map(u => new XReadLines(u)).foreach( u => { + asScalaIterator(u).foreach( x => { + stream.printf("%s%n",x) + }) + }) + + stream.close() + } + } + +} \ No newline at end of file From 45da892ecc9d5da28dd28e7c9d6de1b8aab8cb9d Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Tue, 31 Jan 2012 18:34:53 -0500 Subject: [PATCH 186/356] Better exceptions to catch malformed reads * throw exceptions in LocusIteratorByState when hitting reads starting or ending with deletions --- .../sting/gatk/iterators/LocusIteratorByState.java | 7 +++++-- .../broadinstitute/sting/utils/pileup/PileupElement.java | 8 ++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java b/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java index 53144671ca..316a20a704 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java +++ b/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java @@ -193,6 +193,9 @@ public CigarOperator stepForwardOnGenome() { // we reenter in order to re-check cigarElementCounter against curElement's length return stepForwardOnGenome(); } else { + if (curElement != null && curElement.getOperator() == CigarOperator.D) + throw new UserException.MalformedBAM(read, "read ends with deletion. Cigar: " + read.getCigarString()); + // Reads that contain indels model the genomeOffset as the following base in the reference. Because // we fall into this else block only when indels end the read, increment genomeOffset such that the // current offset of this read is the next ref base after the end of the indel. This position will @@ -228,7 +231,7 @@ public CigarOperator stepForwardOnGenome() { // we see insertions only once, when we step right onto them; the position on the read is scrolled // past the insertion right after that if (eventDelayedFlag > 1) - throw new UserException.MalformedBAM(read, "Adjacent I/D events in read " + read.getReadName()); + throw new UserException.MalformedBAM(read, String.format("Adjacent I/D events in read %s -- cigar: %s", read.getReadName(), read.getCigarString())); insertedBases = Arrays.copyOfRange(read.getReadBases(), readOffset + 1, readOffset + 1 + curElement.getLength()); eventLength = curElement.getLength(); eventStart = readOffset; @@ -247,7 +250,7 @@ public CigarOperator stepForwardOnGenome() { // generate an extended event only if we just stepped into the deletion (i.e. don't // generate the event at every deleted position on the ref, that's what cigarElementCounter==1 is for!) if (eventDelayedFlag > 1) - throw new UserException.MalformedBAM(read, "Adjacent I/D events in read " + read.getReadName()); + throw new UserException.MalformedBAM(read, String.format("Adjacent I/D events in read %s -- cigar: %s", read.getReadName(), read.getCigarString())); eventLength = curElement.getLength(); eventDelayedFlag = 2; // deletion on the ref causes an immediate return, so we have to delay by 1 only eventStart = readOffset; diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java b/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java index d67261ba26..9e2a66f6e0 100755 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java @@ -4,6 +4,7 @@ import com.google.java.contract.Requires; import org.broadinstitute.sting.utils.BaseUtils; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import org.broadinstitute.sting.utils.exceptions.UserException; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; /** @@ -146,9 +147,12 @@ else if (read.getAlignmentStart() > pileupElement.read.getAlignmentStart()) public int getRepresentativeCount() { int representativeCount = 1; - if (read.isReducedRead() && !isInsertionAtBeginningOfRead()) - representativeCount = (isDeletion()) ? Math.round((read.getReducedCount(offset) + read.getReducedCount(offset + 1)) / 2) : read.getReducedCount(offset); + if (read.isReducedRead() && !isInsertionAtBeginningOfRead()) { + if (isDeletion() && (offset + 1 >= read.getReadLength()) ) // deletion in the end of the read + throw new UserException.MalformedBAM(read, String.format("Adjacent I/D events in read %s -- cigar: %s", read.getReadName(), read.getCigarString())); + representativeCount = (isDeletion()) ? Math.round((read.getReducedCount(offset) + read.getReducedCount(offset + 1)) / 2) : read.getReducedCount(offset); + } return representativeCount; } From 87a63d54d62fd639237f7755e7d6bc6498709469 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Wed, 1 Feb 2012 12:05:29 -0500 Subject: [PATCH 187/356] fix the script! --- .../sting/queue/qscripts/lib/VcfToPed.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala index 04f73d562a..1c26204411 100644 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala @@ -63,10 +63,10 @@ class VcfToPed extends QScript { toPed.dbsnp = dbsnp toPed.variant = variants toPed.metaData = meta - lazy val base : String = bed.getName.stripSuffix(".bed")+"_%".format(chunk) - lazy val tBed = new File(tmpdir,base+".bed") - lazy val bim = new File(tmpdir,base+".bim") - lazy val fam = new File(tmpdir,base+".fam") + val base : String = bed.getName.stripSuffix(".bed")+"_%d".format(chunk) + val tBed = new File(tmpdir,base+".bed") + val bim = new File(tmpdir,base+".bim") + val fam = new File(tmpdir,base+".fam") toPed.bed = tBed toPed.bim = bim toPed.fam = fam @@ -89,10 +89,10 @@ class VcfToPed extends QScript { toPed.dbsnp = dbsnp toPed.variant = variants toPed.metaData = meta - lazy val base : String = bed.getName.stripSuffix(".bed")+"_%".format(chunk) - lazy val tBed = new File(tmpdir,base+".bed") - lazy val bim = new File(tmpdir,base+".bim") - lazy val fam = new File(tmpdir,base+".fam") + val base : String = bed.getName.stripSuffix(".bed")+"_%d".format(chunk) + val tBed = new File(tmpdir,base+".bed") + val bim = new File(tmpdir,base+".bim") + val fam = new File(tmpdir,base+".fam") toPed.bed = tBed toPed.bim = bim toPed.fam = fam From b567ed8793ad4af875e4ef4aab41c5ef89352fb5 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Wed, 1 Feb 2012 12:35:18 -0500 Subject: [PATCH 188/356] Use the right reference path :( --- .../org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala index 1c26204411..30b5cb0845 100644 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala @@ -31,7 +31,7 @@ class VcfToPed extends QScript { var intervals : File = _ @Argument(shortName="R",fullName="Ref",required=false,doc="Reference file") - var ref : File = new File("/humgen/1kg/references/human_g1k_v37.fasta") + var ref : File = new File("/humgen/1kg/reference/human_g1k_v37.fasta") @Argument(shortName="D",fullName="dbsnp",required=false,doc="dbsnp file") var dbsnp : File = new File("/humgen/gsa-hpprojects/GATK/data/dbsnp_129_b37.vcf") From a46a29501b9bdc60822e60c74929e0459879a805 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Wed, 1 Feb 2012 13:22:17 -0500 Subject: [PATCH 189/356] Marking unused code in HC for removal. From bc6abc94e9a0c541cf5d0437345e3dc55057a8a5 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Wed, 1 Feb 2012 14:32:19 -0500 Subject: [PATCH 190/356] Bug fix for check of isMateUnmapped. Requires check of mate is mapped. From 2109122cdd0bf8048e7decf9aebe2a3b2b73973f Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Wed, 1 Feb 2012 14:37:24 -0500 Subject: [PATCH 191/356] merging branches From f8c5406084281eacf927585a974c8f73b51d6158 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 2 Feb 2012 09:06:39 -0500 Subject: [PATCH 192/356] Add the ability to extract samples --- .../sting/queue/qscripts/lib/VcfToPed.scala | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala index 30b5cb0845..4995888bb8 100644 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala @@ -6,7 +6,7 @@ import org.broadinstitute.sting.queue.library.ipf.vcf.VCFExtractIntervals import org.broadinstitute.sting.utils.text.XReadLines import collection.JavaConversions._ import java.io._ -import org.broadinstitute.sting.queue.extensions.gatk.VariantsToPed +import org.broadinstitute.sting.queue.extensions.gatk.{SelectVariants, VariantsToPed} /** * Created by IntelliJ IDEA. @@ -36,6 +36,9 @@ class VcfToPed extends QScript { @Argument(shortName="D",fullName="dbsnp",required=false,doc="dbsnp file") var dbsnp : File = new File("/humgen/gsa-hpprojects/GATK/data/dbsnp_129_b37.vcf") + @Argument(shortName="sf",fullName="sampleFile",required=false,doc="sample file") + var samFile : File = _ + val tmpdir : File = System.getProperty("java.io.tmpdir") def script = { @@ -59,9 +62,22 @@ class VcfToPed extends QScript { val toPed : VariantsToPed = new VariantsToPed toPed.memoryLimit = 2 toPed.reference_sequence = ref - toPed.intervals :+= new File(subListFile) + toPed.intervals :+= subListFile toPed.dbsnp = dbsnp - toPed.variant = variants + if ( samFile != null ) { + val base : String = bed.getName.stripSuffix(".bed")+"_%d".format(chunk) + val extract : SelectVariants = new SelectVariants + extract.reference_sequence = ref + extract.memoryLimit = 2 + extract.intervals :+= subListFile + extract.variant = variants + extract.out = new File(tmpdir,base+"_extract%d.vcf".format(chunk)) + extract.sample_file :+= samFile + add(extract) + toPed.variant = extract.out + } else { + toPed.variant = variants + } toPed.metaData = meta val base : String = bed.getName.stripSuffix(".bed")+"_%d".format(chunk) val tBed = new File(tmpdir,base+".bed") From 45bf2562cc2cde70c15868c0e2b14f5c50c5c911 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 2 Feb 2012 09:11:17 -0500 Subject: [PATCH 193/356] . --- .../sting/queue/qscripts/lib/VcfToPed.scala | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala index 4995888bb8..2f691b907f 100644 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala @@ -103,7 +103,20 @@ class VcfToPed extends QScript { toPed.reference_sequence = ref toPed.intervals :+= new File(subListFile) toPed.dbsnp = dbsnp - toPed.variant = variants + if ( samFile != null ) { + val base : String = bed.getName.stripSuffix(".bed")+"_%d".format(chunk) + val extract : SelectVariants = new SelectVariants + extract.reference_sequence = ref + extract.memoryLimit = 2 + extract.intervals :+= subListFile + extract.variant = variants + extract.out = new File(tmpdir,base+"_extract%d.vcf".format(chunk)) + extract.sample_file :+= samFile + add(extract) + toPed.variant = extract.out + } else { + toPed.variant = variants + } toPed.metaData = meta val base : String = bed.getName.stripSuffix(".bed")+"_%d".format(chunk) val tBed = new File(tmpdir,base+".bed") From 48220700eb777df74b290d1e3e9af33c3e9f6084 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Thu, 2 Feb 2012 09:58:03 -0500 Subject: [PATCH 194/356] Adding lftp to the supported download protocols for the 1000G sync. Initial results look very good! Bringing down chrom11 and chrom20 files now. From 4ed06801a72508d64e54cfa59f46ee546b5c74e0 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Thu, 2 Feb 2012 10:17:04 -0500 Subject: [PATCH 195/356] Updating HaplotypeCaller's HMM calc to use GOP as a function of the read instead of a function of the haplotype in preparation for IQSR --- .../gatk/walkers/recalibration/CountCovariatesWalker.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java index bdf25419f2..fdfb29da62 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java @@ -357,11 +357,11 @@ public CountedData map( RefMetaDataTracker tracker, ReferenceContext ref, Alignm final GATKSAMRecord gatkRead = (GATKSAMRecord) p.getRead(); int offset = p.getOffset(); - if( gatkRead.containsTemporaryAttribute( SKIP_RECORD_ATTRIBUTE ) ) { + if( gatkRead.containsTemporaryAttribute( SKIP_RECORD_ATTRIBUTE ) ) { continue; } - if( !gatkRead.containsTemporaryAttribute( SEEN_ATTRIBUTE ) ) + if( !gatkRead.containsTemporaryAttribute( SEEN_ATTRIBUTE ) ) { gatkRead.setTemporaryAttribute( SEEN_ATTRIBUTE, true ); RecalDataManager.parseSAMRecord( gatkRead, RAC ); @@ -377,7 +377,6 @@ public CountedData map( RefMetaDataTracker tracker, ReferenceContext ref, Alignm RecalDataManager.computeCovariates( gatkRead, requestedCovariates )); } - // Skip this position if base quality is zero if( gatkRead.getBaseQualities()[offset] > 0 ) { From 0c562756eb5b96a36485abe40e3907940643b8b7 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 2 Feb 2012 10:30:09 -0500 Subject: [PATCH 196/356] Add a memory limit so this thing doesn't get killed on the farm --- .../org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala index 2f691b907f..913a62e260 100644 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala @@ -118,6 +118,7 @@ class VcfToPed extends QScript { toPed.variant = variants } toPed.metaData = meta + toPed.memoryLimit = 2 val base : String = bed.getName.stripSuffix(".bed")+"_%d".format(chunk) val tBed = new File(tmpdir,base+".bed") val bim = new File(tmpdir,base+".bim") From 0111505ea9e2e3df4c8e5ce3553df45cd0e64eca Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 2 Feb 2012 11:41:16 -0500 Subject: [PATCH 197/356] Terrible. Swapping the paternal and sample ids. --- .../sting/gatk/walkers/variantutils/VariantsToPed.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToPed.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToPed.java index 32b2dd06cc..aab230b69e 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToPed.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToPed.java @@ -114,7 +114,7 @@ public void initialize() { String mid = mVals.containsKey("mom") ? mVals.get("mom") : String.format("dummy_%d",++dummyID); String sex = mVals.containsKey("sex") ? mVals.get("sex") : "3"; String pheno = mVals.get("phenotype"); - outFam.printf("%s\t%s\t%s\t%s\t%s\t%s%n",fid,pid,sample,mid,sex,pheno); + outFam.printf("%s\t%s\t%s\t%s\t%s\t%s%n",fid,sample,pid,mid,sex,pheno); } } } From 27ea6426a43b9a1ad00105953e82d33e7196c217 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 2 Feb 2012 12:29:03 -0500 Subject: [PATCH 198/356] Small script to chunk up a VCF into equal-sized chunks --- .../sting/queue/qscripts/lib/ChunkVCF.scala | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/ChunkVCF.scala diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/ChunkVCF.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/ChunkVCF.scala new file mode 100644 index 0000000000..257fef0215 --- /dev/null +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/ChunkVCF.scala @@ -0,0 +1,88 @@ +package org.broadinstitute.sting.queue.qscripts.lib + +import org.broadinstitute.sting.queue.QScript +import org.broadinstitute.sting.queue.library.ipf.vcf.VCFExtractIntervals +import scala.collection.JavaConversions._ +import org.broadinstitute.sting.utils.text.XReadLines +import java.io.PrintStream +import org.broadinstitute.sting.queue.extensions.gatk.SelectVariants + +/** + * Created by IntelliJ IDEA. + * User: chartl + * Date: 2/2/12 + * Time: 12:13 PM + * To change this template use File | Settings | File Templates. + */ + +class ChunkVCF extends QScript { + + @Input(shortName="V",fullName="VCF",doc="The VCF you want to chunk",required=true) + var inVCF : File = _ + + @Input(shortName="N",fullName="numEntriesInChunk",doc="The number of variants per chunk",required=true) + var numEntries : Int = _ + + @Input(shortName="I",fullName="Intervals",doc="The SNP interval list to chunk. If not provided, one will be created for you to provide in a second run.") + var intervals : File = _ + + @Input(fullName="preserveChromosomes",doc="Restrict chunks to one chromosome (smaller chunk at end of chromosome)",required=false) + var preserve : Boolean = false + + @Input(fullName="reference",doc="The reference file",required=false) + var ref : File = new File("/humgen/1kg/reference/human_g1k_v37.fasta") + + @Input(fullName="samples",doc="A file of sample IDs to condense VCF file to",required=false) + var extractSamples : File = _ + + val tmpdir : File = System.getProperty("java.io.tmpdir") + + def script = { + if ( intervals == null ) { + // create an interval list from the VCF + val ivals : File = swapExt(variants,".vcf",".intervals.list") + val extract : VCFExtractIntervals = new VCFExtractIntervals(variants,ivals,false) + add(extract) + } else { + var chunkNum = 1 + var numLinesInChunk = 0 + var chromosome : String = asScalaIterator(new XReadLines(intervals)).next().split(":")(0) + var chunkFile : File = new File(tmpdir,"ChunkVCF.chunk%d.intervals.list".format(chunkNum)) + var chunkWriter = new PrintStream(chunkFile) + asScalaIterator(new XReadLines(intervals)).foreach( int => { + // check new chromosome or full chunk + if ( ( preserve && ! int.split(":")(0).equals(chromosome) ) || numLinesInChunk > numEntries ) { + chunkWriter.close() + val chunkSelect : SelectVariants = new SelectVariants + chunkSelect.reference_sequence = ref + chunkSelect.memoryLimit = 2 + chunkSelect.intervals :+= chunkFile + if ( extractSamples != null ) + chunkSelect.sample_file = extractSamples + chunkSelect.out = swapExt(inVCF,".vcf",".chunk%d.vcf".format(chunkNum)) + add(chunkSelect) + chunkNum += 1 + numLinesInChunk = 0 + chromosome = int.split(":")(0) + chunkFile = new File(tmpdir,"ChunkVCF.chunk%d.intervals.list".format(chunkNum)) + chunkWriter = new PrintStream(chunkFile) + } + chunkWriter.printf("%s%n",int) + numLinesInChunk += 1 + }) + // last chunk + if ( numLinesInChunk > 0 ) { + // some work to do + val chunkSelect : SelectVariants = new SelectVariants + chunkSelect.reference_sequence = ref + chunkSelect.memoryLimit = 2 + chunkSelect.intervals :+= chunkFile + chunkWriter.close() + if ( extractSamples != null ) + chunkSelect.sample_file = extractSamples + chunkSelect.out = swapExt(inVCF,".vcf",".chunk%d.vcf".format(chunkNum)) + add(chunkSelect) + } + } + } +} \ No newline at end of file From f596377e731ff7509dfd469f7505ecc2f65daf12 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Thu, 2 Feb 2012 12:39:49 -0500 Subject: [PATCH 199/356] Oops. Forgot that some samples are new and the data directory doesn't already exist. Now try to create it if it's not already present. From 974c2499cc9aa9c6855d0543b477f861be012510 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Thu, 2 Feb 2012 12:55:54 -0500 Subject: [PATCH 200/356] Bugfixed to script. --- .../sting/queue/qscripts/lib/ChunkVCF.scala | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/ChunkVCF.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/ChunkVCF.scala index 257fef0215..0184b5d2c0 100644 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/ChunkVCF.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/ChunkVCF.scala @@ -23,7 +23,7 @@ class ChunkVCF extends QScript { @Input(shortName="N",fullName="numEntriesInChunk",doc="The number of variants per chunk",required=true) var numEntries : Int = _ - @Input(shortName="I",fullName="Intervals",doc="The SNP interval list to chunk. If not provided, one will be created for you to provide in a second run.") + @Input(shortName="I",fullName="Intervals",doc="The SNP interval list to chunk. If not provided, one will be created for you to provide in a second run.",required=false) var intervals : File = _ @Input(fullName="preserveChromosomes",doc="Restrict chunks to one chromosome (smaller chunk at end of chromosome)",required=false) @@ -40,8 +40,8 @@ class ChunkVCF extends QScript { def script = { if ( intervals == null ) { // create an interval list from the VCF - val ivals : File = swapExt(variants,".vcf",".intervals.list") - val extract : VCFExtractIntervals = new VCFExtractIntervals(variants,ivals,false) + val ivals : File = swapExt(inVCF,".vcf",".intervals.list") + val extract : VCFExtractIntervals = new VCFExtractIntervals(inVCF,ivals,false) add(extract) } else { var chunkNum = 1 @@ -54,11 +54,12 @@ class ChunkVCF extends QScript { if ( ( preserve && ! int.split(":")(0).equals(chromosome) ) || numLinesInChunk > numEntries ) { chunkWriter.close() val chunkSelect : SelectVariants = new SelectVariants + chunkSelect.variant = inVCF chunkSelect.reference_sequence = ref chunkSelect.memoryLimit = 2 chunkSelect.intervals :+= chunkFile if ( extractSamples != null ) - chunkSelect.sample_file = extractSamples + chunkSelect.sample_file :+= extractSamples chunkSelect.out = swapExt(inVCF,".vcf",".chunk%d.vcf".format(chunkNum)) add(chunkSelect) chunkNum += 1 @@ -74,12 +75,13 @@ class ChunkVCF extends QScript { if ( numLinesInChunk > 0 ) { // some work to do val chunkSelect : SelectVariants = new SelectVariants + chunkSelect.variant = inVCF chunkSelect.reference_sequence = ref chunkSelect.memoryLimit = 2 chunkSelect.intervals :+= chunkFile chunkWriter.close() if ( extractSamples != null ) - chunkSelect.sample_file = extractSamples + chunkSelect.sample_file :+= extractSamples chunkSelect.out = swapExt(inVCF,".vcf",".chunk%d.vcf".format(chunkNum)) add(chunkSelect) } From 601e53d633567ee7411afb9cda84235577ec3e95 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Thu, 2 Feb 2012 16:34:26 -0500 Subject: [PATCH 201/356] Fix when specifying preset active regions with -AR argument --- .../sting/gatk/traversals/TraverseActiveRegions.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java index 769bec720e..ce8cb557b0 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java +++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java @@ -124,7 +124,7 @@ public T traverse( final ActiveRegionWalker walker, // Take the individual isActive calls and integrate them into contiguous active regions and // add these blocks of work to the work queue - final ArrayList activeRegions = integrateActiveList( isActiveList, firstIsActiveStart, activeRegionExtension ); + final ArrayList activeRegions = integrateActiveList( isActiveList, firstIsActiveStart, activeRegionExtension, walker.presetActiveRegions != null ); logger.debug("Integrated " + isActiveList.size() + " isActive calls into " + activeRegions.size() + " regions." ); if( walker.activeRegionOutStream == null ) { workQueue.addAll( activeRegions ); @@ -214,7 +214,7 @@ else if( dataSource == DataSource.REFERENCE_ORDERED_DATA ) } // band-pass filter the list of isActive probabilities and turn into active regions - private ArrayList integrateActiveList( final ArrayList activeList, final GenomeLoc firstIsActiveStart, final int activeRegionExtension ) { + private ArrayList integrateActiveList( final ArrayList activeList, final GenomeLoc firstIsActiveStart, final int activeRegionExtension, final boolean presetRegions ) { final double ACTIVE_PROB_THRESHOLD = 0.2; // BUGBUG: needs to be set-able by the walker author final ArrayList returnList = new ArrayList(); @@ -227,11 +227,11 @@ private ArrayList integrateActiveList( final ArrayList act } else { final Double[] activeProbArray = activeList.toArray(new Double[activeList.size()]); final double[] filteredProbArray = new double[activeProbArray.length]; - final int FILTER_SIZE = 50; // BUGBUG: needs to be set-able by the walker author - final int MAX_ACTIVE_REGION = 425; // BUGBUG: needs to be set-able by the walker author + final int FILTER_SIZE = ( presetRegions ? 0 : 50 ); // BUGBUG: needs to be set-able by the walker author + final int MAX_ACTIVE_REGION = ( presetRegions ? 16001 : 425 ); // BUGBUG: needs to be set-able by the walker author for( int iii = 0; iii < activeProbArray.length; iii++ ) { double maxVal = 0; - for( int jjj = Math.max(0, iii-FILTER_SIZE); jjj < Math.min(activeList.size(), iii+FILTER_SIZE); jjj++ ) { + for( int jjj = Math.max(0, iii-FILTER_SIZE); jjj < Math.min(activeList.size(), iii+FILTER_SIZE+1); jjj++ ) { if( activeProbArray[jjj] > maxVal ) { maxVal = activeProbArray[jjj]; } } filteredProbArray[iii] = maxVal; From 3abfbcbcf2eb81394adf79072232967f8f6b069d Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Fri, 3 Feb 2012 12:23:21 -0500 Subject: [PATCH 202/356] Generalized the TDT for multi-allelic events --- .../TransmissionDisequilibriumTest.java | 54 +++++++++++++------ 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java index 43d5f0b287..34f4bd6079 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java @@ -7,6 +7,7 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatibleWalker; import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ExperimentalAnnotation; import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation; +import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.RodRequiringAnnotation; import org.broadinstitute.sting.utils.MathUtils; import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType; import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine; @@ -17,16 +18,13 @@ /** * Created by IntelliJ IDEA. - * User: rpoplin, lfran + * User: rpoplin, lfran, ebanks * Date: 11/14/11 */ -public class TransmissionDisequilibriumTest extends InfoFieldAnnotation implements ExperimentalAnnotation { +public class TransmissionDisequilibriumTest extends InfoFieldAnnotation implements ExperimentalAnnotation, RodRequiringAnnotation { private Set trios = null; - private final static int REF = 0; - private final static int HET = 1; - private final static int HOM = 2; private final static int MIN_NUM_VALID_TRIOS = 5; // don't calculate this population-level statistic if there are less than X trios with full genotype likelihood information public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc) { @@ -38,10 +36,10 @@ public Map annotate(RefMetaDataTracker tracker, AnnotatorCompati } } - final Map toRet = new HashMap(1); + final Map toRet = new HashMap(1); final HashSet triosToTest = new HashSet(); - for( final Sample child : trios) { + for( final Sample child : trios ) { final boolean hasAppropriateGenotypes = vc.hasGenotype(child.getID()) && vc.getGenotype(child.getID()).hasLikelihoods() && vc.hasGenotype(child.getPaternalID()) && vc.getGenotype(child.getPaternalID()).hasLikelihoods() && vc.hasGenotype(child.getMaternalID()) && vc.getGenotype(child.getMaternalID()).hasLikelihoods(); @@ -65,28 +63,54 @@ public Map annotate(RefMetaDataTracker tracker, AnnotatorCompati // Following derivation in http://en.wikipedia.org/wiki/Transmission_disequilibrium_test#A_modified_version_of_the_TDT private double calculateTDT( final VariantContext vc, final Set triosToTest ) { - final double nABGivenABandBB = calculateNChildren(vc, triosToTest, HET, HET, HOM) + calculateNChildren(vc, triosToTest, HET, HOM, HET); - final double nBBGivenABandBB = calculateNChildren(vc, triosToTest, HOM, HET, HOM) + calculateNChildren(vc, triosToTest, HOM, HOM, HET); - final double nAAGivenABandAB = calculateNChildren(vc, triosToTest, REF, HET, HET); - final double nBBGivenABandAB = calculateNChildren(vc, triosToTest, HOM, HET, HET); - final double nAAGivenAAandAB = calculateNChildren(vc, triosToTest, REF, REF, HET) + calculateNChildren(vc, triosToTest, REF, HET, REF); - final double nABGivenAAandAB = calculateNChildren(vc, triosToTest, HET, REF, HET) + calculateNChildren(vc, triosToTest, HET, HET, REF); + double nABGivenABandBB = 0.0; + double nBBGivenABandBB = 0.0; + double nAAGivenABandAB = 0.0; + double nBBGivenABandAB = 0.0; + double nAAGivenAAandAB = 0.0; + double nABGivenAAandAB = 0.0; + + // for each pair of alleles, add the likelihoods + int numAlleles = vc.getNAlleles(); + for ( int allele1 = 0; allele1 < numAlleles; allele1++ ) { + for ( int allele2 = allele1 + 1; allele2 < numAlleles; allele2++ ) { + + // TODO -- cache these for better performance + final int HOM1index = determineHomIndex(allele1, numAlleles); + final int HETindex = HOM1index + (allele2 - allele1); + final int HOM2index = determineHomIndex(allele2, numAlleles); + + nABGivenABandBB += calculateNChildren(vc, triosToTest, HETindex, HETindex, HOM2index) + calculateNChildren(vc, triosToTest, HETindex, HOM2index, HETindex); + nBBGivenABandBB += calculateNChildren(vc, triosToTest, HOM2index, HETindex, HOM2index) + calculateNChildren(vc, triosToTest, HOM2index, HOM2index, HETindex); + nAAGivenABandAB += calculateNChildren(vc, triosToTest, HOM1index, HETindex, HETindex); + nBBGivenABandAB += calculateNChildren(vc, triosToTest, HOM2index, HETindex, HETindex); + nAAGivenAAandAB += calculateNChildren(vc, triosToTest, HOM1index, HOM1index, HETindex) + calculateNChildren(vc, triosToTest, HOM1index, HETindex, HOM1index); + nABGivenAAandAB += calculateNChildren(vc, triosToTest, HETindex, HOM1index, HETindex) + calculateNChildren(vc, triosToTest, HETindex, HETindex, HOM1index); + } + } final double numer = (nABGivenABandBB - nBBGivenABandBB) + 2.0 * (nAAGivenABandAB - nBBGivenABandAB) + (nAAGivenAAandAB - nABGivenAAandAB); final double denom = (nABGivenABandBB + nBBGivenABandBB) + 4.0 * (nAAGivenABandAB + nBBGivenABandAB) + (nAAGivenAAandAB + nABGivenAAandAB); return (numer * numer) / denom; } - private double calculateNChildren( final VariantContext vc, final Set triosToTest, final int childIdx, final int parent1Idx, final int parent2Idx ) { + private double calculateNChildren( final VariantContext vc, final Set triosToTest, final int childIdx, final int momIdx, final int dadIdx ) { final double likelihoodVector[] = new double[triosToTest.size()]; int iii = 0; for( final Sample child : triosToTest ) { final double[] momGL = vc.getGenotype(child.getMaternalID()).getLikelihoods().getAsVector(); final double[] dadGL = vc.getGenotype(child.getPaternalID()).getLikelihoods().getAsVector(); final double[] childGL = vc.getGenotype(child.getID()).getLikelihoods().getAsVector(); - likelihoodVector[iii++] = momGL[parent1Idx] + dadGL[parent2Idx] + childGL[childIdx]; + likelihoodVector[iii++] = momGL[momIdx] + dadGL[dadIdx] + childGL[childIdx]; } return MathUtils.sumLog10(likelihoodVector); } + + private static int determineHomIndex(final int alleleIndex, int numAlleles) { + int result = 0; + for ( int i = 0; i < alleleIndex; i++ ) + result += numAlleles--; + return result; + } } From 79da5ca2aeab2204855469ee04af1e09b95de39e Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 3 Feb 2012 09:50:26 -0500 Subject: [PATCH 203/356] Parses major and minor GATK versions now. Added unit tests From 53e6d666010eca6cae5e83cc6cdffe97be713c8f Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 3 Feb 2012 12:31:29 -0500 Subject: [PATCH 204/356] analyzeRunReports pushes full stack trace to DB for reporting -- Minor parsing changes to handle this. From 5af3999a2d778231e0321b43203dcf85156fa2e8 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Fri, 3 Feb 2012 14:04:58 -0500 Subject: [PATCH 205/356] updating HaplotypeCaller integration tests From e1d69e4060b4791f126eaf534f12152c5e067b98 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Wed, 1 Feb 2012 19:34:39 -0500 Subject: [PATCH 206/356] make the size of a GenomeLoc int instead of long it will never be bigger than an int and it's actually useful to be an int so we can use it as parameters to array/list/hash size creation. --- .../src/org/broadinstitute/sting/utils/GenomeLoc.java | 2 +- .../sting/utils/interval/IntervalUtils.java | 4 ++-- .../sting/utils/interval/IntervalUtilsUnitTest.java | 9 ++++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java b/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java index ad10b61e7a..41ca58157b 100644 --- a/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java +++ b/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java @@ -436,7 +436,7 @@ public boolean endsAt(GenomeLoc that) { * never be < 1. */ @Ensures("result > 0") - public long size() { + public int size() { return stop - start + 1; } diff --git a/public/java/src/org/broadinstitute/sting/utils/interval/IntervalUtils.java b/public/java/src/org/broadinstitute/sting/utils/interval/IntervalUtils.java index f8655f74a5..ea1eaeb514 100644 --- a/public/java/src/org/broadinstitute/sting/utils/interval/IntervalUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/interval/IntervalUtils.java @@ -314,10 +314,10 @@ else if (file.exists()) * @param reference The reference for the intervals. * @return A map of contig names with their sizes. */ - public static Map getContigSizes(File reference) { + public static Map getContigSizes(File reference) { ReferenceDataSource referenceSource = new ReferenceDataSource(reference); List locs = GenomeLocSortedSet.createSetFromSequenceDictionary(referenceSource.getReference().getSequenceDictionary()).toList(); - Map lengths = new LinkedHashMap(); + Map lengths = new LinkedHashMap(); for (GenomeLoc loc: locs) lengths.put(loc.getContig(), loc.size()); return lengths; diff --git a/public/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsUnitTest.java index a9035ffd92..0a8caa8cc9 100644 --- a/public/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsUnitTest.java @@ -8,13 +8,12 @@ import org.broadinstitute.sting.commandline.IntervalBinding; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.gatk.datasources.reference.ReferenceDataSource; -import org.broadinstitute.sting.utils.GenomeLocSortedSet; -import org.testng.Assert; -import org.broadinstitute.sting.utils.exceptions.UserException; import org.broadinstitute.sting.utils.GenomeLoc; import org.broadinstitute.sting.utils.GenomeLocParser; +import org.broadinstitute.sting.utils.GenomeLocSortedSet; +import org.broadinstitute.sting.utils.exceptions.UserException; import org.broadinstitute.sting.utils.fasta.CachingIndexedFastaSequenceFile; - +import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -341,7 +340,7 @@ public void testOverlappingIntervalsFromSameSourceWithIntersection() { @Test public void testGetContigLengths() { - Map lengths = IntervalUtils.getContigSizes(new File(BaseTest.hg18Reference)); + Map lengths = IntervalUtils.getContigSizes(new File(BaseTest.hg18Reference)); Assert.assertEquals((long)lengths.get("chr1"), 247249719); Assert.assertEquals((long)lengths.get("chr2"), 242951149); Assert.assertEquals((long)lengths.get("chr3"), 199501827); From 3dd6a1f96272a6794108d88ac14ab8f892996ae4 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Wed, 1 Feb 2012 19:35:09 -0500 Subject: [PATCH 207/356] Adding some generic sum and average functions to MathUtils --- .../broadinstitute/sting/utils/MathUtils.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java index 2f2dbd47e4..814cb27656 100644 --- a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java @@ -137,6 +137,10 @@ public static int nonNanSize(Collection numbers) { return size; } + + public static double average(Collection x) { + return (double) sum(x) / x.size(); + } public static double average(Collection numbers, boolean ignoreNan) { if (ignoreNan) { @@ -176,6 +180,13 @@ public static double sum(double[] values) { return s; } + public static long sum(int[] x) { + long total = 0; + for (int v : x) + total += v; + return total; + } + /** * Calculates the log10 cumulative sum of an array with log10 probabilities @@ -722,6 +733,13 @@ public static double average(List vals) { return average(vals, vals.size()); } + public static double average(int[] x) { + int sum = 0; + for (int v : x) + sum += v; + return (double) sum / x.length; + } + public static byte average(byte[] vals) { int sum = 0; for (byte v : vals) { @@ -1079,6 +1097,13 @@ public static byte getQScoreMedian(List reads, List offsets) return getQScoreOrderStatistic(reads, offsets, (int) Math.floor(reads.size() / 2.)); } + public static long sum(Collection x) { + long sum = 0; + for (int v : x) + sum += v; + return sum; + } + /** * A utility class that computes on the fly average and standard deviation for a stream of numbers. * The number of observations does not have to be known in advance, and can be also very big (so that From 4a57add6d0e7591bdbcd059d0195061d3a6a8152 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Wed, 1 Feb 2012 19:35:33 -0500 Subject: [PATCH 208/356] First implementation of DiagnoseTargets * calculates and interprets the coverage of a given interval track * allows to expand intervals by specified number of bases * classifies targets as CALLABLE, LOW_COVERAGE, EXCESSIVE_COVERAGE and POOR_QUALITY. * outputs text file for now (testing purposes only), soon to be VCF. * filters are overly aggressive for now. --- .../diagnostics/targets/CallableStatus.java | 22 ++ .../diagnostics/targets/DiagnoseTargets.java | 172 ++++++++++++ .../targets/IntervalStatisticLocus.java | 34 +++ .../targets/IntervalStatistics.java | 122 +++++++++ .../broadinstitute/sting/utils/MathUtils.java | 255 ++++++++---------- 5 files changed, 458 insertions(+), 147 deletions(-) create mode 100644 public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/targets/CallableStatus.java create mode 100644 public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/targets/DiagnoseTargets.java create mode 100644 public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/targets/IntervalStatisticLocus.java create mode 100644 public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/targets/IntervalStatistics.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/targets/CallableStatus.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/targets/CallableStatus.java new file mode 100644 index 0000000000..60f20074ae --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/targets/CallableStatus.java @@ -0,0 +1,22 @@ +package org.broadinstitute.sting.gatk.walkers.diagnostics.targets; + +/** + * Short one line description of the walker. + * + * @author Mauricio Carneiro + * @since 2/1/12 + */ +public enum CallableStatus { + /** the reference base was an N, which is not considered callable the GATK */ + REF_N, + /** the base satisfied the min. depth for calling but had less than maxDepth to avoid having EXCESSIVE_COVERAGE */ + CALLABLE, + /** absolutely no reads were seen at this locus, regardless of the filtering parameters */ + NO_COVERAGE, + /** there were less than min. depth bases at the locus, after applying filters */ + LOW_COVERAGE, + /** more than -maxDepth read at the locus, indicating some sort of mapping problem */ + EXCESSIVE_COVERAGE, + /** more than --maxFractionOfReadsWithLowMAPQ at the locus, indicating a poor mapping quality of the reads */ + POOR_QUALITY +} diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/targets/DiagnoseTargets.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/targets/DiagnoseTargets.java new file mode 100644 index 0000000000..979fb665f1 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/targets/DiagnoseTargets.java @@ -0,0 +1,172 @@ +package org.broadinstitute.sting.gatk.walkers.diagnostics.targets; + +import org.broad.tribble.Feature; +import org.broadinstitute.sting.commandline.Argument; +import org.broadinstitute.sting.commandline.Input; +import org.broadinstitute.sting.commandline.IntervalBinding; +import org.broadinstitute.sting.commandline.Output; +import org.broadinstitute.sting.gatk.contexts.AlignmentContext; +import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; +import org.broadinstitute.sting.gatk.walkers.By; +import org.broadinstitute.sting.gatk.walkers.DataSource; +import org.broadinstitute.sting.gatk.walkers.LocusWalker; +import org.broadinstitute.sting.utils.GenomeLoc; +import org.broadinstitute.sting.utils.GenomeLocComparator; +import org.broadinstitute.sting.utils.GenomeLocParser; +import org.broadinstitute.sting.utils.exceptions.UserException; + +import java.io.PrintStream; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.TreeSet; + +/** + * Short one line description of the walker. + * + *

+ * [Long description of the walker] + *

+ * + * + *

Input

+ *

+ * [Description of the Input] + *

+ * + *

Output

+ *

+ * [Description of the Output] + *

+ * + *

Examples

+ * 
+ *    java
+ *      -jar GenomeAnalysisTK.jar
+ *      -T [walker name]
+ *
+ * + * @author Mauricio Carneiro + * @since 2/1/12 + */ +@By(value = DataSource.READS) +public class DiagnoseTargets extends LocusWalker { + @Input(fullName = "interval_track", shortName = "int", doc = "", required = true) + private IntervalBinding intervalTrack = null; + + @Output + private PrintStream out = System.out; + + @Argument(fullName = "expand_interval", shortName = "exp", doc = "", required = false) + private int expandInterval = 50; + + @Argument(fullName = "minimum_base_quality", shortName = "mbq", doc = "", required = false) + private int minimumBaseQuality = 20; + + @Argument(fullName = "minimum_mapping_quality", shortName = "mmq", doc = "", required = false) + private int minimumMappingQuality = 20; + + @Argument(fullName = "minimum_coverage", shortName = "mincov", doc = "", required = false) + private int minimumCoverage = 5; + + @Argument(fullName = "maximum_coverage", shortName = "maxcov", doc = "", required = false) + private int maximumCoverage = 700; + + private TreeSet intervalList = null; // The list of intervals of interest (plus expanded intervals if user wants them) + private HashMap intervalMap = null; // interval => statistics + private Iterator intervalListIterator; // An iterator to go over all the intervals provided as we traverse the genome + private GenomeLoc currentInterval = null; // The "current" interval loaded and being filled with statistics + private IntervalStatistics currentIntervalStatistics = null; // The "current" interval loaded and being filled with statistics + + private GenomeLocParser parser; // just an object to allow us to create genome locs (for the expanded intervals) + + @Override + public void initialize() { + super.initialize(); + + if (intervalTrack == null) + throw new UserException("This tool currently only works if you provide an interval track"); + + parser = new GenomeLocParser(getToolkit().getMasterSequenceDictionary()); // Important to initialize the parser before creating the intervals below + + List originalList = intervalTrack.getIntervals(getToolkit()); // The original list of targets provided by the user that will be expanded or not depending on the options provided + intervalList = new TreeSet(new GenomeLocComparator()); + intervalMap = new HashMap(originalList.size() * 2); + for (GenomeLoc interval : originalList) + addAndExpandIntervalToLists(interval); + + intervalListIterator = intervalList.iterator(); + } + + @Override + public Long map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) { + GenomeLoc refLocus = ref.getLocus(); + while (currentInterval == null || currentInterval.isBefore(refLocus)) { + if (!intervalListIterator.hasNext()) + return 0L; + + currentInterval = intervalListIterator.next(); + currentIntervalStatistics = intervalMap.get(currentInterval); + } + + if (currentInterval.isPast(refLocus)) + return 0L; + + byte[] mappingQualities = context.getBasePileup().getMappingQuals(); + byte[] baseQualities = context.getBasePileup().getQuals(); + int coverage = context.getBasePileup().getBaseAndMappingFilteredPileup(minimumBaseQuality, minimumMappingQuality).depthOfCoverage(); + int rawCoverage = context.size(); + + IntervalStatisticLocus locusData = new IntervalStatisticLocus(mappingQualities, baseQualities, coverage, rawCoverage); + currentIntervalStatistics.addLocus(refLocus, locusData); + + return 1L; + } + + @Override + public Long reduceInit() { + return 0L; + } + + @Override + public Long reduce(Long value, Long sum) { + return sum + value; + } + + @Override + public void onTraversalDone(Long result) { + super.onTraversalDone(result); + out.println("Interval\tCallStatus\tCOV\tAVG"); + for (GenomeLoc interval : intervalList) { + IntervalStatistics stats = intervalMap.get(interval); + out.println(String.format("%s\t%s\t%d\t%f", interval, stats.callableStatus(), stats.totalCoverage(), stats.averageCoverage())); + } + } + + private GenomeLoc createIntervalBefore(GenomeLoc interval) { + int start = Math.max(interval.getStart() - expandInterval, 0); + int stop = Math.max(interval.getStart() - 1, 0); + return parser.createGenomeLoc(interval.getContig(), interval.getContigIndex(), start, stop); + } + + private GenomeLoc createIntervalAfter(GenomeLoc interval) { + int contigLimit = getToolkit().getSAMFileHeader().getSequenceDictionary().getSequence(interval.getContigIndex()).getSequenceLength(); + int start = Math.min(interval.getStop() + 1, contigLimit); + int stop = Math.min(interval.getStop() + expandInterval, contigLimit); + return parser.createGenomeLoc(interval.getContig(), interval.getContigIndex(), start, stop); + } + + private void addAndExpandIntervalToLists(GenomeLoc interval) { + if (expandInterval > 0) { + GenomeLoc before = createIntervalBefore(interval); + GenomeLoc after = createIntervalAfter(interval); + intervalList.add(before); + intervalList.add(after); + intervalMap.put(before, new IntervalStatistics(before, minimumCoverage, maximumCoverage, minimumMappingQuality, minimumBaseQuality)); + intervalMap.put(after, new IntervalStatistics(after, minimumCoverage, maximumCoverage, minimumMappingQuality, minimumBaseQuality)); + } + intervalList.add(interval); + intervalMap.put(interval, new IntervalStatistics(interval, minimumCoverage, maximumCoverage, minimumMappingQuality, minimumBaseQuality)); + } +} diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/targets/IntervalStatisticLocus.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/targets/IntervalStatisticLocus.java new file mode 100644 index 0000000000..5620c3902a --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/targets/IntervalStatisticLocus.java @@ -0,0 +1,34 @@ +package org.broadinstitute.sting.gatk.walkers.diagnostics.targets; + +/** + * The definition of a locus for the DiagnoseTargets walker statistics calculation + * + * @author Mauricio Carneiro + * @since 2/3/12 + */ +class IntervalStatisticLocus { + private final byte[] mappingQuality; + private final byte[] baseQuality; + private final int coverage; + private final int rawCoverage; + + public IntervalStatisticLocus(byte[] mappingQuality, byte[] baseQuality, int coverage, int rawCoverage) { + this.mappingQuality = mappingQuality; + this.baseQuality = baseQuality; + this.coverage = coverage; + this.rawCoverage = rawCoverage; + } + + public IntervalStatisticLocus() { + this(new byte[1], new byte[1], 0, 0); + } + + public int getCoverage() { + return coverage; + } + + public int getRawCoverage() { + return rawCoverage; + } + +} diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/targets/IntervalStatistics.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/targets/IntervalStatistics.java new file mode 100644 index 0000000000..8ee5f76fb7 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/targets/IntervalStatistics.java @@ -0,0 +1,122 @@ +package org.broadinstitute.sting.gatk.walkers.diagnostics.targets; + +import org.broadinstitute.sting.utils.GenomeLoc; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; + +import java.util.ArrayList; +import java.util.HashMap; + +/** + * Short one line description of the walker. + * + * @author Mauricio Carneiro + * @since 2/1/12 + */ +class IntervalStatistics { + private final GenomeLoc interval; + private final ArrayList loci; + + private final int minimumCoverageThreshold; + private final int maximumCoverageThreshold; + private final int minimumMappingQuality; + private final int minimumBaseQuality; + + private int preComputedTotalCoverage = -1; // avoids re-calculating the total sum (-1 means we haven't pre-computed it yet) + + private IntervalStatistics(GenomeLoc interval, ArrayList loci, int minimumCoverageThreshold, int maximumCoverageThreshold, int minimumMappingQuality, int minimumBaseQuality) { + this.interval = interval; + this.loci = loci; + this.minimumCoverageThreshold = minimumCoverageThreshold; + this.maximumCoverageThreshold = maximumCoverageThreshold; + this.minimumMappingQuality = minimumMappingQuality; + this.minimumBaseQuality = minimumBaseQuality; + } + + public IntervalStatistics(GenomeLoc interval, int minimumCoverageThreshold, int maximumCoverageThreshold, int minimumMappingQuality, int minimumBaseQuality) { + this(interval, new ArrayList(interval.size()), minimumCoverageThreshold, maximumCoverageThreshold, minimumMappingQuality, minimumBaseQuality); + + // Initialize every loci (this way we don't have to worry about non-existent loci in the object + for (int i = 0; i < interval.size(); i++) + this.loci.add(i, new IntervalStatisticLocus()); + + } + + public long totalCoverage() { + if (preComputedTotalCoverage < 0) + calculateTotalCoverage(); + return preComputedTotalCoverage; + } + + public double averageCoverage() { + if (preComputedTotalCoverage < 0) + calculateTotalCoverage(); + return (double) preComputedTotalCoverage / loci.size(); + } + + /** + * Calculates the callable status of the entire interval + * + * @return the callable status of the entire interval + */ + public CallableStatus callableStatus() { + long max = -1; + CallableStatus maxCallableStatus = null; + HashMap statusCounts = new HashMap(CallableStatus.values().length); + + // initialize the statusCounts with all callable states + for (CallableStatus key : CallableStatus.values()) + statusCounts.put(key, 0); + + // calculate the callable status for each locus + for (int i = 0; i < loci.size(); i++) { + CallableStatus status = callableStatus(i); + int count = statusCounts.get(status) + 1; + statusCounts.put(status, count); + + if (count > max) { + max = count; + maxCallableStatus = status; + } + } + + return maxCallableStatus; + } + + public void addLocus(GenomeLoc locus, IntervalStatisticLocus locusData) { + if (!interval.containsP(locus)) + throw new ReviewedStingException(String.format("Locus %s is not part of the Interval", locus)); + + int locusIndex = locus.getStart() - interval.getStart(); + + loci.add(locusIndex, locusData); + } + + /** + * returns the callable status of this locus without taking the reference base into account. + * + * @param locusIndex location in the genome to inquire (only one locus) + * @return the callable status of a locus + */ + private CallableStatus callableStatus(int locusIndex) { + if (loci.get(locusIndex).getCoverage() > maximumCoverageThreshold) + return CallableStatus.EXCESSIVE_COVERAGE; + + if (loci.get(locusIndex).getCoverage() >= minimumCoverageThreshold) + return CallableStatus.CALLABLE; + + if (loci.get(locusIndex).getRawCoverage() >= minimumCoverageThreshold) + return CallableStatus.POOR_QUALITY; + + if (loci.get(locusIndex).getRawCoverage() > 0) + return CallableStatus.LOW_COVERAGE; + + return CallableStatus.NO_COVERAGE; + } + + private void calculateTotalCoverage() { + preComputedTotalCoverage = 0; + for (IntervalStatisticLocus locus : loci) + preComputedTotalCoverage += locus.getCoverage(); + } + +} diff --git a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java index 814cb27656..a4e9fc7ed0 100644 --- a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java @@ -49,7 +49,6 @@ public class MathUtils { * high precision */ - /** * Private constructor. No instantiating this class! */ @@ -60,48 +59,48 @@ private MathUtils() { // under/overflow checking, so this shouldn't be used in the general case (but is fine // if one is already make those checks before calling in to the rounding). public static int fastRound(double d) { - return (d > 0) ? (int)(d + 0.5d) : (int)(d - 0.5d); + return (d > 0) ? (int) (d + 0.5d) : (int) (d - 0.5d); } public static double approximateLog10SumLog10(final double[] vals) { - return approximateLog10SumLog10(vals, vals.length); + return approximateLog10SumLog10(vals, vals.length); } public static double approximateLog10SumLog10(final double[] vals, final int endIndex) { - final int maxElementIndex = MathUtils.maxElementIndex(vals, endIndex); - double approxSum = vals[maxElementIndex]; - if ( approxSum == Double.NEGATIVE_INFINITY ) + final int maxElementIndex = MathUtils.maxElementIndex(vals, endIndex); + double approxSum = vals[maxElementIndex]; + if (approxSum == Double.NEGATIVE_INFINITY) return approxSum; - for ( int i = 0; i < endIndex; i++ ) { - if ( i == maxElementIndex || vals[i] == Double.NEGATIVE_INFINITY ) - continue; - - final double diff = approxSum - vals[i]; - if ( diff < MathUtils.MAX_JACOBIAN_TOLERANCE ) { - // See notes from the 2-inout implementation below - final int ind = fastRound(diff / MathUtils.JACOBIAN_LOG_TABLE_STEP); // hard rounding - approxSum += MathUtils.jacobianLogTable[ind]; - } - } + for (int i = 0; i < endIndex; i++) { + if (i == maxElementIndex || vals[i] == Double.NEGATIVE_INFINITY) + continue; + + final double diff = approxSum - vals[i]; + if (diff < MathUtils.MAX_JACOBIAN_TOLERANCE) { + // See notes from the 2-inout implementation below + final int ind = fastRound(diff / MathUtils.JACOBIAN_LOG_TABLE_STEP); // hard rounding + approxSum += MathUtils.jacobianLogTable[ind]; + } + } return approxSum; } public static double approximateLog10SumLog10(double small, double big) { // make sure small is really the smaller value - if ( small > big ) { + if (small > big) { final double t = big; big = small; small = t; } - if ( small == Double.NEGATIVE_INFINITY || big == Double.NEGATIVE_INFINITY ) + if (small == Double.NEGATIVE_INFINITY || big == Double.NEGATIVE_INFINITY) return big; - final double diff = big - small; - if ( diff >= MathUtils.MAX_JACOBIAN_TOLERANCE ) + final double diff = big - small; + if (diff >= MathUtils.MAX_JACOBIAN_TOLERANCE) return big; // OK, so |y-x| < tol: we use the following identity then: @@ -137,7 +136,7 @@ public static int nonNanSize(Collection numbers) { return size; } - + public static double average(Collection x) { return (double) sum(x) / x.size(); } @@ -145,7 +144,8 @@ public static double average(Collection x) { public static double average(Collection numbers, boolean ignoreNan) { if (ignoreNan) { return sum(numbers, true) / nonNanSize(numbers); - } else { + } + else { return sum(numbers, false) / nonNanSize(numbers); } } @@ -176,7 +176,8 @@ public static double variance(Collection numbers) { public static double sum(double[] values) { double s = 0.0; - for (double v : values) s += v; + for (double v : values) + s += v; return s; } @@ -187,7 +188,6 @@ public static long sum(int[] x) { return total; } - /** * Calculates the log10 cumulative sum of an array with log10 probabilities * @@ -229,21 +229,23 @@ public static double log10sumLog10(double[] log10p, int start, int finish) { public static double sumDoubles(List values) { double s = 0.0; - for (double v : values) s += v; + for (double v : values) + s += v; return s; } public static int sumIntegers(List values) { int s = 0; - for (int v : values) s += v; + for (int v : values) + s += v; return s; } public static double sumLog10(double[] log10values) { return Math.pow(10.0, log10sumLog10(log10values)); -// double s = 0.0; -// for ( double v : log10values) s += Math.pow(10.0, v); -// return s; + // double s = 0.0; + // for ( double v : log10values) s += Math.pow(10.0, v); + // return s; } public static double log10sumLog10(double[] log10values) { @@ -456,7 +458,6 @@ public static double rms(byte[] x) { return Math.sqrt(rms); } - /** * calculate the Root Mean Square of an array of integers * @@ -517,7 +518,6 @@ public static double round(double num, int digits) { return result; } - /** * normalizes the log10-based array. ASSUMES THAT ALL ARRAY ENTRIES ARE <= 0 (<= 1 IN REAL-SPACE). * @@ -554,7 +554,8 @@ public static double[] normalizeFromLog10(double[] array, boolean takeLog10OfOut sum += normalized[i]; for (int i = 0; i < array.length; i++) { double x = normalized[i] / sum; - if (takeLog10OfOutput) x = Math.log10(x); + if (takeLog10OfOutput) + x = Math.log10(x); normalized[i] = x; } @@ -576,7 +577,8 @@ public static double[] normalizeFromLog10(List array, boolean takeLog10O sum += normalized[i]; for (int i = 0; i < array.size(); i++) { double x = normalized[i] / sum; - if (takeLog10OfOutput) x = Math.log10(x); + if (takeLog10OfOutput) + x = Math.log10(x); normalized[i] = x; } @@ -598,11 +600,12 @@ public static double[] normalizeFromLog10(List array) { } public static int maxElementIndex(final double[] array) { - return maxElementIndex(array, array.length); + return maxElementIndex(array, array.length); } public static int maxElementIndex(final double[] array, final int endIndex) { - if (array == null) throw new IllegalArgumentException("Array cannot be null!"); + if (array == null) + throw new IllegalArgumentException("Array cannot be null!"); int maxI = -1; for (int i = 0; i < endIndex; i++) { @@ -614,11 +617,12 @@ public static int maxElementIndex(final double[] array, final int endIndex) { } public static int maxElementIndex(final int[] array) { - return maxElementIndex(array, array.length); + return maxElementIndex(array, array.length); } public static int maxElementIndex(final int[] array, int endIndex) { - if (array == null) throw new IllegalArgumentException("Array cannot be null!"); + if (array == null) + throw new IllegalArgumentException("Array cannot be null!"); int maxI = -1; for (int i = 0; i < endIndex; i++) { @@ -646,7 +650,8 @@ public static byte arrayMin(byte[] array) { } public static int minElementIndex(double[] array) { - if (array == null) throw new IllegalArgumentException("Array cannot be null!"); + if (array == null) + throw new IllegalArgumentException("Array cannot be null!"); int minI = -1; for (int i = 0; i < array.length; i++) { @@ -658,7 +663,8 @@ public static int minElementIndex(double[] array) { } public static int minElementIndex(byte[] array) { - if (array == null) throw new IllegalArgumentException("Array cannot be null!"); + if (array == null) + throw new IllegalArgumentException("Array cannot be null!"); int minI = -1; for (int i = 0; i < array.length; i++) { @@ -670,7 +676,8 @@ public static int minElementIndex(byte[] array) { } public static int minElementIndex(int[] array) { - if (array == null) throw new IllegalArgumentException("Array cannot be null!"); + if (array == null) + throw new IllegalArgumentException("Array cannot be null!"); int minI = -1; for (int i = 0; i < array.length; i++) { @@ -682,20 +689,26 @@ public static int minElementIndex(int[] array) { } public static int arrayMaxInt(List array) { - if (array == null) throw new IllegalArgumentException("Array cannot be null!"); - if (array.size() == 0) throw new IllegalArgumentException("Array size cannot be 0!"); + if (array == null) + throw new IllegalArgumentException("Array cannot be null!"); + if (array.size() == 0) + throw new IllegalArgumentException("Array size cannot be 0!"); int m = array.get(0); - for (int e : array) m = Math.max(m, e); + for (int e : array) + m = Math.max(m, e); return m; } public static double arrayMaxDouble(List array) { - if (array == null) throw new IllegalArgumentException("Array cannot be null!"); - if (array.size() == 0) throw new IllegalArgumentException("Array size cannot be 0!"); + if (array == null) + throw new IllegalArgumentException("Array cannot be null!"); + if (array.size() == 0) + throw new IllegalArgumentException("Array size cannot be 0!"); double m = array.get(0); - for (double e : array) m = Math.max(m, e); + for (double e : array) + m = Math.max(m, e); return m; } @@ -816,7 +829,6 @@ public int compare(Integer a, Integer b) { return permutation; } - public static int[] permuteArray(int[] array, Integer[] permutation) { int[] output = new int[array.length]; for (int i = 0; i < output.length; i++) { @@ -857,7 +869,6 @@ public static List permuteList(List list, Integer[] permutation) { return output; } - /** * Draw N random elements from list. */ @@ -923,7 +934,8 @@ public static int countOccurrences(char c, String s) { public static int countOccurrences(T x, List l) { int count = 0; for (T y : l) { - if (x.equals(y)) count++; + if (x.equals(y)) + count++; } return count; @@ -1031,9 +1043,11 @@ public static Comparable orderStatisticSearch(int orderStat, List li for (Comparable y : list) { if (x.compareTo(y) > 0) { lessThanX.add(y); - } else if (x.compareTo(y) < 0) { + } + else if (x.compareTo(y) < 0) { greaterThanX.add(y); - } else + } + else equalToX.add(y); } @@ -1046,7 +1060,6 @@ else if (lessThanX.size() + equalToX.size() >= orderStat) } - public static Object getMedian(List list) { return orderStatisticSearch((int) Math.ceil(list.size() / 2), list); } @@ -1076,10 +1089,12 @@ public static byte getQScoreOrderStatistic(List reads, List if (quality < qk) { lessThanQReads.add(read); lessThanQOffsets.add(offset); - } else if (quality > qk) { + } + else if (quality > qk) { greaterThanQReads.add(read); greaterThanQOffsets.add(offset); - } else { + } + else { equalToQReads.add(reads.get(iter)); } } @@ -1100,7 +1115,7 @@ public static byte getQScoreMedian(List reads, List offsets) public static long sum(Collection x) { long sum = 0; for (int v : x) - sum += v; + sum += v; return sum; } @@ -1209,8 +1224,7 @@ public static double ratio(long num, long denom) { log10Cache[k] = Math.log10(k); for (int k = 0; k < JACOBIAN_LOG_TABLE_SIZE; k++) { - jacobianLogTable[k] = Math.log10(1.0 + Math.pow(10.0, -((double) k) - * JACOBIAN_LOG_TABLE_STEP)); + jacobianLogTable[k] = Math.log10(1.0 + Math.pow(10.0, -((double) k) * JACOBIAN_LOG_TABLE_STEP)); } } @@ -1257,7 +1271,8 @@ else if (diff < -MAX_JACOBIAN_TOLERANCE) else if (diff >= 0) { int ind = (int) (diff * INV_JACOBIAN_LOG_TABLE_STEP + 0.5); return x + jacobianLogTable[ind]; - } else { + } + else { int ind = (int) (-diff * INV_JACOBIAN_LOG_TABLE_STEP + 0.5); return y + jacobianLogTable[ind]; } @@ -1298,71 +1313,7 @@ public static double lnToLog10(double ln) { /** * Constants to simplify the log gamma function calculation. */ - private static final double - zero = 0.0, - one = 1.0, - half = .5, - a0 = 7.72156649015328655494e-02, - a1 = 3.22467033424113591611e-01, - a2 = 6.73523010531292681824e-02, - a3 = 2.05808084325167332806e-02, - a4 = 7.38555086081402883957e-03, - a5 = 2.89051383673415629091e-03, - a6 = 1.19270763183362067845e-03, - a7 = 5.10069792153511336608e-04, - a8 = 2.20862790713908385557e-04, - a9 = 1.08011567247583939954e-04, - a10 = 2.52144565451257326939e-05, - a11 = 4.48640949618915160150e-05, - tc = 1.46163214496836224576e+00, - tf = -1.21486290535849611461e-01, - tt = -3.63867699703950536541e-18, - t0 = 4.83836122723810047042e-01, - t1 = -1.47587722994593911752e-01, - t2 = 6.46249402391333854778e-02, - t3 = -3.27885410759859649565e-02, - t4 = 1.79706750811820387126e-02, - t5 = -1.03142241298341437450e-02, - t6 = 6.10053870246291332635e-03, - t7 = -3.68452016781138256760e-03, - t8 = 2.25964780900612472250e-03, - t9 = -1.40346469989232843813e-03, - t10 = 8.81081882437654011382e-04, - t11 = -5.38595305356740546715e-04, - t12 = 3.15632070903625950361e-04, - t13 = -3.12754168375120860518e-04, - t14 = 3.35529192635519073543e-04, - u0 = -7.72156649015328655494e-02, - u1 = 6.32827064025093366517e-01, - u2 = 1.45492250137234768737e+00, - u3 = 9.77717527963372745603e-01, - u4 = 2.28963728064692451092e-01, - u5 = 1.33810918536787660377e-02, - v1 = 2.45597793713041134822e+00, - v2 = 2.12848976379893395361e+00, - v3 = 7.69285150456672783825e-01, - v4 = 1.04222645593369134254e-01, - v5 = 3.21709242282423911810e-03, - s0 = -7.72156649015328655494e-02, - s1 = 2.14982415960608852501e-01, - s2 = 3.25778796408930981787e-01, - s3 = 1.46350472652464452805e-01, - s4 = 2.66422703033638609560e-02, - s5 = 1.84028451407337715652e-03, - s6 = 3.19475326584100867617e-05, - r1 = 1.39200533467621045958e+00, - r2 = 7.21935547567138069525e-01, - r3 = 1.71933865632803078993e-01, - r4 = 1.86459191715652901344e-02, - r5 = 7.77942496381893596434e-04, - r6 = 7.32668430744625636189e-06, - w0 = 4.18938533204672725052e-01, - w1 = 8.33333333333329678849e-02, - w2 = -2.77777777728775536470e-03, - w3 = 7.93650558643019558500e-04, - w4 = -5.95187557450339963135e-04, - w5 = 8.36339918996282139126e-04, - w6 = -1.63092934096575273989e-03; + private static final double zero = 0.0, one = 1.0, half = .5, a0 = 7.72156649015328655494e-02, a1 = 3.22467033424113591611e-01, a2 = 6.73523010531292681824e-02, a3 = 2.05808084325167332806e-02, a4 = 7.38555086081402883957e-03, a5 = 2.89051383673415629091e-03, a6 = 1.19270763183362067845e-03, a7 = 5.10069792153511336608e-04, a8 = 2.20862790713908385557e-04, a9 = 1.08011567247583939954e-04, a10 = 2.52144565451257326939e-05, a11 = 4.48640949618915160150e-05, tc = 1.46163214496836224576e+00, tf = -1.21486290535849611461e-01, tt = -3.63867699703950536541e-18, t0 = 4.83836122723810047042e-01, t1 = -1.47587722994593911752e-01, t2 = 6.46249402391333854778e-02, t3 = -3.27885410759859649565e-02, t4 = 1.79706750811820387126e-02, t5 = -1.03142241298341437450e-02, t6 = 6.10053870246291332635e-03, t7 = -3.68452016781138256760e-03, t8 = 2.25964780900612472250e-03, t9 = -1.40346469989232843813e-03, t10 = 8.81081882437654011382e-04, t11 = -5.38595305356740546715e-04, t12 = 3.15632070903625950361e-04, t13 = -3.12754168375120860518e-04, t14 = 3.35529192635519073543e-04, u0 = -7.72156649015328655494e-02, u1 = 6.32827064025093366517e-01, u2 = 1.45492250137234768737e+00, u3 = 9.77717527963372745603e-01, u4 = 2.28963728064692451092e-01, u5 = 1.33810918536787660377e-02, v1 = 2.45597793713041134822e+00, v2 = 2.12848976379893395361e+00, v3 = 7.69285150456672783825e-01, v4 = 1.04222645593369134254e-01, v5 = 3.21709242282423911810e-03, s0 = -7.72156649015328655494e-02, s1 = 2.14982415960608852501e-01, s2 = 3.25778796408930981787e-01, s3 = 1.46350472652464452805e-01, s4 = 2.66422703033638609560e-02, s5 = 1.84028451407337715652e-03, s6 = 3.19475326584100867617e-05, r1 = 1.39200533467621045958e+00, r2 = 7.21935547567138069525e-01, r3 = 1.71933865632803078993e-01, r4 = 1.86459191715652901344e-02, r5 = 7.77942496381893596434e-04, r6 = 7.32668430744625636189e-06, w0 = 4.18938533204672725052e-01, w1 = 8.33333333333329678849e-02, w2 = -2.77777777728775536470e-03, w3 = 7.93650558643019558500e-04, w4 = -5.95187557450339963135e-04, w5 = 8.36339918996282139126e-04, w6 = -1.63092934096575273989e-03; /** * Efficient rounding functions to simplify the log gamma function calculation @@ -1393,14 +1344,17 @@ private static double lnGamma(double x) { /* purge off +-inf, NaN, +-0, and negative arguments */ int ix = hx & 0x7fffffff; - if (ix >= 0x7ff00000) return Double.POSITIVE_INFINITY; - if ((ix | lx) == 0 || hx < 0) return Double.NaN; + if (ix >= 0x7ff00000) + return Double.POSITIVE_INFINITY; + if ((ix | lx) == 0 || hx < 0) + return Double.NaN; if (ix < 0x3b900000) { /* |x|<2**-70, return -log(|x|) */ return -Math.log(x); } /* purge off 1 and 2 */ - if ((((ix - 0x3ff00000) | lx) == 0) || (((ix - 0x40000000) | lx) == 0)) r = 0; + if ((((ix - 0x3ff00000) | lx) == 0) || (((ix - 0x40000000) | lx) == 0)) + r = 0; /* for x < 2.0 */ else if (ix < 0x40000000) { if (ix <= 0x3feccccc) { /* lgamma(x) = lgamma(x+1)-log(x) */ @@ -1408,22 +1362,27 @@ else if (ix < 0x40000000) { if (ix >= 0x3FE76944) { y = one - x; i = 0; - } else if (ix >= 0x3FCDA661) { + } + else if (ix >= 0x3FCDA661) { y = x - (tc - one); i = 1; - } else { + } + else { y = x; i = 2; } - } else { + } + else { r = zero; if (ix >= 0x3FFBB4C3) { y = 2.0 - x; i = 0; - } /* [1.7316,2] */ else if (ix >= 0x3FF3B4C4) { + } /* [1.7316,2] */ + else if (ix >= 0x3FF3B4C4) { y = x - tc; i = 1; - } /* [1.23,1.73] */ else { + } /* [1.23,1.73] */ + else { y = x - one; i = 2; } @@ -1451,7 +1410,8 @@ else if (ix < 0x40000000) { p2 = one + y * (v1 + y * (v2 + y * (v3 + y * (v4 + y * v5)))); r += (-0.5 * y + p1 / p2); } - } else if (ix < 0x40200000) { /* x < 8.0 */ + } + else if (ix < 0x40200000) { /* x < 8.0 */ i = (int) x; t = zero; y = x - (double) i; @@ -1474,13 +1434,15 @@ else if (ix < 0x40000000) { break; } /* 8.0 <= x < 2**58 */ - } else if (ix < 0x43900000) { + } + else if (ix < 0x43900000) { t = Math.log(x); z = one / x; y = z * z; w = w0 + z * (w1 + y * (w2 + y * (w3 + y * (w4 + y * (w5 + y * w6))))); r = (x - half) * (t - one) + w; - } else + } + else /* 2**58 <= x <= inf */ r = x * (Math.log(x) - one); return r; @@ -1515,7 +1477,6 @@ public static double log10BinomialProbability(int n, int k, double log10p) { return log10BinomialCoefficient(n, k) + log10p * k + log10OneMinusP * (n - k); } - /** * Calculates the log10 of the multinomial coefficient. Designed to prevent * overflows even with very large numbers. @@ -1559,7 +1520,6 @@ public static double log10Factorial(int x) { return log10Gamma(x + 1); } - /** * Adds two arrays together and returns a new array with the sum. * @@ -1597,17 +1557,18 @@ public static Object[] arrayShuffle(Object[] array) { /** * Vector operations + * * @param v1 first numerical array * @param v2 second numerical array - * @return a new array with the elements added + * @return a new array with the elements added */ public static Double[] vectorSum(E v1[], E v2[]) { if (v1.length != v2.length) throw new UserException("BUG: vectors v1, v2 of different size in vectorSum()"); Double[] result = new Double[v1.length]; - for (int k=0; k < v1.length; k++) - result[k] = v1[k].doubleValue()+v2[k].doubleValue(); + for (int k = 0; k < v1.length; k++) + result[k] = v1[k].doubleValue() + v2[k].doubleValue(); return result; } @@ -1615,19 +1576,19 @@ public static Double[] vectorSum(E v1[], E v2[]) { public static Double[] scalarTimesVector(E a, E[] v1) { Double result[] = new Double[v1.length]; - for (int k=0; k < v1.length; k++) - result[k] = a.doubleValue()*v1[k].doubleValue(); + for (int k = 0; k < v1.length; k++) + result[k] = a.doubleValue() * v1[k].doubleValue(); return result; } - public static Double dotProduct(E[] v1, E[] v2) { + public static Double dotProduct(E[] v1, E[] v2) { if (v1.length != v2.length) throw new UserException("BUG: vectors v1, v2 of different size in vectorSum()"); Double result = 0.0; - for (int k=0; k < v1.length; k++) - result += v1[k].doubleValue() *v2[k].doubleValue(); + for (int k = 0; k < v1.length; k++) + result += v1[k].doubleValue() * v2[k].doubleValue(); return result; @@ -1635,7 +1596,7 @@ public static Double dotProduct(E[] v1, E[] v2) { public static double[] vectorLog10(double v1[]) { double result[] = new double[v1.length]; - for (int k=0; k < v1.length; k++) + for (int k = 0; k < v1.length; k++) result[k] = Math.log10(v1[k]); return result; @@ -1645,7 +1606,7 @@ public static double[] vectorLog10(double v1[]) { // todo - silly overloading, just because Java can't unbox/box arrays of primitive types, and we can't do generics with primitive types! public static Double[] vectorLog10(Double v1[]) { Double result[] = new Double[v1.length]; - for (int k=0; k < v1.length; k++) + for (int k = 0; k < v1.length; k++) result[k] = Math.log10(v1[k]); return result; From 894d3340be2a95a15c6c91f4785e8d8f1ef5776b Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Fri, 3 Feb 2012 17:13:52 -0500 Subject: [PATCH 209/356] Active Region Traversal should use GATKSAMRecords everywhere instead of SAMRecords. misc cleanup. --- .../analyzecovariates/AnalyzeCovariates.java | 7 +++---- .../traversals/TraverseActiveRegions.java | 21 +++++++++---------- .../recalibration/CountCovariatesWalker.java | 2 +- .../TableRecalibrationWalker.java | 2 +- .../broadinstitute/sting/utils/baq/BAQ.java | 2 +- 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java b/public/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java index a399867fa7..a999593413 100755 --- a/public/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java +++ b/public/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java @@ -139,11 +139,11 @@ public class AnalyzeCovariates extends CommandLineProgram { */ @Argument(fullName="max_histogram_value", shortName="maxHist", required = false, doc="If supplied, this value will be the max value of the histogram plots") private int MAX_HISTOGRAM_VALUE = 0; + @Hidden @Argument(fullName="do_indel_quality", shortName="indels", required = false, doc="If supplied, do indel quality plotting") private boolean DO_INDEL_QUALITY = false; - ///////////////////////////// // Private Member Variables ///////////////////////////// @@ -274,7 +274,6 @@ private void addCSVData(String line) { RecalDatum datum = new RecalDatum( Long.parseLong( vals[iii] ), Long.parseLong( vals[iii + 1] ), Double.parseDouble( vals[1] ), 0.0 ); // Add that datum to all the collapsed tables which will be used in the sequential calculation dataManager.addToAllTables( key, datum, IGNORE_QSCORES_LESS_THAN ); - } private void writeDataTables() { @@ -341,7 +340,7 @@ private void callRScripts() { // for each covariate for( int iii = 1; iii < requestedCovariates.size(); iii++ ) { - Covariate cov = requestedCovariates.get(iii); + final Covariate cov = requestedCovariates.get(iii); final File outputFile = new File(OUTPUT_DIR, readGroup + "." + cov.getClass().getSimpleName()+ ".dat"); if (DO_INDEL_QUALITY) { RScriptExecutor executor = new RScriptExecutor(); @@ -349,7 +348,7 @@ private void callRScripts() { // The second argument is the name of the covariate in order to make the plots look nice executor.addArgs(outputFile, cov.getClass().getSimpleName().split("Covariate")[0]); executor.exec(); - } else { + } else { if( iii == 1 ) { // Analyze reported quality RScriptExecutor executor = new RScriptExecutor(); diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java index ce8cb557b0..58c2df877e 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java +++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java @@ -1,6 +1,5 @@ package org.broadinstitute.sting.gatk.traversals; -import net.sf.samtools.SAMRecord; import org.apache.log4j.Logger; import org.broadinstitute.sting.gatk.WalkerManager; import org.broadinstitute.sting.gatk.contexts.AlignmentContext; @@ -29,7 +28,7 @@ public class TraverseActiveRegions extends TraversalEngine workQueue = new LinkedList(); - private final LinkedHashSet myReads = new LinkedHashSet(); + private final LinkedHashSet myReads = new LinkedHashSet(); @Override protected String getTraversalType() { @@ -101,7 +100,7 @@ public T traverse( final ActiveRegionWalker walker, // Grab all the previously unseen reads from this pileup and add them to the massive read list for( final PileupElement p : locus.getBasePileup() ) { - final SAMRecord read = p.getRead(); + final GATKSAMRecord read = p.getRead(); if( !myReads.contains(read) ) { myReads.add(read); } @@ -111,7 +110,7 @@ public T traverse( final ActiveRegionWalker walker, // which active regions in the work queue are now safe to process if( !locusView.hasNext() ) { for( final PileupElement p : locus.getBasePileup() ) { - final SAMRecord read = p.getRead(); + final GATKSAMRecord read = p.getRead(); if( !myReads.contains(read) ) { myReads.add(read); } @@ -156,9 +155,9 @@ public T endTraversal( final Walker walker, T sum) { return sum; } - private T processActiveRegion( final ActiveRegion activeRegion, final LinkedHashSet reads, final Queue workQueue, final T sum, final ActiveRegionWalker walker ) { - final ArrayList placedReads = new ArrayList(); - for( final SAMRecord read : reads ) { + private T processActiveRegion( final ActiveRegion activeRegion, final LinkedHashSet reads, final Queue workQueue, final T sum, final ActiveRegionWalker walker ) { + final ArrayList placedReads = new ArrayList(); + for( final GATKSAMRecord read : reads ) { final GenomeLoc readLoc = this.engine.getGenomeLocParser().createGenomeLoc( read ); if( activeRegion.getLocation().overlapsP( readLoc ) ) { // The region which the highest amount of overlap is chosen as the primary region for the read (tie breaking is done as right most region) @@ -170,22 +169,22 @@ private T processActiveRegion( final ActiveRegion activeRegion, final LinkedHash bestRegion = otherRegionToTest; } } - bestRegion.add( (GATKSAMRecord) read ); + bestRegion.add( read ); // The read is also added to all other regions in which it overlaps but marked as non-primary if( walker.wantsNonPrimaryReads() ) { if( !bestRegion.equals(activeRegion) ) { - activeRegion.add( (GATKSAMRecord) read ); + activeRegion.add( read ); } for( final ActiveRegion otherRegionToTest : workQueue ) { if( !bestRegion.equals(otherRegionToTest) && otherRegionToTest.getExtendedLoc().overlapsP( readLoc ) ) { - otherRegionToTest.add( (GATKSAMRecord) read ); + otherRegionToTest.add( read ); } } } placedReads.add( read ); } else if( activeRegion.getExtendedLoc().overlapsP( readLoc ) && walker.wantsNonPrimaryReads() ) { - activeRegion.add( (GATKSAMRecord) read ); + activeRegion.add( read ); } } reads.removeAll( placedReads ); // remove all the reads which have been placed into their active region diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java index fdfb29da62..f6f05d39c5 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java @@ -354,7 +354,7 @@ public CountedData map( RefMetaDataTracker tracker, ReferenceContext ref, Alignm if( tracker.getValues(knownSites).size() == 0 ) { // If something here is in one of the knownSites tracks then skip over it, otherwise proceed // For each read at this locus for( final PileupElement p : context.getBasePileup() ) { - final GATKSAMRecord gatkRead = (GATKSAMRecord) p.getRead(); + final GATKSAMRecord gatkRead = p.getRead(); int offset = p.getOffset(); if( gatkRead.containsTemporaryAttribute( SKIP_RECORD_ATTRIBUTE ) ) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java index 1ce02a3cf3..6e214c6bb9 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java @@ -399,7 +399,7 @@ public SAMRecord map( ReferenceContext refBases, GATKSAMRecord read, ReadMetaDat //compute all covariate values for this read final Comparable[][] covariateValues_offset_x_covar = - RecalDataManager.computeCovariates((GATKSAMRecord) read, requestedCovariates); + RecalDataManager.computeCovariates(read, requestedCovariates); // For each base in the read for( int offset = 0; offset < read.getReadLength(); offset++ ) { diff --git a/public/java/src/org/broadinstitute/sting/utils/baq/BAQ.java b/public/java/src/org/broadinstitute/sting/utils/baq/BAQ.java index 4f096f86e1..1864522942 100644 --- a/public/java/src/org/broadinstitute/sting/utils/baq/BAQ.java +++ b/public/java/src/org/broadinstitute/sting/utils/baq/BAQ.java @@ -673,7 +673,7 @@ public byte[] baqRead(SAMRecord read, IndexedFastaSequenceFile refReader, Calcul } /** - * Returns true if we don't think this read is eligable for the BAQ calculation. Examples include non-PF reads, + * Returns true if we don't think this read is eligible for the BAQ calculation. Examples include non-PF reads, * duplicates, or unmapped reads. Used by baqRead to determine if a read should fall through the calculation. * * @param read From 2cd33b2f1f20abc481e7ab77d9a6650933a6e249 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sat, 4 Feb 2012 08:22:12 -0500 Subject: [PATCH 210/356] Better display of LSF usage for gsafolk From 5343f8ba67cccb90fd91ad77e09f8bd9a2a2d7f5 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Sun, 5 Feb 2012 13:09:03 -0500 Subject: [PATCH 211/356] Initial version of on-the-fly, lazy loading base quality score recalibration. It isn't completely hooked up yet but I'm committing so Mauricio and Mark can see how I envision it will fit together. Look it over and give any feedback. With the exception of the Solid specific code we are very very close to being able to remove TableRecalibrationWalker from the code base and just replace it with PrintReads -BQSR recal.csv --- .../sting/gatk/GenomeAnalysisEngine.java | 17 +- .../arguments/GATKArgumentCollection.java | 9 + .../recalibration/ContextCovariate.java | 62 ++++ .../recalibration/RecalDataManager.java | 5 +- .../TableRecalibrationWalker.java | 1 - .../sting/utils/QualityUtils.java | 8 + .../sting/utils/pileup/PileupElement.java | 17 +- .../recalibration/BaseRecalibration.java | 293 ++++++++++++++++++ .../sting/utils/sam/GATKSAMRecord.java | 61 ++++ 9 files changed, 466 insertions(+), 7 deletions(-) create mode 100644 public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java create mode 100644 public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java b/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java index 6140d543a9..97d1de1fae 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java @@ -53,6 +53,7 @@ import org.broadinstitute.sting.utils.exceptions.UserException; import org.broadinstitute.sting.utils.interval.IntervalSetRule; import org.broadinstitute.sting.utils.interval.IntervalUtils; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; import java.io.File; import java.util.*; @@ -179,10 +180,18 @@ public void setReferenceMetaDataFiles(Collection referenceMetaDataFi */ private static final long GATK_RANDOM_SEED = 47382911L; private static Random randomGenerator = new Random(GATK_RANDOM_SEED); - public static Random getRandomGenerator() { return randomGenerator; } public static void resetRandomGenerator() { randomGenerator.setSeed(GATK_RANDOM_SEED); } public static void resetRandomGenerator(long seed) { randomGenerator.setSeed(seed); } + + /** + * Static base quality score recalibration helper object + */ + private static BaseRecalibration baseRecalibration = null; + public static BaseRecalibration getBaseRecalibration() { return baseRecalibration; } + public static boolean hasBaseRecalibration() { return baseRecalibration != null; } + public static void setBaseRecalibration(File recalFile) { baseRecalibration = new BaseRecalibration(recalFile); } + /** * Actually run the GATK with the specified walker. * @@ -205,6 +214,10 @@ public Object execute() { if (this.getArguments().nonDeterministicRandomSeed) resetRandomGenerator(System.currentTimeMillis()); + // if the use specified an input BQSR recalibration table then enable on the fly recalibration + if (this.getArguments().RECAL_FILE != null) + setBaseRecalibration(this.getArguments().RECAL_FILE); + // Determine how the threads should be divided between CPU vs. IO. determineThreadAllocation(); @@ -224,7 +237,7 @@ public Object execute() { // create temp directories as necessary initializeTempDirectory(); - // create the output streams " + // create the output streams initializeOutputStreams(microScheduler.getOutputTracker()); Iterable shardStrategy = getShardStrategy(readsDataSource,microScheduler.getReference(),intervals); diff --git a/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java index 08d2c1ad15..206fa5765f 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java +++ b/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java @@ -185,6 +185,15 @@ public static DownsamplingMethod getDefaultDownsamplingMethod() { @Argument(fullName="useOriginalQualities", shortName = "OQ", doc = "If set, use the original base quality scores from the OQ tag when present instead of the standard scores", required=false) public Boolean useOriginalBaseQualities = false; + /** + * After the header, data records occur one per line until the end of the file. The first several items on a line are the + * values of the individual covariates and will change depending on which covariates were specified at runtime. The last + * three items are the data- that is, number of observations for this combination of covariates, number of reference mismatches, + * and the raw empirical quality score calculated by phred-scaling the mismatch rate. + */ + @Input(fullName="BQSR", shortName="BQSR", required=false, doc="Filename for the input covariates table recalibration .csv file which enables on the fly base quality score recalibration") + public File RECAL_FILE = null; // BUGBUG: need a better argument name once we decide how BQSRs v1 and v2 will live in the code base simultaneously + @Argument(fullName="defaultBaseQualities", shortName = "DBQ", doc = "If reads are missing some or all base quality scores, this value will be used for all base quality scores", required=false) public byte defaultBaseQualities = -1; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java new file mode 100644 index 0000000000..837062dd2a --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2011 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR + * THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.walkers.recalibration; + +import net.sf.samtools.SAMRecord; + +import java.util.Arrays; + +/** + * Created by IntelliJ IDEA. + * User: rpoplin + * Date: 9/26/11 + */ + +public class ContextCovariate implements Covariate { + + final int CONTEXT_SIZE = 8; + String allN = ""; + + // Initialize any member variables using the command-line arguments passed to the walkers + public void initialize( final RecalibrationArgumentCollection RAC ) { + for( int iii = 0; iii < CONTEXT_SIZE; iii++ ) { + allN += "N"; + } + } + + public void getValues(SAMRecord read, Comparable[] comparable) { + byte[] bases = read.getReadBases(); + for(int i = 0; i < read.getReadLength(); i++) { + comparable[i] = ( i-CONTEXT_SIZE < 0 ? allN : new String(Arrays.copyOfRange(bases,i-CONTEXT_SIZE,i)) ); + } + } + + // Used to get the covariate's value from input csv file in TableRecalibrationWalker + public final Comparable getValue( final String str ) { + return str; + } + +} diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java index a0c928afa0..66ad1fb9c8 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java @@ -25,7 +25,6 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; -import net.sf.samtools.SAMReadGroupRecord; import net.sf.samtools.SAMRecord; import net.sf.samtools.SAMUtils; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; @@ -86,14 +85,14 @@ public enum SOLID_NOCALL_STRATEGY { PURGE_READ } - RecalDataManager() { + public RecalDataManager() { data = new NestedHashMap(); dataCollapsedReadGroup = null; dataCollapsedQualityScore = null; dataCollapsedByCovariate = null; } - RecalDataManager( final boolean createCollapsedTables, final int numCovariates ) { + public RecalDataManager( final boolean createCollapsedTables, final int numCovariates ) { if( createCollapsedTables ) { // Initialize all the collapsed tables, only used by TableRecalibrationWalker data = null; dataCollapsedReadGroup = new NestedHashMap(); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java index 6e214c6bb9..a569aefd25 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java @@ -181,7 +181,6 @@ public class TableRecalibrationWalker extends ReadWalker 127; -128 -> 128; -1 -> 255; etc. } + static public double[] qualArrayToLog10ErrorProb(byte[] quals) { + double[] returnArray = new double[quals.length]; + for( int iii = 0; iii < quals.length; iii++ ) { + returnArray[iii] = ((double) quals[iii])/-10.0; + } + return returnArray; + } + /** * Convert a probability to a quality score. Note, this is capped at Q40. * diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java b/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java index 9e2a66f6e0..a4830223e2 100755 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java @@ -27,7 +27,6 @@ public class PileupElement implements Comparable { protected final boolean isBeforeInsertion; protected final boolean isNextToSoftClip; - /** * Creates a new pileup element. * @@ -89,6 +88,14 @@ public int getBaseIndex() { public byte getQual() { return getQual(offset); } + + public byte getBaseInsertionQual() { + return getBaseInsertionQual(offset); + } + + public byte getBaseDeletionQual() { + return getBaseDeletionQual(offset); + } public int getMappingQual() { return read.getMappingQuality(); @@ -111,6 +118,14 @@ protected byte getQual(final int offset) { return (isDeletion() || isInsertionAtBeginningOfRead()) ? DELETION_QUAL : read.getBaseQualities()[offset]; } + protected byte getBaseInsertionQual(final int offset) { + return (isDeletion() || isInsertionAtBeginningOfRead()) ? DELETION_QUAL : read.getBaseInsertionQualities()[offset]; + } + + protected byte getBaseDeletionQual(final int offset) { + return (isDeletion() || isInsertionAtBeginningOfRead()) ? DELETION_QUAL : read.getBaseDeletionQualities()[offset]; + } + @Override public int compareTo(final PileupElement pileupElement) { if (offset < pileupElement.offset) diff --git a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java new file mode 100644 index 0000000000..2e785043d8 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java @@ -0,0 +1,293 @@ +/* + * Copyright (c) 2012 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR + * THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.recalibration; + +import org.broadinstitute.sting.gatk.walkers.recalibration.Covariate; +import org.broadinstitute.sting.gatk.walkers.recalibration.RecalDataManager; +import org.broadinstitute.sting.gatk.walkers.recalibration.RecalDatum; +import org.broadinstitute.sting.utils.QualityUtils; +import org.broadinstitute.sting.utils.classloader.PluginManager; +import org.broadinstitute.sting.utils.collections.NestedHashMap; +import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException; +import org.broadinstitute.sting.utils.exceptions.UserException; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; +import org.broadinstitute.sting.utils.text.XReadLines; + +import java.io.File; +import java.io.FileNotFoundException; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +/** + * Utility methods to facilitate on-the-fly base quality score recalibration. + * + * User: rpoplin + * Date: 2/4/12 + */ + +public class BaseRecalibration { + + public enum BaseRecalibrationType { + BASE_SUBSTITUTION, + BASE_INSERTION, + BASE_DELETION + } + + private RecalDataManager dataManager; // Holds the data HashMap, mostly used by TableRecalibrationWalker to create collapsed data hashmaps + private final ArrayList requestedCovariates = new ArrayList(); // List of covariates to be used in this calculation + public static final Pattern COMMENT_PATTERN = Pattern.compile("^#.*"); + public static final Pattern COVARIATE_PATTERN = Pattern.compile("^ReadGroup,QualityScore,.*"); + public static final String EOF_MARKER = "EOF"; + private static final int MAX_QUALITY_SCORE = 65; //BUGBUG: what value to use here? + private NestedHashMap qualityScoreByFullCovariateKey = new NestedHashMap(); // Caches the result of performSequentialQualityCalculation(..) for all sets of covariate values. + + public BaseRecalibration( final File RECAL_FILE ) { + // Get a list of all available covariates + final List> classes = new PluginManager(Covariate.class).getPlugins(); + + int lineNumber = 0; + boolean foundAllCovariates = false; + + // Read in the data from the csv file and populate the data map and covariates list + boolean sawEOF = false; + try { + for ( String line : new XReadLines(RECAL_FILE) ) { + lineNumber++; + if ( EOF_MARKER.equals(line) ) { + sawEOF = true; + } else if( COMMENT_PATTERN.matcher(line).matches() ) { + ; // Skip over the comment lines, (which start with '#') + } + // Read in the covariates that were used from the input file + else if( COVARIATE_PATTERN.matcher(line).matches() ) { // The line string is either specifying a covariate or is giving csv data + if( foundAllCovariates ) { + throw new UserException.MalformedFile( RECAL_FILE, "Malformed input recalibration file. Found covariate names intermingled with data in file: " + RECAL_FILE ); + } else { // Found the covariate list in input file, loop through all of them and instantiate them + String[] vals = line.split(","); + for( int iii = 0; iii < vals.length - 3; iii++ ) { // There are n-3 covariates. The last three items are nObservations, nMismatch, and Qempirical + boolean foundClass = false; + for( Class covClass : classes ) { + if( (vals[iii] + "Covariate").equalsIgnoreCase( covClass.getSimpleName() ) ) { + foundClass = true; + try { + Covariate covariate = (Covariate)covClass.newInstance(); + requestedCovariates.add( covariate ); + } catch (Exception e) { + throw new DynamicClassResolutionException(covClass, e); + } + + } + } + + if( !foundClass ) { + throw new UserException.MalformedFile(RECAL_FILE, "Malformed input recalibration file. The requested covariate type (" + (vals[iii] + "Covariate") + ") isn't a valid covariate option." ); + } + } + } + + } else { // Found a line of data + if( !foundAllCovariates ) { + foundAllCovariates = true; + + // At this point all the covariates should have been found and initialized + if( requestedCovariates.size() < 2 ) { + throw new UserException.MalformedFile(RECAL_FILE, "Malformed input recalibration csv file. Covariate names can't be found in file: " + RECAL_FILE ); + } + + final boolean createCollapsedTables = true; + + // Initialize any covariate member variables using the shared argument collection + for( Covariate cov : requestedCovariates ) { + cov.initialize( null ); // BUGBUG: do any of the used covariates actually need the RecalibrationArgumentCollection? + } + // Initialize the data hashMaps + dataManager = new RecalDataManager( createCollapsedTables, requestedCovariates.size() ); + + } + addCSVData(RECAL_FILE, line); // Parse the line and add the data to the HashMap + } + } + + } catch ( FileNotFoundException e ) { + throw new UserException.CouldNotReadInputFile(RECAL_FILE, "Can not find input file", e); + } catch ( NumberFormatException e ) { + throw new UserException.MalformedFile(RECAL_FILE, "Error parsing recalibration data at line " + lineNumber + ". Perhaps your table was generated by an older version of CovariateCounterWalker."); + } + + if ( !sawEOF ) { + final String errorMessage = "No EOF marker was present in the recal covariates table; this could mean that the file is corrupted or was generated with an old version of the CountCovariates tool."; + throw new UserException.MalformedFile(RECAL_FILE, errorMessage); + } + + if( dataManager == null ) { + throw new UserException.MalformedFile(RECAL_FILE, "Can't initialize the data manager. Perhaps the recal csv file contains no data?"); + } + + dataManager.generateEmpiricalQualities( 1, MAX_QUALITY_SCORE ); + } + + /** + * For each covariate read in a value and parse it. Associate those values with the data itself (num observation and num mismatches) + * @param line A line of CSV data read from the recalibration table data file + */ + private void addCSVData(final File file, final String line) { + final String[] vals = line.split(","); + + // Check if the data line is malformed, for example if the read group string contains a comma then it won't be parsed correctly + if( vals.length != requestedCovariates.size() + 3 ) { // +3 because of nObservations, nMismatch, and Qempirical + throw new UserException.MalformedFile(file, "Malformed input recalibration file. Found data line with too many fields: " + line + + " --Perhaps the read group string contains a comma and isn't being parsed correctly."); + } + + final Object[] key = new Object[requestedCovariates.size()]; + Covariate cov; + int iii; + for( iii = 0; iii < requestedCovariates.size(); iii++ ) { + cov = requestedCovariates.get( iii ); + key[iii] = cov.getValue( vals[iii] ); + } + + // Create a new datum using the number of observations, number of mismatches, and reported quality score + final RecalDatum datum = new RecalDatum( Long.parseLong( vals[iii] ), Long.parseLong( vals[iii + 1] ), Double.parseDouble( vals[1] ), 0.0 ); + // Add that datum to all the collapsed tables which will be used in the sequential calculation + dataManager.addToAllTables( key, datum, QualityUtils.MIN_USABLE_Q_SCORE ); //BUGBUG: used to be Q5 now is Q6, probably doesn't matter + } + + public byte[] recalibrateRead( final GATKSAMRecord read, final byte[] originalQuals ) { + + final byte[] recalQuals = originalQuals.clone(); + + //compute all covariate values for this read + final Comparable[][] covariateValues_offset_x_covar = + RecalDataManager.computeCovariates(read, requestedCovariates); + + // For each base in the read + for( int offset = 0; offset < read.getReadLength(); offset++ ) { + + final Object[] fullCovariateKey = covariateValues_offset_x_covar[offset]; + + Byte qualityScore = (Byte) qualityScoreByFullCovariateKey.get(fullCovariateKey); + if(qualityScore == null) + { + qualityScore = performSequentialQualityCalculation( fullCovariateKey ); + qualityScoreByFullCovariateKey.put(qualityScore, fullCovariateKey); + } + + recalQuals[offset] = qualityScore; + } + + preserveQScores( originalQuals, recalQuals ); // Overwrite the work done if original quality score is too low + + return recalQuals; + } + + /** + * Implements a serial recalibration of the reads using the combinational table. + * First, we perform a positional recalibration, and then a subsequent dinuc correction. + * + * Given the full recalibration table, we perform the following preprocessing steps: + * + * - calculate the global quality score shift across all data [DeltaQ] + * - calculate for each of cycle and dinuc the shift of the quality scores relative to the global shift + * -- i.e., DeltaQ(dinuc) = Sum(pos) Sum(Qual) Qempirical(pos, qual, dinuc) - Qreported(pos, qual, dinuc) / Npos * Nqual + * - The final shift equation is: + * + * Qrecal = Qreported + DeltaQ + DeltaQ(pos) + DeltaQ(dinuc) + DeltaQ( ... any other covariate ... ) + * @param key The list of Comparables that were calculated from the covariates + * @return A recalibrated quality score as a byte + */ + private byte performSequentialQualityCalculation( final Object... key ) { + + final byte qualFromRead = (byte)Integer.parseInt(key[1].toString()); + final Object[] readGroupCollapsedKey = new Object[1]; + final Object[] qualityScoreCollapsedKey = new Object[2]; + final Object[] covariateCollapsedKey = new Object[3]; + + // The global quality shift (over the read group only) + readGroupCollapsedKey[0] = key[0]; + final RecalDatum globalRecalDatum = ((RecalDatum)dataManager.getCollapsedTable(0).get( readGroupCollapsedKey )); + double globalDeltaQ = 0.0; + if( globalRecalDatum != null ) { + final double globalDeltaQEmpirical = globalRecalDatum.getEmpiricalQuality(); + final double aggregrateQReported = globalRecalDatum.getEstimatedQReported(); + globalDeltaQ = globalDeltaQEmpirical - aggregrateQReported; + } + + // The shift in quality between reported and empirical + qualityScoreCollapsedKey[0] = key[0]; + qualityScoreCollapsedKey[1] = key[1]; + final RecalDatum qReportedRecalDatum = ((RecalDatum)dataManager.getCollapsedTable(1).get( qualityScoreCollapsedKey )); + double deltaQReported = 0.0; + if( qReportedRecalDatum != null ) { + final double deltaQReportedEmpirical = qReportedRecalDatum.getEmpiricalQuality(); + deltaQReported = deltaQReportedEmpirical - qualFromRead - globalDeltaQ; + } + + // The shift in quality due to each covariate by itself in turn + double deltaQCovariates = 0.0; + double deltaQCovariateEmpirical; + covariateCollapsedKey[0] = key[0]; + covariateCollapsedKey[1] = key[1]; + for( int iii = 2; iii < key.length; iii++ ) { + covariateCollapsedKey[2] = key[iii]; // The given covariate + final RecalDatum covariateRecalDatum = ((RecalDatum)dataManager.getCollapsedTable(iii).get( covariateCollapsedKey )); + if( covariateRecalDatum != null ) { + deltaQCovariateEmpirical = covariateRecalDatum.getEmpiricalQuality(); + deltaQCovariates += ( deltaQCovariateEmpirical - qualFromRead - (globalDeltaQ + deltaQReported) ); + } + } + + final double newQuality = qualFromRead + globalDeltaQ + deltaQReported + deltaQCovariates; + return QualityUtils.boundQual( (int)Math.round(newQuality), (byte)MAX_QUALITY_SCORE ); + + // Verbose printouts used to validate with old recalibrator + //if(key.contains(null)) { + // System.out.println( key + String.format(" => %d + %.2f + %.2f + %.2f + %.2f = %d", + // qualFromRead, globalDeltaQ, deltaQReported, deltaQPos, deltaQDinuc, newQualityByte)); + //} + //else { + // System.out.println( String.format("%s %s %s %s => %d + %.2f + %.2f + %.2f + %.2f = %d", + // key.get(0).toString(), key.get(3).toString(), key.get(2).toString(), key.get(1).toString(), qualFromRead, globalDeltaQ, deltaQReported, deltaQPos, deltaQDinuc, newQualityByte) ); + //} + + //return newQualityByte; + } + + /** + * Loop over the list of qualities and overwrite the newly recalibrated score to be the original score if it was less than some threshold + * @param originalQuals The list of original base quality scores + * @param recalQuals A list of the new recalibrated quality scores + */ + private void preserveQScores( final byte[] originalQuals, final byte[] recalQuals ) { + for( int iii = 0; iii < recalQuals.length; iii++ ) { + if( originalQuals[iii] < QualityUtils.MIN_USABLE_Q_SCORE ) { //BUGBUG: used to be Q5 now is Q6, probably doesn't matter + recalQuals[iii] = originalQuals[iii]; + } + } + } + +} diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java index 03b794ae35..e9b46ac244 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java @@ -25,8 +25,10 @@ package org.broadinstitute.sting.utils.sam; import net.sf.samtools.*; +import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.utils.NGSPlatform; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -48,6 +50,11 @@ public class GATKSAMRecord extends BAMRecord { public static final String REDUCED_READ_ORIGINAL_ALIGNMENT_START_SHIFT = "OP"; // reads that are clipped may use this attribute to keep track of their original alignment start public static final String REDUCED_READ_ORIGINAL_ALIGNMENT_END_SHIFT = "OE"; // reads that are clipped may use this attribute to keep track of their original alignment end + // Base Quality Score Recalibrator specific attribute tags + public static final String BQSR_BASE_INSERTION_QUALITIES = "BI"; + public static final String BQSR_BASE_DELETION_QUALITIES = "BD"; + public static final String BQSR_BASES_HAVE_BEEN_RECALIBRATED_TAG = "BR"; + // the SAMRecord data we're caching private String mReadString = null; private GATKSAMReadGroupRecord mReadGroup = null; @@ -155,6 +162,60 @@ public boolean equals(Object o) { return super.equals(o); } + /* + @Override + public byte[] getBaseQualities() { + if( getAttribute( BQSR_BASES_HAVE_BEEN_RECALIBRATED_TAG ) != null ) { + return super.getBaseQualities(); + } else { + // if the recal data was populated in the engine then recalibrate the quality scores on the fly + if( GenomeAnalysisEngine.hasBaseRecalibration() ) { + final byte[] quals = GenomeAnalysisEngine.getBaseRecalibration().recalibrateRead( this, super.getBaseQualities() ); + setBaseQualities(quals); + setAttribute( BQSR_BASES_HAVE_BEEN_RECALIBRATED_TAG, true ); + return quals; + } else { // just use the qualities that are in the read since we don't have the sufficient information to recalibrate on the fly + return super.getBaseQualities(); + } + } + } + */ + + /** + * Accessors for base insertion and base deletion quality scores + */ + public byte[] getBaseInsertionQualities() { + byte[] quals = getByteArrayAttribute( BQSR_BASE_INSERTION_QUALITIES ); + if( quals == null ) { + quals = new byte[getBaseQualities().length]; + Arrays.fill(quals, (byte) 45); // allow for differing default values between BaseInsertions and BaseDeletions + // if the recal data was populated in the engine then recalibrate the quality scores on the fly + // else give default values which are flat Q45 + if( GenomeAnalysisEngine.hasBaseRecalibration() ) { + quals = GenomeAnalysisEngine.getBaseRecalibration().recalibrateRead( this, quals ); // the original quals here are the flat base insertion/deletion quals, NOT the original base qualities + } + // add the qual array to the read so that we don't have to do the recalibration work again + setAttribute( BQSR_BASE_INSERTION_QUALITIES, quals ); + } + return quals; + } + + public byte[] getBaseDeletionQualities() { + byte[] quals = getByteArrayAttribute( BQSR_BASE_DELETION_QUALITIES ); + if( quals == null ) { + quals = new byte[getBaseQualities().length]; + Arrays.fill(quals, (byte) 45); + // if the recal data was populated in the engine then recalibrate the quality scores on the fly + // else give default values which are flat Q45 + if( GenomeAnalysisEngine.hasBaseRecalibration() ) { + quals = GenomeAnalysisEngine.getBaseRecalibration().recalibrateRead( this, quals ); // the original quals here are the flat base insertion/deletion quals, NOT the original base qualities + } + // add the qual array to the read so that we don't have to do the recalibration work again + setAttribute( BQSR_BASE_DELETION_QUALITIES, quals ); + } + return quals; + } + /** * Efficient caching accessor that returns the GATK NGSPlatform of this read * @return From cef550903ee0af5f151ec96f7f56693774f5e85a Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Mon, 6 Feb 2012 00:48:00 -0500 Subject: [PATCH 212/356] Minor optimization --- .../gatk/walkers/annotator/TransmissionDisequilibriumTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java index 34f4bd6079..d84ba44bc1 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java @@ -73,10 +73,11 @@ private double calculateTDT( final VariantContext vc, final Set triosToT // for each pair of alleles, add the likelihoods int numAlleles = vc.getNAlleles(); for ( int allele1 = 0; allele1 < numAlleles; allele1++ ) { + final int HOM1index = determineHomIndex(allele1, numAlleles); + for ( int allele2 = allele1 + 1; allele2 < numAlleles; allele2++ ) { // TODO -- cache these for better performance - final int HOM1index = determineHomIndex(allele1, numAlleles); final int HETindex = HOM1index + (allele2 - allele1); final int HOM2index = determineHomIndex(allele2, numAlleles); From b7ffd144e852ef0ebf479d1e7388cde76c37d33a Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Mon, 6 Feb 2012 08:54:42 -0500 Subject: [PATCH 213/356] Cleaning up the covariate classes and removing unused code from the bqsr optimizations in 2009. --- .../sting/gatk/GenomeAnalysisEngine.java | 4 +- .../arguments/GATKArgumentCollection.java | 2 +- .../recalibration/ContextCovariate.java | 4 +- .../walkers/recalibration/CycleCovariate.java | 3 ++ .../walkers/recalibration/DinucCovariate.java | 39 ++----------------- .../recalibration/GCContentCovariate.java | 10 ++--- .../recalibration/HomopolymerCovariate.java | 6 ++- .../MappingQualityCovariate.java | 6 ++- .../recalibration/MinimumNQSCovariate.java | 15 ++++--- .../recalibration/PositionCovariate.java | 14 ++++--- .../recalibration/PrimerRoundCovariate.java | 15 ++++--- .../recalibration/QualityScoreCovariate.java | 11 ++---- .../recalibration/ReadGroupCovariate.java | 11 ++---- .../sting/utils/sam/GATKSAMRecord.java | 6 ++- 14 files changed, 61 insertions(+), 85 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java b/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java index 97d1de1fae..c0db75aa9c 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java @@ -215,8 +215,8 @@ public Object execute() { resetRandomGenerator(System.currentTimeMillis()); // if the use specified an input BQSR recalibration table then enable on the fly recalibration - if (this.getArguments().RECAL_FILE != null) - setBaseRecalibration(this.getArguments().RECAL_FILE); + if (this.getArguments().BQSR_RECAL_FILE != null) + setBaseRecalibration(this.getArguments().BQSR_RECAL_FILE); // Determine how the threads should be divided between CPU vs. IO. determineThreadAllocation(); diff --git a/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java index 206fa5765f..b3a1e24887 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java +++ b/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java @@ -192,7 +192,7 @@ public static DownsamplingMethod getDefaultDownsamplingMethod() { * and the raw empirical quality score calculated by phred-scaling the mismatch rate. */ @Input(fullName="BQSR", shortName="BQSR", required=false, doc="Filename for the input covariates table recalibration .csv file which enables on the fly base quality score recalibration") - public File RECAL_FILE = null; // BUGBUG: need a better argument name once we decide how BQSRs v1 and v2 will live in the code base simultaneously + public File BQSR_RECAL_FILE = null; // BUGBUG: need a better argument name once we decide how BQSRs v1 and v2 will live in the code base simultaneously @Argument(fullName="defaultBaseQualities", shortName = "DBQ", doc = "If reads are missing some or all base quality scores, this value will be used for all base quality scores", required=false) public byte defaultBaseQualities = -1; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java index 837062dd2a..8b8f2cee91 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java @@ -41,12 +41,14 @@ public class ContextCovariate implements Covariate { String allN = ""; // Initialize any member variables using the command-line arguments passed to the walkers + @Override public void initialize( final RecalibrationArgumentCollection RAC ) { for( int iii = 0; iii < CONTEXT_SIZE; iii++ ) { allN += "N"; } } + @Override public void getValues(SAMRecord read, Comparable[] comparable) { byte[] bases = read.getReadBases(); for(int i = 0; i < read.getReadLength(); i++) { @@ -55,8 +57,8 @@ public void getValues(SAMRecord read, Comparable[] comparable) { } // Used to get the covariate's value from input csv file in TableRecalibrationWalker + @Override public final Comparable getValue( final String str ) { return str; } - } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java index b0819ee691..e72b426d0c 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java @@ -51,6 +51,7 @@ public class CycleCovariate implements StandardCovariate { private final static EnumSet FLOW_CYCLE_PLATFORMS = EnumSet.of(NGSPlatform.LS454, NGSPlatform.ION_TORRENT); // Initialize any member variables using the command-line arguments passed to the walkers + @Override public void initialize( final RecalibrationArgumentCollection RAC ) { if( RAC.DEFAULT_PLATFORM != null ) { if( RAC.DEFAULT_PLATFORM.equalsIgnoreCase( "SLX" ) || RAC.DEFAULT_PLATFORM.equalsIgnoreCase( "ILLUMINA" ) || @@ -63,6 +64,7 @@ public void initialize( final RecalibrationArgumentCollection RAC ) { } // Used to pick out the covariate's value from attributes of the read + @Override public void getValues(SAMRecord read, Comparable[] comparable) { //----------------------------- @@ -164,6 +166,7 @@ else if( FLOW_CYCLE_PLATFORMS.contains(ngsPlatform) ) { } // Used to get the covariate's value from input csv file in TableRecalibrationWalker + @Override public final Comparable getValue( final String str ) { return Integer.parseInt( str ); } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/DinucCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/DinucCovariate.java index a7717161a7..90768fe90d 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/DinucCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/DinucCovariate.java @@ -48,6 +48,7 @@ public class DinucCovariate implements StandardCovariate { private HashMap dinucHashMap; // Initialize any member variables using the command-line arguments passed to the walkers + @Override public void initialize( final RecalibrationArgumentCollection RAC ) { final byte[] BASES = { (byte)'A', (byte)'C', (byte)'G', (byte)'T' }; dinucHashMap = new HashMap(); @@ -60,44 +61,10 @@ public void initialize( final RecalibrationArgumentCollection RAC ) { dinucHashMap.put( Dinuc.hashBytes(NO_CALL, NO_CALL), NO_DINUC ); } - /* - // Used to pick out the covariate's value from attributes of the read - public final Comparable getValue( final SAMRecord read, final int offset ) { - - byte base; - byte prevBase; - final byte[] bases = read.getReadBases(); - // If this is a negative strand read then we need to reverse the direction for our previous base - if( read.getReadNegativeStrandFlag() ) { - // No dinuc at the beginning of the read - if( offset == bases.length-1 ) { - return NO_DINUC; - } - base = (byte)BaseUtils.simpleComplement( (char)(bases[offset]) ); - // Note: We are using the previous base in the read, not the previous base in the reference. This is done in part to be consistent with unmapped reads. - prevBase = (byte)BaseUtils.simpleComplement( (char)(bases[offset + 1]) ); - } else { - // No dinuc at the beginning of the read - if( offset == 0 ) { - return NO_DINUC; - } - base = bases[offset]; - // Note: We are using the previous base in the read, not the previous base in the reference. This is done in part to be consistent with unmapped reads. - prevBase = bases[offset - 1]; - } - - // Make sure the previous base is good - if( !BaseUtils.isRegularBase( prevBase ) ) { - return NO_DINUC; - } - - return dinucHashMap.get( Dinuc.hashBytes( prevBase, base ) ); - } - */ - /** * Takes an array of size (at least) read.getReadLength() and fills it with the covariate values for each position in the read. */ + @Override public void getValues( SAMRecord read, Comparable[] result ) { final HashMap dinucHashMapRef = this.dinucHashMap; //optimize access to dinucHashMap final int readLength = read.getReadLength(); @@ -134,6 +101,7 @@ public void getValues( SAMRecord read, Comparable[] result ) { } // Used to get the covariate's value from input csv file in TableRecalibrationWalker + @Override public final Comparable getValue( final String str ) { byte[] bytes = str.getBytes(); final Dinuc returnDinuc = dinucHashMap.get( Dinuc.hashBytes( bytes[0], bytes[1] ) ); @@ -143,7 +111,6 @@ public final Comparable getValue( final String str ) { return returnDinuc; } - /** * Reverses the given array in place. * diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/GCContentCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/GCContentCovariate.java index be4e4ebfcb..1a085d5c03 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/GCContentCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/GCContentCovariate.java @@ -41,12 +41,13 @@ public class GCContentCovariate implements ExperimentalCovariate { int numBack = 7; // Initialize any member variables using the command-line arguments passed to the walkers + @Override public void initialize( final RecalibrationArgumentCollection RAC ) { numBack = RAC.HOMOPOLYMER_NBACK; } // Used to pick out the covariate's value from attributes of the read - public final Comparable getValue( final SAMRecord read, final int offset ) { + private final Comparable getValue( final SAMRecord read, final int offset ) { // ATTGCCCCGTAAAAAAAGAGAA // 0000123456654321001122 @@ -75,7 +76,8 @@ public final Comparable getValue( final SAMRecord read, final int offset ) { return -1; } } - + + @Override public void getValues(SAMRecord read, Comparable[] comparable) { for(int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized @@ -83,10 +85,8 @@ public void getValues(SAMRecord read, Comparable[] comparable) { } // Used to get the covariate's value from input csv file in TableRecalibrationWalker + @Override public final Comparable getValue( final String str ) { return Integer.parseInt( str ); } - - - } \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/HomopolymerCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/HomopolymerCovariate.java index f9a75de6f9..a54f9597b3 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/HomopolymerCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/HomopolymerCovariate.java @@ -43,12 +43,13 @@ public class HomopolymerCovariate implements ExperimentalCovariate { int numBack = 7; // Initialize any member variables using the command-line arguments passed to the walkers + @Override public void initialize( final RecalibrationArgumentCollection RAC ) { numBack = RAC.HOMOPOLYMER_NBACK; } // Used to pick out the covariate's value from attributes of the read - public final Comparable getValue( final SAMRecord read, final int offset ) { + private final Comparable getValue( final SAMRecord read, final int offset ) { // This block of code is for if you don't want to only count consecutive bases // ATTGCCCCGTAAAAAAAAATA @@ -90,6 +91,7 @@ public final Comparable getValue( final SAMRecord read, final int offset ) { return numAgree; } + @Override public void getValues(SAMRecord read, Comparable[] comparable) { for(int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized @@ -97,8 +99,8 @@ public void getValues(SAMRecord read, Comparable[] comparable) { } // Used to get the covariate's value from input csv file in TableRecalibrationWalker + @Override public final Comparable getValue( final String str ) { return Integer.parseInt( str ); } - } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MappingQualityCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MappingQualityCovariate.java index f9149a528b..ad64844288 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MappingQualityCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MappingQualityCovariate.java @@ -38,23 +38,25 @@ public class MappingQualityCovariate implements ExperimentalCovariate { // Initialize any member variables using the command-line arguments passed to the walkers + @Override public void initialize( final RecalibrationArgumentCollection RAC ) { } // Used to pick out the covariate's value from attributes of the read - public final Comparable getValue( final SAMRecord read, final int offset ) { + private final Comparable getValue( final SAMRecord read, final int offset ) { return read.getMappingQuality(); } // Used to get the covariate's value from input csv file in TableRecalibrationWalker + @Override public final Comparable getValue( final String str ) { return Integer.parseInt( str ); } + @Override public void getValues(SAMRecord read, Comparable[] comparable) { for(int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } } - } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MinimumNQSCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MinimumNQSCovariate.java index 64cae2b623..0c1c66a5f8 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MinimumNQSCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MinimumNQSCovariate.java @@ -41,12 +41,13 @@ public class MinimumNQSCovariate implements ExperimentalCovariate { private int windowReach; // How far in each direction from the current base to look // Initialize any member variables using the command-line arguments passed to the walkers + @Override public void initialize( final RecalibrationArgumentCollection RAC ) { windowReach = RAC.WINDOW_SIZE / 2; // integer division } // Used to pick out the covariate's value from attributes of the read - public final Comparable getValue( final SAMRecord read, final int offset ) { + private final Comparable getValue( final SAMRecord read, final int offset ) { // Loop over the list of base quality scores in the window and find the minimum final byte[] quals = read.getBaseQualities(); @@ -61,14 +62,16 @@ public final Comparable getValue( final SAMRecord read, final int offset ) { return minQual; } - // Used to get the covariate's value from input csv file in TableRecalibrationWalker - public final Comparable getValue( final String str ) { - return Integer.parseInt( str ); - } - + @Override public void getValues(SAMRecord read, Comparable[] comparable) { for(int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } } + + // Used to get the covariate's value from input csv file in TableRecalibrationWalker + @Override + public final Comparable getValue( final String str ) { + return Integer.parseInt( str ); + } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PositionCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PositionCovariate.java index 2495df57ac..2a4497b0d8 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PositionCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PositionCovariate.java @@ -39,11 +39,12 @@ public class PositionCovariate implements ExperimentalCovariate { // Initialize any member variables using the command-line arguments passed to the walkers + @Override public void initialize( final RecalibrationArgumentCollection RAC ) { } // Used to pick out the covariate's value from attributes of the read - public final Comparable getValue( final SAMRecord read, final int offset ) { + private final Comparable getValue( final SAMRecord read, final int offset ) { int cycle = offset; if( read.getReadNegativeStrandFlag() ) { cycle = read.getReadLength() - (offset + 1); @@ -51,15 +52,16 @@ public final Comparable getValue( final SAMRecord read, final int offset ) { return cycle; } - // Used to get the covariate's value from input csv file in TableRecalibrationWalker - public final Comparable getValue( final String str ) { - return Integer.parseInt( str ); - } - + @Override public void getValues(SAMRecord read, Comparable[] comparable) { for(int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } } + // Used to get the covariate's value from input csv file in TableRecalibrationWalker + @Override + public final Comparable getValue( final String str ) { + return Integer.parseInt( str ); + } } \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PrimerRoundCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PrimerRoundCovariate.java index 23fdeebe3d..4a96292349 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PrimerRoundCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PrimerRoundCovariate.java @@ -40,11 +40,12 @@ public class PrimerRoundCovariate implements ExperimentalCovariate { // Initialize any member variables using the command-line arguments passed to the walkers + @Override public void initialize( final RecalibrationArgumentCollection RAC ) { } // Used to pick out the covariate's value from attributes of the read - public final Comparable getValue( final SAMRecord read, final int offset ) { + private final Comparable getValue( final SAMRecord read, final int offset ) { if( read.getReadGroup().getPlatform().equalsIgnoreCase( "SOLID" ) || read.getReadGroup().getPlatform().equalsIgnoreCase( "ABI_SOLID" ) ) { int pos = offset; if( read.getReadNegativeStrandFlag() ) { @@ -57,14 +58,16 @@ public final Comparable getValue( final SAMRecord read, final int offset ) { } - // Used to get the covariate's value from input csv file in TableRecalibrationWalker - public final Comparable getValue( final String str ) { - return Integer.parseInt( str ); - } - + @Override public void getValues(SAMRecord read, Comparable[] comparable) { for(int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } } + + // Used to get the covariate's value from input csv file in TableRecalibrationWalker + @Override + public final Comparable getValue( final String str ) { + return Integer.parseInt( str ); + } } \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/QualityScoreCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/QualityScoreCovariate.java index df0101e18d..de6d5065bf 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/QualityScoreCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/QualityScoreCovariate.java @@ -38,16 +38,11 @@ public class QualityScoreCovariate implements RequiredCovariate { // Initialize any member variables using the command-line arguments passed to the walkers + @Override public void initialize( final RecalibrationArgumentCollection RAC ) { } - /* - // Used to pick out the covariate's value from attributes of the read - public final Comparable getValue( final SAMRecord read, final int offset ) { - return (int)(read.getBaseQualities()[offset]); - } - */ - + @Override public void getValues(SAMRecord read, Comparable[] comparable) { byte[] baseQualities = read.getBaseQualities(); for(int i = 0; i < read.getReadLength(); i++) { @@ -56,8 +51,8 @@ public void getValues(SAMRecord read, Comparable[] comparable) { } // Used to get the covariate's value from input csv file in TableRecalibrationWalker + @Override public final Comparable getValue( final String str ) { return Integer.parseInt( str ); } - } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ReadGroupCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ReadGroupCovariate.java index 0c853c349d..cb108feb8c 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ReadGroupCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ReadGroupCovariate.java @@ -40,16 +40,11 @@ public class ReadGroupCovariate implements RequiredCovariate{ public static final String defaultReadGroup = "DefaultReadGroup"; // Initialize any member variables using the command-line arguments passed to the walkers + @Override public void initialize( final RecalibrationArgumentCollection RAC ) { } - /* - // Used to pick out the covariate's value from attributes of the read - public final Comparable getValue( final SAMRecord read, final int offset ) { - return read.getReadGroup().getReadGroupId(); - } - */ - + @Override public void getValues(SAMRecord read, Comparable[] comparable) { final String readGroupId = read.getReadGroup().getReadGroupId(); for(int i = 0; i < read.getReadLength(); i++) { @@ -58,10 +53,10 @@ public void getValues(SAMRecord read, Comparable[] comparable) { } // Used to get the covariate's value from input csv file in TableRecalibrationWalker + @Override public final Comparable getValue( final String str ) { return str; } - } diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java index e9b46ac244..66e957a41c 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java @@ -162,9 +162,11 @@ public boolean equals(Object o) { return super.equals(o); } - /* + @Override public byte[] getBaseQualities() { + return super.getBaseQualities(); + /* if( getAttribute( BQSR_BASES_HAVE_BEEN_RECALIBRATED_TAG ) != null ) { return super.getBaseQualities(); } else { @@ -178,8 +180,8 @@ public byte[] getBaseQualities() { return super.getBaseQualities(); } } + */ } - */ /** * Accessors for base insertion and base deletion quality scores From 9d94f310f15d0a99dca19d83e1de6c6ff195bda2 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Mon, 6 Feb 2012 09:01:19 -0500 Subject: [PATCH 214/356] Break AF histogram into max and min AFs --- .../evaluators/MultiallelicSummary.java | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java index 9113e75382..82b6656af6 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java @@ -90,8 +90,11 @@ public enum Type { @DataPoint(description = "Multi-allelic Indel Novelty Rate") public String indelNoveltyRate = "NA"; - @DataPoint(description="Histogram of allele frequencies") - AFHistogram AFhistogram = new AFHistogram(); + @DataPoint(description="Histogram of allele frequencies for most common alternate allele") + AFHistogram AFhistogramMax = new AFHistogram(); + + @DataPoint(description="Histogram of allele frequencies for less common alternate alleles") + AFHistogram AFhistogramMin = new AFHistogram(); /* * AF histogram table object @@ -130,18 +133,10 @@ private static Object[] initColKeys() { public String getName() { return "AFHistTable"; } - public void update(VariantContext vc) { - final Object obj = vc.getAttribute(VCFConstants.ALLELE_FREQUENCY_KEY, null); - if ( obj == null || !(obj instanceof List) ) - return; - - List list = (List)obj; - for ( String str : list ) { - final double AF = Double.valueOf(str); - final int bin = (int)(numBins * MathUtils.round(AF, 2)); - AFhistogram[bin]++; - } - } + public void update(final double AF) { + final int bin = (int)(numBins * MathUtils.round(AF, 2)); + AFhistogram[bin]++; + } } public void initialize(VariantEvalWalker walker) {} @@ -180,7 +175,7 @@ public String update2(VariantContext eval, VariantContext comp, RefMetaDataTrack default: throw new UserException.BadInput("Unexpected variant context type: " + eval); } - AFhistogram.update(eval); + updateAFhistogram(eval); return null; // we don't capture any interesting sites } @@ -213,6 +208,24 @@ else if ( knownAlleles > 0 ) private void calculateIndelPairwiseNovelty(VariantContext eval, VariantContext comp) { } + private void updateAFhistogram(VariantContext vc) { + + final Object obj = vc.getAttribute(VCFConstants.ALLELE_FREQUENCY_KEY, null); + if ( obj == null || !(obj instanceof List) ) + return; + + List list = (List)obj; + ArrayList AFs = new ArrayList(list.size()); + for ( String str : list ) { + AFs.add(Double.valueOf(str)); + } + + Collections.sort(AFs); + AFhistogramMax.update(AFs.get(AFs.size()-1)); + for ( int i = 0; i < AFs.size() - 1; i++ ) + AFhistogramMin.update(AFs.get(i)); + } + private final String noveltyRate(final int all, final int known) { final int novel = all - known; final double rate = (novel / (1.0 * all)); From dc05b71e39b1e0124a5954a9c561d4556269117d Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Mon, 6 Feb 2012 11:10:24 -0500 Subject: [PATCH 215/356] Updating Covariate interface with Mauricio to include an errorModel parameter. On the fly recalibration of base insertion and base deletion quals is live for the HaplotypeCaller --- .../walkers/recalibration/ContextCovariate.java | 5 +++-- .../recalibration/CountCovariatesWalker.java | 3 ++- .../gatk/walkers/recalibration/Covariate.java | 10 ++++++---- .../walkers/recalibration/CycleCovariate.java | 3 ++- .../walkers/recalibration/DinucCovariate.java | 11 ++++++----- .../recalibration/GCContentCovariate.java | 3 ++- .../recalibration/HomopolymerCovariate.java | 3 ++- .../recalibration/MappingQualityCovariate.java | 3 ++- .../recalibration/MinimumNQSCovariate.java | 3 ++- .../walkers/recalibration/PositionCovariate.java | 3 ++- .../recalibration/PrimerRoundCovariate.java | 3 ++- .../recalibration/QualityScoreCovariate.java | 16 ++++++++++++---- .../recalibration/ReadGroupCovariate.java | 5 +++-- .../walkers/recalibration/RecalDataManager.java | 5 +++-- .../recalibration/TableRecalibrationWalker.java | 3 ++- .../utils/recalibration/BaseRecalibration.java | 4 ++-- .../sting/utils/sam/GATKSAMRecord.java | 5 +++-- 17 files changed, 56 insertions(+), 32 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java index 8b8f2cee91..0edd5d03b2 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java @@ -26,6 +26,7 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; import net.sf.samtools.SAMRecord; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; import java.util.Arrays; @@ -35,7 +36,7 @@ * Date: 9/26/11 */ -public class ContextCovariate implements Covariate { +public class ContextCovariate implements ExperimentalCovariate { final int CONTEXT_SIZE = 8; String allN = ""; @@ -49,7 +50,7 @@ public void initialize( final RecalibrationArgumentCollection RAC ) { } @Override - public void getValues(SAMRecord read, Comparable[] comparable) { + public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { byte[] bases = read.getReadBases(); for(int i = 0; i < read.getReadLength(); i++) { comparable[i] = ( i-CONTEXT_SIZE < 0 ? allN : new String(Arrays.copyOfRange(bases,i-CONTEXT_SIZE,i)) ); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java index f6f05d39c5..4e3d4048bc 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java @@ -41,6 +41,7 @@ import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException; import org.broadinstitute.sting.utils.exceptions.UserException; import org.broadinstitute.sting.utils.pileup.PileupElement; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import java.io.PrintStream; @@ -374,7 +375,7 @@ public CountedData map( RefMetaDataTracker tracker, ReferenceContext ref, Alignm RecalDataManager.parseColorSpace( gatkRead ); gatkRead.setTemporaryAttribute( COVARS_ATTRIBUTE, - RecalDataManager.computeCovariates( gatkRead, requestedCovariates )); + RecalDataManager.computeCovariates( gatkRead, requestedCovariates, BaseRecalibration.BaseRecalibrationType.BASE_SUBSTITUTION )); } // Skip this position if base quality is zero diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/Covariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/Covariate.java index 46ce006ee2..2e32dbb8ca 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/Covariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/Covariate.java @@ -1,6 +1,7 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; import net.sf.samtools.SAMRecord; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; /* * Copyright (c) 2009 The Broad Institute @@ -32,7 +33,7 @@ * User: rpoplin * Date: Oct 30, 2009 * - * The Covariate interface. A Covariate is a feature used in the recalibration that can be picked out of the read, offset, and corresponding reference bases + * The Covariate interface. A Covariate is a feature used in the recalibration that can be picked out of the read. * In general most error checking and adjustments to the data are done before the call to the covariates getValue methods in order to speed up the code. * This unfortunately muddies the code, but most of these corrections can be done per read while the covariates get called per base, resulting in a big speed up. */ @@ -40,9 +41,10 @@ public interface Covariate { public void initialize( RecalibrationArgumentCollection RAC ); // Initialize any member variables using the command-line arguments passed to the walkers public Comparable getValue( String str ); // Used to get the covariate's value from input csv file in TableRecalibrationWalker - public void getValues( SAMRecord read, Comparable[] comparable ); //Takes an array of size (at least) read.getReadLength() and fills it with covariate - //values for each position in the read. This method was created as an optimization over calling getValue( read, offset ) for each offset and allows - //read-specific calculations to be done just once rather than for each offset. + public void getValues( SAMRecord read, Comparable[] comparable, BaseRecalibration.BaseRecalibrationType modelType ); + //Takes an array of size (at least) read.getReadLength() and fills it with covariate + //values for each position in the read. This method was created as an optimization over calling getValue( read, offset ) for each offset and allows + //read-specific calculations to be done just once rather than for each offset. } interface RequiredCovariate extends Covariate { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java index e72b426d0c..00490d8983 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java @@ -4,6 +4,7 @@ import org.broadinstitute.sting.utils.BaseUtils; import org.broadinstitute.sting.utils.NGSPlatform; import org.broadinstitute.sting.utils.exceptions.UserException; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import java.util.EnumSet; @@ -65,7 +66,7 @@ public void initialize( final RecalibrationArgumentCollection RAC ) { // Used to pick out the covariate's value from attributes of the read @Override - public void getValues(SAMRecord read, Comparable[] comparable) { + public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { //----------------------------- // Illumina, Solid, PacBio, and Complete Genomics diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/DinucCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/DinucCovariate.java index 90768fe90d..e60b1f795a 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/DinucCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/DinucCovariate.java @@ -2,6 +2,7 @@ import net.sf.samtools.SAMRecord; import org.broadinstitute.sting.utils.BaseUtils; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; import java.util.HashMap; @@ -65,7 +66,7 @@ public void initialize( final RecalibrationArgumentCollection RAC ) { * Takes an array of size (at least) read.getReadLength() and fills it with the covariate values for each position in the read. */ @Override - public void getValues( SAMRecord read, Comparable[] result ) { + public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { final HashMap dinucHashMapRef = this.dinucHashMap; //optimize access to dinucHashMap final int readLength = read.getReadLength(); final boolean negativeStrand = read.getReadNegativeStrandFlag(); @@ -78,7 +79,7 @@ public void getValues( SAMRecord read, Comparable[] result ) { if(negativeStrand) { bases = BaseUtils.simpleReverseComplement(bases); //this is NOT in-place } - result[0] = NO_DINUC; // No dinuc at the beginning of the read + comparable[0] = NO_DINUC; // No dinuc at the beginning of the read prevBase = bases[0]; offset++; @@ -87,16 +88,16 @@ public void getValues( SAMRecord read, Comparable[] result ) { // previous base in the reference. This is done in part to be consistent with unmapped reads. base = bases[offset]; if( BaseUtils.isRegularBase( prevBase ) ) { - result[offset] = dinucHashMapRef.get( Dinuc.hashBytes( prevBase, base ) ); + comparable[offset] = dinucHashMapRef.get( Dinuc.hashBytes( prevBase, base ) ); } else { - result[offset] = NO_DINUC; + comparable[offset] = NO_DINUC; } offset++; prevBase = base; } if(negativeStrand) { - reverse( result ); + reverse( comparable ); } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/GCContentCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/GCContentCovariate.java index 1a085d5c03..e4ff415fe4 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/GCContentCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/GCContentCovariate.java @@ -1,6 +1,7 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; import net.sf.samtools.SAMRecord; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; /* * Copyright (c) 2010 The Broad Institute @@ -78,7 +79,7 @@ private final Comparable getValue( final SAMRecord read, final int offset ) { } @Override - public void getValues(SAMRecord read, Comparable[] comparable) { + public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { for(int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/HomopolymerCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/HomopolymerCovariate.java index a54f9597b3..24cb98a8d9 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/HomopolymerCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/HomopolymerCovariate.java @@ -1,6 +1,7 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; import net.sf.samtools.SAMRecord; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; /* * Copyright (c) 2009 The Broad Institute @@ -92,7 +93,7 @@ private final Comparable getValue( final SAMRecord read, final int offset ) { } @Override - public void getValues(SAMRecord read, Comparable[] comparable) { + public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { for(int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MappingQualityCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MappingQualityCovariate.java index ad64844288..ec5b357a43 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MappingQualityCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MappingQualityCovariate.java @@ -1,6 +1,7 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; import net.sf.samtools.SAMRecord; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; /* * Copyright (c) 2009 The Broad Institute @@ -54,7 +55,7 @@ public final Comparable getValue( final String str ) { } @Override - public void getValues(SAMRecord read, Comparable[] comparable) { + public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { for(int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MinimumNQSCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MinimumNQSCovariate.java index 0c1c66a5f8..21fd14e0c4 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MinimumNQSCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MinimumNQSCovariate.java @@ -1,6 +1,7 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; import net.sf.samtools.SAMRecord; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; /* * Copyright (c) 2009 The Broad Institute @@ -63,7 +64,7 @@ private final Comparable getValue( final SAMRecord read, final int offset ) { } @Override - public void getValues(SAMRecord read, Comparable[] comparable) { + public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { for(int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PositionCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PositionCovariate.java index 2a4497b0d8..5c410ce5f8 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PositionCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PositionCovariate.java @@ -1,6 +1,7 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; import net.sf.samtools.SAMRecord; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; /* * Copyright (c) 2009 The Broad Institute @@ -53,7 +54,7 @@ private final Comparable getValue( final SAMRecord read, final int offset ) { } @Override - public void getValues(SAMRecord read, Comparable[] comparable) { + public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { for(int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PrimerRoundCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PrimerRoundCovariate.java index 4a96292349..e6aa44226e 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PrimerRoundCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PrimerRoundCovariate.java @@ -1,6 +1,7 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; import net.sf.samtools.SAMRecord; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; /* * Copyright (c) 2009 The Broad Institute @@ -59,7 +60,7 @@ private final Comparable getValue( final SAMRecord read, final int offset ) { } @Override - public void getValues(SAMRecord read, Comparable[] comparable) { + public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { for(int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/QualityScoreCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/QualityScoreCovariate.java index de6d5065bf..f85b523501 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/QualityScoreCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/QualityScoreCovariate.java @@ -1,6 +1,9 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; import net.sf.samtools.SAMRecord; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; + +import java.util.Arrays; /* * Copyright (c) 2009 The Broad Institute @@ -43,10 +46,15 @@ public void initialize( final RecalibrationArgumentCollection RAC ) { } @Override - public void getValues(SAMRecord read, Comparable[] comparable) { - byte[] baseQualities = read.getBaseQualities(); - for(int i = 0; i < read.getReadLength(); i++) { - comparable[i] = (int) baseQualities[i]; + public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { + if( modelType == BaseRecalibration.BaseRecalibrationType.BASE_SUBSTITUTION ) { + byte[] baseQualities = read.getBaseQualities(); + for(int i = 0; i < read.getReadLength(); i++) { + comparable[i] = (int) baseQualities[i]; + } + } else { // model == BASE_INSERTION || model == BASE_DELETION + Arrays.fill(comparable, 45); // Some day in the future when base insertion and base deletion quals exist the samtools API will + // be updated and the original quals will be pulled here, but for now we assume the original quality is a flat Q45 } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ReadGroupCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ReadGroupCovariate.java index cb108feb8c..e270771288 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ReadGroupCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ReadGroupCovariate.java @@ -1,6 +1,7 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; import net.sf.samtools.SAMRecord; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; /* * Copyright (c) 2009 The Broad Institute @@ -35,7 +36,7 @@ * The Read Group covariate. */ -public class ReadGroupCovariate implements RequiredCovariate{ +public class ReadGroupCovariate implements RequiredCovariate { public static final String defaultReadGroup = "DefaultReadGroup"; @@ -45,7 +46,7 @@ public void initialize( final RecalibrationArgumentCollection RAC ) { } @Override - public void getValues(SAMRecord read, Comparable[] comparable) { + public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { final String readGroupId = read.getReadGroup().getReadGroupId(); for(int i = 0; i < read.getReadLength(); i++) { comparable[i] = readGroupId; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java index 66ad1fb9c8..be02063dec 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java @@ -33,6 +33,7 @@ import org.broadinstitute.sting.utils.collections.NestedHashMap; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; import org.broadinstitute.sting.utils.sam.AlignmentUtils; import org.broadinstitute.sting.utils.sam.GATKSAMReadGroupRecord; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; @@ -571,7 +572,7 @@ public static boolean isInconsistentColorSpace( final SAMRecord read, final int * value for the ith position in the read and the jth covariate in * reqeustedCovariates list. */ - public static Comparable[][] computeCovariates(final GATKSAMRecord gatkRead, final List requestedCovariates) { + public static Comparable[][] computeCovariates( final GATKSAMRecord gatkRead, final List requestedCovariates, final BaseRecalibration.BaseRecalibrationType modelType ) { //compute all covariates for this read final List requestedCovariatesRef = requestedCovariates; final int numRequestedCovariates = requestedCovariatesRef.size(); @@ -582,7 +583,7 @@ public static Comparable[][] computeCovariates(final GATKSAMRecord gatkRead, fin // Loop through the list of requested covariates and compute the values of each covariate for all positions in this read for( int i = 0; i < numRequestedCovariates; i++ ) { - requestedCovariatesRef.get(i).getValues( gatkRead, tempCovariateValuesHolder ); + requestedCovariatesRef.get(i).getValues( gatkRead, tempCovariateValuesHolder, modelType ); for(int j = 0; j < readLength; j++) { //copy values into a 2D array that allows all covar types to be extracted at once for //an offset j by doing covariateValues_offset_x_covar[j]. This avoids the need to later iterate over covar types. diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java index a569aefd25..a8006d506f 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java @@ -39,6 +39,7 @@ import org.broadinstitute.sting.utils.collections.NestedHashMap; import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException; import org.broadinstitute.sting.utils.exceptions.UserException; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import org.broadinstitute.sting.utils.text.TextFormattingUtils; import org.broadinstitute.sting.utils.text.XReadLines; @@ -398,7 +399,7 @@ public SAMRecord map( ReferenceContext refBases, GATKSAMRecord read, ReadMetaDat //compute all covariate values for this read final Comparable[][] covariateValues_offset_x_covar = - RecalDataManager.computeCovariates(read, requestedCovariates); + RecalDataManager.computeCovariates(read, requestedCovariates, BaseRecalibration.BaseRecalibrationType.BASE_SUBSTITUTION); // For each base in the read for( int offset = 0; offset < read.getReadLength(); offset++ ) { diff --git a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java index 2e785043d8..ce52f09a28 100644 --- a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java +++ b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java @@ -177,13 +177,13 @@ private void addCSVData(final File file, final String line) { dataManager.addToAllTables( key, datum, QualityUtils.MIN_USABLE_Q_SCORE ); //BUGBUG: used to be Q5 now is Q6, probably doesn't matter } - public byte[] recalibrateRead( final GATKSAMRecord read, final byte[] originalQuals ) { + public byte[] recalibrateRead( final GATKSAMRecord read, final byte[] originalQuals, final BaseRecalibrationType modelType ) { final byte[] recalQuals = originalQuals.clone(); //compute all covariate values for this read final Comparable[][] covariateValues_offset_x_covar = - RecalDataManager.computeCovariates(read, requestedCovariates); + RecalDataManager.computeCovariates(read, requestedCovariates, modelType); // For each base in the read for( int offset = 0; offset < read.getReadLength(); offset++ ) { diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java index 66e957a41c..bdcf2b210b 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java @@ -27,6 +27,7 @@ import net.sf.samtools.*; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.utils.NGSPlatform; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; import java.util.Arrays; import java.util.HashMap; @@ -194,7 +195,7 @@ public byte[] getBaseInsertionQualities() { // if the recal data was populated in the engine then recalibrate the quality scores on the fly // else give default values which are flat Q45 if( GenomeAnalysisEngine.hasBaseRecalibration() ) { - quals = GenomeAnalysisEngine.getBaseRecalibration().recalibrateRead( this, quals ); // the original quals here are the flat base insertion/deletion quals, NOT the original base qualities + quals = GenomeAnalysisEngine.getBaseRecalibration().recalibrateRead( this, quals, BaseRecalibration.BaseRecalibrationType.BASE_INSERTION ); // the original quals here are the flat base insertion/deletion quals, NOT the original base qualities } // add the qual array to the read so that we don't have to do the recalibration work again setAttribute( BQSR_BASE_INSERTION_QUALITIES, quals ); @@ -210,7 +211,7 @@ public byte[] getBaseDeletionQualities() { // if the recal data was populated in the engine then recalibrate the quality scores on the fly // else give default values which are flat Q45 if( GenomeAnalysisEngine.hasBaseRecalibration() ) { - quals = GenomeAnalysisEngine.getBaseRecalibration().recalibrateRead( this, quals ); // the original quals here are the flat base insertion/deletion quals, NOT the original base qualities + quals = GenomeAnalysisEngine.getBaseRecalibration().recalibrateRead( this, quals, BaseRecalibration.BaseRecalibrationType.BASE_DELETION ); // the original quals here are the flat base insertion/deletion quals, NOT the original base qualities } // add the qual array to the read so that we don't have to do the recalibration work again setAttribute( BQSR_BASE_DELETION_QUALITIES, quals ); From edb4edc08fb0dea2aeea61afcfe4fd39faa7ada1 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Mon, 6 Feb 2012 11:53:15 -0500 Subject: [PATCH 216/356] Commented out unused metrics for now --- .../varianteval/evaluators/MultiallelicSummary.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java index 82b6656af6..eef73c190a 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java @@ -83,11 +83,12 @@ public enum Type { @DataPoint(description = "Multi-allelic SNP Novelty Rate") public String SNPNoveltyRate = "NA"; - @DataPoint(description = "Multi-allelic Indels partially known") + //TODO -- implement me + //@DataPoint(description = "Multi-allelic Indels partially known") public int knownIndelsPartial = 0; - @DataPoint(description = "Multi-allelic Indels completely known") + //@DataPoint(description = "Multi-allelic Indels completely known") public int knownIndelsComplete = 0; - @DataPoint(description = "Multi-allelic Indel Novelty Rate") + //@DataPoint(description = "Multi-allelic Indel Novelty Rate") public String indelNoveltyRate = "NA"; @DataPoint(description="Histogram of allele frequencies for most common alternate allele") From 91897f5fe78e452916e2e786890773bea536e970 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Mon, 6 Feb 2012 16:23:32 -0500 Subject: [PATCH 217/356] Transpose rows/cols in AF table to make it molten (so I can plot easily in R) --- .../evaluators/MultiallelicSummary.java | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java index eef73c190a..97aebc376a 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/MultiallelicSummary.java @@ -91,25 +91,31 @@ public enum Type { //@DataPoint(description = "Multi-allelic Indel Novelty Rate") public String indelNoveltyRate = "NA"; - @DataPoint(description="Histogram of allele frequencies for most common alternate allele") - AFHistogram AFhistogramMax = new AFHistogram(); + @DataPoint(description="Histogram of allele frequencies for most common SNP alternate allele") + AFHistogram AFhistogramMaxSnp = new AFHistogram(); - @DataPoint(description="Histogram of allele frequencies for less common alternate alleles") - AFHistogram AFhistogramMin = new AFHistogram(); + @DataPoint(description="Histogram of allele frequencies for less common SNP alternate alleles") + AFHistogram AFhistogramMinSnp = new AFHistogram(); + + @DataPoint(description="Histogram of allele frequencies for most common Indel alternate allele") + AFHistogram AFhistogramMaxIndel = new AFHistogram(); + + @DataPoint(description="Histogram of allele frequencies for less common Indel alternate alleles") + AFHistogram AFhistogramMinIndel = new AFHistogram(); /* * AF histogram table object */ static class AFHistogram implements TableType { - private Object[] colKeys, rowKeys = {"pairwise_AF"}; + private Object[] rowKeys, colKeys = {"count"}; private int[] AFhistogram; private static final double AFincrement = 0.01; private static final int numBins = (int)(1.00 / AFincrement); public AFHistogram() { - colKeys = initColKeys(); - AFhistogram = new int[colKeys.length]; + rowKeys = initRowKeys(); + AFhistogram = new int[rowKeys.length]; } public Object[] getColumnKeys() { @@ -121,10 +127,10 @@ public Object[] getRowKeys() { } public Object getCell(int row, int col) { - return AFhistogram[col]; + return AFhistogram[row]; } - private static Object[] initColKeys() { + private static Object[] initRowKeys() { ArrayList keyList = new ArrayList(numBins + 1); for ( double a = 0.00; a <= 1.01; a += AFincrement ) { keyList.add(String.format("%.2f", a)); @@ -164,6 +170,7 @@ public String update2(VariantContext eval, VariantContext comp, RefMetaDataTrack nMultiSNPs++; calculatePairwiseTiTv(eval); calculateSNPPairwiseNovelty(eval, comp); + updateAFhistogram(eval, AFhistogramMaxSnp, AFhistogramMinSnp); } break; case INDEL: @@ -171,13 +178,13 @@ public String update2(VariantContext eval, VariantContext comp, RefMetaDataTrack if ( !eval.isBiallelic() ) { nMultiIndels++; calculateIndelPairwiseNovelty(eval, comp); + updateAFhistogram(eval, AFhistogramMaxIndel, AFhistogramMinIndel); } break; default: throw new UserException.BadInput("Unexpected variant context type: " + eval); } - updateAFhistogram(eval); - + return null; // we don't capture any interesting sites } @@ -209,7 +216,7 @@ else if ( knownAlleles > 0 ) private void calculateIndelPairwiseNovelty(VariantContext eval, VariantContext comp) { } - private void updateAFhistogram(VariantContext vc) { + private void updateAFhistogram(VariantContext vc, AFHistogram max, AFHistogram min) { final Object obj = vc.getAttribute(VCFConstants.ALLELE_FREQUENCY_KEY, null); if ( obj == null || !(obj instanceof List) ) @@ -222,9 +229,9 @@ private void updateAFhistogram(VariantContext vc) { } Collections.sort(AFs); - AFhistogramMax.update(AFs.get(AFs.size()-1)); + max.update(AFs.get(AFs.size()-1)); for ( int i = 0; i < AFs.size() - 1; i++ ) - AFhistogramMin.update(AFs.get(i)); + min.update(AFs.get(i)); } private final String noveltyRate(final int all, final int known) { From 6e6f0f10e1073ed4adf6aa05631f3a65c08503ee Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Mon, 6 Feb 2012 12:31:20 -0500 Subject: [PATCH 218/356] BaseQualityScoreRecalibration walker (bqsr v2) first commit includes * Adding the context covariate standard in both modes (including old CountCovariates) with parameters * Updating all covariates and modules to use GATKSAMRecord throughout the code. * BQSR now processes indels in the pileup (but doesn't do anything with them yet) --- .../recalibration/ContextCovariate.java | 27 +- .../gatk/walkers/recalibration/Covariate.java | 25 +- .../walkers/recalibration/CycleCovariate.java | 134 +++-- .../walkers/recalibration/DinucCovariate.java | 57 +- .../recalibration/GCContentCovariate.java | 33 +- .../recalibration/HomopolymerCovariate.java | 24 +- .../MappingQualityCovariate.java | 16 +- .../recalibration/MinimumNQSCovariate.java | 17 +- .../recalibration/PositionCovariate.java | 15 +- .../recalibration/PrimerRoundCovariate.java | 24 +- .../recalibration/QualityScoreCovariate.java | 19 +- .../recalibration/ReadGroupCovariate.java | 12 +- .../recalibration/RecalDataManager.java | 543 ++++++++++-------- .../RecalibrationArgumentCollection.java | 25 +- 14 files changed, 550 insertions(+), 421 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java index 0edd5d03b2..875782fdc0 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java @@ -25,8 +25,9 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; -import net.sf.samtools.SAMRecord; +import org.broadinstitute.sting.utils.exceptions.UserException; import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import java.util.Arrays; @@ -38,28 +39,32 @@ public class ContextCovariate implements ExperimentalCovariate { - final int CONTEXT_SIZE = 8; - String allN = ""; + private int CONTEXT_SIZE; + private String allN = ""; // Initialize any member variables using the command-line arguments passed to the walkers @Override - public void initialize( final RecalibrationArgumentCollection RAC ) { - for( int iii = 0; iii < CONTEXT_SIZE; iii++ ) { + public void initialize(final RecalibrationArgumentCollection RAC) { + CONTEXT_SIZE = RAC.CONTEXT_SIZE; + + if (CONTEXT_SIZE <= 0) + throw new UserException("Context Size must be positive, if you don't want to use the context covariate, just turn it off instead"); + + // initialize allN given the size of the context + for (int i = 0; i < CONTEXT_SIZE; i++) allN += "N"; - } } @Override - public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { byte[] bases = read.getReadBases(); - for(int i = 0; i < read.getReadLength(); i++) { - comparable[i] = ( i-CONTEXT_SIZE < 0 ? allN : new String(Arrays.copyOfRange(bases,i-CONTEXT_SIZE,i)) ); - } + for (int i = 0; i < read.getReadLength(); i++) + comparable[i] = (i < CONTEXT_SIZE) ? allN : new String(Arrays.copyOfRange(bases, i - CONTEXT_SIZE, i)); } // Used to get the covariate's value from input csv file in TableRecalibrationWalker @Override - public final Comparable getValue( final String str ) { + public final Comparable getValue(final String str) { return str; } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/Covariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/Covariate.java index 2e32dbb8ca..e4edb8ca68 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/Covariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/Covariate.java @@ -1,7 +1,7 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; -import net.sf.samtools.SAMRecord; import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; /* * Copyright (c) 2009 The Broad Institute @@ -39,19 +39,18 @@ */ public interface Covariate { - public void initialize( RecalibrationArgumentCollection RAC ); // Initialize any member variables using the command-line arguments passed to the walkers - public Comparable getValue( String str ); // Used to get the covariate's value from input csv file in TableRecalibrationWalker - public void getValues( SAMRecord read, Comparable[] comparable, BaseRecalibration.BaseRecalibrationType modelType ); - //Takes an array of size (at least) read.getReadLength() and fills it with covariate - //values for each position in the read. This method was created as an optimization over calling getValue( read, offset ) for each offset and allows - //read-specific calculations to be done just once rather than for each offset. -} + public void initialize(RecalibrationArgumentCollection RAC); // Initialize any member variables using the command-line arguments passed to the walkers -interface RequiredCovariate extends Covariate { -} + public Comparable getValue(String str); // Used to get the covariate's value from input csv file in TableRecalibrationWalker -interface StandardCovariate extends Covariate { + public void getValues(GATKSAMRecord read, Comparable[] comparable, BaseRecalibration.BaseRecalibrationType modelType); + //Takes an array of size (at least) read.getReadLength() and fills it with covariate + //values for each position in the read. This method was created as an optimization over calling getValue( read, offset ) for each offset and allows + //read-specific calculations to be done just once rather than for each offset. } -interface ExperimentalCovariate extends Covariate { -} +interface RequiredCovariate extends Covariate {} + +interface StandardCovariate extends Covariate {} + +interface ExperimentalCovariate extends Covariate {} diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java index 00490d8983..4244af7d11 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java @@ -1,6 +1,5 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; -import net.sf.samtools.SAMRecord; import org.broadinstitute.sting.utils.BaseUtils; import org.broadinstitute.sting.utils.NGSPlatform; import org.broadinstitute.sting.utils.exceptions.UserException; @@ -40,69 +39,69 @@ * Date: Oct 30, 2009 * * The Cycle covariate. - * For Solexa the cycle is simply the position in the read (counting backwards if it is a negative strand read) - * For 454 the cycle is the TACG flow cycle, that is, each flow grabs all the TACG's in order in a single cycle - * For example, for the read: AAACCCCGAAATTTTTACTG - * the cycle would be 11111111222333333344 - * For SOLiD the cycle is a more complicated mixture of ligation cycle and primer round + * For Solexa the cycle is simply the position in the read (counting backwards if it is a negative strand read) + * For 454 the cycle is the TACG flow cycle, that is, each flow grabs all the TACG's in order in a single cycle + * For example, for the read: AAACCCCGAAATTTTTACTG + * the cycle would be 11111111222333333344 + * For SOLiD the cycle is a more complicated mixture of ligation cycle and primer round */ public class CycleCovariate implements StandardCovariate { private final static EnumSet DISCRETE_CYCLE_PLATFORMS = EnumSet.of(NGSPlatform.ILLUMINA, NGSPlatform.SOLID, NGSPlatform.PACBIO, NGSPlatform.COMPLETE_GENOMICS); - private final static EnumSet FLOW_CYCLE_PLATFORMS = EnumSet.of(NGSPlatform.LS454, NGSPlatform.ION_TORRENT); + private final static EnumSet FLOW_CYCLE_PLATFORMS = EnumSet.of(NGSPlatform.LS454, NGSPlatform.ION_TORRENT); // Initialize any member variables using the command-line arguments passed to the walkers @Override - public void initialize( final RecalibrationArgumentCollection RAC ) { - if( RAC.DEFAULT_PLATFORM != null ) { - if( RAC.DEFAULT_PLATFORM.equalsIgnoreCase( "SLX" ) || RAC.DEFAULT_PLATFORM.equalsIgnoreCase( "ILLUMINA" ) || - RAC.DEFAULT_PLATFORM.contains( "454" ) || RAC.DEFAULT_PLATFORM.equalsIgnoreCase( "SOLID" ) || RAC.DEFAULT_PLATFORM.equalsIgnoreCase( "ABI_SOLID" ) ) { + public void initialize(final RecalibrationArgumentCollection RAC) { + if (RAC.DEFAULT_PLATFORM != null) { + if (RAC.DEFAULT_PLATFORM.equalsIgnoreCase("SLX") || RAC.DEFAULT_PLATFORM.equalsIgnoreCase("ILLUMINA") || + RAC.DEFAULT_PLATFORM.contains("454") || RAC.DEFAULT_PLATFORM.equalsIgnoreCase("SOLID") || RAC.DEFAULT_PLATFORM.equalsIgnoreCase("ABI_SOLID")) { // nothing to do - } else { - throw new UserException.CommandLineException("The requested default platform (" + RAC.DEFAULT_PLATFORM +") is not a recognized platform. Implemented options are illumina, 454, and solid"); + } + else { + throw new UserException.CommandLineException("The requested default platform (" + RAC.DEFAULT_PLATFORM + ") is not a recognized platform. Implemented options are illumina, 454, and solid"); } } } // Used to pick out the covariate's value from attributes of the read @Override - public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { //----------------------------- // Illumina, Solid, PacBio, and Complete Genomics //----------------------------- - final NGSPlatform ngsPlatform = ((GATKSAMRecord)read).getNGSPlatform(); - if( DISCRETE_CYCLE_PLATFORMS.contains(ngsPlatform) ) { + final NGSPlatform ngsPlatform = read.getNGSPlatform(); + if (DISCRETE_CYCLE_PLATFORMS.contains(ngsPlatform)) { final int init; final int increment; - if( !read.getReadNegativeStrandFlag() ) { + if (!read.getReadNegativeStrandFlag()) { // Differentiate between first and second of pair. // The sequencing machine cycle keeps incrementing for the second read in a pair. So it is possible for a read group // to have an error affecting quality at a particular cycle on the first of pair which carries over to the second of pair. // Therefore the cycle covariate must differentiate between first and second of pair reads. // This effect can not be corrected by pulling out the first of pair and second of pair flags into a separate covariate because // the current sequential model would consider the effects independently instead of jointly. - if( read.getReadPairedFlag() && read.getSecondOfPairFlag() ) { + if (read.getReadPairedFlag() && read.getSecondOfPairFlag()) { //second of pair, positive strand init = -1; increment = -1; } - else - { + else { //first of pair, positive strand init = 1; increment = 1; } - } else { - if( read.getReadPairedFlag() && read.getSecondOfPairFlag() ) { + } + else { + if (read.getReadPairedFlag() && read.getSecondOfPairFlag()) { //second of pair, negative strand init = -read.getReadLength(); increment = 1; } - else - { + else { //first of pair, negative strand init = read.getReadLength(); increment = -1; @@ -110,7 +109,7 @@ public void getValues( final SAMRecord read, final Comparable[] comparable, fina } int cycle = init; - for(int i = 0; i < read.getReadLength(); i++) { + for (int i = 0; i < read.getReadLength(); i++) { comparable[i] = cycle; cycle += increment; } @@ -119,7 +118,7 @@ public void getValues( final SAMRecord read, final Comparable[] comparable, fina //----------------------------- // 454 and Ion Torrent //----------------------------- - else if( FLOW_CYCLE_PLATFORMS.contains(ngsPlatform) ) { + else if (FLOW_CYCLE_PLATFORMS.contains(ngsPlatform)) { final int readLength = read.getReadLength(); final byte[] bases = read.getReadBases(); @@ -136,39 +135,78 @@ else if( FLOW_CYCLE_PLATFORMS.contains(ngsPlatform) ) { // BUGBUG: Consider looking at degradation of base quality scores in homopolymer runs to detect when the cycle incremented even though the nucleotide didn't change // For example, AAAAAAA was probably read in two flow cycles but here we count it as one - if( !read.getReadNegativeStrandFlag() ) { // Forward direction + if (!read.getReadNegativeStrandFlag()) { // Forward direction int iii = 0; - while( iii < readLength ) - { - while( iii < readLength && bases[iii] == (byte)'T' ) { comparable[iii] = cycle; iii++; } - while( iii < readLength && bases[iii] == (byte)'A' ) { comparable[iii] = cycle; iii++; } - while( iii < readLength && bases[iii] == (byte)'C' ) { comparable[iii] = cycle; iii++; } - while( iii < readLength && bases[iii] == (byte)'G' ) { comparable[iii] = cycle; iii++; } - if( iii < readLength ) { if (multiplyByNegative1) cycle--; else cycle++; } - if( iii < readLength && !BaseUtils.isRegularBase(bases[iii]) ) { comparable[iii] = cycle; iii++; } + while (iii < readLength) { + while (iii < readLength && bases[iii] == (byte) 'T') { + comparable[iii] = cycle; + iii++; + } + while (iii < readLength && bases[iii] == (byte) 'A') { + comparable[iii] = cycle; + iii++; + } + while (iii < readLength && bases[iii] == (byte) 'C') { + comparable[iii] = cycle; + iii++; + } + while (iii < readLength && bases[iii] == (byte) 'G') { + comparable[iii] = cycle; + iii++; + } + if (iii < readLength) { + if (multiplyByNegative1) + cycle--; + else + cycle++; + } + if (iii < readLength && !BaseUtils.isRegularBase(bases[iii])) { + comparable[iii] = cycle; + iii++; + } } - } else { // Negative direction - int iii = readLength-1; - while( iii >= 0 ) - { - while( iii >= 0 && bases[iii] == (byte)'T' ) { comparable[iii] = cycle; iii--; } - while( iii >= 0 && bases[iii] == (byte)'A' ) { comparable[iii] = cycle; iii--; } - while( iii >= 0 && bases[iii] == (byte)'C' ) { comparable[iii] = cycle; iii--; } - while( iii >= 0 && bases[iii] == (byte)'G' ) { comparable[iii] = cycle; iii--; } - if( iii >= 0 ) { if (multiplyByNegative1) cycle--; else cycle++; } - if( iii >= 0 && !BaseUtils.isRegularBase(bases[iii]) ) { comparable[iii] = cycle; iii--; } + } + else { // Negative direction + int iii = readLength - 1; + while (iii >= 0) { + while (iii >= 0 && bases[iii] == (byte) 'T') { + comparable[iii] = cycle; + iii--; + } + while (iii >= 0 && bases[iii] == (byte) 'A') { + comparable[iii] = cycle; + iii--; + } + while (iii >= 0 && bases[iii] == (byte) 'C') { + comparable[iii] = cycle; + iii--; + } + while (iii >= 0 && bases[iii] == (byte) 'G') { + comparable[iii] = cycle; + iii--; + } + if (iii >= 0) { + if (multiplyByNegative1) + cycle--; + else + cycle++; + } + if (iii >= 0 && !BaseUtils.isRegularBase(bases[iii])) { + comparable[iii] = cycle; + iii--; + } } } } - else { + else { throw new UserException("The platform (" + read.getReadGroup().getPlatform() + ") associated with read group " + read.getReadGroup() + " is not a recognized platform. Implemented options are e.g. illumina, 454, and solid"); } } // Used to get the covariate's value from input csv file in TableRecalibrationWalker @Override - public final Comparable getValue( final String str ) { - return Integer.parseInt( str ); + public final Comparable getValue(final String str) { + return Integer.parseInt(str); } } \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/DinucCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/DinucCovariate.java index e60b1f795a..2fa1b33cab 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/DinucCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/DinucCovariate.java @@ -1,8 +1,8 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; -import net.sf.samtools.SAMRecord; import org.broadinstitute.sting.utils.BaseUtils; import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import java.util.HashMap; @@ -43,30 +43,30 @@ public class DinucCovariate implements StandardCovariate { - private static final byte NO_CALL = (byte)'N'; + private static final byte NO_CALL = (byte) 'N'; private static final Dinuc NO_DINUC = new Dinuc(NO_CALL, NO_CALL); private HashMap dinucHashMap; // Initialize any member variables using the command-line arguments passed to the walkers @Override - public void initialize( final RecalibrationArgumentCollection RAC ) { - final byte[] BASES = { (byte)'A', (byte)'C', (byte)'G', (byte)'T' }; + public void initialize(final RecalibrationArgumentCollection RAC) { + final byte[] BASES = {(byte) 'A', (byte) 'C', (byte) 'G', (byte) 'T'}; dinucHashMap = new HashMap(); - for( byte byte1 : BASES ) { - for( byte byte2: BASES ) { - dinucHashMap.put( Dinuc.hashBytes(byte1, byte2), new Dinuc(byte1, byte2) ); // This might seem silly, but Strings are too slow + for (byte byte1 : BASES) { + for (byte byte2 : BASES) { + dinucHashMap.put(Dinuc.hashBytes(byte1, byte2), new Dinuc(byte1, byte2)); // This might seem silly, but Strings are too slow } } // Add the "no dinuc" entry too - dinucHashMap.put( Dinuc.hashBytes(NO_CALL, NO_CALL), NO_DINUC ); + dinucHashMap.put(Dinuc.hashBytes(NO_CALL, NO_CALL), NO_DINUC); } /** * Takes an array of size (at least) read.getReadLength() and fills it with the covariate values for each position in the read. */ @Override - public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { final HashMap dinucHashMapRef = this.dinucHashMap; //optimize access to dinucHashMap final int readLength = read.getReadLength(); final boolean negativeStrand = read.getReadNegativeStrandFlag(); @@ -76,37 +76,38 @@ public void getValues( final SAMRecord read, final Comparable[] comparable, fina int offset = 0; // If this is a negative strand read then we need to reverse the direction for our previous base - if(negativeStrand) { + if (negativeStrand) { bases = BaseUtils.simpleReverseComplement(bases); //this is NOT in-place } comparable[0] = NO_DINUC; // No dinuc at the beginning of the read prevBase = bases[0]; offset++; - while(offset < readLength) { - // Note: We are using the previous base in the read, not the - // previous base in the reference. This is done in part to be consistent with unmapped reads. - base = bases[offset]; - if( BaseUtils.isRegularBase( prevBase ) ) { - comparable[offset] = dinucHashMapRef.get( Dinuc.hashBytes( prevBase, base ) ); - } else { - comparable[offset] = NO_DINUC; - } + while (offset < readLength) { + // Note: We are using the previous base in the read, not the + // previous base in the reference. This is done in part to be consistent with unmapped reads. + base = bases[offset]; + if (BaseUtils.isRegularBase(prevBase)) { + comparable[offset] = dinucHashMapRef.get(Dinuc.hashBytes(prevBase, base)); + } + else { + comparable[offset] = NO_DINUC; + } - offset++; - prevBase = base; + offset++; + prevBase = base; } - if(negativeStrand) { - reverse( comparable ); + if (negativeStrand) { + reverse(comparable); } } // Used to get the covariate's value from input csv file in TableRecalibrationWalker @Override - public final Comparable getValue( final String str ) { + public final Comparable getValue(final String str) { byte[] bytes = str.getBytes(); - final Dinuc returnDinuc = dinucHashMap.get( Dinuc.hashBytes( bytes[0], bytes[1] ) ); - if( returnDinuc.compareTo(NO_DINUC) == 0 ) { + final Dinuc returnDinuc = dinucHashMap.get(Dinuc.hashBytes(bytes[0], bytes[1])); + if (returnDinuc.compareTo(NO_DINUC) == 0) { return null; } return returnDinuc; @@ -115,11 +116,11 @@ public final Comparable getValue( final String str ) { /** * Reverses the given array in place. * - * @param array + * @param array any array */ private static void reverse(final Comparable[] array) { final int arrayLength = array.length; - for(int l = 0, r = arrayLength - 1; l < r; l++, r--) { + for (int l = 0, r = arrayLength - 1; l < r; l++, r--) { final Comparable temp = array[l]; array[l] = array[r]; array[r] = temp; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/GCContentCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/GCContentCovariate.java index e4ff415fe4..7b209ae5cf 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/GCContentCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/GCContentCovariate.java @@ -2,6 +2,7 @@ import net.sf.samtools.SAMRecord; import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; /* * Copyright (c) 2010 The Broad Institute @@ -39,55 +40,57 @@ public class GCContentCovariate implements ExperimentalCovariate { - int numBack = 7; + private int numBack = 7; // Initialize any member variables using the command-line arguments passed to the walkers @Override - public void initialize( final RecalibrationArgumentCollection RAC ) { + public void initialize(final RecalibrationArgumentCollection RAC) { numBack = RAC.HOMOPOLYMER_NBACK; } // Used to pick out the covariate's value from attributes of the read - private final Comparable getValue( final SAMRecord read, final int offset ) { + private Comparable getValue(final SAMRecord read, final int offset) { // ATTGCCCCGTAAAAAAAGAGAA // 0000123456654321001122 - if( read.getReadGroup().getPlatform().equalsIgnoreCase( "ILLUMINA" ) || read.getReadGroup().getPlatform().equalsIgnoreCase( "SLX" ) ) { + if (read.getReadGroup().getPlatform().equalsIgnoreCase("ILLUMINA") || read.getReadGroup().getPlatform().equalsIgnoreCase("SLX")) { int numGC = 0; - int startPos = 0; - int stopPos = 0; + int startPos; + int stopPos; final byte[] bases = read.getReadBases(); - if( !read.getReadNegativeStrandFlag() ) { // Forward direction + if (!read.getReadNegativeStrandFlag()) { // Forward direction startPos = Math.max(offset - numBack, 0); stopPos = Math.max(offset - 1, 0); - } else { // Negative direction + } + else { // Negative direction startPos = Math.min(offset + 2, bases.length); stopPos = Math.min(offset + numBack + 1, bases.length); } - for( int iii = startPos; iii < stopPos; iii++ ) { - if( bases[iii] == (byte)'G' || bases[iii] == (byte)'C' ) { + for (int iii = startPos; iii < stopPos; iii++) { + if (bases[iii] == (byte) 'G' || bases[iii] == (byte) 'C') { numGC++; } } return numGC; - } else { // This effect is specific to the Illumina platform + } + else { // This effect is specific to the Illumina platform return -1; } } @Override - public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { - for(int iii = 0; iii < read.getReadLength(); iii++) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { + for (int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } } // Used to get the covariate's value from input csv file in TableRecalibrationWalker @Override - public final Comparable getValue( final String str ) { - return Integer.parseInt( str ); + public final Comparable getValue(final String str) { + return Integer.parseInt(str); } } \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/HomopolymerCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/HomopolymerCovariate.java index 24cb98a8d9..fd67edc3b3 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/HomopolymerCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/HomopolymerCovariate.java @@ -2,6 +2,7 @@ import net.sf.samtools.SAMRecord; import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; /* * Copyright (c) 2009 The Broad Institute @@ -41,16 +42,16 @@ public class HomopolymerCovariate implements ExperimentalCovariate { - int numBack = 7; + private int numBack; // Initialize any member variables using the command-line arguments passed to the walkers @Override - public void initialize( final RecalibrationArgumentCollection RAC ) { + public void initialize(final RecalibrationArgumentCollection RAC) { numBack = RAC.HOMOPOLYMER_NBACK; } // Used to pick out the covariate's value from attributes of the read - private final Comparable getValue( final SAMRecord read, final int offset ) { + private Comparable getValue(final SAMRecord read, final int offset) { // This block of code is for if you don't want to only count consecutive bases // ATTGCCCCGTAAAAAAAAATA @@ -77,13 +78,14 @@ private final Comparable getValue( final SAMRecord read, final int offset ) { int numAgree = 0; // The number of consecutive bases that agree with you in the previous numBack bases of the read final byte[] bases = read.getReadBases(); int iii = offset; - if( !read.getReadNegativeStrandFlag() ) { // Forward direction - while( iii <= bases.length-2 && bases[iii] == bases[iii+1] && numAgree < numBack ) { + if (!read.getReadNegativeStrandFlag()) { // Forward direction + while (iii <= bases.length - 2 && bases[iii] == bases[iii + 1] && numAgree < numBack) { numAgree++; iii++; } - } else { // Negative direction - while( iii >= 1 && bases[iii] == bases[iii-1] && numAgree < numBack ) { + } + else { // Negative direction + while (iii >= 1 && bases[iii] == bases[iii - 1] && numAgree < numBack) { numAgree++; iii--; } @@ -93,15 +95,15 @@ private final Comparable getValue( final SAMRecord read, final int offset ) { } @Override - public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { - for(int iii = 0; iii < read.getReadLength(); iii++) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { + for (int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } } // Used to get the covariate's value from input csv file in TableRecalibrationWalker @Override - public final Comparable getValue( final String str ) { - return Integer.parseInt( str ); + public final Comparable getValue(final String str) { + return Integer.parseInt(str); } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MappingQualityCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MappingQualityCovariate.java index ec5b357a43..e22049890c 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MappingQualityCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MappingQualityCovariate.java @@ -1,7 +1,7 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; -import net.sf.samtools.SAMRecord; import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; /* * Copyright (c) 2009 The Broad Institute @@ -40,24 +40,24 @@ public class MappingQualityCovariate implements ExperimentalCovariate { // Initialize any member variables using the command-line arguments passed to the walkers @Override - public void initialize( final RecalibrationArgumentCollection RAC ) { + public void initialize(final RecalibrationArgumentCollection RAC) { } // Used to pick out the covariate's value from attributes of the read - private final Comparable getValue( final SAMRecord read, final int offset ) { + private Comparable getValue(final GATKSAMRecord read) { return read.getMappingQuality(); } // Used to get the covariate's value from input csv file in TableRecalibrationWalker @Override - public final Comparable getValue( final String str ) { - return Integer.parseInt( str ); + public final Comparable getValue(final String str) { + return Integer.parseInt(str); } @Override - public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { - for(int iii = 0; iii < read.getReadLength(); iii++) { - comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized + public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { + for (int iii = 0; iii < read.getReadLength(); iii++) { + comparable[iii] = getValue(read); // BUGBUG: this can be optimized } } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MinimumNQSCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MinimumNQSCovariate.java index 21fd14e0c4..1dfb915b9a 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MinimumNQSCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MinimumNQSCovariate.java @@ -2,6 +2,7 @@ import net.sf.samtools.SAMRecord; import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; /* * Copyright (c) 2009 The Broad Institute @@ -43,20 +44,20 @@ public class MinimumNQSCovariate implements ExperimentalCovariate { // Initialize any member variables using the command-line arguments passed to the walkers @Override - public void initialize( final RecalibrationArgumentCollection RAC ) { + public void initialize(final RecalibrationArgumentCollection RAC) { windowReach = RAC.WINDOW_SIZE / 2; // integer division } // Used to pick out the covariate's value from attributes of the read - private final Comparable getValue( final SAMRecord read, final int offset ) { + private Comparable getValue(final SAMRecord read, final int offset) { // Loop over the list of base quality scores in the window and find the minimum final byte[] quals = read.getBaseQualities(); int minQual = quals[offset]; final int minIndex = Math.max(offset - windowReach, 0); final int maxIndex = Math.min(offset + windowReach, quals.length - 1); - for ( int iii = minIndex; iii < maxIndex; iii++ ) { - if( quals[iii] < minQual ) { + for (int iii = minIndex; iii < maxIndex; iii++) { + if (quals[iii] < minQual) { minQual = quals[iii]; } } @@ -64,15 +65,15 @@ private final Comparable getValue( final SAMRecord read, final int offset ) { } @Override - public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { - for(int iii = 0; iii < read.getReadLength(); iii++) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { + for (int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } } // Used to get the covariate's value from input csv file in TableRecalibrationWalker @Override - public final Comparable getValue( final String str ) { - return Integer.parseInt( str ); + public final Comparable getValue(final String str) { + return Integer.parseInt(str); } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PositionCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PositionCovariate.java index 5c410ce5f8..fbd1efc47b 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PositionCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PositionCovariate.java @@ -2,6 +2,7 @@ import net.sf.samtools.SAMRecord; import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; /* * Copyright (c) 2009 The Broad Institute @@ -41,28 +42,28 @@ public class PositionCovariate implements ExperimentalCovariate { // Initialize any member variables using the command-line arguments passed to the walkers @Override - public void initialize( final RecalibrationArgumentCollection RAC ) { + public void initialize(final RecalibrationArgumentCollection RAC) { } // Used to pick out the covariate's value from attributes of the read - private final Comparable getValue( final SAMRecord read, final int offset ) { + private Comparable getValue(final SAMRecord read, final int offset) { int cycle = offset; - if( read.getReadNegativeStrandFlag() ) { + if (read.getReadNegativeStrandFlag()) { cycle = read.getReadLength() - (offset + 1); } return cycle; } @Override - public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { - for(int iii = 0; iii < read.getReadLength(); iii++) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { + for (int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } } // Used to get the covariate's value from input csv file in TableRecalibrationWalker @Override - public final Comparable getValue( final String str ) { - return Integer.parseInt( str ); + public final Comparable getValue(final String str) { + return Integer.parseInt(str); } } \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PrimerRoundCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PrimerRoundCovariate.java index e6aa44226e..8dfa118849 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PrimerRoundCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PrimerRoundCovariate.java @@ -2,6 +2,7 @@ import net.sf.samtools.SAMRecord; import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; /* * Copyright (c) 2009 The Broad Institute @@ -34,41 +35,42 @@ * Date: Nov 13, 2009 * * The Primer Round covariate. - * For Solexa and 454 this is the same value of the length of the read. - * For SOLiD this is different for each position according to http://www3.appliedbiosystems.com/cms/groups/mcb_marketing/documents/generaldocuments/cms_057511.pdf + * For Solexa and 454 this is the same value of the length of the read. + * For SOLiD this is different for each position according to http://www3.appliedbiosystems.com/cms/groups/mcb_marketing/documents/generaldocuments/cms_057511.pdf */ public class PrimerRoundCovariate implements ExperimentalCovariate { // Initialize any member variables using the command-line arguments passed to the walkers @Override - public void initialize( final RecalibrationArgumentCollection RAC ) { + public void initialize(final RecalibrationArgumentCollection RAC) { } // Used to pick out the covariate's value from attributes of the read - private final Comparable getValue( final SAMRecord read, final int offset ) { - if( read.getReadGroup().getPlatform().equalsIgnoreCase( "SOLID" ) || read.getReadGroup().getPlatform().equalsIgnoreCase( "ABI_SOLID" ) ) { + private Comparable getValue(final SAMRecord read, final int offset) { + if (read.getReadGroup().getPlatform().equalsIgnoreCase("SOLID") || read.getReadGroup().getPlatform().equalsIgnoreCase("ABI_SOLID")) { int pos = offset; - if( read.getReadNegativeStrandFlag() ) { + if (read.getReadNegativeStrandFlag()) { pos = read.getReadLength() - (offset + 1); } return pos % 5; // the primer round according to http://www3.appliedbiosystems.com/cms/groups/mcb_marketing/documents/generaldocuments/cms_057511.pdf - } else { + } + else { return 1; // nothing to do here because it is always the same } } @Override - public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { - for(int iii = 0; iii < read.getReadLength(); iii++) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { + for (int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } } // Used to get the covariate's value from input csv file in TableRecalibrationWalker @Override - public final Comparable getValue( final String str ) { - return Integer.parseInt( str ); + public final Comparable getValue(final String str) { + return Integer.parseInt(str); } } \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/QualityScoreCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/QualityScoreCovariate.java index f85b523501..1ed4a6fe85 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/QualityScoreCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/QualityScoreCovariate.java @@ -1,7 +1,7 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; -import net.sf.samtools.SAMRecord; import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import java.util.Arrays; @@ -42,25 +42,26 @@ public class QualityScoreCovariate implements RequiredCovariate { // Initialize any member variables using the command-line arguments passed to the walkers @Override - public void initialize( final RecalibrationArgumentCollection RAC ) { + public void initialize(final RecalibrationArgumentCollection RAC) { } @Override - public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { - if( modelType == BaseRecalibration.BaseRecalibrationType.BASE_SUBSTITUTION ) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { + if (modelType == BaseRecalibration.BaseRecalibrationType.BASE_SUBSTITUTION) { byte[] baseQualities = read.getBaseQualities(); - for(int i = 0; i < read.getReadLength(); i++) { + for (int i = 0; i < read.getReadLength(); i++) { comparable[i] = (int) baseQualities[i]; } - } else { // model == BASE_INSERTION || model == BASE_DELETION + } + else { // model == BASE_INSERTION || model == BASE_DELETION Arrays.fill(comparable, 45); // Some day in the future when base insertion and base deletion quals exist the samtools API will - // be updated and the original quals will be pulled here, but for now we assume the original quality is a flat Q45 + // be updated and the original quals will be pulled here, but for now we assume the original quality is a flat Q45 } } // Used to get the covariate's value from input csv file in TableRecalibrationWalker @Override - public final Comparable getValue( final String str ) { - return Integer.parseInt( str ); + public final Comparable getValue(final String str) { + return Integer.parseInt(str); } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ReadGroupCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ReadGroupCovariate.java index e270771288..27e1d82635 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ReadGroupCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ReadGroupCovariate.java @@ -1,7 +1,7 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; -import net.sf.samtools.SAMRecord; import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; /* * Copyright (c) 2009 The Broad Institute @@ -38,24 +38,22 @@ public class ReadGroupCovariate implements RequiredCovariate { - public static final String defaultReadGroup = "DefaultReadGroup"; - // Initialize any member variables using the command-line arguments passed to the walkers @Override - public void initialize( final RecalibrationArgumentCollection RAC ) { + public void initialize(final RecalibrationArgumentCollection RAC) { } @Override - public void getValues( final SAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType ) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { final String readGroupId = read.getReadGroup().getReadGroupId(); - for(int i = 0; i < read.getReadLength(); i++) { + for (int i = 0; i < read.getReadLength(); i++) { comparable[i] = readGroupId; } } // Used to get the covariate's value from input csv file in TableRecalibrationWalker @Override - public final Comparable getValue( final String str ) { + public final Comparable getValue(final String str) { return str; } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java index be02063dec..18b33c0e80 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java @@ -25,7 +25,6 @@ package org.broadinstitute.sting.gatk.walkers.recalibration; -import net.sf.samtools.SAMRecord; import net.sf.samtools.SAMUtils; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.utils.BaseUtils; @@ -67,22 +66,36 @@ public class RecalDataManager { private static boolean warnUserNullPlatform = false; public enum SOLID_RECAL_MODE { - /** Treat reference inserted bases as reference matching bases. Very unsafe! */ + /** + * Treat reference inserted bases as reference matching bases. Very unsafe! + */ DO_NOTHING, - /** Set reference inserted bases and the previous base (because of color space alignment details) to Q0. This is the default option. */ + /** + * Set reference inserted bases and the previous base (because of color space alignment details) to Q0. This is the default option. + */ SET_Q_ZERO, - /** In addition to setting the quality scores to zero, also set the base itself to 'N'. This is useful to visualize in IGV. */ + /** + * In addition to setting the quality scores to zero, also set the base itself to 'N'. This is useful to visualize in IGV. + */ SET_Q_ZERO_BASE_N, - /** Look at the color quality scores and probabilistically decide to change the reference inserted base to be the base which is implied by the original color space instead of the reference. */ + /** + * Look at the color quality scores and probabilistically decide to change the reference inserted base to be the base which is implied by the original color space instead of the reference. + */ REMOVE_REF_BIAS } public enum SOLID_NOCALL_STRATEGY { - /** When a no call is detected throw an exception to alert the user that recalibrating this SOLiD data is unsafe. This is the default option. */ + /** + * When a no call is detected throw an exception to alert the user that recalibrating this SOLiD data is unsafe. This is the default option. + */ THROW_EXCEPTION, - /** Leave the read in the output bam completely untouched. This mode is only okay if the no calls are very rare. */ + /** + * Leave the read in the output bam completely untouched. This mode is only okay if the no calls are very rare. + */ LEAVE_READ_UNRECALIBRATED, - /** Mark these reads as failing vendor quality checks so they can be filtered out by downstream analyses. */ + /** + * Mark these reads as failing vendor quality checks so they can be filtered out by downstream analyses. + */ PURGE_READ } @@ -93,16 +106,17 @@ public RecalDataManager() { dataCollapsedByCovariate = null; } - public RecalDataManager( final boolean createCollapsedTables, final int numCovariates ) { - if( createCollapsedTables ) { // Initialize all the collapsed tables, only used by TableRecalibrationWalker + public RecalDataManager(final boolean createCollapsedTables, final int numCovariates) { + if (createCollapsedTables) { // Initialize all the collapsed tables, only used by TableRecalibrationWalker data = null; dataCollapsedReadGroup = new NestedHashMap(); dataCollapsedQualityScore = new NestedHashMap(); dataCollapsedByCovariate = new ArrayList(); - for( int iii = 0; iii < numCovariates - 2; iii++ ) { // readGroup and QualityScore aren't counted here, their tables are separate - dataCollapsedByCovariate.add( new NestedHashMap() ); + for (int iii = 0; iii < numCovariates - 2; iii++) { // readGroup and QualityScore aren't counted here, their tables are separate + dataCollapsedByCovariate.add(new NestedHashMap()); } - } else { + } + else { data = new NestedHashMap(); dataCollapsedReadGroup = null; dataCollapsedQualityScore = null; @@ -112,54 +126,58 @@ public RecalDataManager( final boolean createCollapsedTables, final int numCovar /** * Add the given mapping to all of the collapsed hash tables - * @param key The list of comparables that is the key for this mapping - * @param fullDatum The RecalDatum which is the data for this mapping + * + * @param key The list of comparables that is the key for this mapping + * @param fullDatum The RecalDatum which is the data for this mapping * @param PRESERVE_QSCORES_LESS_THAN The threshold in report quality for adding to the aggregate collapsed table */ - public final void addToAllTables( final Object[] key, final RecalDatum fullDatum, final int PRESERVE_QSCORES_LESS_THAN ) { + public final void addToAllTables(final Object[] key, final RecalDatum fullDatum, final int PRESERVE_QSCORES_LESS_THAN) { // The full dataset isn't actually ever used for anything because of the sequential calculation so no need to keep the full data HashMap around //data.put(key, thisDatum); // add the mapping to the main table - final int qualityScore = Integer.parseInt( key[1].toString() ); + final int qualityScore = Integer.parseInt(key[1].toString()); final Object[] readGroupCollapsedKey = new Object[1]; final Object[] qualityScoreCollapsedKey = new Object[2]; final Object[] covariateCollapsedKey = new Object[3]; RecalDatum collapsedDatum; // Create dataCollapsedReadGroup, the table where everything except read group has been collapsed - if( qualityScore >= PRESERVE_QSCORES_LESS_THAN ) { + if (qualityScore >= PRESERVE_QSCORES_LESS_THAN) { readGroupCollapsedKey[0] = key[0]; // Make a new key with just the read group - collapsedDatum = (RecalDatum) dataCollapsedReadGroup.get( readGroupCollapsedKey ); - if( collapsedDatum == null ) { - dataCollapsedReadGroup.put( new RecalDatum(fullDatum), readGroupCollapsedKey ); - } else { - collapsedDatum.combine( fullDatum ); // using combine instead of increment in order to calculate overall aggregateQReported + collapsedDatum = (RecalDatum) dataCollapsedReadGroup.get(readGroupCollapsedKey); + if (collapsedDatum == null) { + dataCollapsedReadGroup.put(new RecalDatum(fullDatum), readGroupCollapsedKey); + } + else { + collapsedDatum.combine(fullDatum); // using combine instead of increment in order to calculate overall aggregateQReported } } // Create dataCollapsedQuality, the table where everything except read group and quality score has been collapsed qualityScoreCollapsedKey[0] = key[0]; // Make a new key with the read group ... qualityScoreCollapsedKey[1] = key[1]; // and quality score - collapsedDatum = (RecalDatum) dataCollapsedQualityScore.get( qualityScoreCollapsedKey ); - if( collapsedDatum == null ) { - dataCollapsedQualityScore.put( new RecalDatum(fullDatum), qualityScoreCollapsedKey ); - } else { - collapsedDatum.increment( fullDatum ); + collapsedDatum = (RecalDatum) dataCollapsedQualityScore.get(qualityScoreCollapsedKey); + if (collapsedDatum == null) { + dataCollapsedQualityScore.put(new RecalDatum(fullDatum), qualityScoreCollapsedKey); + } + else { + collapsedDatum.increment(fullDatum); } // Create dataCollapsedByCovariate's, the tables where everything except read group, quality score, and given covariate has been collapsed - for( int iii = 0; iii < dataCollapsedByCovariate.size(); iii++ ) { + for (int iii = 0; iii < dataCollapsedByCovariate.size(); iii++) { covariateCollapsedKey[0] = key[0]; // Make a new key with the read group ... covariateCollapsedKey[1] = key[1]; // and quality score ... final Object theCovariateElement = key[iii + 2]; // and the given covariate - if( theCovariateElement != null ) { + if (theCovariateElement != null) { covariateCollapsedKey[2] = theCovariateElement; - collapsedDatum = (RecalDatum) dataCollapsedByCovariate.get(iii).get( covariateCollapsedKey ); - if( collapsedDatum == null ) { - dataCollapsedByCovariate.get(iii).put( new RecalDatum(fullDatum), covariateCollapsedKey ); - } else { - collapsedDatum.increment( fullDatum ); + collapsedDatum = (RecalDatum) dataCollapsedByCovariate.get(iii).get(covariateCollapsedKey); + if (collapsedDatum == null) { + dataCollapsedByCovariate.get(iii).put(new RecalDatum(fullDatum), covariateCollapsedKey); + } + else { + collapsedDatum.increment(fullDatum); } } } @@ -167,150 +185,162 @@ public final void addToAllTables( final Object[] key, final RecalDatum fullDatum /** * Loop over all the collapsed tables and turn the recalDatums found there into an empirical quality score - * that will be used in the sequential calculation in TableRecalibrationWalker + * that will be used in the sequential calculation in TableRecalibrationWalker + * * @param smoothing The smoothing parameter that goes into empirical quality score calculation - * @param maxQual At which value to cap the quality scores + * @param maxQual At which value to cap the quality scores */ - public final void generateEmpiricalQualities( final int smoothing, final int maxQual ) { + public final void generateEmpiricalQualities(final int smoothing, final int maxQual) { recursivelyGenerateEmpiricalQualities(dataCollapsedReadGroup.data, smoothing, maxQual); recursivelyGenerateEmpiricalQualities(dataCollapsedQualityScore.data, smoothing, maxQual); - for( NestedHashMap map : dataCollapsedByCovariate ) { + for (NestedHashMap map : dataCollapsedByCovariate) { recursivelyGenerateEmpiricalQualities(map.data, smoothing, maxQual); checkForSingletons(map.data); } } - private void recursivelyGenerateEmpiricalQualities( final Map data, final int smoothing, final int maxQual ) { + private void recursivelyGenerateEmpiricalQualities(final Map data, final int smoothing, final int maxQual) { - for( Object comp : data.keySet() ) { + for (Object comp : data.keySet()) { final Object val = data.get(comp); - if( val instanceof RecalDatum ) { // We are at the end of the nested hash maps - ((RecalDatum)val).calcCombinedEmpiricalQuality(smoothing, maxQual); - } else { // Another layer in the nested hash map - recursivelyGenerateEmpiricalQualities( (Map) val, smoothing, maxQual); + if (val instanceof RecalDatum) { // We are at the end of the nested hash maps + ((RecalDatum) val).calcCombinedEmpiricalQuality(smoothing, maxQual); + } + else { // Another layer in the nested hash map + recursivelyGenerateEmpiricalQualities((Map) val, smoothing, maxQual); } } } - private void checkForSingletons( final Map data ) { + private void checkForSingletons(final Map data) { // todo -- this looks like it's better just as a data.valueSet() call? - for( Object comp : data.keySet() ) { + for (Object comp : data.keySet()) { final Object val = data.get(comp); - if( val instanceof RecalDatum ) { // We are at the end of the nested hash maps - if( data.keySet().size() == 1) { + if (val instanceof RecalDatum) { // We are at the end of the nested hash maps + if (data.keySet().size() == 1) { data.clear(); // don't TableRecalibrate a non-required covariate if it only has one element because that correction has already been done ... - // in a previous step of the sequential calculation model + // in a previous step of the sequential calculation model } - } else { // Another layer in the nested hash map - checkForSingletons( (Map) val ); + } + else { // Another layer in the nested hash map + checkForSingletons((Map) val); } } } /** * Get the appropriate collapsed table out of the set of all the tables held by this Object + * * @param covariate Which covariate indexes the desired collapsed HashMap * @return The desired collapsed HashMap */ - public final NestedHashMap getCollapsedTable( final int covariate ) { - if( covariate == 0) { + public final NestedHashMap getCollapsedTable(final int covariate) { + if (covariate == 0) { return dataCollapsedReadGroup; // Table where everything except read group has been collapsed - } else if( covariate == 1 ) { + } + else if (covariate == 1) { return dataCollapsedQualityScore; // Table where everything except read group and quality score has been collapsed - } else { - return dataCollapsedByCovariate.get( covariate - 2 ); // Table where everything except read group, quality score, and given covariate has been collapsed + } + else { + return dataCollapsedByCovariate.get(covariate - 2); // Table where everything except read group, quality score, and given covariate has been collapsed } } /** * Section of code shared between the two recalibration walkers which uses the command line arguments to adjust attributes of the read such as quals or platform string + * * @param read The read to adjust - * @param RAC The list of shared command line arguments + * @param RAC The list of shared command line arguments */ - public static void parseSAMRecord( final SAMRecord read, final RecalibrationArgumentCollection RAC ) { - GATKSAMReadGroupRecord readGroup = ((GATKSAMRecord)read).getReadGroup(); + public static void parseSAMRecord(final GATKSAMRecord read, final RecalibrationArgumentCollection RAC) { + GATKSAMReadGroupRecord readGroup = ((GATKSAMRecord) read).getReadGroup(); // If there are no read groups we have to default to something, and that something could be specified by the user using command line arguments - if( readGroup == null ) { - if( RAC.DEFAULT_READ_GROUP != null && RAC.DEFAULT_PLATFORM != null) { - if( !warnUserNullReadGroup && RAC.FORCE_READ_GROUP == null ) { + if (readGroup == null) { + if (RAC.DEFAULT_READ_GROUP != null && RAC.DEFAULT_PLATFORM != null) { + if (!warnUserNullReadGroup && RAC.FORCE_READ_GROUP == null) { Utils.warnUser("The input .bam file contains reads with no read group. " + - "Defaulting to read group ID = " + RAC.DEFAULT_READ_GROUP + " and platform = " + RAC.DEFAULT_PLATFORM + ". " + - "First observed at read with name = " + read.getReadName() ); + "Defaulting to read group ID = " + RAC.DEFAULT_READ_GROUP + " and platform = " + RAC.DEFAULT_PLATFORM + ". " + + "First observed at read with name = " + read.getReadName()); warnUserNullReadGroup = true; } // There is no readGroup so defaulting to these values - readGroup = new GATKSAMReadGroupRecord( RAC.DEFAULT_READ_GROUP ); - readGroup.setPlatform( RAC.DEFAULT_PLATFORM ); - ((GATKSAMRecord)read).setReadGroup( readGroup ); - } else { - throw new UserException.MalformedBAM(read, "The input .bam file contains reads with no read group. First observed at read with name = " + read.getReadName() ); + readGroup = new GATKSAMReadGroupRecord(RAC.DEFAULT_READ_GROUP); + readGroup.setPlatform(RAC.DEFAULT_PLATFORM); + ((GATKSAMRecord) read).setReadGroup(readGroup); + } + else { + throw new UserException.MalformedBAM(read, "The input .bam file contains reads with no read group. First observed at read with name = " + read.getReadName()); } } - if( RAC.FORCE_READ_GROUP != null && !readGroup.getReadGroupId().equals(RAC.FORCE_READ_GROUP) ) { // Collapse all the read groups into a single common String provided by the user + if (RAC.FORCE_READ_GROUP != null && !readGroup.getReadGroupId().equals(RAC.FORCE_READ_GROUP)) { // Collapse all the read groups into a single common String provided by the user final String oldPlatform = readGroup.getPlatform(); - readGroup = new GATKSAMReadGroupRecord( RAC.FORCE_READ_GROUP ); - readGroup.setPlatform( oldPlatform ); - ((GATKSAMRecord)read).setReadGroup( readGroup ); + readGroup = new GATKSAMReadGroupRecord(RAC.FORCE_READ_GROUP); + readGroup.setPlatform(oldPlatform); + ((GATKSAMRecord) read).setReadGroup(readGroup); } - if( RAC.FORCE_PLATFORM != null && (readGroup.getPlatform() == null || !readGroup.getPlatform().equals(RAC.FORCE_PLATFORM))) { - readGroup.setPlatform( RAC.FORCE_PLATFORM ); + if (RAC.FORCE_PLATFORM != null && (readGroup.getPlatform() == null || !readGroup.getPlatform().equals(RAC.FORCE_PLATFORM))) { + readGroup.setPlatform(RAC.FORCE_PLATFORM); } - if ( readGroup.getPlatform() == null ) { - if( RAC.DEFAULT_PLATFORM != null ) { - if( !warnUserNullPlatform ) { + if (readGroup.getPlatform() == null) { + if (RAC.DEFAULT_PLATFORM != null) { + if (!warnUserNullPlatform) { Utils.warnUser("The input .bam file contains reads with no platform information. " + - "Defaulting to platform = " + RAC.DEFAULT_PLATFORM + ". " + - "First observed at read with name = " + read.getReadName() ); + "Defaulting to platform = " + RAC.DEFAULT_PLATFORM + ". " + + "First observed at read with name = " + read.getReadName()); warnUserNullPlatform = true; } - readGroup.setPlatform( RAC.DEFAULT_PLATFORM ); - } else { - throw new UserException.MalformedBAM(read, "The input .bam file contains reads with no platform information. First observed at read with name = " + read.getReadName() ); + readGroup.setPlatform(RAC.DEFAULT_PLATFORM); + } + else { + throw new UserException.MalformedBAM(read, "The input .bam file contains reads with no platform information. First observed at read with name = " + read.getReadName()); } } } /** * Parse through the color space of the read and add a new tag to the SAMRecord that says which bases are inconsistent with the color space + * * @param read The SAMRecord to parse */ - public static void parseColorSpace( final SAMRecord read ) { + public static void parseColorSpace(final GATKSAMRecord read) { // If this is a SOLID read then we have to check if the color space is inconsistent. This is our only sign that SOLID has inserted the reference base - if( read.getReadGroup().getPlatform().toUpperCase().contains("SOLID") ) { - if( read.getAttribute(RecalDataManager.COLOR_SPACE_INCONSISTENCY_TAG) == null ) { // Haven't calculated the inconsistency array yet for this read + if (read.getReadGroup().getPlatform().toUpperCase().contains("SOLID")) { + if (read.getAttribute(RecalDataManager.COLOR_SPACE_INCONSISTENCY_TAG) == null) { // Haven't calculated the inconsistency array yet for this read final Object attr = read.getAttribute(RecalDataManager.COLOR_SPACE_ATTRIBUTE_TAG); - if( attr != null ) { + if (attr != null) { byte[] colorSpace; - if( attr instanceof String ) { - colorSpace = ((String)attr).getBytes(); - } else { + if (attr instanceof String) { + colorSpace = ((String) attr).getBytes(); + } + else { throw new UserException.MalformedBAM(read, String.format("Value encoded by %s in %s isn't a string!", RecalDataManager.COLOR_SPACE_ATTRIBUTE_TAG, read.getReadName())); } // Loop over the read and calculate first the inferred bases from the color and then check if it is consistent with the read byte[] readBases = read.getReadBases(); - if( read.getReadNegativeStrandFlag() ) { - readBases = BaseUtils.simpleReverseComplement( read.getReadBases() ); + if (read.getReadNegativeStrandFlag()) { + readBases = BaseUtils.simpleReverseComplement(read.getReadBases()); } final byte[] inconsistency = new byte[readBases.length]; int iii; byte prevBase = colorSpace[0]; // The sentinel - for( iii = 0; iii < readBases.length; iii++ ) { - final byte thisBase = getNextBaseFromColor( read, prevBase, colorSpace[iii + 1] ); - inconsistency[iii] = (byte)( thisBase == readBases[iii] ? 0 : 1 ); + for (iii = 0; iii < readBases.length; iii++) { + final byte thisBase = getNextBaseFromColor(read, prevBase, colorSpace[iii + 1]); + inconsistency[iii] = (byte) (thisBase == readBases[iii] ? 0 : 1); prevBase = readBases[iii]; } - read.setAttribute( RecalDataManager.COLOR_SPACE_INCONSISTENCY_TAG, inconsistency ); + read.setAttribute(RecalDataManager.COLOR_SPACE_INCONSISTENCY_TAG, inconsistency); - } else { + } + else { throw new UserException.MalformedBAM(read, "Unable to find color space information in SOLiD read. First observed at read with name = " + read.getReadName() + - " Unfortunately this .bam file can not be recalibrated without color space information because of potential reference bias."); + " Unfortunately this .bam file can not be recalibrated without color space information because of potential reference bias."); } } } @@ -319,52 +349,57 @@ public static void parseColorSpace( final SAMRecord read ) { /** * Parse through the color space of the read and apply the desired --solid_recal_mode correction to the bases * This method doesn't add the inconsistent tag to the read like parseColorSpace does - * @param read The SAMRecord to parse + * + * @param read The SAMRecord to parse * @param originalQualScores The array of original quality scores to modify during the correction - * @param solidRecalMode Which mode of solid recalibration to apply - * @param refBases The reference for this read + * @param solidRecalMode Which mode of solid recalibration to apply + * @param refBases The reference for this read * @return A new array of quality scores that have been ref bias corrected */ - public static byte[] calcColorSpace( final SAMRecord read, byte[] originalQualScores, final SOLID_RECAL_MODE solidRecalMode, final byte[] refBases ) { + public static byte[] calcColorSpace(final GATKSAMRecord read, byte[] originalQualScores, final SOLID_RECAL_MODE solidRecalMode, final byte[] refBases) { final Object attr = read.getAttribute(RecalDataManager.COLOR_SPACE_ATTRIBUTE_TAG); - if( attr != null ) { + if (attr != null) { byte[] colorSpace; - if( attr instanceof String ) { - colorSpace = ((String)attr).getBytes(); - } else { + if (attr instanceof String) { + colorSpace = ((String) attr).getBytes(); + } + else { throw new ReviewedStingException(String.format("Value encoded by %s in %s isn't a string!", RecalDataManager.COLOR_SPACE_ATTRIBUTE_TAG, read.getReadName())); } // Loop over the read and calculate first the inferred bases from the color and then check if it is consistent with the read byte[] readBases = read.getReadBases(); final byte[] colorImpliedBases = readBases.clone(); - byte[] refBasesDirRead = AlignmentUtils.alignmentToByteArray( read.getCigar(), read.getReadBases(), refBases ); //BUGBUG: This needs to change when read walkers are changed to give the aligned refBases - if( read.getReadNegativeStrandFlag() ) { - readBases = BaseUtils.simpleReverseComplement( read.getReadBases() ); - refBasesDirRead = BaseUtils.simpleReverseComplement( refBasesDirRead.clone() ); + byte[] refBasesDirRead = AlignmentUtils.alignmentToByteArray(read.getCigar(), read.getReadBases(), refBases); //BUGBUG: This needs to change when read walkers are changed to give the aligned refBases + if (read.getReadNegativeStrandFlag()) { + readBases = BaseUtils.simpleReverseComplement(read.getReadBases()); + refBasesDirRead = BaseUtils.simpleReverseComplement(refBasesDirRead.clone()); } final int[] inconsistency = new int[readBases.length]; byte prevBase = colorSpace[0]; // The sentinel - for( int iii = 0; iii < readBases.length; iii++ ) { - final byte thisBase = getNextBaseFromColor( read, prevBase, colorSpace[iii + 1] ); + for (int iii = 0; iii < readBases.length; iii++) { + final byte thisBase = getNextBaseFromColor(read, prevBase, colorSpace[iii + 1]); colorImpliedBases[iii] = thisBase; - inconsistency[iii] = ( thisBase == readBases[iii] ? 0 : 1 ); + inconsistency[iii] = (thisBase == readBases[iii] ? 0 : 1); prevBase = readBases[iii]; } // Now that we have the inconsistency array apply the desired correction to the inconsistent bases - if( solidRecalMode == SOLID_RECAL_MODE.SET_Q_ZERO ) { // Set inconsistent bases and the one before it to Q0 + if (solidRecalMode == SOLID_RECAL_MODE.SET_Q_ZERO) { // Set inconsistent bases and the one before it to Q0 final boolean setBaseN = false; originalQualScores = solidRecalSetToQZero(read, readBases, inconsistency, originalQualScores, refBasesDirRead, setBaseN); - } else if( solidRecalMode == SOLID_RECAL_MODE.SET_Q_ZERO_BASE_N ) { + } + else if (solidRecalMode == SOLID_RECAL_MODE.SET_Q_ZERO_BASE_N) { final boolean setBaseN = true; originalQualScores = solidRecalSetToQZero(read, readBases, inconsistency, originalQualScores, refBasesDirRead, setBaseN); - } else if( solidRecalMode == SOLID_RECAL_MODE.REMOVE_REF_BIAS ) { // Use the color space quality to probabilistically remove ref bases at inconsistent color space bases + } + else if (solidRecalMode == SOLID_RECAL_MODE.REMOVE_REF_BIAS) { // Use the color space quality to probabilistically remove ref bases at inconsistent color space bases solidRecalRemoveRefBias(read, readBases, inconsistency, colorImpliedBases, refBasesDirRead); } - } else { + } + else { throw new UserException.MalformedBAM(read, "Unable to find color space information in SOLiD read. First observed at read with name = " + read.getReadName() + " Unfortunately this .bam file can not be recalibrated without color space information because of potential reference bias."); } @@ -372,26 +407,28 @@ public static byte[] calcColorSpace( final SAMRecord read, byte[] originalQualSc return originalQualScores; } - public static boolean checkNoCallColorSpace( final SAMRecord read ) { - if( read.getReadGroup().getPlatform().toUpperCase().contains("SOLID") ) { + public static boolean checkNoCallColorSpace(final GATKSAMRecord read) { + if (read.getReadGroup().getPlatform().toUpperCase().contains("SOLID")) { final Object attr = read.getAttribute(RecalDataManager.COLOR_SPACE_ATTRIBUTE_TAG); - if( attr != null ) { + if (attr != null) { byte[] colorSpace; - if( attr instanceof String ) { - colorSpace = ((String)attr).substring(1).getBytes(); // trim off the Sentinel - } else { + if (attr instanceof String) { + colorSpace = ((String) attr).substring(1).getBytes(); // trim off the Sentinel + } + else { throw new ReviewedStingException(String.format("Value encoded by %s in %s isn't a string!", RecalDataManager.COLOR_SPACE_ATTRIBUTE_TAG, read.getReadName())); } - for( byte color : colorSpace ) { - if( color != (byte)'0' && color != (byte)'1' && color != (byte)'2' && color != (byte)'3' ) { + for (byte color : colorSpace) { + if (color != (byte) '0' && color != (byte) '1' && color != (byte) '2' && color != (byte) '3') { return true; // There is a bad color in this SOLiD read and the user wants to skip over it } } - } else { + } + else { throw new UserException.MalformedBAM(read, "Unable to find color space information in SOLiD read. First observed at read with name = " + read.getReadName() + - " Unfortunately this .bam file can not be recalibrated without color space information because of potential reference bias."); + " Unfortunately this .bam file can not be recalibrated without color space information because of potential reference bias."); } } @@ -400,90 +437,105 @@ public static boolean checkNoCallColorSpace( final SAMRecord read ) { /** * Perform the SET_Q_ZERO solid recalibration. Inconsistent color space bases and their previous base are set to quality zero - * @param read The SAMRecord to recalibrate - * @param readBases The bases in the read which have been RC'd if necessary - * @param inconsistency The array of 1/0 that says if this base is inconsistent with its color + * + * @param read The SAMRecord to recalibrate + * @param readBases The bases in the read which have been RC'd if necessary + * @param inconsistency The array of 1/0 that says if this base is inconsistent with its color * @param originalQualScores The array of original quality scores to set to zero if needed - * @param refBases The reference which has been RC'd if necessary - * @param setBaseN Should we also set the base to N as well as quality zero in order to visualize in IGV or something similar + * @param refBases The reference which has been RC'd if necessary + * @param setBaseN Should we also set the base to N as well as quality zero in order to visualize in IGV or something similar * @return The byte array of original quality scores some of which might have been set to zero */ - private static byte[] solidRecalSetToQZero( final SAMRecord read, byte[] readBases, final int[] inconsistency, final byte[] originalQualScores, - final byte[] refBases, final boolean setBaseN ) { + private static byte[] solidRecalSetToQZero(final GATKSAMRecord read, byte[] readBases, final int[] inconsistency, final byte[] originalQualScores, final byte[] refBases, final boolean setBaseN) { final boolean negStrand = read.getReadNegativeStrandFlag(); - for( int iii = 1; iii < originalQualScores.length; iii++ ) { - if( inconsistency[iii] == 1 ) { - if( readBases[iii] == refBases[iii] ) { - if( negStrand ) { originalQualScores[originalQualScores.length-(iii+1)] = (byte)0; } - else { originalQualScores[iii] = (byte)0; } - if( setBaseN ) { readBases[iii] = (byte)'N'; } + for (int iii = 1; iii < originalQualScores.length; iii++) { + if (inconsistency[iii] == 1) { + if (readBases[iii] == refBases[iii]) { + if (negStrand) { + originalQualScores[originalQualScores.length - (iii + 1)] = (byte) 0; + } + else { + originalQualScores[iii] = (byte) 0; + } + if (setBaseN) { + readBases[iii] = (byte) 'N'; + } } // Set the prev base to Q0 as well - if( readBases[iii-1] == refBases[iii-1] ) { - if( negStrand ) { originalQualScores[originalQualScores.length-iii] = (byte)0; } - else { originalQualScores[iii-1] = (byte)0; } - if( setBaseN ) { readBases[iii-1] = (byte)'N'; } + if (readBases[iii - 1] == refBases[iii - 1]) { + if (negStrand) { + originalQualScores[originalQualScores.length - iii] = (byte) 0; + } + else { + originalQualScores[iii - 1] = (byte) 0; + } + if (setBaseN) { + readBases[iii - 1] = (byte) 'N'; + } } } } - if( negStrand ) { - readBases = BaseUtils.simpleReverseComplement( readBases.clone() ); // Put the bases back in reverse order to stuff them back in the read + if (negStrand) { + readBases = BaseUtils.simpleReverseComplement(readBases.clone()); // Put the bases back in reverse order to stuff them back in the read } - read.setReadBases( readBases ); + read.setReadBases(readBases); return originalQualScores; } /** * Peform the REMOVE_REF_BIAS solid recalibration. Look at the color space qualities and probabilistically decide if the base should be change to match the color or left as reference - * @param read The SAMRecord to recalibrate - * @param readBases The bases in the read which have been RC'd if necessary - * @param inconsistency The array of 1/0 that says if this base is inconsistent with its color + * + * @param read The SAMRecord to recalibrate + * @param readBases The bases in the read which have been RC'd if necessary + * @param inconsistency The array of 1/0 that says if this base is inconsistent with its color * @param colorImpliedBases The bases implied by the color space, RC'd if necessary - * @param refBases The reference which has been RC'd if necessary + * @param refBases The reference which has been RC'd if necessary */ - private static void solidRecalRemoveRefBias( final SAMRecord read, byte[] readBases, final int[] inconsistency, final byte[] colorImpliedBases, - final byte[] refBases) { + private static void solidRecalRemoveRefBias(final GATKSAMRecord read, byte[] readBases, final int[] inconsistency, final byte[] colorImpliedBases, final byte[] refBases) { final Object attr = read.getAttribute(RecalDataManager.COLOR_SPACE_QUAL_ATTRIBUTE_TAG); - if( attr != null ) { + if (attr != null) { byte[] colorSpaceQuals; - if( attr instanceof String ) { - String x = (String)attr; + if (attr instanceof String) { + String x = (String) attr; colorSpaceQuals = x.getBytes(); SAMUtils.fastqToPhred(colorSpaceQuals); - } else { + } + else { throw new ReviewedStingException(String.format("Value encoded by %s in %s isn't a string!", RecalDataManager.COLOR_SPACE_QUAL_ATTRIBUTE_TAG, read.getReadName())); } - for( int iii = 1; iii < inconsistency.length - 1; iii++ ) { - if( inconsistency[iii] == 1 ) { - for( int jjj = iii - 1; jjj <= iii; jjj++ ) { // Correct this base and the one before it along the direction of the read - if( jjj == iii || inconsistency[jjj] == 0 ) { // Don't want to correct the previous base a second time if it was already corrected in the previous step - if( readBases[jjj] == refBases[jjj] ) { - if( colorSpaceQuals[jjj] == colorSpaceQuals[jjj+1] ) { // Equal evidence for the color implied base and the reference base, so flip a coin - final int rand = GenomeAnalysisEngine.getRandomGenerator().nextInt( 2 ); - if( rand == 0 ) { // The color implied base won the coin flip + for (int iii = 1; iii < inconsistency.length - 1; iii++) { + if (inconsistency[iii] == 1) { + for (int jjj = iii - 1; jjj <= iii; jjj++) { // Correct this base and the one before it along the direction of the read + if (jjj == iii || inconsistency[jjj] == 0) { // Don't want to correct the previous base a second time if it was already corrected in the previous step + if (readBases[jjj] == refBases[jjj]) { + if (colorSpaceQuals[jjj] == colorSpaceQuals[jjj + 1]) { // Equal evidence for the color implied base and the reference base, so flip a coin + final int rand = GenomeAnalysisEngine.getRandomGenerator().nextInt(2); + if (rand == 0) { // The color implied base won the coin flip readBases[jjj] = colorImpliedBases[jjj]; } - } else { - final int maxQuality = Math.max((int)colorSpaceQuals[jjj], (int)colorSpaceQuals[jjj+1]); - final int minQuality = Math.min((int)colorSpaceQuals[jjj], (int)colorSpaceQuals[jjj+1]); + } + else { + final int maxQuality = Math.max((int) colorSpaceQuals[jjj], (int) colorSpaceQuals[jjj + 1]); + final int minQuality = Math.min((int) colorSpaceQuals[jjj], (int) colorSpaceQuals[jjj + 1]); int diffInQuality = maxQuality - minQuality; int numLow = minQuality; - if( numLow == 0 ) { + if (numLow == 0) { numLow++; diffInQuality++; } - final int numHigh = Math.round( numLow * (float)Math.pow(10.0f, (float) diffInQuality / 10.0f) ); // The color with higher quality is exponentially more likely - final int rand = GenomeAnalysisEngine.getRandomGenerator().nextInt( numLow + numHigh ); - if( rand >= numLow ) { // higher q score won - if( maxQuality == (int)colorSpaceQuals[jjj] ) { + final int numHigh = Math.round(numLow * (float) Math.pow(10.0f, (float) diffInQuality / 10.0f)); // The color with higher quality is exponentially more likely + final int rand = GenomeAnalysisEngine.getRandomGenerator().nextInt(numLow + numHigh); + if (rand >= numLow) { // higher q score won + if (maxQuality == (int) colorSpaceQuals[jjj]) { readBases[jjj] = colorImpliedBases[jjj]; } // else ref color had higher q score, and won out, so nothing to do here - } else { // lower q score won - if( minQuality == (int)colorSpaceQuals[jjj] ) { + } + else { // lower q score won + if (minQuality == (int) colorSpaceQuals[jjj]) { readBases[jjj] = colorImpliedBases[jjj]; } // else ref color had lower q score, and won out, so nothing to do here } @@ -494,52 +546,56 @@ private static void solidRecalRemoveRefBias( final SAMRecord read, byte[] readBa } } - if( read.getReadNegativeStrandFlag() ) { - readBases = BaseUtils.simpleReverseComplement( readBases.clone() ); // Put the bases back in reverse order to stuff them back in the read + if (read.getReadNegativeStrandFlag()) { + readBases = BaseUtils.simpleReverseComplement(readBases.clone()); // Put the bases back in reverse order to stuff them back in the read } - read.setReadBases( readBases ); - } else { // No color space quality tag in file + read.setReadBases(readBases); + } + else { // No color space quality tag in file throw new UserException.MalformedBAM(read, "REMOVE_REF_BIAS recal mode requires color space qualities but they can't be found for read: " + read.getReadName()); } } /** * Given the base and the color calculate the next base in the sequence + * * @param prevBase The base - * @param color The color + * @param color The color * @return The next base in the sequence */ - private static byte getNextBaseFromColor( SAMRecord read, final byte prevBase, final byte color ) { - switch(color) { + private static byte getNextBaseFromColor(GATKSAMRecord read, final byte prevBase, final byte color) { + switch (color) { case '0': return prevBase; case '1': - return performColorOne( prevBase ); + return performColorOne(prevBase); case '2': - return performColorTwo( prevBase ); + return performColorTwo(prevBase); case '3': - return performColorThree( prevBase ); + return performColorThree(prevBase); default: - throw new UserException.MalformedBAM(read, "Unrecognized color space in SOLID read, color = " + (char)color + - " Unfortunately this bam file can not be recalibrated without full color space information because of potential reference bias."); + throw new UserException.MalformedBAM(read, "Unrecognized color space in SOLID read, color = " + (char) color + + " Unfortunately this bam file can not be recalibrated without full color space information because of potential reference bias."); } } /** * Check if this base is inconsistent with its color space. If it is then SOLID inserted the reference here and we should reduce the quality - * @param read The read which contains the color space to check against + * + * @param read The read which contains the color space to check against * @param offset The offset in the read at which to check * @return Returns true if the base was inconsistent with the color space */ - public static boolean isInconsistentColorSpace( final SAMRecord read, final int offset ) { + public static boolean isInconsistentColorSpace(final GATKSAMRecord read, final int offset) { final Object attr = read.getAttribute(RecalDataManager.COLOR_SPACE_INCONSISTENCY_TAG); - if( attr != null ) { - final byte[] inconsistency = (byte[])attr; + if (attr != null) { + final byte[] inconsistency = (byte[]) attr; // NOTE: The inconsistency array is in the direction of the read, not aligned to the reference! - if( read.getReadNegativeStrandFlag() ) { // Negative direction - return inconsistency[inconsistency.length - offset - 1] != (byte)0; - } else { // Forward direction - return inconsistency[offset] != (byte)0; + if (read.getReadNegativeStrandFlag()) { // Negative direction + return inconsistency[inconsistency.length - offset - 1] != (byte) 0; + } + else { // Forward direction + return inconsistency[offset] != (byte) 0; } // This block of code is for if you want to check both the offset and the next base for color space inconsistency @@ -557,7 +613,8 @@ public static boolean isInconsistentColorSpace( final SAMRecord read, final int // } //} - } else { // No inconsistency array, so nothing is inconsistent + } + else { // No inconsistency array, so nothing is inconsistent return false; } } @@ -566,33 +623,32 @@ public static boolean isInconsistentColorSpace( final SAMRecord read, final int * Computes all requested covariates for every offset in the given read * by calling covariate.getValues(..). * - * @param gatkRead The read for which to compute covariate values. + * @param gatkRead The read for which to compute covariate values. * @param requestedCovariates The list of requested covariates. * @return An array of covariate values where result[i][j] is the covariate - * value for the ith position in the read and the jth covariate in - * reqeustedCovariates list. + * value for the ith position in the read and the jth covariate in + * reqeustedCovariates list. */ - public static Comparable[][] computeCovariates( final GATKSAMRecord gatkRead, final List requestedCovariates, final BaseRecalibration.BaseRecalibrationType modelType ) { - //compute all covariates for this read - final List requestedCovariatesRef = requestedCovariates; - final int numRequestedCovariates = requestedCovariatesRef.size(); - final int readLength = gatkRead.getReadLength(); - - final Comparable[][] covariateValues_offset_x_covar = new Comparable[readLength][numRequestedCovariates]; - final Comparable[] tempCovariateValuesHolder = new Comparable[readLength]; - - // Loop through the list of requested covariates and compute the values of each covariate for all positions in this read - for( int i = 0; i < numRequestedCovariates; i++ ) { - requestedCovariatesRef.get(i).getValues( gatkRead, tempCovariateValuesHolder, modelType ); - for(int j = 0; j < readLength; j++) { - //copy values into a 2D array that allows all covar types to be extracted at once for - //an offset j by doing covariateValues_offset_x_covar[j]. This avoids the need to later iterate over covar types. - covariateValues_offset_x_covar[j][i] = tempCovariateValuesHolder[j]; - } - } - - return covariateValues_offset_x_covar; - } + public static Comparable[][] computeCovariates(final GATKSAMRecord gatkRead, final List requestedCovariates, final BaseRecalibration.BaseRecalibrationType modelType) { + //compute all covariates for this read + final int numRequestedCovariates = requestedCovariates.size(); + final int readLength = gatkRead.getReadLength(); + + final Comparable[][] covariateValues_offset_x_covar = new Comparable[readLength][numRequestedCovariates]; + final Comparable[] tempCovariateValuesHolder = new Comparable[readLength]; + + // Loop through the list of requested covariates and compute the values of each covariate for all positions in this read + for (int i = 0; i < numRequestedCovariates; i++) { + requestedCovariates.get(i).getValues(gatkRead, tempCovariateValuesHolder, modelType); + for (int j = 0; j < readLength; j++) { + //copy values into a 2D array that allows all covar types to be extracted at once for + //an offset j by doing covariateValues_offset_x_covar[j]. This avoids the need to later iterate over covar types. + covariateValues_offset_x_covar[j][i] = tempCovariateValuesHolder[j]; + } + } + + return covariateValues_offset_x_covar; + } /** * Perform a ceratin transversion (A <-> C or G <-> T) on the base. @@ -603,14 +659,19 @@ public static Comparable[][] computeCovariates( final GATKSAMRecord gatkRead, fi private static byte performColorOne(byte base) { switch (base) { case 'A': - case 'a': return 'C'; + case 'a': + return 'C'; case 'C': - case 'c': return 'A'; + case 'c': + return 'A'; case 'G': - case 'g': return 'T'; + case 'g': + return 'T'; case 'T': - case 't': return 'G'; - default: return base; + case 't': + return 'G'; + default: + return base; } } @@ -623,14 +684,19 @@ private static byte performColorOne(byte base) { private static byte performColorTwo(byte base) { switch (base) { case 'A': - case 'a': return 'G'; + case 'a': + return 'G'; case 'C': - case 'c': return 'T'; + case 'c': + return 'T'; case 'G': - case 'g': return 'A'; + case 'g': + return 'A'; case 'T': - case 't': return 'C'; - default: return base; + case 't': + return 'C'; + default: + return base; } } @@ -643,14 +709,19 @@ private static byte performColorTwo(byte base) { private static byte performColorThree(byte base) { switch (base) { case 'A': - case 'a': return 'T'; + case 'a': + return 'T'; case 'C': - case 'c': return 'G'; + case 'c': + return 'G'; case 'G': - case 'g': return 'C'; + case 'g': + return 'C'; case 'T': - case 't': return 'A'; - default: return base; + case 't': + return 'A'; + default: + return base; } } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationArgumentCollection.java index 75de84cb40..ffdb0cca7a 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationArgumentCollection.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationArgumentCollection.java @@ -43,36 +43,36 @@ public class RecalibrationArgumentCollection { // Shared Command Line Arguments ////////////////////////////////// @Hidden - @Argument(fullName="default_read_group", shortName="dRG", required=false, doc="If a read has no read group then default to the provided String.") + @Argument(fullName = "default_read_group", shortName = "dRG", required = false, doc = "If a read has no read group then default to the provided String.") public String DEFAULT_READ_GROUP = null; @Hidden - @Argument(fullName="default_platform", shortName="dP", required=false, doc="If a read has no platform then default to the provided String. Valid options are illumina, 454, and solid.") + @Argument(fullName = "default_platform", shortName = "dP", required = false, doc = "If a read has no platform then default to the provided String. Valid options are illumina, 454, and solid.") public String DEFAULT_PLATFORM = null; @Hidden - @Argument(fullName="force_read_group", shortName="fRG", required=false, doc="If provided, the read group ID of EVERY read will be forced to be the provided String. This is useful to collapse all data into a single read group.") + @Argument(fullName = "force_read_group", shortName = "fRG", required = false, doc = "If provided, the read group ID of EVERY read will be forced to be the provided String. This is useful to collapse all data into a single read group.") public String FORCE_READ_GROUP = null; @Hidden - @Argument(fullName="force_platform", shortName="fP", required=false, doc="If provided, the platform of EVERY read will be forced to be the provided String. Valid options are illumina, 454, and solid.") + @Argument(fullName = "force_platform", shortName = "fP", required = false, doc = "If provided, the platform of EVERY read will be forced to be the provided String. Valid options are illumina, 454, and solid.") public String FORCE_PLATFORM = null; @Hidden - @Argument(fullName = "window_size_nqs", shortName="nqs", doc="The window size used by MinimumNQSCovariate for its calculation", required=false) + @Argument(fullName = "window_size_nqs", shortName = "nqs", doc = "The window size used by MinimumNQSCovariate for its calculation", required = false) public int WINDOW_SIZE = 5; /** * This window size tells the module in how big of a neighborhood around the current base it should look for the minimum base quality score. */ @Hidden - @Argument(fullName = "homopolymer_nback", shortName="nback", doc="The number of previous bases to look at in HomopolymerCovariate", required=false) + @Argument(fullName = "homopolymer_nback", shortName = "nback", doc = "The number of previous bases to look at in HomopolymerCovariate", required = false) public int HOMOPOLYMER_NBACK = 7; @Hidden - @Argument(fullName = "exception_if_no_tile", shortName="throwTileException", doc="If provided, TileCovariate will throw an exception when no tile can be found. The default behavior is to use tile = -1", required=false) + @Argument(fullName = "exception_if_no_tile", shortName = "throwTileException", doc = "If provided, TileCovariate will throw an exception when no tile can be found. The default behavior is to use tile = -1", required = false) public boolean EXCEPTION_IF_NO_TILE = false; /** * CountCovariates and TableRecalibration accept a --solid_recal_mode flag which governs how the recalibrator handles the * reads which have had the reference inserted because of color space inconsistencies. */ - @Argument(fullName="solid_recal_mode", shortName="sMode", required = false, doc="How should we recalibrate solid bases in which the reference was inserted? Options = DO_NOTHING, SET_Q_ZERO, SET_Q_ZERO_BASE_N, or REMOVE_REF_BIAS") + @Argument(fullName = "solid_recal_mode", shortName = "sMode", required = false, doc = "How should we recalibrate solid bases in which the reference was inserted? Options = DO_NOTHING, SET_Q_ZERO, SET_Q_ZERO_BASE_N, or REMOVE_REF_BIAS") public RecalDataManager.SOLID_RECAL_MODE SOLID_RECAL_MODE = RecalDataManager.SOLID_RECAL_MODE.SET_Q_ZERO; /** @@ -80,6 +80,13 @@ public class RecalibrationArgumentCollection { * no calls in the color space tag. Unfortunately because of the reference inserted bases mentioned above, reads with no calls in * their color space tag can not be recalibrated. */ - @Argument(fullName = "solid_nocall_strategy", shortName="solid_nocall_strategy", doc="Defines the behavior of the recalibrator when it encounters no calls in the color space. Options = THROW_EXCEPTION, LEAVE_READ_UNRECALIBRATED, or PURGE_READ", required=false) + @Argument(fullName = "solid_nocall_strategy", shortName = "solid_nocall_strategy", doc = "Defines the behavior of the recalibrator when it encounters no calls in the color space. Options = THROW_EXCEPTION, LEAVE_READ_UNRECALIBRATED, or PURGE_READ", required = false) public RecalDataManager.SOLID_NOCALL_STRATEGY SOLID_NOCALL_STRATEGY = RecalDataManager.SOLID_NOCALL_STRATEGY.THROW_EXCEPTION; + + /** + * The context covariate will use a context of this size to calculate it's covariate value + */ + @Argument(fullName = "context_size", shortName = "cs", doc = "size of the k-mer context to be used", required = false) + int CONTEXT_SIZE = 8; + } From 5961868a7fe4463549fdf43045315f962510e1bd Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Mon, 6 Feb 2012 22:47:27 -0500 Subject: [PATCH 219/356] fixup for BQSR (HC integration tests) In the new BQSR implementation, covariates do depend on the RecalibrationArgumentCollection. --- .../recalibration/RecalibrationArgumentCollection.java | 2 +- .../sting/utils/recalibration/BaseRecalibration.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationArgumentCollection.java index ffdb0cca7a..7f3035f1e2 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationArgumentCollection.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationArgumentCollection.java @@ -87,6 +87,6 @@ public class RecalibrationArgumentCollection { * The context covariate will use a context of this size to calculate it's covariate value */ @Argument(fullName = "context_size", shortName = "cs", doc = "size of the k-mer context to be used", required = false) - int CONTEXT_SIZE = 8; + public int CONTEXT_SIZE = 8; } diff --git a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java index ce52f09a28..75d4b1e170 100644 --- a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java +++ b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java @@ -28,6 +28,7 @@ import org.broadinstitute.sting.gatk.walkers.recalibration.Covariate; import org.broadinstitute.sting.gatk.walkers.recalibration.RecalDataManager; import org.broadinstitute.sting.gatk.walkers.recalibration.RecalDatum; +import org.broadinstitute.sting.gatk.walkers.recalibration.RecalibrationArgumentCollection; import org.broadinstitute.sting.utils.QualityUtils; import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.collections.NestedHashMap; @@ -121,8 +122,9 @@ else if( COVARIATE_PATTERN.matcher(line).matches() ) { // The line string is eit final boolean createCollapsedTables = true; // Initialize any covariate member variables using the shared argument collection + RecalibrationArgumentCollection RAC = new RecalibrationArgumentCollection(); for( Covariate cov : requestedCovariates ) { - cov.initialize( null ); // BUGBUG: do any of the used covariates actually need the RecalibrationArgumentCollection? + cov.initialize( RAC ); } // Initialize the data hashMaps dataManager = new RecalDataManager( createCollapsedTables, requestedCovariates.size() ); From 9d1a19bbaab27a419efaa8b906084378961f4af0 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Mon, 6 Feb 2012 22:49:29 -0500 Subject: [PATCH 220/356] Multi-allelic indels were not being printed out correctly in VariantsToTable; fixed. --- .../walkers/variantutils/VariantsToTable.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java index 4b3aa4864b..e43d54e144 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java @@ -272,12 +272,11 @@ public static abstract class Getter { public abstract String get(VariantContext getters.put("POS", new Getter() { public String get(VariantContext vc) { return Integer.toString(vc.getStart()); } }); getters.put("REF", new Getter() { public String get(VariantContext vc) { - String x = ""; - if ( vc.hasReferenceBaseForIndel() && !vc.isSNP() ) { - Byte refByte = vc.getReferenceBaseForIndel(); - x=x+new String(new byte[]{refByte}); - } - return x+vc.getReference().getDisplayString(); + StringBuilder x = new StringBuilder(); + if ( vc.hasReferenceBaseForIndel() && !vc.isSNP() ) + x.append((char)vc.getReferenceBaseForIndel().byteValue()); + x.append(vc.getReference().getDisplayString()); + return x.toString(); } }); getters.put("ALT", new Getter() { @@ -285,13 +284,11 @@ public String get(VariantContext vc) { StringBuilder x = new StringBuilder(); int n = vc.getAlternateAlleles().size(); if ( n == 0 ) return "."; - if ( vc.hasReferenceBaseForIndel() && !vc.isSNP() ) { - Byte refByte = vc.getReferenceBaseForIndel(); - x.append(new String(new byte[]{refByte})); - } for ( int i = 0; i < n; i++ ) { if ( i != 0 ) x.append(","); + if ( vc.hasReferenceBaseForIndel() && !vc.isSNP() ) + x.append((char)vc.getReferenceBaseForIndel().byteValue()); x.append(vc.getAlternateAllele(i).getDisplayString()); } return x.toString(); From a6477e558a141480d86236e6e5f28cb09b7a96f5 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Tue, 7 Feb 2012 09:37:32 -0500 Subject: [PATCH 221/356] adding docs to HaplotypeCaller From 718da7757e75a2367da8b1e6351ceef60bf76c47 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Tue, 7 Feb 2012 13:15:58 -0500 Subject: [PATCH 222/356] Fixes to ValidateVariants as per GS post: ref base of mixed alleles were sometimes wrong, error print out of bad ACs was throwing a RuntimeException, don't validate ACs if there are no genotypes. --- .../gatk/walkers/variantutils/ValidateVariants.java | 11 +++++------ .../sting/utils/variantcontext/VariantContext.java | 8 +++++++- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariants.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariants.java index fdfca982c9..530258fe07 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariants.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariants.java @@ -128,13 +128,13 @@ private void validate(VariantContext vc, RefMetaDataTracker tracker, ReferenceCo // get the true reference allele Allele reportedRefAllele = vc.getReference(); - Allele observedRefAllele; + Allele observedRefAllele = null; // insertions if ( vc.isSimpleInsertion() ) { observedRefAllele = Allele.create(Allele.NULL_ALLELE_STRING); } // deletions - else if ( vc.isSimpleDeletion() || vc.isMixed() || vc.isMNP() ) { + else if ( vc.isSimpleDeletion() || vc.isMNP() ) { // we can't validate arbitrarily long deletions if ( reportedRefAllele.length() > 100 ) { logger.info(String.format("Reference allele is too long (%d) at position %s:%d; skipping that record.", reportedRefAllele.length(), vc.getChr(), vc.getStart())); @@ -143,16 +143,15 @@ else if ( vc.isSimpleDeletion() || vc.isMixed() || vc.isMNP() ) { // deletions are associated with the (position of) the last (preceding) non-deleted base; // hence to get actually deleted bases we need offset = 1 - int offset = 1 ; - if ( vc.isMNP() ) offset = 0; // if it's an MNP, the reported position IS the first modified base + int offset = vc.isMNP() ? 0 : 1; byte[] refBytes = ref.getBases(); byte[] trueRef = new byte[reportedRefAllele.length()]; for (int i = 0; i < reportedRefAllele.length(); i++) trueRef[i] = refBytes[i+offset]; observedRefAllele = Allele.create(trueRef, true); } - // SNPs, etc. - else { + // SNPs, etc. but not mixed types because they are too difficult + else if ( !vc.isMixed() ) { byte[] refByte = new byte[1]; refByte[0] = ref.getBase(); observedRefAllele = Allele.create(refByte, true); diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java index 247e412ddb..27721be95b 100755 --- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java +++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java @@ -920,6 +920,9 @@ public void extraStrictValidation(Allele reference, Byte paddedRefBase, Set Date: Tue, 7 Feb 2012 13:30:54 -0500 Subject: [PATCH 223/356] Document -L unmapped --- .../sting/gatk/arguments/GATKArgumentCollection.java | 1 + 1 file changed, 1 insertion(+) diff --git a/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java index b3a1e24887..8ec7078010 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java +++ b/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java @@ -75,6 +75,7 @@ public GATKArgumentCollection() { * Using this option one can instruct the GATK engine to traverse over only part of the genome. This argument can be specified multiple times. * One may use samtools-style intervals either explicitly (e.g. -L chr1 or -L chr1:100-200) or listed in a file (e.g. -L myFile.intervals). * Additionally, one may specify a rod file to traverse over the positions for which there is a record in the file (e.g. -L file.vcf). + * To specify the completely unmapped reads in the BAM file (i.e. those without a reference contig) use -L unmapped. */ @Input(fullName = "intervals", shortName = "L", doc = "One or more genomic intervals over which to operate. Can be explicitly specified on the command line or in a file (including a rod file)", required = false) public List> intervals = null; From 0d3ea0401c0afb82487a1c2018750350fad790ca Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Tue, 7 Feb 2012 13:22:46 -0500 Subject: [PATCH 224/356] BQSR Parameter cleanup * get rid of 320C argument that nobody uses. * get rid of DEFAULT_READ_GROUP parameter and functionality (later to become an engine argument). --- .../recalibration/CountCovariatesWalker.java | 278 +++++++++--------- .../recalibration/RecalDataManager.java | 26 -- .../RecalibrationArgumentCollection.java | 22 +- .../TableRecalibrationWalker.java | 247 ++++++++-------- 4 files changed, 280 insertions(+), 293 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java index 4e3d4048bc..626460be6c 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java @@ -77,20 +77,20 @@ *

Output

*

* A recalibration table file in CSV format that is used by the TableRecalibration walker. - * It is a comma-separated text file relating the desired covariates to the number of such bases and their rate of mismatch in the genome, and its implied empirical quality score. + * It is a comma-separated text file relating the desired covariates to the number of such bases and their rate of mismatch in the genome, and its implied empirical quality score. * - * The first 20 lines of such a file is shown below. + * The first 20 lines of such a file is shown below. * * The file begins with a series of comment lines describing: * ** The number of counted loci * ** The number of counted bases * ** The number of skipped loci and the fraction skipped, due to presence in dbSNP or bad reference bases - * - * * After the comments appears a header line indicating which covariates were used as well as the ordering of elements in the subsequent records. + * + * * After the comments appears a header line indicating which covariates were used as well as the ordering of elements in the subsequent records. * * * After the header, data records occur one per line until the end of the file. The first several items on a line are the values of the individual covariates and will change - * depending on which covariates were specified at runtime. The last three items are the data- that is, number of observations for this combination of covariates, number of + * depending on which covariates were specified at runtime. The last three items are the data- that is, number of observations for this combination of covariates, number of * reference mismatches, and the raw empirical quality score calculated by phred-scaling the mismatch rate. - * + * * 

  * # Counted Sites    19451059
  * # Counted Bases    56582018
@@ -129,13 +129,14 @@
  *   -cov DinucCovariate \
  *   -recalFile my_reads.recal_data.csv
  *
- * */ @BAQMode(ApplicationTime = BAQ.ApplicationTime.FORBIDDEN) -@By( DataSource.READS ) // Only look at covered loci, not every loci of the reference file -@ReadFilters( {MappingQualityZeroFilter.class, MappingQualityUnavailableFilter.class} ) // Filter out all reads with zero or unavailable mapping quality -@Requires( {DataSource.READS, DataSource.REFERENCE, DataSource.REFERENCE_BASES} ) // This walker requires both -I input.bam and -R reference.fasta +@By(DataSource.READS) // Only look at covered loci, not every loci of the reference file +@ReadFilters({MappingQualityZeroFilter.class, MappingQualityUnavailableFilter.class}) +// Filter out all reads with zero or unavailable mapping quality +@Requires({DataSource.READS, DataSource.REFERENCE, DataSource.REFERENCE_BASES}) +// This walker requires both -I input.bam and -R reference.fasta @PartitionBy(PartitionType.LOCUS) public class CountCovariatesWalker extends LocusWalker implements TreeReducible { @@ -149,7 +150,8 @@ public class CountCovariatesWalker extends LocusWalker> knownSites = Collections.emptyList(); /** @@ -169,31 +171,31 @@ public class CountCovariatesWalker extends LocusWalker> covariateClasses = new PluginManager( Covariate.class ).getPlugins(); - final List> requiredClasses = new PluginManager( RequiredCovariate.class ).getPlugins(); - final List> standardClasses = new PluginManager( StandardCovariate.class ).getPlugins(); + final List> covariateClasses = new PluginManager(Covariate.class).getPlugins(); + final List> requiredClasses = new PluginManager(RequiredCovariate.class).getPlugins(); + final List> standardClasses = new PluginManager(StandardCovariate.class).getPlugins(); // Print and exit if that's what was requested - if ( LIST_ONLY ) { - logger.info( "Available covariates:" ); - for( Class covClass : covariateClasses ) { - logger.info( covClass.getSimpleName() ); + if (LIST_ONLY) { + logger.info("Available covariates:"); + for (Class covClass : covariateClasses) { + logger.info(covClass.getSimpleName()); } logger.info(""); - System.exit( 0 ); // Early exit here because user requested it + System.exit(0); // Early exit here because user requested it } // Warn the user if no dbSNP file or other variant mask was specified - if( knownSites.isEmpty() && !RUN_WITHOUT_DBSNP ) { + if (knownSites.isEmpty() && !RUN_WITHOUT_DBSNP) { throw new UserException.CommandLineException("This calculation is critically dependent on being able to skip over known variant sites. Please provide a VCF file containing known sites of genetic variation."); } // Initialize the requested covariates by parsing the -cov argument // First add the required covariates - if( requiredClasses.size() == 2) { // readGroup and reported quality score - requestedCovariates.add( new ReadGroupCovariate() ); // Order is important here - requestedCovariates.add( new QualityScoreCovariate() ); - } else { + if (requiredClasses.size() == 2) { // readGroup and reported quality score + requestedCovariates.add(new ReadGroupCovariate()); // Order is important here + requestedCovariates.add(new QualityScoreCovariate()); + } + else { throw new UserException.CommandLineException("There are more required covariates than expected. The instantiation list needs to be updated with the new required covariate and in the correct order."); } // Next add the standard covariates if -standard was specified by the user - if( USE_STANDARD_COVARIATES ) { + if (USE_STANDARD_COVARIATES) { // We want the standard covariates to appear in a consistent order but the packageUtils method gives a random order // A list of Classes can't be sorted, but a list of Class names can be final List standardClassNames = new ArrayList(); - for( Class covClass : standardClasses ) { - standardClassNames.add( covClass.getName() ); + for (Class covClass : standardClasses) { + standardClassNames.add(covClass.getName()); } Collections.sort(standardClassNames); // Sort the list of class names - for( String className : standardClassNames ) { - for( Class covClass : standardClasses ) { // Find the class that matches this class name - if( covClass.getName().equals( className ) ) { + for (String className : standardClassNames) { + for (Class covClass : standardClasses) { // Find the class that matches this class name + if (covClass.getName().equals(className)) { try { - final Covariate covariate = (Covariate)covClass.newInstance(); - requestedCovariates.add( covariate ); + final Covariate covariate = (Covariate) covClass.newInstance(); + requestedCovariates.add(covariate); } catch (Exception e) { throw new DynamicClassResolutionException(covClass, e); } @@ -302,17 +307,17 @@ public void initialize() { } } // Finally parse the -cov arguments that were provided, skipping over the ones already specified - if( COVARIATES != null ) { - for( String requestedCovariateString : COVARIATES ) { + if (COVARIATES != null) { + for (String requestedCovariateString : COVARIATES) { boolean foundClass = false; - for( Class covClass : covariateClasses ) { - if( requestedCovariateString.equalsIgnoreCase( covClass.getSimpleName() ) ) { // -cov argument matches the class name for an implementing class + for (Class covClass : covariateClasses) { + if (requestedCovariateString.equalsIgnoreCase(covClass.getSimpleName())) { // -cov argument matches the class name for an implementing class foundClass = true; - if( !requiredClasses.contains( covClass ) && (!USE_STANDARD_COVARIATES || !standardClasses.contains( covClass )) ) { + if (!requiredClasses.contains(covClass) && (!USE_STANDARD_COVARIATES || !standardClasses.contains(covClass))) { try { // Now that we've found a matching class, try to instantiate it - final Covariate covariate = (Covariate)covClass.newInstance(); - requestedCovariates.add( covariate ); + final Covariate covariate = (Covariate) covClass.newInstance(); + requestedCovariates.add(covariate); } catch (Exception e) { throw new DynamicClassResolutionException(covClass, e); } @@ -320,20 +325,19 @@ public void initialize() { } } - if( !foundClass ) { - throw new UserException.CommandLineException( "The requested covariate type (" + requestedCovariateString + ") isn't a valid covariate option. Use --list to see possible covariates." ); + if (!foundClass) { + throw new UserException.CommandLineException("The requested covariate type (" + requestedCovariateString + ") isn't a valid covariate option. Use --list to see possible covariates."); } } } - logger.info( "The covariates being used here: " ); - for( Covariate cov : requestedCovariates ) { - logger.info( "\t" + cov.getClass().getSimpleName() ); - cov.initialize( RAC ); // Initialize any covariate member variables using the shared argument collection + logger.info("The covariates being used here: "); + for (Covariate cov : requestedCovariates) { + logger.info("\t" + cov.getClass().getSimpleName()); + cov.initialize(RAC); // Initialize any covariate member variables using the shared argument collection } } - //--------------------------------------------------------------------------------------------------------------- // // map @@ -342,62 +346,63 @@ public void initialize() { /** * For each read at this locus get the various covariate values and increment that location in the map based on - * whether or not the base matches the reference at this particular location + * whether or not the base matches the reference at this particular location + * * @param tracker The reference metadata tracker - * @param ref The reference context + * @param ref The reference context * @param context The alignment context * @return Returns 1, but this value isn't used in the reduce step */ - public CountedData map( RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context ) { + public CountedData map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) { // Only use data from non-dbsnp sites // Assume every mismatch at a non-dbsnp site is indicative of poor quality CountedData counter = new CountedData(); - if( tracker.getValues(knownSites).size() == 0 ) { // If something here is in one of the knownSites tracks then skip over it, otherwise proceed + if (tracker.getValues(knownSites).size() == 0) { // If something here is in one of the knownSites tracks then skip over it, otherwise proceed // For each read at this locus - for( final PileupElement p : context.getBasePileup() ) { + for (final PileupElement p : context.getBasePileup()) { final GATKSAMRecord gatkRead = p.getRead(); int offset = p.getOffset(); - if( gatkRead.containsTemporaryAttribute( SKIP_RECORD_ATTRIBUTE ) ) { + if (gatkRead.containsTemporaryAttribute(SKIP_RECORD_ATTRIBUTE)) { continue; } - if( !gatkRead.containsTemporaryAttribute( SEEN_ATTRIBUTE ) ) - { - gatkRead.setTemporaryAttribute( SEEN_ATTRIBUTE, true ); - RecalDataManager.parseSAMRecord( gatkRead, RAC ); + if (!gatkRead.containsTemporaryAttribute(SEEN_ATTRIBUTE)) { + gatkRead.setTemporaryAttribute(SEEN_ATTRIBUTE, true); + RecalDataManager.parseSAMRecord(gatkRead, RAC); // Skip over reads with no calls in the color space if the user requested it - if( !(RAC.SOLID_NOCALL_STRATEGY == RecalDataManager.SOLID_NOCALL_STRATEGY.THROW_EXCEPTION) && RecalDataManager.checkNoCallColorSpace( gatkRead ) ) { - gatkRead.setTemporaryAttribute( SKIP_RECORD_ATTRIBUTE, true); + if (!(RAC.SOLID_NOCALL_STRATEGY == RecalDataManager.SOLID_NOCALL_STRATEGY.THROW_EXCEPTION) && RecalDataManager.checkNoCallColorSpace(gatkRead)) { + gatkRead.setTemporaryAttribute(SKIP_RECORD_ATTRIBUTE, true); continue; } - RecalDataManager.parseColorSpace( gatkRead ); - gatkRead.setTemporaryAttribute( COVARS_ATTRIBUTE, - RecalDataManager.computeCovariates( gatkRead, requestedCovariates, BaseRecalibration.BaseRecalibrationType.BASE_SUBSTITUTION )); + RecalDataManager.parseColorSpace(gatkRead); + gatkRead.setTemporaryAttribute(COVARS_ATTRIBUTE, RecalDataManager.computeCovariates(gatkRead, requestedCovariates, BaseRecalibration.BaseRecalibrationType.BASE_SUBSTITUTION)); } // Skip this position if base quality is zero - if( gatkRead.getBaseQualities()[offset] > 0 ) { + if (gatkRead.getBaseQualities()[offset] > 0) { byte[] bases = gatkRead.getReadBases(); byte refBase = ref.getBase(); // Skip if this base is an 'N' or etc. - if( BaseUtils.isRegularBase( bases[offset] ) ) { + if (BaseUtils.isRegularBase(bases[offset])) { // SOLID bams have inserted the reference base into the read if the color space in inconsistent with the read base so skip it - if( !gatkRead.getReadGroup().getPlatform().toUpperCase().contains("SOLID") || RAC.SOLID_RECAL_MODE == RecalDataManager.SOLID_RECAL_MODE.DO_NOTHING || - !RecalDataManager.isInconsistentColorSpace( gatkRead, offset ) ) { + if (!gatkRead.getReadGroup().getPlatform().toUpperCase().contains("SOLID") || RAC.SOLID_RECAL_MODE == RecalDataManager.SOLID_RECAL_MODE.DO_NOTHING || + !RecalDataManager.isInconsistentColorSpace(gatkRead, offset)) { // This base finally passed all the checks for a good base, so add it to the big data hashmap - updateDataFromRead( counter, gatkRead, offset, refBase ); + updateDataFromRead(counter, gatkRead, offset, refBase); - } else { // calculate SOLID reference insertion rate - if( refBase == bases[offset] ) { + } + else { // calculate SOLID reference insertion rate + if (refBase == bases[offset]) { counter.solidInsertedReferenceBases++; - } else { + } + else { counter.otherColorSpaceInconsistency++; } } @@ -405,7 +410,8 @@ public CountedData map( RefMetaDataTracker tracker, ReferenceContext ref, Alignm } } counter.countedSites++; - } else { // We skipped over the dbSNP site, and we are only processing every Nth locus + } + else { // We skipped over the dbSNP site, and we are only processing every Nth locus counter.skippedSites++; updateMismatchCounts(counter, context, ref.getBase()); // For sanity check to ensure novel mismatch rate vs dnsnp mismatch rate is reasonable } @@ -413,7 +419,7 @@ public CountedData map( RefMetaDataTracker tracker, ReferenceContext ref, Alignm return counter; } - /** + /** * Update the mismatch / total_base counts for a given class of loci. * * @param counter The CountedData to be updated @@ -421,13 +427,13 @@ public CountedData map( RefMetaDataTracker tracker, ReferenceContext ref, Alignm * @param refBase The reference base */ private static void updateMismatchCounts(CountedData counter, final AlignmentContext context, final byte refBase) { - for( PileupElement p : context.getBasePileup() ) { + for (PileupElement p : context.getBasePileup()) { final byte readBase = p.getBase(); final int readBaseIndex = BaseUtils.simpleBaseToBaseIndex(readBase); - final int refBaseIndex = BaseUtils.simpleBaseToBaseIndex(refBase); + final int refBaseIndex = BaseUtils.simpleBaseToBaseIndex(refBase); - if( readBaseIndex != -1 && refBaseIndex != -1 ) { - if( readBaseIndex != refBaseIndex ) { + if (readBaseIndex != -1 && refBaseIndex != -1) { + if (readBaseIndex != refBaseIndex) { counter.novelCountsMM++; } counter.novelCountsBases++; @@ -439,13 +445,14 @@ private static void updateMismatchCounts(CountedData counter, final AlignmentCon * Major workhorse routine for this walker. * Loop through the list of requested covariates and pick out the value from the read, offset, and reference * Using the list of covariate values as a key, pick out the RecalDatum and increment, - * adding one to the number of observations and potentially one to the number of mismatches + * adding one to the number of observations and potentially one to the number of mismatches * Lots of things are passed as parameters to this method as a strategy for optimizing the covariate.getValue calls - * because pulling things out of the SAMRecord is an expensive operation. - * @param counter Data structure which holds the counted bases + * because pulling things out of the SAMRecord is an expensive operation. + * + * @param counter Data structure which holds the counted bases * @param gatkRead The SAMRecord holding all the data for this read - * @param offset The offset in the read for this locus - * @param refBase The reference base at this locus + * @param offset The offset in the read for this locus + * @param refBase The reference base at this locus */ private void updateDataFromRead(CountedData counter, final GATKSAMRecord gatkRead, final int offset, final byte refBase) { final Object[][] covars = (Comparable[][]) gatkRead.getTemporaryAttribute(COVARS_ATTRIBUTE); @@ -453,10 +460,10 @@ private void updateDataFromRead(CountedData counter, final GATKSAMRecord gatkRea // Using the list of covariate values as a key, pick out the RecalDatum from the data HashMap final NestedHashMap data = dataManager.data; //optimization - create local reference - RecalDatumOptimized datum = (RecalDatumOptimized) data.get( key ); - if( datum == null ) { // key doesn't exist yet in the map so make a new bucket and add it + RecalDatumOptimized datum = (RecalDatumOptimized) data.get(key); + if (datum == null) { // key doesn't exist yet in the map so make a new bucket and add it // initialized with zeros, will be incremented at end of method - datum = (RecalDatumOptimized)data.put( new RecalDatumOptimized(), true, (Object[])key ); + datum = (RecalDatumOptimized) data.put(new RecalDatumOptimized(), true, (Object[]) key); } // Need the bases to determine whether or not we have a mismatch @@ -464,13 +471,12 @@ private void updateDataFromRead(CountedData counter, final GATKSAMRecord gatkRea final long curMismatches = datum.getNumMismatches(); // Add one to the number of observations and potentially one to the number of mismatches - datum.incrementBaseCounts( base, refBase ); + datum.incrementBaseCounts(base, refBase); counter.countedBases++; counter.novelCountsBases++; counter.novelCountsMM += datum.getNumMismatches() - curMismatches; // For sanity check to ensure novel mismatch rate vs dnsnp mismatch rate is reasonable } - //--------------------------------------------------------------------------------------------------------------- // // reduce @@ -479,6 +485,7 @@ private void updateDataFromRead(CountedData counter, final GATKSAMRecord gatkRea /** * Initialize the reduce step by creating a PrintStream from the filename specified as an argument to the walker. + * * @return returns A PrintStream created from the -recalFile filename argument specified to the walker */ public CountedData reduceInit() { @@ -487,11 +494,12 @@ public CountedData reduceInit() { /** * The Reduce method doesn't do anything for this walker. + * * @param mapped Result of the map. This value is immediately ignored. - * @param sum The summing CountedData used to output the CSV data + * @param sum The summing CountedData used to output the CSV data * @return returns The sum used to output the CSV data */ - public CountedData reduce( CountedData mapped, CountedData sum ) { + public CountedData reduce(CountedData mapped, CountedData sum) { // Do a dbSNP sanity check every so often return validatingDbsnpMismatchRate(sum.add(mapped)); } @@ -500,16 +508,15 @@ public CountedData reduce( CountedData mapped, CountedData sum ) { * Validate the dbSNP reference mismatch rates. */ private CountedData validatingDbsnpMismatchRate(CountedData counter) { - if( ++counter.lociSinceLastDbsnpCheck >= DBSNP_VALIDATION_CHECK_FREQUENCY ) { + if (++counter.lociSinceLastDbsnpCheck >= DBSNP_VALIDATION_CHECK_FREQUENCY) { counter.lociSinceLastDbsnpCheck = 0; - if( counter.novelCountsBases != 0L && counter.dbSNPCountsBases != 0L ) { - final double fractionMM_novel = (double)counter.novelCountsMM / (double)counter.novelCountsBases; - final double fractionMM_dbsnp = (double)counter.dbSNPCountsMM / (double)counter.dbSNPCountsBases; + if (counter.novelCountsBases != 0L && counter.dbSNPCountsBases != 0L) { + final double fractionMM_novel = (double) counter.novelCountsMM / (double) counter.novelCountsBases; + final double fractionMM_dbsnp = (double) counter.dbSNPCountsMM / (double) counter.dbSNPCountsBases; - if( fractionMM_dbsnp < DBSNP_VS_NOVEL_MISMATCH_RATE * fractionMM_novel ) { - Utils.warnUser("The variation rate at the supplied list of known variant sites seems suspiciously low. Please double-check that the correct ROD is being used. " + - String.format("[dbSNP variation rate = %.4f, novel variation rate = %.4f]", fractionMM_dbsnp, fractionMM_novel) ); + if (fractionMM_dbsnp < DBSNP_VS_NOVEL_MISMATCH_RATE * fractionMM_novel) { + Utils.warnUser("The variation rate at the supplied list of known variant sites seems suspiciously low. Please double-check that the correct ROD is being used. " + String.format("[dbSNP variation rate = %.4f, novel variation rate = %.4f]", fractionMM_dbsnp, fractionMM_novel)); DBSNP_VALIDATION_CHECK_FREQUENCY *= 2; // Don't annoyingly output the warning message every megabase of a large file } } @@ -518,47 +525,50 @@ private CountedData validatingDbsnpMismatchRate(CountedData counter) { return counter; } - public CountedData treeReduce( CountedData sum1, CountedData sum2 ) { + public CountedData treeReduce(CountedData sum1, CountedData sum2) { return validatingDbsnpMismatchRate(sum1.add(sum2)); } /** * Write out the full data hashmap to disk in CSV format + * * @param sum The CountedData to write out to RECAL_FILE */ - public void onTraversalDone( CountedData sum ) { - logger.info( "Writing raw recalibration data..." ); - if( sum.countedBases == 0L ) { + public void onTraversalDone(CountedData sum) { + logger.info("Writing raw recalibration data..."); + if (sum.countedBases == 0L) { throw new UserException.BadInput("Could not find any usable data in the input BAM file(s)."); } - outputToCSV( sum, RECAL_FILE ); - logger.info( "...done!" ); + outputToCSV(sum, RECAL_FILE); + logger.info("...done!"); } /** * For each entry (key-value pair) in the data hashmap output the Covariate's values as well as the RecalDatum's data in CSV format + * * @param recalTableStream The PrintStream to write out to */ - private void outputToCSV( CountedData sum, final PrintStream recalTableStream ) { + private void outputToCSV(CountedData sum, final PrintStream recalTableStream) { recalTableStream.printf("# Counted Sites %d%n", sum.countedSites); recalTableStream.printf("# Counted Bases %d%n", sum.countedBases); recalTableStream.printf("# Skipped Sites %d%n", sum.skippedSites); - recalTableStream.printf("# Fraction Skipped 1 / %.0f bp%n", (double)sum.countedSites / sum.skippedSites); + recalTableStream.printf("# Fraction Skipped 1 / %.0f bp%n", (double) sum.countedSites / sum.skippedSites); - if( sum.solidInsertedReferenceBases != 0 ) { + if (sum.solidInsertedReferenceBases != 0) { recalTableStream.printf("# Fraction SOLiD inserted reference 1 / %.0f bases%n", (double) sum.countedBases / sum.solidInsertedReferenceBases); recalTableStream.printf("# Fraction other color space inconsistencies 1 / %.0f bases%n", (double) sum.countedBases / sum.otherColorSpaceInconsistency); } // Output header saying which covariates were used and in what order - for( Covariate cov : requestedCovariates ) { - recalTableStream.print( cov.getClass().getSimpleName().split("Covariate")[0] + "," ); + for (Covariate cov : requestedCovariates) { + recalTableStream.print(cov.getClass().getSimpleName().split("Covariate")[0] + ","); } recalTableStream.println("nObservations,nMismatches,Qempirical"); - if( DONT_SORT_OUTPUT ) { + if (DONT_SORT_OUTPUT) { printMappings(recalTableStream, 0, new Object[requestedCovariates.size()], dataManager.data.data); - } else { + } + else { printMappingsSorted(recalTableStream, 0, new Object[requestedCovariates.size()], dataManager.data.data); } @@ -566,45 +576,47 @@ private void outputToCSV( CountedData sum, final PrintStream recalTableStream ) recalTableStream.println(TableRecalibrationWalker.EOF_MARKER); } - private void printMappingsSorted( final PrintStream recalTableStream, final int curPos, final Object[] key, final Map data) { + private void printMappingsSorted(final PrintStream recalTableStream, final int curPos, final Object[] key, final Map data) { final ArrayList keyList = new ArrayList(); - for( Object comp : data.keySet() ) { + for (Object comp : data.keySet()) { keyList.add((Comparable) comp); } Collections.sort(keyList); - for( Comparable comp : keyList ) { + for (Comparable comp : keyList) { key[curPos] = comp; final Object val = data.get(comp); - if( val instanceof RecalDatumOptimized ) { // We are at the end of the nested hash maps + if (val instanceof RecalDatumOptimized) { // We are at the end of the nested hash maps // For each Covariate in the key - for( Object compToPrint : key ) { + for (Object compToPrint : key) { // Output the Covariate's value - recalTableStream.print( compToPrint + "," ); + recalTableStream.print(compToPrint + ","); } // Output the RecalDatum entry - recalTableStream.println( ((RecalDatumOptimized)val).outputToCSV() ); - } else { // Another layer in the nested hash map - printMappingsSorted( recalTableStream, curPos + 1, key, (Map) val ); + recalTableStream.println(((RecalDatumOptimized) val).outputToCSV()); + } + else { // Another layer in the nested hash map + printMappingsSorted(recalTableStream, curPos + 1, key, (Map) val); } } } - private void printMappings( final PrintStream recalTableStream, final int curPos, final Object[] key, final Map data) { - for( Object comp : data.keySet() ) { + private void printMappings(final PrintStream recalTableStream, final int curPos, final Object[] key, final Map data) { + for (Object comp : data.keySet()) { key[curPos] = comp; final Object val = data.get(comp); - if( val instanceof RecalDatumOptimized ) { // We are at the end of the nested hash maps + if (val instanceof RecalDatumOptimized) { // We are at the end of the nested hash maps // For each Covariate in the key - for( Object compToPrint : key ) { + for (Object compToPrint : key) { // Output the Covariate's value - recalTableStream.print( compToPrint + "," ); + recalTableStream.print(compToPrint + ","); } // Output the RecalDatum entry - recalTableStream.println( ((RecalDatumOptimized)val).outputToCSV() ); - } else { // Another layer in the nested hash map - printMappings( recalTableStream, curPos + 1, key, (Map) val ); + recalTableStream.println(((RecalDatumOptimized) val).outputToCSV()); + } + else { // Another layer in the nested hash map + printMappings(recalTableStream, curPos + 1, key, (Map) val); } } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java index 18b33c0e80..72c2b28299 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java @@ -256,32 +256,6 @@ else if (covariate == 1) { public static void parseSAMRecord(final GATKSAMRecord read, final RecalibrationArgumentCollection RAC) { GATKSAMReadGroupRecord readGroup = ((GATKSAMRecord) read).getReadGroup(); - // If there are no read groups we have to default to something, and that something could be specified by the user using command line arguments - if (readGroup == null) { - if (RAC.DEFAULT_READ_GROUP != null && RAC.DEFAULT_PLATFORM != null) { - if (!warnUserNullReadGroup && RAC.FORCE_READ_GROUP == null) { - Utils.warnUser("The input .bam file contains reads with no read group. " + - "Defaulting to read group ID = " + RAC.DEFAULT_READ_GROUP + " and platform = " + RAC.DEFAULT_PLATFORM + ". " + - "First observed at read with name = " + read.getReadName()); - warnUserNullReadGroup = true; - } - // There is no readGroup so defaulting to these values - readGroup = new GATKSAMReadGroupRecord(RAC.DEFAULT_READ_GROUP); - readGroup.setPlatform(RAC.DEFAULT_PLATFORM); - ((GATKSAMRecord) read).setReadGroup(readGroup); - } - else { - throw new UserException.MalformedBAM(read, "The input .bam file contains reads with no read group. First observed at read with name = " + read.getReadName()); - } - } - - if (RAC.FORCE_READ_GROUP != null && !readGroup.getReadGroupId().equals(RAC.FORCE_READ_GROUP)) { // Collapse all the read groups into a single common String provided by the user - final String oldPlatform = readGroup.getPlatform(); - readGroup = new GATKSAMReadGroupRecord(RAC.FORCE_READ_GROUP); - readGroup.setPlatform(oldPlatform); - ((GATKSAMRecord) read).setReadGroup(readGroup); - } - if (RAC.FORCE_PLATFORM != null && (readGroup.getPlatform() == null || !readGroup.getPlatform().equals(RAC.FORCE_PLATFORM))) { readGroup.setPlatform(RAC.FORCE_PLATFORM); } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationArgumentCollection.java index 7f3035f1e2..9752b1deee 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationArgumentCollection.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalibrationArgumentCollection.java @@ -43,31 +43,15 @@ public class RecalibrationArgumentCollection { // Shared Command Line Arguments ////////////////////////////////// @Hidden - @Argument(fullName = "default_read_group", shortName = "dRG", required = false, doc = "If a read has no read group then default to the provided String.") - public String DEFAULT_READ_GROUP = null; - @Hidden @Argument(fullName = "default_platform", shortName = "dP", required = false, doc = "If a read has no platform then default to the provided String. Valid options are illumina, 454, and solid.") public String DEFAULT_PLATFORM = null; @Hidden - @Argument(fullName = "force_read_group", shortName = "fRG", required = false, doc = "If provided, the read group ID of EVERY read will be forced to be the provided String. This is useful to collapse all data into a single read group.") - public String FORCE_READ_GROUP = null; - @Hidden @Argument(fullName = "force_platform", shortName = "fP", required = false, doc = "If provided, the platform of EVERY read will be forced to be the provided String. Valid options are illumina, 454, and solid.") public String FORCE_PLATFORM = null; @Hidden @Argument(fullName = "window_size_nqs", shortName = "nqs", doc = "The window size used by MinimumNQSCovariate for its calculation", required = false) public int WINDOW_SIZE = 5; - /** - * This window size tells the module in how big of a neighborhood around the current base it should look for the minimum base quality score. - */ - @Hidden - @Argument(fullName = "homopolymer_nback", shortName = "nback", doc = "The number of previous bases to look at in HomopolymerCovariate", required = false) - public int HOMOPOLYMER_NBACK = 7; - @Hidden - @Argument(fullName = "exception_if_no_tile", shortName = "throwTileException", doc = "If provided, TileCovariate will throw an exception when no tile can be found. The default behavior is to use tile = -1", required = false) - public boolean EXCEPTION_IF_NO_TILE = false; - /** * CountCovariates and TableRecalibration accept a --solid_recal_mode flag which governs how the recalibrator handles the * reads which have had the reference inserted because of color space inconsistencies. @@ -89,4 +73,10 @@ public class RecalibrationArgumentCollection { @Argument(fullName = "context_size", shortName = "cs", doc = "size of the k-mer context to be used", required = false) public int CONTEXT_SIZE = 8; + /** + * This window size tells the module in how big of a neighborhood around the current base it should look for the minimum base quality score. + */ + @Argument(fullName = "homopolymer_nback", shortName = "nback", doc = "The number of previous bases to look at in HomopolymerCovariate", required = false) + public int HOMOPOLYMER_NBACK = 7; + } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java index a8006d506f..cd848cd9e0 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java @@ -86,12 +86,12 @@ * -o my_reads.recal.bam \ * -recalFile my_reads.recal_data.csv * - * */ @BAQMode(QualityMode = BAQ.QualityMode.ADD_TAG, ApplicationTime = BAQ.ApplicationTime.ON_OUTPUT) @WalkerName("TableRecalibration") -@Requires({ DataSource.READS, DataSource.REFERENCE, DataSource.REFERENCE_BASES }) // This walker requires -I input.bam, it also requires -R reference.fasta +@Requires({DataSource.READS, DataSource.REFERENCE, DataSource.REFERENCE_BASES}) +// This walker requires -I input.bam, it also requires -R reference.fasta public class TableRecalibrationWalker extends ReadWalker { public static final String PROGRAM_RECORD_NAME = "GATK TableRecalibration"; @@ -99,7 +99,8 @@ public class TableRecalibrationWalker extends ReadWalker> classes = new PluginManager(Covariate.class).getPlugins(); @@ -205,31 +206,33 @@ public void initialize() { boolean foundAllCovariates = false; // Read in the data from the csv file and populate the data map and covariates list - logger.info( "Reading in the data from input csv file..." ); + logger.info("Reading in the data from input csv file..."); boolean sawEOF = false; try { - for ( String line : new XReadLines(RECAL_FILE) ) { + for (String line : new XReadLines(RECAL_FILE)) { lineNumber++; - if ( EOF_MARKER.equals(line) ) { + if (EOF_MARKER.equals(line)) { sawEOF = true; - } else if( COMMENT_PATTERN.matcher(line).matches() || OLD_RECALIBRATOR_HEADER.matcher(line).matches() ) { + } + else if (COMMENT_PATTERN.matcher(line).matches() || OLD_RECALIBRATOR_HEADER.matcher(line).matches()) { ; // Skip over the comment lines, (which start with '#') } // Read in the covariates that were used from the input file - else if( COVARIATE_PATTERN.matcher(line).matches() ) { // The line string is either specifying a covariate or is giving csv data - if( foundAllCovariates ) { - throw new UserException.MalformedFile( RECAL_FILE, "Malformed input recalibration file. Found covariate names intermingled with data in file: " + RECAL_FILE ); - } else { // Found the covariate list in input file, loop through all of them and instantiate them + else if (COVARIATE_PATTERN.matcher(line).matches()) { // The line string is either specifying a covariate or is giving csv data + if (foundAllCovariates) { + throw new UserException.MalformedFile(RECAL_FILE, "Malformed input recalibration file. Found covariate names intermingled with data in file: " + RECAL_FILE); + } + else { // Found the covariate list in input file, loop through all of them and instantiate them String[] vals = line.split(","); - for( int iii = 0; iii < vals.length - 3; iii++ ) { // There are n-3 covariates. The last three items are nObservations, nMismatch, and Qempirical + for (int iii = 0; iii < vals.length - 3; iii++) { // There are n-3 covariates. The last three items are nObservations, nMismatch, and Qempirical boolean foundClass = false; - for( Class covClass : classes ) { - if( (vals[iii] + "Covariate").equalsIgnoreCase( covClass.getSimpleName() ) ) { + for (Class covClass : classes) { + if ((vals[iii] + "Covariate").equalsIgnoreCase(covClass.getSimpleName())) { foundClass = true; try { - Covariate covariate = (Covariate)covClass.newInstance(); - requestedCovariates.add( covariate ); + Covariate covariate = (Covariate) covClass.newInstance(); + requestedCovariates.add(covariate); } catch (Exception e) { throw new DynamicClassResolutionException(covClass, e); } @@ -237,107 +240,110 @@ else if( COVARIATE_PATTERN.matcher(line).matches() ) { // The line string is eit } } - if( !foundClass ) { - throw new UserException.MalformedFile(RECAL_FILE, "Malformed input recalibration file. The requested covariate type (" + (vals[iii] + "Covariate") + ") isn't a valid covariate option." ); + if (!foundClass) { + throw new UserException.MalformedFile(RECAL_FILE, "Malformed input recalibration file. The requested covariate type (" + (vals[iii] + "Covariate") + ") isn't a valid covariate option."); } } } - } else { // Found a line of data - if( !foundAllCovariates ) { + } + else { // Found a line of data + if (!foundAllCovariates) { foundAllCovariates = true; // At this point all the covariates should have been found and initialized - if( requestedCovariates.size() < 2 ) { - throw new UserException.MalformedFile(RECAL_FILE, "Malformed input recalibration csv file. Covariate names can't be found in file: " + RECAL_FILE ); + if (requestedCovariates.size() < 2) { + throw new UserException.MalformedFile(RECAL_FILE, "Malformed input recalibration csv file. Covariate names can't be found in file: " + RECAL_FILE); } final boolean createCollapsedTables = true; // Initialize any covariate member variables using the shared argument collection - for( Covariate cov : requestedCovariates ) { - cov.initialize( RAC ); + for (Covariate cov : requestedCovariates) { + cov.initialize(RAC); } // Initialize the data hashMaps - dataManager = new RecalDataManager( createCollapsedTables, requestedCovariates.size() ); + dataManager = new RecalDataManager(createCollapsedTables, requestedCovariates.size()); } addCSVData(RECAL_FILE, line); // Parse the line and add the data to the HashMap } } - } catch ( FileNotFoundException e ) { + } catch (FileNotFoundException e) { throw new UserException.CouldNotReadInputFile(RECAL_FILE, "Can not find input file", e); - } catch ( NumberFormatException e ) { + } catch (NumberFormatException e) { throw new UserException.MalformedFile(RECAL_FILE, "Error parsing recalibration data at line " + lineNumber + ". Perhaps your table was generated by an older version of CovariateCounterWalker."); } - logger.info( "...done!" ); + logger.info("...done!"); - if ( !sawEOF ) { + if (!sawEOF) { final String errorMessage = "No EOF marker was present in the recal covariates table; this could mean that the file is corrupted or was generated with an old version of the CountCovariates tool."; - if ( REQUIRE_EOF ) + if (REQUIRE_EOF) throw new UserException.MalformedFile(RECAL_FILE, errorMessage); logger.warn(errorMessage); } - logger.info( "The covariates being used here: " ); - for( Covariate cov : requestedCovariates ) { - logger.info( "\t" + cov.getClass().getSimpleName() ); + logger.info("The covariates being used here: "); + for (Covariate cov : requestedCovariates) { + logger.info("\t" + cov.getClass().getSimpleName()); } - if( dataManager == null ) { + if (dataManager == null) { throw new UserException.MalformedFile(RECAL_FILE, "Can't initialize the data manager. Perhaps the recal csv file contains no data?"); } // Create the tables of empirical quality scores that will be used in the sequential calculation - logger.info( "Generating tables of empirical qualities for use in sequential calculation..." ); - dataManager.generateEmpiricalQualities( SMOOTHING, MAX_QUALITY_SCORE ); - logger.info( "...done!" ); + logger.info("Generating tables of empirical qualities for use in sequential calculation..."); + dataManager.generateEmpiricalQualities(SMOOTHING, MAX_QUALITY_SCORE); + logger.info("...done!"); // Take the header of the input SAM file and tweak it by adding in a new programRecord with the version number and list of covariates that were used final SAMFileHeader header = getToolkit().getSAMFileHeader().clone(); - if( !NO_PG_TAG ) { + if (!NO_PG_TAG) { final SAMProgramRecord programRecord = new SAMProgramRecord(PROGRAM_RECORD_NAME); final ResourceBundle headerInfo = TextFormattingUtils.loadResourceBundle("StingText"); try { final String version = headerInfo.getString("org.broadinstitute.sting.gatk.version"); programRecord.setProgramVersion(version); - } catch (MissingResourceException e) {} + } catch (MissingResourceException e) { + } StringBuffer sb = new StringBuffer(); sb.append(getToolkit().createApproximateCommandLineArgumentString(getToolkit(), this)); sb.append(" Covariates=["); - for( Covariate cov : requestedCovariates ) { + for (Covariate cov : requestedCovariates) { sb.append(cov.getClass().getSimpleName()); sb.append(", "); } - sb.setCharAt(sb.length()-2, ']'); - sb.setCharAt(sb.length()-1, ' '); + sb.setCharAt(sb.length() - 2, ']'); + sb.setCharAt(sb.length() - 1, ' '); programRecord.setCommandLine(sb.toString()); List oldRecords = header.getProgramRecords(); - List newRecords = new ArrayList(oldRecords.size()+1); - for ( SAMProgramRecord record : oldRecords ) { - if ( !record.getId().startsWith(PROGRAM_RECORD_NAME) ) + List newRecords = new ArrayList(oldRecords.size() + 1); + for (SAMProgramRecord record : oldRecords) { + if (!record.getId().startsWith(PROGRAM_RECORD_NAME)) newRecords.add(record); } newRecords.add(programRecord); header.setProgramRecords(newRecords); // Write out the new header - OUTPUT_BAM.writeHeader( header ); + OUTPUT_BAM.writeHeader(header); } } /** * For each covariate read in a value and parse it. Associate those values with the data itself (num observation and num mismatches) + * * @param line A line of CSV data read from the recalibration table data file */ private void addCSVData(final File file, final String line) { final String[] vals = line.split(","); // Check if the data line is malformed, for example if the read group string contains a comma then it won't be parsed correctly - if( vals.length != requestedCovariates.size() + 3 ) { // +3 because of nObservations, nMismatch, and Qempirical + if (vals.length != requestedCovariates.size() + 3) { // +3 because of nObservations, nMismatch, and Qempirical throw new UserException.MalformedFile(file, "Malformed input recalibration file. Found data line with too many fields: " + line + " --Perhaps the read group string contains a comma and isn't being parsed correctly."); } @@ -345,15 +351,15 @@ private void addCSVData(final File file, final String line) { final Object[] key = new Object[requestedCovariates.size()]; Covariate cov; int iii; - for( iii = 0; iii < requestedCovariates.size(); iii++ ) { - cov = requestedCovariates.get( iii ); - key[iii] = cov.getValue( vals[iii] ); + for (iii = 0; iii < requestedCovariates.size(); iii++) { + cov = requestedCovariates.get(iii); + key[iii] = cov.getValue(vals[iii]); } // Create a new datum using the number of observations, number of mismatches, and reported quality score - final RecalDatum datum = new RecalDatum( Long.parseLong( vals[iii] ), Long.parseLong( vals[iii + 1] ), Double.parseDouble( vals[1] ), 0.0 ); + final RecalDatum datum = new RecalDatum(Long.parseLong(vals[iii]), Long.parseLong(vals[iii + 1]), Double.parseDouble(vals[1]), 0.0); // Add that datum to all the collapsed tables which will be used in the sequential calculation - dataManager.addToAllTables( key, datum, PRESERVE_QSCORES_LESS_THAN ); + dataManager.addToAllTables(key, datum, PRESERVE_QSCORES_LESS_THAN); } //--------------------------------------------------------------------------------------------------------------- @@ -366,64 +372,63 @@ private void addCSVData(final File file, final String line) { * For each base in the read calculate a new recalibrated quality score and replace the quality scores in the read * * @param refBases References bases over the length of the read - * @param read The read to be recalibrated + * @param read The read to be recalibrated * @return The read with quality scores replaced */ - public SAMRecord map( ReferenceContext refBases, GATKSAMRecord read, ReadMetaDataTracker metaDataTracker ) { + public SAMRecord map(ReferenceContext refBases, GATKSAMRecord read, ReadMetaDataTracker metaDataTracker) { - if( read.getReadLength() == 0 ) { // Some reads have '*' as the SEQ field and samtools returns length zero. We don't touch these reads. + if (read.getReadLength() == 0) { // Some reads have '*' as the SEQ field and samtools returns length zero. We don't touch these reads. return read; } - RecalDataManager.parseSAMRecord( read, RAC ); + RecalDataManager.parseSAMRecord(read, RAC); byte[] originalQuals = read.getBaseQualities(); final byte[] recalQuals = originalQuals.clone(); final String platform = read.getReadGroup().getPlatform(); - if( platform.toUpperCase().contains("SOLID") && !(RAC.SOLID_RECAL_MODE == RecalDataManager.SOLID_RECAL_MODE.DO_NOTHING) ) { - if( !(RAC.SOLID_NOCALL_STRATEGY == RecalDataManager.SOLID_NOCALL_STRATEGY.THROW_EXCEPTION) ) { - final boolean badColor = RecalDataManager.checkNoCallColorSpace( read ); - if( badColor ) { + if (platform.toUpperCase().contains("SOLID") && !(RAC.SOLID_RECAL_MODE == RecalDataManager.SOLID_RECAL_MODE.DO_NOTHING)) { + if (!(RAC.SOLID_NOCALL_STRATEGY == RecalDataManager.SOLID_NOCALL_STRATEGY.THROW_EXCEPTION)) { + final boolean badColor = RecalDataManager.checkNoCallColorSpace(read); + if (badColor) { numReadsWithMalformedColorSpace++; - if( RAC.SOLID_NOCALL_STRATEGY == RecalDataManager.SOLID_NOCALL_STRATEGY.LEAVE_READ_UNRECALIBRATED ) { + if (RAC.SOLID_NOCALL_STRATEGY == RecalDataManager.SOLID_NOCALL_STRATEGY.LEAVE_READ_UNRECALIBRATED) { return read; // can't recalibrate a SOLiD read with no calls in the color space, and the user wants to skip over them - } else if ( RAC.SOLID_NOCALL_STRATEGY == RecalDataManager.SOLID_NOCALL_STRATEGY.PURGE_READ ) { + } + else if (RAC.SOLID_NOCALL_STRATEGY == RecalDataManager.SOLID_NOCALL_STRATEGY.PURGE_READ) { read.setReadFailsVendorQualityCheckFlag(true); return read; } } } - originalQuals = RecalDataManager.calcColorSpace( read, originalQuals, RAC.SOLID_RECAL_MODE, refBases == null ? null : refBases.getBases() ); + originalQuals = RecalDataManager.calcColorSpace(read, originalQuals, RAC.SOLID_RECAL_MODE, refBases == null ? null : refBases.getBases()); } //compute all covariate values for this read - final Comparable[][] covariateValues_offset_x_covar = - RecalDataManager.computeCovariates(read, requestedCovariates, BaseRecalibration.BaseRecalibrationType.BASE_SUBSTITUTION); + final Comparable[][] covariateValues_offset_x_covar = RecalDataManager.computeCovariates(read, requestedCovariates, BaseRecalibration.BaseRecalibrationType.BASE_SUBSTITUTION); // For each base in the read - for( int offset = 0; offset < read.getReadLength(); offset++ ) { + for (int offset = 0; offset < read.getReadLength(); offset++) { final Object[] fullCovariateKey = covariateValues_offset_x_covar[offset]; Byte qualityScore = (Byte) qualityScoreByFullCovariateKey.get(fullCovariateKey); - if(qualityScore == null) - { - qualityScore = performSequentialQualityCalculation( fullCovariateKey ); + if (qualityScore == null) { + qualityScore = performSequentialQualityCalculation(fullCovariateKey); qualityScoreByFullCovariateKey.put(qualityScore, fullCovariateKey); } recalQuals[offset] = qualityScore; } - preserveQScores( originalQuals, recalQuals ); // Overwrite the work done if original quality score is too low + preserveQScores(originalQuals, recalQuals); // Overwrite the work done if original quality score is too low - read.setBaseQualities( recalQuals ); // Overwrite old qualities with new recalibrated qualities - if ( !DO_NOT_WRITE_OQ && read.getAttribute(RecalDataManager.ORIGINAL_QUAL_ATTRIBUTE_TAG) == null ) { // Save the old qualities if the tag isn't already taken in the read + read.setBaseQualities(recalQuals); // Overwrite old qualities with new recalibrated qualities + if (!DO_NOT_WRITE_OQ && read.getAttribute(RecalDataManager.ORIGINAL_QUAL_ATTRIBUTE_TAG) == null) { // Save the old qualities if the tag isn't already taken in the read read.setAttribute(RecalDataManager.ORIGINAL_QUAL_ATTRIBUTE_TAG, SAMUtils.phredToFastq(originalQuals)); } - if (! skipUQUpdate && refBases != null && read.getAttribute(SAMTag.UQ.name()) != null) { + if (!skipUQUpdate && refBases != null && read.getAttribute(SAMTag.UQ.name()) != null) { read.setAttribute(SAMTag.UQ.name(), SequenceUtil.sumQualitiesOfMismatches(read, refBases.getBases(), read.getAlignmentStart() - 1, false)); } @@ -440,27 +445,28 @@ public SAMRecord map( ReferenceContext refBases, GATKSAMRecord read, ReadMetaDat * * Given the full recalibration table, we perform the following preprocessing steps: * - * - calculate the global quality score shift across all data [DeltaQ] - * - calculate for each of cycle and dinuc the shift of the quality scores relative to the global shift - * -- i.e., DeltaQ(dinuc) = Sum(pos) Sum(Qual) Qempirical(pos, qual, dinuc) - Qreported(pos, qual, dinuc) / Npos * Nqual - * - The final shift equation is: + * - calculate the global quality score shift across all data [DeltaQ] + * - calculate for each of cycle and dinuc the shift of the quality scores relative to the global shift + * -- i.e., DeltaQ(dinuc) = Sum(pos) Sum(Qual) Qempirical(pos, qual, dinuc) - Qreported(pos, qual, dinuc) / Npos * Nqual + * - The final shift equation is: + * + * Qrecal = Qreported + DeltaQ + DeltaQ(pos) + DeltaQ(dinuc) + DeltaQ( ... any other covariate ... ) * - * Qrecal = Qreported + DeltaQ + DeltaQ(pos) + DeltaQ(dinuc) + DeltaQ( ... any other covariate ... ) * @param key The list of Comparables that were calculated from the covariates * @return A recalibrated quality score as a byte */ - private byte performSequentialQualityCalculation( final Object... key ) { + private byte performSequentialQualityCalculation(final Object... key) { - final byte qualFromRead = (byte)Integer.parseInt(key[1].toString()); + final byte qualFromRead = (byte) Integer.parseInt(key[1].toString()); final Object[] readGroupCollapsedKey = new Object[1]; final Object[] qualityScoreCollapsedKey = new Object[2]; final Object[] covariateCollapsedKey = new Object[3]; // The global quality shift (over the read group only) readGroupCollapsedKey[0] = key[0]; - final RecalDatum globalRecalDatum = ((RecalDatum)dataManager.getCollapsedTable(0).get( readGroupCollapsedKey )); + final RecalDatum globalRecalDatum = ((RecalDatum) dataManager.getCollapsedTable(0).get(readGroupCollapsedKey)); double globalDeltaQ = 0.0; - if( globalRecalDatum != null ) { + if (globalRecalDatum != null) { final double globalDeltaQEmpirical = globalRecalDatum.getEmpiricalQuality(); final double aggregrateQReported = globalRecalDatum.getEstimatedQReported(); globalDeltaQ = globalDeltaQEmpirical - aggregrateQReported; @@ -469,9 +475,9 @@ private byte performSequentialQualityCalculation( final Object... key ) { // The shift in quality between reported and empirical qualityScoreCollapsedKey[0] = key[0]; qualityScoreCollapsedKey[1] = key[1]; - final RecalDatum qReportedRecalDatum = ((RecalDatum)dataManager.getCollapsedTable(1).get( qualityScoreCollapsedKey )); + final RecalDatum qReportedRecalDatum = ((RecalDatum) dataManager.getCollapsedTable(1).get(qualityScoreCollapsedKey)); double deltaQReported = 0.0; - if( qReportedRecalDatum != null ) { + if (qReportedRecalDatum != null) { final double deltaQReportedEmpirical = qReportedRecalDatum.getEmpiricalQuality(); deltaQReported = deltaQReportedEmpirical - qualFromRead - globalDeltaQ; } @@ -481,17 +487,17 @@ private byte performSequentialQualityCalculation( final Object... key ) { double deltaQCovariateEmpirical; covariateCollapsedKey[0] = key[0]; covariateCollapsedKey[1] = key[1]; - for( int iii = 2; iii < key.length; iii++ ) { - covariateCollapsedKey[2] = key[iii]; // The given covariate - final RecalDatum covariateRecalDatum = ((RecalDatum)dataManager.getCollapsedTable(iii).get( covariateCollapsedKey )); - if( covariateRecalDatum != null ) { + for (int iii = 2; iii < key.length; iii++) { + covariateCollapsedKey[2] = key[iii]; // The given covariate + final RecalDatum covariateRecalDatum = ((RecalDatum) dataManager.getCollapsedTable(iii).get(covariateCollapsedKey)); + if (covariateRecalDatum != null) { deltaQCovariateEmpirical = covariateRecalDatum.getEmpiricalQuality(); - deltaQCovariates += ( deltaQCovariateEmpirical - qualFromRead - (globalDeltaQ + deltaQReported) ); + deltaQCovariates += (deltaQCovariateEmpirical - qualFromRead - (globalDeltaQ + deltaQReported)); } } final double newQuality = qualFromRead + globalDeltaQ + deltaQReported + deltaQCovariates; - return QualityUtils.boundQual( (int)Math.round(newQuality), (byte)MAX_QUALITY_SCORE ); + return QualityUtils.boundQual((int) Math.round(newQuality), (byte) MAX_QUALITY_SCORE); // Verbose printouts used to validate with old recalibrator //if(key.contains(null)) { @@ -508,12 +514,13 @@ private byte performSequentialQualityCalculation( final Object... key ) { /** * Loop over the list of qualities and overwrite the newly recalibrated score to be the original score if it was less than some threshold + * * @param originalQuals The list of original base quality scores - * @param recalQuals A list of the new recalibrated quality scores + * @param recalQuals A list of the new recalibrated quality scores */ - private void preserveQScores( final byte[] originalQuals, final byte[] recalQuals ) { - for( int iii = 0; iii < recalQuals.length; iii++ ) { - if( originalQuals[iii] < PRESERVE_QSCORES_LESS_THAN ) { + private void preserveQScores(final byte[] originalQuals, final byte[] recalQuals) { + for (int iii = 0; iii < recalQuals.length; iii++) { + if (originalQuals[iii] < PRESERVE_QSCORES_LESS_THAN) { recalQuals[iii] = originalQuals[iii]; } } @@ -527,6 +534,7 @@ private void preserveQScores( final byte[] originalQuals, final byte[] recalQual /** * Start the reduce with a handle to the output bam file + * * @return A FileWriter pointing to a new bam file */ public SAMFileWriter reduceInit() { @@ -535,12 +543,13 @@ public SAMFileWriter reduceInit() { /** * Output each read to disk - * @param read The read to output + * + * @param read The read to output * @param output The FileWriter to write the read to * @return The FileWriter */ - public SAMFileWriter reduce( SAMRecord read, SAMFileWriter output ) { - if( output != null ) { + public SAMFileWriter reduce(SAMRecord read, SAMFileWriter output) { + if (output != null) { output.addAlignment(read); } return output; @@ -548,20 +557,22 @@ public SAMFileWriter reduce( SAMRecord read, SAMFileWriter output ) { /** * Do nothing + * * @param output The SAMFileWriter that outputs the bam file */ public void onTraversalDone(SAMFileWriter output) { - if( numReadsWithMalformedColorSpace != 0 ) { - if( RAC.SOLID_NOCALL_STRATEGY == RecalDataManager.SOLID_NOCALL_STRATEGY.LEAVE_READ_UNRECALIBRATED ) { + if (numReadsWithMalformedColorSpace != 0) { + if (RAC.SOLID_NOCALL_STRATEGY == RecalDataManager.SOLID_NOCALL_STRATEGY.LEAVE_READ_UNRECALIBRATED) { Utils.warnUser("Discovered " + numReadsWithMalformedColorSpace + " SOLiD reads with no calls in the color space. Unfortunately these reads cannot be recalibrated with this recalibration algorithm " + - "because we use reference mismatch rate as the only indication of a base's true quality. These reads have had reference bases inserted as a way of correcting " + - "for color space misalignments and there is now no way of knowing how often it mismatches the reference and therefore no way to recalibrate the quality score. " + - "These reads remain in the output bam file but haven't been corrected for reference bias. !!! USE AT YOUR OWN RISK !!!"); - } else if ( RAC.SOLID_NOCALL_STRATEGY == RecalDataManager.SOLID_NOCALL_STRATEGY.PURGE_READ ) { + "because we use reference mismatch rate as the only indication of a base's true quality. These reads have had reference bases inserted as a way of correcting " + + "for color space misalignments and there is now no way of knowing how often it mismatches the reference and therefore no way to recalibrate the quality score. " + + "These reads remain in the output bam file but haven't been corrected for reference bias. !!! USE AT YOUR OWN RISK !!!"); + } + else if (RAC.SOLID_NOCALL_STRATEGY == RecalDataManager.SOLID_NOCALL_STRATEGY.PURGE_READ) { Utils.warnUser("Discovered " + numReadsWithMalformedColorSpace + " SOLiD reads with no calls in the color space. Unfortunately these reads cannot be recalibrated with this recalibration algorithm " + - "because we use reference mismatch rate as the only indication of a base's true quality. These reads have had reference bases inserted as a way of correcting " + - "for color space misalignments and there is now no way of knowing how often it mismatches the reference and therefore no way to recalibrate the quality score. " + - "These reads were completely removed from the output bam file."); + "because we use reference mismatch rate as the only indication of a base's true quality. These reads have had reference bases inserted as a way of correcting " + + "for color space misalignments and there is now no way of knowing how often it mismatches the reference and therefore no way to recalibrate the quality score. " + + "These reads were completely removed from the output bam file."); } } From e89887cd8e69a9dc8ecdf0df120c298a6994d808 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Tue, 7 Feb 2012 18:11:53 -0500 Subject: [PATCH 225/356] laying groundwork to have insertions and deletions going through the system. --- .../recalibration/RecalDataManager.java | 17 +- .../broadinstitute/sting/utils/BaseUtils.java | 282 ++++++++++-------- 2 files changed, 163 insertions(+), 136 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java index 72c2b28299..311e33f8a5 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java @@ -36,6 +36,7 @@ import org.broadinstitute.sting.utils.sam.AlignmentUtils; import org.broadinstitute.sting.utils.sam.GATKSAMReadGroupRecord; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; +import org.broadinstitute.sting.utils.sam.ReadUtils; import java.util.ArrayList; import java.util.List; @@ -284,7 +285,7 @@ public static void parseSAMRecord(final GATKSAMRecord read, final RecalibrationA public static void parseColorSpace(final GATKSAMRecord read) { // If this is a SOLID read then we have to check if the color space is inconsistent. This is our only sign that SOLID has inserted the reference base - if (read.getReadGroup().getPlatform().toUpperCase().contains("SOLID")) { + if (ReadUtils.isSOLiDRead(read)) { if (read.getAttribute(RecalDataManager.COLOR_SPACE_INCONSISTENCY_TAG) == null) { // Haven't calculated the inconsistency array yet for this read final Object attr = read.getAttribute(RecalDataManager.COLOR_SPACE_ATTRIBUTE_TAG); if (attr != null) { @@ -382,7 +383,7 @@ else if (solidRecalMode == SOLID_RECAL_MODE.REMOVE_REF_BIAS) { // Use the color } public static boolean checkNoCallColorSpace(final GATKSAMRecord read) { - if (read.getReadGroup().getPlatform().toUpperCase().contains("SOLID")) { + if (ReadUtils.isSOLiDRead(read)) { final Object attr = read.getAttribute(RecalDataManager.COLOR_SPACE_ATTRIBUTE_TAG); if (attr != null) { byte[] colorSpace; @@ -611,21 +612,17 @@ public static Comparable[][] computeCovariates(final GATKSAMRecord gatkRead, fin final Comparable[][] covariateValues_offset_x_covar = new Comparable[readLength][numRequestedCovariates]; final Comparable[] tempCovariateValuesHolder = new Comparable[readLength]; - // Loop through the list of requested covariates and compute the values of each covariate for all positions in this read - for (int i = 0; i < numRequestedCovariates; i++) { + for (int i = 0; i < numRequestedCovariates; i++) { // Loop through the list of requested covariates and compute the values of each covariate for all positions in this read requestedCovariates.get(i).getValues(gatkRead, tempCovariateValuesHolder, modelType); - for (int j = 0; j < readLength; j++) { - //copy values into a 2D array that allows all covar types to be extracted at once for - //an offset j by doing covariateValues_offset_x_covar[j]. This avoids the need to later iterate over covar types. - covariateValues_offset_x_covar[j][i] = tempCovariateValuesHolder[j]; - } + for (int j = 0; j < readLength; j++) + covariateValues_offset_x_covar[j][i] = tempCovariateValuesHolder[j]; // copy values into a 2D array that allows all covar types to be extracted at once for an offset j by doing covariateValues_offset_x_covar[j]. This avoids the need to later iterate over covar types. } return covariateValues_offset_x_covar; } /** - * Perform a ceratin transversion (A <-> C or G <-> T) on the base. + * Perform a certain transversion (A <-> C or G <-> T) on the base. * * @param base the base [AaCcGgTt] * @return the transversion of the base, or the input base if it's not one of the understood ones diff --git a/public/java/src/org/broadinstitute/sting/utils/BaseUtils.java b/public/java/src/org/broadinstitute/sting/utils/BaseUtils.java index 673b1524da..61812629ce 100644 --- a/public/java/src/org/broadinstitute/sting/utils/BaseUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/BaseUtils.java @@ -2,57 +2,59 @@ import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; - /** * BaseUtils contains some basic utilities for manipulating nucleotides. */ public class BaseUtils { - public final static byte A = (byte)'A'; - public final static byte C = (byte)'C'; - public final static byte G = (byte)'G'; - public final static byte T = (byte)'T'; + public final static byte A = (byte) 'A'; + public final static byte C = (byte) 'C'; + public final static byte G = (byte) 'G'; + public final static byte T = (byte) 'T'; - public final static byte N = (byte)'N'; - public final static byte D = (byte)'D'; + public final static byte N = (byte) 'N'; + public final static byte D = (byte) 'D'; // // todo -- we need a generalized base abstraction using the Base enum. // - public final static byte[] BASES = { 'A', 'C', 'G', 'T' }; - public final static byte[] EXTENDED_BASES = { 'A', 'C', 'G', 'T', 'N', 'D' }; + public final static byte[] BASES = {'A', 'C', 'G', 'T'}; + public final static byte[] EXTENDED_BASES = {'A', 'C', 'G', 'T', 'N', 'D'}; public enum Base { - A ( 'A', 0 ), - C ( 'C', 1 ), - G ( 'G', 2 ), - T ( 'T', 3 ); + A('A', 0), + C('C', 1), + G('G', 2), + T('T', 3); byte b; int index; + private Base(char base, int index) { - this.b = (byte)base; + this.b = (byte) base; this.index = index; } public byte getBase() { return b; } - public char getBaseAsChar() { return (char)b; } + + public char getBaseAsChar() { return (char) b; } + public int getIndex() { return index; } public boolean sameBase(byte o) { return b == o; } - public boolean sameBase(char o) { return b == (byte)o; } - public boolean sameBase(int i) { return index == i; } - } + public boolean sameBase(char o) { return b == (byte) o; } + + public boolean sameBase(int i) { return index == i; } + } // todo -- fix me (enums?) public static final byte DELETION_INDEX = 4; public static final byte NO_CALL_INDEX = 5; // (this is 'N') - public static int gIndex = BaseUtils.simpleBaseToBaseIndex((byte)'G'); - public static int cIndex = BaseUtils.simpleBaseToBaseIndex((byte)'C'); - public static int aIndex = BaseUtils.simpleBaseToBaseIndex((byte)'A'); - public static int tIndex = BaseUtils.simpleBaseToBaseIndex((byte)'T'); - + public static int gIndex = BaseUtils.simpleBaseToBaseIndex((byte) 'G'); + public static int cIndex = BaseUtils.simpleBaseToBaseIndex((byte) 'C'); + public static int aIndex = BaseUtils.simpleBaseToBaseIndex((byte) 'A'); + public static int tIndex = BaseUtils.simpleBaseToBaseIndex((byte) 'T'); /// In genetics, a transition is a mutation changing a purine to another purine nucleotide (A <-> G) or // a pyrimidine to another pyrimidine nucleotide (C <-> T). @@ -64,28 +66,31 @@ public enum BaseSubstitutionType { /** * Returns the base substitution type of the 2 state SNP + * * @param base1 * @param base2 * @return */ - public static BaseSubstitutionType SNPSubstitutionType( byte base1, byte base2 ) { + public static BaseSubstitutionType SNPSubstitutionType(byte base1, byte base2) { BaseSubstitutionType t = isTransition(base1, base2) ? BaseSubstitutionType.TRANSITION : BaseSubstitutionType.TRANSVERSION; //System.out.printf("SNPSubstitutionType( char %c, char %c ) => %s%n", base1, base2, t); return t; } - public static boolean isTransition( byte base1, byte base2 ) { + public static boolean isTransition(byte base1, byte base2) { int b1 = simpleBaseToBaseIndex(base1); int b2 = simpleBaseToBaseIndex(base2); return b1 == 0 && b2 == 2 || b1 == 2 && b2 == 0 || - b1 == 1 && b2 == 3 || b1 == 3 && b2 == 1; + b1 == 1 && b2 == 3 || b1 == 3 && b2 == 1; } - public static boolean isTransversion( byte base1, byte base2 ) { - return ! isTransition(base1, base2); + public static boolean isTransversion(byte base1, byte base2) { + return !isTransition(base1, base2); } - /** Private constructor. No instantiating this class! */ + /** + * Private constructor. No instantiating this class! + */ private BaseUtils() {} static public boolean basesAreEqual(byte base1, byte base2) { @@ -96,7 +101,6 @@ static public boolean extendedBasesAreEqual(byte base1, byte base2) { return extendedBaseToBaseIndex(base1) == extendedBaseToBaseIndex(base2); } - /** * Converts a IUPAC nucleotide code to a pair of bases * @@ -163,33 +167,37 @@ static public char[] iupacToBases(char code) { /** * Converts a simple base to a base index * - * @param base [AaCcGgTt] + * @param base [AaCcGgTt] * @return 0, 1, 2, 3, or -1 if the base can't be understood */ static public int simpleBaseToBaseIndex(byte base) { switch (base) { case '*': // the wildcard character counts as an A case 'A': - case 'a': return 0; + case 'a': + return 0; case 'C': - case 'c': return 1; + case 'c': + return 1; case 'G': - case 'g': return 2; + case 'g': + return 2; case 'T': - case 't': return 3; + case 't': + return 3; - default: return -1; + default: + return -1; } } - /** * Converts a simple base to a base index * - * @param base [AaCcGgTt] + * @param base [AaCcGgTt] * @return 0, 1, 2, 3, or -1 if the base can't be understood */ @Deprecated @@ -197,29 +205,37 @@ static public int simpleBaseToBaseIndex(char base) { switch (base) { case '*': // the wildcard character counts as an A case 'A': - case 'a': return 0; + case 'a': + return 0; case 'C': - case 'c': return 1; + case 'c': + return 1; case 'G': - case 'g': return 2; + case 'g': + return 2; case 'T': - case 't': return 3; + case 't': + return 3; - default: return -1; + default: + return -1; } } static public int extendedBaseToBaseIndex(byte base) { switch (base) { case 'd': - case 'D': return DELETION_INDEX; + case 'D': + return DELETION_INDEX; case 'n': - case 'N': return NO_CALL_INDEX; + case 'N': + return NO_CALL_INDEX; - default: return simpleBaseToBaseIndex(base); + default: + return simpleBaseToBaseIndex(base); } } @@ -232,11 +248,6 @@ static public boolean isRegularBase(byte base) { return simpleBaseToBaseIndex(base) != -1; } - @Deprecated - static public boolean isNBase(char base) { - return isNBase((byte)base); - } - static public boolean isNBase(byte base) { return base == 'N' || base == 'n'; } @@ -244,68 +255,83 @@ static public boolean isNBase(byte base) { /** * Converts a base index to a simple base * - * @param baseIndex 0, 1, 2, 3 + * @param baseIndex 0, 1, 2, 3 * @return A, C, G, T, or '.' if the index can't be understood */ static public byte baseIndexToSimpleBase(int baseIndex) { switch (baseIndex) { - case 0: return 'A'; - case 1: return 'C'; - case 2: return 'G'; - case 3: return 'T'; - default: return '.'; + case 0: + return 'A'; + case 1: + return 'C'; + case 2: + return 'G'; + case 3: + return 'T'; + default: + return '.'; } } @Deprecated static public char baseIndexToSimpleBaseAsChar(int baseIndex) { - return (char)baseIndexToSimpleBase(baseIndex); + return (char) baseIndexToSimpleBase(baseIndex); } /** * Converts a base index to a base index representing its cross-talk partner * - * @param baseIndex 0, 1, 2, 3 + * @param baseIndex 0, 1, 2, 3 * @return 1, 0, 3, 2, or -1 if the index can't be understood */ static public int crossTalkPartnerIndex(int baseIndex) { switch (baseIndex) { - case 0: return 1; // A -> C - case 1: return 0; // C -> A - case 2: return 3; // G -> T - case 3: return 2; // T -> G - default: return -1; + case 0: + return 1; // A -> C + case 1: + return 0; // C -> A + case 2: + return 3; // G -> T + case 3: + return 2; // T -> G + default: + return -1; } } /** * Converts a base to the base representing its cross-talk partner * - * @param base [AaCcGgTt] + * @param base [AaCcGgTt] * @return C, A, T, G, or '.' if the base can't be understood */ @Deprecated static public char crossTalkPartnerBase(char base) { - return (char)baseIndexToSimpleBase(crossTalkPartnerIndex(simpleBaseToBaseIndex(base))); + return (char) baseIndexToSimpleBase(crossTalkPartnerIndex(simpleBaseToBaseIndex(base))); } /** * Return the complement of a base index. * - * @param baseIndex the base index (0:A, 1:C, 2:G, 3:T) + * @param baseIndex the base index (0:A, 1:C, 2:G, 3:T) * @return the complementary base index */ static public byte complementIndex(int baseIndex) { switch (baseIndex) { - case 0: return 3; // a -> t - case 1: return 2; // c -> g - case 2: return 1; // g -> c - case 3: return 0; // t -> a - default: return -1; // wtf? + case 0: + return 3; // a -> t + case 1: + return 2; // c -> g + case 2: + return 1; // g -> c + case 3: + return 0; // t -> a + default: + return -1; // wtf? } } - /** + /** * Return the complement (A <-> T or C <-> G) of a base, or the specified base if it can't be complemented (i.e. an ambiguous base). * * @param base the base [AaCcGgTt] @@ -314,20 +340,25 @@ static public byte complementIndex(int baseIndex) { static public byte simpleComplement(byte base) { switch (base) { case 'A': - case 'a': return 'T'; + case 'a': + return 'T'; case 'C': - case 'c': return 'G'; + case 'c': + return 'G'; case 'G': - case 'g': return 'C'; + case 'g': + return 'C'; case 'T': - case 't': return 'A'; - default: return base; + case 't': + return 'A'; + default: + return base; } } @Deprecated static public char simpleComplement(char base) { - return (char)simpleComplement((byte)base); + return (char) simpleComplement((byte) base); } /** @@ -349,7 +380,7 @@ static public byte[] simpleReverseComplement(byte[] bases) { /** * Complement a byte array of bases (that is, chars casted to bytes, *not* base indices in byte form) * - * @param bases the byte array of bases + * @param bases the byte array of bases * @return the complement of the base byte array */ static public byte[] simpleComplement(byte[] bases) { @@ -382,7 +413,7 @@ static public char[] simpleReverseComplement(char[] bases) { /** * Complement a char array of bases * - * @param bases the char array of bases + * @param bases the char array of bases * @return the complement of the base char array */ @Deprecated @@ -399,7 +430,7 @@ static public char[] simpleComplement(char[] bases) { /** * Reverse complement a String of bases. Preserves ambiguous bases. * - * @param bases the String of bases + * @param bases the String of bases * @return the reverse complement of the String */ @Deprecated @@ -407,11 +438,10 @@ static public String simpleReverseComplement(String bases) { return new String(simpleReverseComplement(bases.getBytes())); } - /** * Complement a String of bases. Preserves ambiguous bases. * - * @param bases the String of bases + * @param bases the String of bases * @return the complement of the String */ @Deprecated @@ -451,7 +481,7 @@ static public int mostFrequentBaseIndexNotRef(int[] baseCounts, byte refSimpleBa /** * Returns the most common base in the basecounts array. To be used with pileup.getBaseCounts. * - * @param baseCounts counts of a,c,g,t in order. + * @param baseCounts counts of a,c,g,t in order. * @return the most common base */ static public byte mostFrequentSimpleBase(int[] baseCounts) { @@ -461,13 +491,13 @@ static public byte mostFrequentSimpleBase(int[] baseCounts) { /** * For the most frequent base in the sequence, return the percentage of the read it constitutes. * - * @param sequence the read sequence - * @return the percentage of the read that's made up of the most frequent base + * @param sequence the read sequence + * @return the percentage of the read that's made up of the most frequent base */ static public double mostFrequentBaseFraction(byte[] sequence) { int[] baseCounts = new int[4]; - for ( byte base : sequence ) { + for (byte base : sequence) { int baseIndex = simpleBaseToBaseIndex(base); if (baseIndex >= 0) { @@ -477,7 +507,7 @@ static public double mostFrequentBaseFraction(byte[] sequence) { int mostFrequentBaseIndex = mostFrequentBaseIndex(baseCounts); - return ((double) baseCounts[mostFrequentBaseIndex])/((double) sequence.length); + return ((double) baseCounts[mostFrequentBaseIndex]) / ((double) sequence.length); } // -------------------------------------------------------------------------------- @@ -531,50 +561,50 @@ static public byte getRandomBase() { static public byte getRandomBase(char excludeBase) { return BaseUtils.baseIndexToSimpleBase(getRandomBaseIndex(BaseUtils.simpleBaseToBaseIndex(excludeBase))); } - - - /** Computes the smallest period >= minPeriod for the specified string. The period is defined as such p, + + /** + * Computes the smallest period >= minPeriod for the specified string. The period is defined as such p, * that for all i = 0... seq.length-1, seq[ i % p ] = seq[i] (or equivalently seq[i] = seq[i+p] for i=0...seq.length-1-p). - * The sequence does not have to contain whole number of periods. For instance, "ACACACAC" has a period - * of 2 (it has a period of 4 as well), and so does - * "ACACA"; similarly, smallest periods of "CTCCTC", "CTCCT", and "CTCC" are all equal to 3. The "trivial" period is + * The sequence does not have to contain whole number of periods. For instance, "ACACACAC" has a period + * of 2 (it has a period of 4 as well), and so does + * "ACACA"; similarly, smallest periods of "CTCCTC", "CTCCT", and "CTCC" are all equal to 3. The "trivial" period is * the length of the string itself, and it will always be returned if no smaller period can be found in the specified period range * or if specified minPeriod is greater than the sequence length. - * + * * @param seq * @return */ public static int sequencePeriod(byte[] seq, int minPeriod) { - int period = ( minPeriod > seq.length ? seq.length : minPeriod ); - // we assume that bases [0,period-1] repeat themselves and check this assumption - // until we find correct period - - for ( int pos = period ; pos < seq.length ; pos++ ) { - - int offset = pos % period; // we are currenlty 'offset' bases into the putative repeat of period 'period' - // if our current hypothesis holds, base[pos] must be the same as base[offset] - - if ( Character.toUpperCase( seq[pos] ) != - Character.toUpperCase( seq[offset] ) - ) { - - // period we have been trying so far does not work. - // two possibilities: - // A) offset = 0, i.e. current position pos must be start of the next repeat, but it is not; - // in this case only bases from start up to the current one, inclusive, may form a repeat, if at all; - // so period is at least pos+1 (remember, pos is 0-based), then on the next loop re-entrance - // pos will be autoincremented and we will be checking next base - // B) offset != 0, i.e. the current base breaks the repeat, but maybe it starts a new one? - // hence we should first check if it matches the first base of the sequence, and to do that - // we set period to pos (thus trying the hypothesis that bases from start up to the current one, - // non-inclusive are repeated hereafter), and decrement pos (this will re-test current base against the first base - // on the next loop re-entrance after pos is autoincremented) - if ( offset == 0 ) period = pos+1; - else period = pos-- ; - - } - } - return period; + int period = (minPeriod > seq.length ? seq.length : minPeriod); + // we assume that bases [0,period-1] repeat themselves and check this assumption + // until we find correct period + + for (int pos = period; pos < seq.length; pos++) { + + int offset = pos % period; // we are currenlty 'offset' bases into the putative repeat of period 'period' + // if our current hypothesis holds, base[pos] must be the same as base[offset] + + if (Character.toUpperCase(seq[pos]) != Character.toUpperCase(seq[offset])) { + + // period we have been trying so far does not work. + // two possibilities: + // A) offset = 0, i.e. current position pos must be start of the next repeat, but it is not; + // in this case only bases from start up to the current one, inclusive, may form a repeat, if at all; + // so period is at least pos+1 (remember, pos is 0-based), then on the next loop re-entrance + // pos will be autoincremented and we will be checking next base + // B) offset != 0, i.e. the current base breaks the repeat, but maybe it starts a new one? + // hence we should first check if it matches the first base of the sequence, and to do that + // we set period to pos (thus trying the hypothesis that bases from start up to the current one, + // non-inclusive are repeated hereafter), and decrement pos (this will re-test current base against the first base + // on the next loop re-entrance after pos is autoincremented) + if (offset == 0) + period = pos + 1; + else + period = pos--; + + } + } + return period; } } From c0c676590b49d3d384aebc716a0ac2388f9c850f Mon Sep 17 00:00:00 2001 From: Roger Zurawicki Date: Mon, 9 Jan 2012 23:46:48 -0500 Subject: [PATCH 226/356] First implementation of GATKReportGatherer - Added the GATKReportGatherer - Added private methods in GATKReport to combine Tables and Reports - It is very conservative and it will only gather if the table columns, match. - At the column level it uses the (redundant) row ids to add new rows. It will throw an exception if it is overwriting data. Added the gatherer functions to CoverageByRG Also added the scatterCount parameter in the Interval Coverage script Made some more GATKReport methods public The UnitTest included shows that the merging methods work Added a getter for the PrimaryKeyName Fixed bugs that prevented the gatherer form working Working GATKReportGatherer Has only the functional to addLines The input file parser assumes that the first column is the primary key Signed-off-by: Mauricio Carneiro --- .../sting/gatk/report/GATKReport.java | 65 +++-- .../sting/gatk/report/GATKReportGatherer.java | 46 ++++ .../sting/gatk/report/GATKReportTable.java | 252 +++++++++++------- .../sting/gatk/report/GATKReportUnitTest.java | 128 +++++++-- 4 files changed, 357 insertions(+), 134 deletions(-) create mode 100644 public/java/src/org/broadinstitute/sting/gatk/report/GATKReportGatherer.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReport.java b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReport.java index 608b5d1d0a..c0abe74500 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReport.java +++ b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReport.java @@ -24,7 +24,8 @@ public GATKReport() { /** * Create a new GATKReport with the contents of a GATKReport on disk. - * @param filename the path to the file to load + * + * @param filename the path to the file to load */ public GATKReport(String filename) { this(new File(filename)); @@ -32,7 +33,8 @@ public GATKReport(String filename) { /** * Create a new GATKReport with the contents of a GATKReport on disk. - * @param file the file to load + * + * @param file the file to load */ public GATKReport(File file) { loadReport(file); @@ -40,7 +42,8 @@ public GATKReport(File file) { /** * Load a GATKReport file from disk - * @param file the file to load + * + * @param file the file to load */ private void loadReport(File file) { try { @@ -48,12 +51,11 @@ private void loadReport(File file) { GATKReportTable table = null; String[] header = null; - int id = 0; GATKReportVersion version = null; List columnStarts = null; String line; - while ( (line = reader.readLine()) != null ) { + while ((line = reader.readLine()) != null) { if (line.startsWith(GATKREPORT_HEADER_PREFIX)) { @@ -71,7 +73,7 @@ private void loadReport(File file) { header = null; columnStarts = null; - } else if ( line.trim().isEmpty() ) { + } else if (line.trim().isEmpty()) { // do nothing } else { if (table != null) { @@ -97,19 +99,22 @@ private void loadReport(File file) { if (header == null) { header = splitLine; - table.addPrimaryKey("id", false); - - for ( String columnName : header ) { - table.addColumn(columnName, ""); + // Set the first column as the primary key + table.addPrimaryKey(header[0]); + // Set every other column as column + for (int i = 1; i < header.length; i++) { + table.addColumn(header[i], ""); } - id = 0; } else { - for (int columnIndex = 0; columnIndex < header.length; columnIndex++) { - table.set(id, header[columnIndex], splitLine[columnIndex]); + //Get primary key Value from the current line array + String primaryKey = splitLine[0]; + //Input all the remaining values + for (int columnIndex = 1; columnIndex < header.length; columnIndex++) { + table.set(primaryKey, header[columnIndex], splitLine[columnIndex]); } - id++; + } } } @@ -124,8 +129,8 @@ private void loadReport(File file) { /** * Add a new table to the collection * - * @param tableName the name of the table - * @param tableDescription the description of the table + * @param tableName the name of the table + * @param tableDescription the description of the table */ public void addTable(String tableName, String tableDescription) { addTable(tableName, tableDescription, true); @@ -139,7 +144,7 @@ public void addTable(String tableName, String tableDescription, boolean sortByPr /** * Return true if table with a given name exists * - * @param tableName the name of the table + * @param tableName the name of the table * @return true if the table exists, false otherwise */ public boolean hasTable(String tableName) { @@ -149,8 +154,8 @@ public boolean hasTable(String tableName) { /** * Return a table with a given name * - * @param tableName the name of the table - * @return the table object + * @param tableName the name of the table + * @return the table object */ public GATKReportTable getTable(String tableName) { GATKReportTable table = tables.get(tableName); @@ -162,7 +167,7 @@ public GATKReportTable getTable(String tableName) { /** * Print all tables contained within this container to a PrintStream * - * @param out the PrintStream to which the tables should be written + * @param out the PrintStream to which the tables should be written */ public void print(PrintStream out) { for (GATKReportTable table : tables.values()) { @@ -175,4 +180,24 @@ public void print(PrintStream out) { public Collection getTables() { return tables.values(); } + + public void combineWith(GATKReport input) { + + // For every input table, add values + System.out.println("This.tables: keySet"); + for (String s : tables.keySet()) + System.out.println(s); + + // todo test tables exist + + + for (String tableName : input.tables.keySet()) { + System.out.println("Input table key: " + tableName); + if (tables.containsKey(tableName)) + tables.get(tableName).mergeRows(input.getTable(tableName)); + else + throw new ReviewedStingException("Failed to combine GATKReport, tables don't match!"); + } + + } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportGatherer.java b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportGatherer.java new file mode 100644 index 0000000000..0d15971ae1 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportGatherer.java @@ -0,0 +1,46 @@ +package org.broadinstitute.sting.gatk.report; + +import org.broadinstitute.sting.commandline.Gatherer; +import org.broadinstitute.sting.utils.exceptions.UserException; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.PrintStream; +import java.util.List; + +/** + * Created by IntelliJ IDEA. + * User: roger + * Date: 1/9/12 + * Time: 11:17 PM + * To change this template use File | Settings | File Templates. + */ +public class GATKReportGatherer extends Gatherer { + @Override + public void gather(List inputs, File output) { + //Combines inputs GATKReport to one output + + PrintStream o; + try { + o = new PrintStream(output); + } catch (FileNotFoundException e) { + throw new UserException("File to be output by CoverageByRG Gather function was not found"); + } + + GATKReport current = new GATKReport(); + boolean isFirst = true; + for (File input : inputs) { + + // If the table is empty + if (isFirst) { + current = new GATKReport(input); + isFirst = false; + } else { + GATKReport toAdd = new GATKReport(input); + current.combineWith(toAdd); + } + } + + current.print(o); + } +} diff --git a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java index b72b20e0b7..ac18891d78 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java +++ b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java @@ -4,7 +4,10 @@ import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import java.io.PrintStream; -import java.util.*; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -12,12 +15,12 @@ * A data structure that allows data to be collected over the course of a walker's computation, then have that data * written to a PrintStream such that it's human-readable, AWK-able, and R-friendly (given that you load it using the * GATKReport loader module). - * + *

* The goal of this object is to use the same data structure for both accumulating data during a walker's computation * and emitting that data to a file for easy analysis in R (or any other program/language that can take in a table of * results). Thus, all of the infrastructure below is designed simply to make printing the following as easy as * possible: - * + *

* ##:GATKReport.v0.1 ErrorRatePerCycle : The error rate per sequenced position in the reads * cycle errorrate.61PA8.7 qualavg.61PA8.7 * 0 0.007451835696110506 25.474613284804366 @@ -29,60 +32,60 @@ * 6 5.452562704471102E-4 36.1217248908297 * 7 5.452562704471102E-4 36.1910480349345 * 8 5.452562704471102E-4 36.00345705967977 - * + *

* Here, we have a GATKReport table - a well-formatted, easy to read representation of some tabular data. Every single * table has this same GATKReport.v0.1 header, which permits multiple files from different sources to be cat-ed * together, which makes it very easy to pull tables from different programs into R via a single file. - * + *

* ------------ * Definitions: - * + *

* Table info: - * The first line, structured as - * ##:

:
- * + * The first line, structured as + * ##:
:
+ *

* Table header: - * The second line, specifying a unique name for each column in the table. - * - * The first column mentioned in the table header is the "primary key" column - a column that provides the unique - * identifier for each row in the table. Once this column is created, any element in the table can be referenced by - * the row-column coordinate, i.e. "primary key"-"column name" coordinate. - * - * When a column is added to a table, a default value must be specified (usually 0). This is the initial value for - * an element in a column. This permits operations like increment() and decrement() to work properly on columns that - * are effectively counters for a particular event. - * - * Finally, the display property for each column can be set during column creation. This is useful when a given - * column stores an intermediate result that will be used later on, perhaps to calculate the value of another column. - * In these cases, it's obviously necessary to store the value required for further computation, but it's not - * necessary to actually print the intermediate column. - * + * The second line, specifying a unique name for each column in the table. + *

+ * The first column mentioned in the table header is the "primary key" column - a column that provides the unique + * identifier for each row in the table. Once this column is created, any element in the table can be referenced by + * the row-column coordinate, i.e. "primary key"-"column name" coordinate. + *

+ * When a column is added to a table, a default value must be specified (usually 0). This is the initial value for + * an element in a column. This permits operations like increment() and decrement() to work properly on columns that + * are effectively counters for a particular event. + *

+ * Finally, the display property for each column can be set during column creation. This is useful when a given + * column stores an intermediate result that will be used later on, perhaps to calculate the value of another column. + * In these cases, it's obviously necessary to store the value required for further computation, but it's not + * necessary to actually print the intermediate column. + *

* Table body: - * The values of the table itself. - * + * The values of the table itself. + *

* --------------- * Implementation: - * + *

* The implementation of this table has two components: - * 1. A TreeSet that stores all the values ever specified for the primary key. Any get() operation that - * refers to an element where the primary key object does not exist will result in its implicit creation. I - * haven't yet decided if this is a good idea... - * - * 2. A HashMap that stores a mapping from column name to column contents. Each - * GATKReportColumn is effectively a map (in fact, GATKReportColumn extends TreeMap) between - * primary key and the column value. This means that, given N columns, the primary key information is stored - * N+1 times. This is obviously wasteful and can likely be handled much more elegantly in future implementations. - * + * 1. A TreeSet that stores all the values ever specified for the primary key. Any get() operation that + * refers to an element where the primary key object does not exist will result in its implicit creation. I + * haven't yet decided if this is a good idea... + *

+ * 2. A HashMap that stores a mapping from column name to column contents. Each + * GATKReportColumn is effectively a map (in fact, GATKReportColumn extends TreeMap) between + * primary key and the column value. This means that, given N columns, the primary key information is stored + * N+1 times. This is obviously wasteful and can likely be handled much more elegantly in future implementations. + *

* ------------------------------ * Element and column operations: - * + *

* In addition to simply getting and setting values, this object also permits some simple operations to be applied to * individual elements or to whole columns. For instance, an element can be easily incremented without the hassle of * calling get(), incrementing the obtained value by 1, and then calling set() with the new value. Also, some vector * operations are supported. For instance, two whole columns can be divided and have the result be set to a third * column. This is especially useful when aggregating counts in two intermediate columns that will eventually need to * be manipulated row-by-row to compute the final column. - * + *

* Note: I've made no attempt whatsoever to make these operations efficient. Right now, some of the methods check the * type of the stored object using an instanceof call and attempt to do the right thing. Others cast the contents of * the cell to a Number, call the Number.toDouble() method and compute a result. This is clearly not the ideal design, @@ -92,7 +95,9 @@ * @author Khalid Shakir */ public class GATKReportTable { - /** REGEX that matches any table with an invalid name */ + /** + * REGEX that matches any table with an invalid name + */ public final static String INVALID_TABLE_NAME_REGEX = "[^a-zA-Z0-9_\\-\\.]"; private static final GATKReportVersion LATEST_REPORT_VERSION = GATKReportVersion.V0_2; private String tableName; @@ -109,8 +114,8 @@ public class GATKReportTable { /** * Verifies that a table or column name has only alphanumeric characters - no spaces or special characters allowed * - * @param name the name of the table or column - * @return true if the name is valid, false if otherwise + * @param name the name of the table or column + * @return true if the name is valid, false if otherwise */ private boolean isValidName(String name) { Pattern p = Pattern.compile(INVALID_TABLE_NAME_REGEX); @@ -122,8 +127,8 @@ private boolean isValidName(String name) { /** * Verifies that a table or column name has only alphanumeric characters - no spaces or special characters allowed * - * @param description the name of the table or column - * @return true if the name is valid, false if otherwise + * @param description the name of the table or column + * @return true if the name is valid, false if otherwise */ private boolean isValidDescription(String description) { Pattern p = Pattern.compile("\\r|\\n"); @@ -135,15 +140,15 @@ private boolean isValidDescription(String description) { /** * Construct a new GATK report table with the specified name and description * - * @param tableName the name of the table - * @param tableDescription the description of the table + * @param tableName the name of the table + * @param tableDescription the description of the table */ public GATKReportTable(String tableName, String tableDescription) { this(tableName, tableDescription, true); } public GATKReportTable(String tableName, String tableDescription, boolean sortByPrimaryKey) { - if (!isValidName(tableName)) { + if (!isValidName(tableName)) { throw new ReviewedStingException("Attempted to set a GATKReportTable name of '" + tableName + "'. GATKReportTable names must be purely alphanumeric - no spaces or special characters are allowed."); } @@ -169,7 +174,7 @@ protected void setVersion(GATKReportVersion version) { /** * Add a primary key column. This becomes the unique identifier for every column in the table. * - * @param primaryKeyName the name of the primary key column + * @param primaryKeyName the name of the primary key column */ public void addPrimaryKey(String primaryKeyName) { addPrimaryKey(primaryKeyName, true); @@ -178,8 +183,8 @@ public void addPrimaryKey(String primaryKeyName) { /** * Add an optionally visible primary key column. This becomes the unique identifier for every column in the table, and will always be printed as the first column. * - * @param primaryKeyName the name of the primary key column - * @param display should this primary key be displayed? + * @param primaryKeyName the name of the primary key column + * @param display should this primary key be displayed? */ public void addPrimaryKey(String primaryKeyName, boolean display) { if (!isValidName(primaryKeyName)) { @@ -195,6 +200,7 @@ public void addPrimaryKey(String primaryKeyName, boolean display) { /** * Returns the first primary key matching the dotted column values. * Ex: dbsnp.eval.called.all.novel.all + * * @param dottedColumnValues Period concatenated values. * @return The first primary key matching the column values or throws an exception. */ @@ -208,6 +214,7 @@ public Object getPrimaryKey(String dottedColumnValues) { /** * Returns true if there is at least on row with the dotted column values. * Ex: dbsnp.eval.called.all.novel.all + * * @param dottedColumnValues Period concatenated values. * @return true if there is at least one row matching the columns. */ @@ -218,6 +225,7 @@ public boolean containsPrimaryKey(String dottedColumnValues) { /** * Returns the first primary key matching the dotted column values. * Ex: dbsnp.eval.called.all.novel.all + * * @param dottedColumnValues Period concatenated values. * @return The first primary key matching the column values or null. */ @@ -228,6 +236,7 @@ private Object findPrimaryKey(String dottedColumnValues) { /** * Returns the first primary key matching the column values. * Ex: new String[] { "dbsnp", "eval", "called", "all", "novel", "all" } + * * @param columnValues column values. * @return The first primary key matching the column values. */ @@ -235,7 +244,7 @@ private Object findPrimaryKey(Object[] columnValues) { for (Object primaryKey : primaryKeyColumn) { boolean matching = true; for (int i = 0; matching && i < columnValues.length; i++) { - matching = ObjectUtils.equals(columnValues[i], get(primaryKey, i+1)); + matching = ObjectUtils.equals(columnValues[i], get(primaryKey, i + 1)); } if (matching) return primaryKey; @@ -246,8 +255,8 @@ private Object findPrimaryKey(Object[] columnValues) { /** * Add a column to the report and specify the default value that should be supplied if a given position in the table is never explicitly set. * - * @param columnName the name of the column - * @param defaultValue the default value for the column + * @param columnName the name of the column + * @param defaultValue the default value for the column */ public void addColumn(String columnName, Object defaultValue) { addColumn(columnName, defaultValue, null); @@ -256,12 +265,13 @@ public void addColumn(String columnName, Object defaultValue) { public void addColumn(String columnName, Object defaultValue, String format) { addColumn(columnName, defaultValue, true, format); } + /** * Add a column to the report, specify the default column value, and specify whether the column should be displayed in the final output (useful when intermediate columns are necessary for later calculations, but are not required to be in the output file. * - * @param columnName the name of the column - * @param defaultValue the default value of the column - * @param display if true - the column will be displayed; if false - the column will be hidden + * @param columnName the name of the column + * @param defaultValue the default value of the column + * @param display if true - the column will be displayed; if false - the column will be hidden */ public void addColumn(String columnName, Object defaultValue, boolean display) { addColumn(columnName, defaultValue, display, null); @@ -277,8 +287,8 @@ public void addColumn(String columnName, Object defaultValue, boolean display, S /** * Check if the requested element exists, and if not, create it. * - * @param primaryKey the primary key value - * @param columnName the name of the column + * @param primaryKey the primary key value + * @param columnName the name of the column */ private void verifyEntry(Object primaryKey, String columnName) { if (!columns.containsKey(columnName)) { @@ -299,9 +309,9 @@ public boolean containsKey(Object primaryKey) { /** * Set the value for a given position in the table * - * @param primaryKey the primary key value - * @param columnName the name of the column - * @param value the value to set + * @param primaryKey the primary key value + * @param columnName the name of the column + * @param value the value to set */ public void set(Object primaryKey, String columnName, Object value) { verifyEntry(primaryKey, columnName); @@ -312,13 +322,13 @@ public void set(Object primaryKey, String columnName, Object value) { /** * Get a value from the given position in the table * - * @param primaryKey the primary key value - * @param columnName the name of the column - * @return the value stored at the specified position in the table + * @param primaryKey the primary key value + * @param columnName the name of the column + * @return the value stored at the specified position in the table */ public Object get(Object primaryKey, String columnName) { verifyEntry(primaryKey, columnName); - + return columns.get(columnName).get(primaryKey); } @@ -327,7 +337,7 @@ public Object get(Object primaryKey, String columnName) { * * @param primaryKey the primary key value * @param columnIndex the index of the column - * @return the value stored at the specified position in the table + * @return the value stored at the specified position in the table */ private Object get(Object primaryKey, int columnIndex) { return columns.getByIndex(columnIndex).get(primaryKey); @@ -336,8 +346,8 @@ private Object get(Object primaryKey, int columnIndex) { /** * Increment an element in the table. This implementation is awful - a functor would probably be better. * - * @param primaryKey the primary key value - * @param columnName the name of the column + * @param primaryKey the primary key value + * @param columnName the name of the column */ public void increment(Object primaryKey, String columnName) { Object oldValue = get(primaryKey, columnName); @@ -365,8 +375,8 @@ public void increment(Object primaryKey, String columnName) { /** * Decrement an element in the table. This implementation is awful - a functor would probably be better. * - * @param primaryKey the primary key value - * @param columnName the name of the column + * @param primaryKey the primary key value + * @param columnName the name of the column */ public void decrement(Object primaryKey, String columnName) { Object oldValue = get(primaryKey, columnName); @@ -394,9 +404,9 @@ public void decrement(Object primaryKey, String columnName) { /** * Add the specified value to an element in the table * - * @param primaryKey the primary key value - * @param columnName the name of the column - * @param valueToAdd the value to add + * @param primaryKey the primary key value + * @param columnName the name of the column + * @param valueToAdd the value to add */ public void add(Object primaryKey, String columnName, Object valueToAdd) { Object oldValue = get(primaryKey, columnName); @@ -424,8 +434,8 @@ public void add(Object primaryKey, String columnName, Object valueToAdd) { /** * Subtract the specified value from an element in the table * - * @param primaryKey the primary key value - * @param columnName the name of the column + * @param primaryKey the primary key value + * @param columnName the name of the column * @param valueToSubtract the value to subtract */ public void subtract(Object primaryKey, String columnName, Object valueToSubtract) { @@ -454,9 +464,9 @@ public void subtract(Object primaryKey, String columnName, Object valueToSubtrac /** * Multiply the specified value to an element in the table * - * @param primaryKey the primary key value - * @param columnName the name of the column - * @param valueToMultiply the value to multiply by + * @param primaryKey the primary key value + * @param columnName the name of the column + * @param valueToMultiply the value to multiply by */ public void multiply(Object primaryKey, String columnName, Object valueToMultiply) { Object oldValue = get(primaryKey, columnName); @@ -484,9 +494,9 @@ public void multiply(Object primaryKey, String columnName, Object valueToMultipl /** * Divide the specified value from an element in the table * - * @param primaryKey the primary key value - * @param columnName the name of the column - * @param valueToDivide the value to divide by + * @param primaryKey the primary key value + * @param columnName the name of the column + * @param valueToDivide the value to divide by */ public void divide(Object primaryKey, String columnName, Object valueToDivide) { Object oldValue = get(primaryKey, columnName); @@ -514,9 +524,9 @@ public void divide(Object primaryKey, String columnName, Object valueToDivide) { /** * Add two columns to each other and set the results to a third column * - * @param columnToSet the column that should hold the results - * @param augend the column that shall be the augend - * @param addend the column that shall be the addend + * @param columnToSet the column that should hold the results + * @param augend the column that shall be the augend + * @param addend the column that shall be the addend */ public void addColumns(String columnToSet, String augend, String addend) { for (Object primaryKey : primaryKeyColumn) { @@ -532,8 +542,8 @@ public void addColumns(String columnToSet, String augend, String addend) { /** * Subtract one column from another and set the results to a third column * - * @param columnToSet the column that should hold the results - * @param minuend the column that shall be the minuend (the a in a - b) + * @param columnToSet the column that should hold the results + * @param minuend the column that shall be the minuend (the a in a - b) * @param subtrahend the column that shall be the subtrahend (the b in a - b) */ public void subtractColumns(String columnToSet, String minuend, String subtrahend) { @@ -551,8 +561,8 @@ public void subtractColumns(String columnToSet, String minuend, String subtrahen * Multiply two columns by each other and set the results to a third column * * @param columnToSet the column that should hold the results - * @param multiplier the column that shall be the multiplier - * @param multiplicand the column that shall be the multiplicand + * @param multiplier the column that shall be the multiplier + * @param multiplicand the column that shall be the multiplicand */ public void multiplyColumns(String columnToSet, String multiplier, String multiplicand) { for (Object primaryKey : primaryKeyColumn) { @@ -568,9 +578,9 @@ public void multiplyColumns(String columnToSet, String multiplier, String multip /** * Divide two columns by each other and set the results to a third column * - * @param columnToSet the column that should hold the results - * @param numeratorColumn the column that shall be the numerator - * @param denominatorColumn the column that shall be the denominator + * @param columnToSet the column that should hold the results + * @param numeratorColumn the column that shall be the numerator + * @param denominatorColumn the column that shall be the denominator */ public void divideColumns(String columnToSet, String numeratorColumn, String denominatorColumn) { for (Object primaryKey : primaryKeyColumn) { @@ -585,10 +595,11 @@ public void divideColumns(String columnToSet, String numeratorColumn, String den /** * Return the print width of the primary key column - * @return the width of the primary key column + * + * @return the width of the primary key column */ public int getPrimaryKeyColumnWidth() { - int maxWidth = primaryKeyName.length(); + int maxWidth = getPrimaryKeyName().length(); for (Object primaryKey : primaryKeyColumn) { int width = primaryKey.toString().length(); @@ -604,7 +615,7 @@ public int getPrimaryKeyColumnWidth() { /** * Write the table to the PrintStream, formatted nicely to be human-readable, AWK-able, and R-friendly. * - * @param out the PrintStream to which the table should be written + * @param out the PrintStream to which the table should be written */ public void write(PrintStream out) { // Get the column widths for everything @@ -620,13 +631,15 @@ public void write(PrintStream out) { // Emit the table header, taking into account the padding requirement if the primary key is a hidden column boolean needsPadding = false; if (primaryKeyDisplay) { - out.printf(primaryKeyFormat, primaryKeyName); + out.printf(primaryKeyFormat, getPrimaryKeyName()); needsPadding = true; } for (String columnName : columns.keySet()) { if (columns.get(columnName).isDisplayable()) { - if (needsPadding) { out.printf(" "); } + if (needsPadding) { + out.printf(" "); + } out.printf(columnFormats.get(columnName).getNameFormat(), columnName); needsPadding = true; @@ -645,7 +658,9 @@ public void write(PrintStream out) { for (String columnName : columns.keySet()) { if (columns.get(columnName).isDisplayable()) { - if (needsPadding) { out.printf(" "); } + if (needsPadding) { + out.printf(" "); + } String value = columns.get(columnName).getStringValue(primaryKey); out.printf(columnFormats.get(columnName).getValueFormat(), value); @@ -675,4 +690,49 @@ public String getTableDescription() { public GATKReportColumns getColumns() { return columns; } + + public void mergeRows(GATKReportTable input) { + /* + * This function is different from addRowsFrom because we will add the ability to sum,average, etc rows + * TODO: Add other combining algorithms + */ + + // Make sure the columns match AND the Primary Key + if (input.getColumns().keySet().equals(this.getColumns().keySet()) && + input.getPrimaryKeyName().equals(this.getPrimaryKeyName())) { + this.addRowsFrom(input); + } else + throw new ReviewedStingException("Failed to combine GATKReportTable, columns don't match!"); + } + + public void addRowsFrom(GATKReportTable input) { + // add column by column + + // For every column + for (String columnKey : input.getColumns().keySet()) { + GATKReportColumn current = this.getColumns().get(columnKey); + GATKReportColumn toAdd = input.getColumns().get(columnKey); + // We want to take the current column and add all the values from input + + // The column is a map of values + for (Object rowKey : toAdd.keySet()) { + // We add every value from toAdd to the current + if (!current.containsKey(rowKey)) { + this.set(rowKey, columnKey, toAdd.get(rowKey)); + System.out.printf("Putting row with PK: %s \n", rowKey); + } else { + + // TODO we should be able to handle combining data by adding, averaging, etc. + this.set(rowKey, columnKey, toAdd.get(rowKey)); + + System.out.printf("OVERWRITING Row with PK: %s \n", rowKey); + } + } + } + + } + + public String getPrimaryKeyName() { + return primaryKeyName; + } } diff --git a/public/java/test/org/broadinstitute/sting/gatk/report/GATKReportUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/report/GATKReportUnitTest.java index c9b81a9d35..77ed6972db 100644 --- a/public/java/test/org/broadinstitute/sting/gatk/report/GATKReportUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/report/GATKReportUnitTest.java @@ -49,23 +49,23 @@ public void testParse() throws Exception { @DataProvider(name = "rightAlignValues") public Object[][] getRightAlignValues() { - return new Object[][] { - new Object[] {null, true}, - new Object[] {"null", true}, - new Object[] {"NA", true}, - new Object[] {"0", true}, - new Object[] {"0.0", true}, - new Object[] {"-0", true}, - new Object[] {"-0.0", true}, - new Object[] {String.valueOf(Long.MAX_VALUE), true}, - new Object[] {String.valueOf(Long.MIN_VALUE), true}, - new Object[] {String.valueOf(Float.MIN_NORMAL), true}, - new Object[] {String.valueOf(Double.MAX_VALUE), true}, - new Object[] {String.valueOf(Double.MIN_VALUE), true}, - new Object[] {String.valueOf(Double.POSITIVE_INFINITY), true}, - new Object[] {String.valueOf(Double.NEGATIVE_INFINITY), true}, - new Object[] {String.valueOf(Double.NaN), true}, - new Object[] {"hello", false} + return new Object[][]{ + new Object[]{null, true}, + new Object[]{"null", true}, + new Object[]{"NA", true}, + new Object[]{"0", true}, + new Object[]{"0.0", true}, + new Object[]{"-0", true}, + new Object[]{"-0.0", true}, + new Object[]{String.valueOf(Long.MAX_VALUE), true}, + new Object[]{String.valueOf(Long.MIN_VALUE), true}, + new Object[]{String.valueOf(Float.MIN_NORMAL), true}, + new Object[]{String.valueOf(Double.MAX_VALUE), true}, + new Object[]{String.valueOf(Double.MIN_VALUE), true}, + new Object[]{String.valueOf(Double.POSITIVE_INFINITY), true}, + new Object[]{String.valueOf(Double.NEGATIVE_INFINITY), true}, + new Object[]{String.valueOf(Double.NaN), true}, + new Object[]{"hello", false} }; } @@ -73,4 +73,96 @@ public Object[][] getRightAlignValues() { public void testIsRightAlign(String value, boolean expected) { Assert.assertEquals(GATKReportColumn.isRightAlign(value), expected, "right align of '" + value + "'"); } -} + + @Test + public void testGATKReportGatherer() { + + /* + GATKReportTable actual1 = new GATKReportTable("TableName", "Description"); + actual1.addPrimaryKey("key"); + actual1.addColumn("colA", 0); + actual1.addColumn("colB", 0); + actual1.set("row1", "colA", 1); + actual1.set("row1", "colB", 2); + + GATKReportTable actual2 = new GATKReportTable("TableName", "Description"); + actual2.addPrimaryKey("key"); + actual2.addColumn("colA", 0); + actual2.addColumn("colB", 0); + actual2.set("row2", "colA", 3); + actual2.set("row2", "colB", 4); + + GATKReportTable actual3 = new GATKReportTable("TableName", "Description"); + actual3.addPrimaryKey("key"); + actual3.addColumn("colA", 0); + actual3.addColumn("colB", 0); + actual3.set("row3", "colA", 5); + actual3.set("row3", "colB", 6); + + actual1.mergeRows(actual2); + actual1.mergeRows(actual3); + actual1.write(System.out); + */ + + GATKReportTable expected = new GATKReportTable("TableName", "Description"); + expected.addPrimaryKey("key"); + expected.addColumn("colA", 0); + expected.addColumn("colB", 0); + expected.set("row1", "colA", 1); + expected.set("row1", "colB", 2); + expected.set("row2", "colA", 3); + expected.set("row2", "colB", 4); + expected.set("row3", "colA", 5); + expected.set("row3", "colB", 6); + expected.write(System.out); + + GATKReport report1, report2, report3; + report1 = new GATKReport(); + report1.addTable("TableName", "Description"); + report1.getTable("TableName").addPrimaryKey("key"); + report1.getTable("TableName").addColumn("colA", 0); + report1.getTable("TableName").addColumn("colB", 0); + report1.getTable("TableName").set("row1", "colA", 1); + report1.getTable("TableName").set("row1", "colB", 2); + + report2 = new GATKReport(); + report2.addTable("TableName", "Description"); + report2.getTable("TableName").addPrimaryKey("key"); + report2.getTable("TableName").addColumn("colA", 0); + report2.getTable("TableName").addColumn("colB", 0); + report2.getTable("TableName").set("row2", "colA", 3); + report2.getTable("TableName").set("row2", "colB", 4); + + report3 = new GATKReport(); + report3.addTable("TableName", "Description"); + report3.getTable("TableName").addPrimaryKey("key"); + report3.getTable("TableName").addColumn("colA", 0); + report3.getTable("TableName").addColumn("colB", 0); + report3.getTable("TableName").set("row3", "colA", 5); + report3.getTable("TableName").set("row3", "colB", 6); + + report1.combineWith(report2); + report1.combineWith(report3); + + report1.print(System.out); + /* + File a = new File("/home/roger/tbls/a.tbl"); + File b = new File("/home/roger/tbls/b.tbl"); + File c = new File("/home/roger/tbls/c.tbl"); + File out = new File("/home/roger/tbls/out.tbl"); + + + List FileList = new ArrayList(); + FileList.add(a); + FileList.add(b); + FileList.add(c); + + GATKReportGatherer gatherer = new GATKReportGatherer(); + gatherer.gather(FileList, out); + System.out.print(out); + */ + + //Assert.assertEquals(1,1); + + } +} \ No newline at end of file From 337819e79176a3fd2cd41283251c926ac3046ac9 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Tue, 7 Feb 2012 19:22:32 -0500 Subject: [PATCH 227/356] disabling the test while we fix it --- .../broadinstitute/sting/gatk/report/GATKReportUnitTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/java/test/org/broadinstitute/sting/gatk/report/GATKReportUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/report/GATKReportUnitTest.java index 77ed6972db..b9a89fcfe7 100644 --- a/public/java/test/org/broadinstitute/sting/gatk/report/GATKReportUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/report/GATKReportUnitTest.java @@ -30,7 +30,7 @@ import org.testng.annotations.Test; public class GATKReportUnitTest extends BaseTest { - @Test + @Test(enabled = false) public void testParse() throws Exception { String reportPath = validationDataLocation + "exampleGATKReport.eval"; GATKReport report = new GATKReport(reportPath); From f30731f19b7f64debe6622047e8cf372eef5d230 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Tue, 7 Feb 2012 21:38:39 -0500 Subject: [PATCH 228/356] these were not supposed to be committed. Pulling it out From cda1e1b2079bf0d99c77c974ac0d3cdf883e2634 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Wed, 8 Feb 2012 02:24:54 -0500 Subject: [PATCH 229/356] Minor manual merge update for List class to Seq interface usage. --- .../sting/queue/extensions/gatk/GATKIntervalsUnitTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/scala/test/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervalsUnitTest.scala b/public/scala/test/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervalsUnitTest.scala index 115b2021d6..b233505575 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervalsUnitTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervalsUnitTest.scala @@ -89,6 +89,6 @@ class GATKIntervalsUnitTest { } private def testSortAndMergeIntervals(actual: Seq[String], expected: Seq[String]) { - Assert.assertEquals(new GATKIntervals(hg18Reference, actual.toList).locs.toSeq, expected.map(hg18GenomeLocParser.parseGenomeLoc(_))) + Assert.assertEquals(new GATKIntervals(hg18Reference, actual).locs.toSeq, expected.map(hg18GenomeLocParser.parseGenomeLoc(_))) } } From 5b58fe741ad532156283822ab3e4dcccfdf1738c Mon Sep 17 00:00:00 2001 From: Matt Hanna Date: Fri, 3 Feb 2012 16:43:00 -0500 Subject: [PATCH 230/356] Retiring Picard customizations for async I/O and cleaning up parts of the code to use common Picard utilities I recently discovered. Also embedded bug fix for issues reading sparse shards and did some cleanup based on comments during BAM reading code transition meetings. --- .../src/net/sf/samtools/BAMFileReader.java | 762 ------------------ .../java/src/net/sf/samtools/GATKChunk.java | 4 + .../net/sf/samtools/PicardNamespaceUtils.java | 39 + .../net/sf/samtools/util/BAMInputStream.java | 72 -- .../util/BlockCompressedInputStream.java | 483 ----------- ...ReaderPosition.java => BAMAccessPlan.java} | 62 +- .../reads/BGZFBlockLoadingDispatcher.java | 8 +- .../datasources/reads/BlockInputStream.java | 198 ++--- .../gatk/datasources/reads/BlockLoader.java | 22 +- .../gatk/datasources/reads/ReadShard.java | 2 +- .../gatk/datasources/reads/SAMDataSource.java | 61 +- 11 files changed, 236 insertions(+), 1477 deletions(-) delete mode 100644 public/java/src/net/sf/samtools/BAMFileReader.java create mode 100644 public/java/src/net/sf/samtools/PicardNamespaceUtils.java delete mode 100644 public/java/src/net/sf/samtools/util/BAMInputStream.java delete mode 100755 public/java/src/net/sf/samtools/util/BlockCompressedInputStream.java rename public/java/src/org/broadinstitute/sting/gatk/datasources/reads/{SAMReaderPosition.java => BAMAccessPlan.java} (58%) diff --git a/public/java/src/net/sf/samtools/BAMFileReader.java b/public/java/src/net/sf/samtools/BAMFileReader.java deleted file mode 100644 index 5005b6265f..0000000000 --- a/public/java/src/net/sf/samtools/BAMFileReader.java +++ /dev/null @@ -1,762 +0,0 @@ -/* - * Copyright (c) 2011, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ -package net.sf.samtools; - - -import net.sf.samtools.util.*; -import net.sf.samtools.SAMFileReader.ValidationStringency; - -import java.io.*; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.NoSuchElementException; - -/** - * Internal class for reading and querying BAM files. - */ -class BAMFileReader extends SAMFileReader.ReaderImplementation { - // True if reading from a File rather than an InputStream - private boolean mIsSeekable = false; - - // For converting bytes into other primitive types - private BinaryCodec mStream = null; - - // Underlying compressed data stream. - private final BAMInputStream mInputStream; - private SAMFileHeader mFileHeader = null; - - // Populated if the file is seekable and an index exists - private File mIndexFile; - private BAMIndex mIndex = null; - private long mFirstRecordPointer = 0; - private CloseableIterator mCurrentIterator = null; - - // If true, all SAMRecords are fully decoded as they are read. - private final boolean eagerDecode; - - // For error-checking. - private ValidationStringency mValidationStringency; - - // For creating BAMRecords - private SAMRecordFactory samRecordFactory; - - /** - * Use the caching index reader implementation rather than the disk-hit-per-file model. - */ - private boolean mEnableIndexCaching = false; - - /** - * Use the traditional memory-mapped implementation for BAM file indexes rather than regular I/O. - */ - private boolean mEnableIndexMemoryMapping = true; - - /** - * Add information about the origin (reader and position) to SAM records. - */ - private SAMFileReader mFileReader = null; - - /** - * Prepare to read BAM from a stream (not seekable) - * @param stream source of bytes. - * @param eagerDecode if true, decode all BAM fields as reading rather than lazily. - * @param validationStringency Controls how to handle invalidate reads or header lines. - */ - BAMFileReader(final InputStream stream, - final File indexFile, - final boolean eagerDecode, - final ValidationStringency validationStringency, - final SAMRecordFactory factory) - throws IOException { - mIndexFile = indexFile; - mIsSeekable = false; - mInputStream = stream instanceof BAMInputStream ? (BAMInputStream)stream : new BlockCompressedInputStream(stream); - mStream = new BinaryCodec(new DataInputStream((InputStream)mInputStream)); - this.eagerDecode = eagerDecode; - this.mValidationStringency = validationStringency; - this.samRecordFactory = factory; - readHeader(null); - } - - /** - * Prepare to read BAM from a file (seekable) - * @param file source of bytes. - * @param eagerDecode if true, decode all BAM fields as reading rather than lazily. - * @param validationStringency Controls how to handle invalidate reads or header lines. - */ - BAMFileReader(final File file, - final File indexFile, - final boolean eagerDecode, - final ValidationStringency validationStringency, - final SAMRecordFactory factory) - throws IOException { - this(new BlockCompressedInputStream(file), indexFile!=null ? indexFile : findIndexFile(file), eagerDecode, file.getAbsolutePath(), validationStringency, factory); - if (mIndexFile != null && mIndexFile.lastModified() < file.lastModified()) { - System.err.println("WARNING: BAM index file " + mIndexFile.getAbsolutePath() + - " is older than BAM " + file.getAbsolutePath()); - } - } - - BAMFileReader(final SeekableStream strm, - final File indexFile, - final boolean eagerDecode, - final ValidationStringency validationStringency, - final SAMRecordFactory factory) - throws IOException { - this(strm instanceof BAMInputStream ? (BAMInputStream)strm : new BlockCompressedInputStream(strm), - indexFile, - eagerDecode, - strm.getSource(), - validationStringency, - factory); - } - - private BAMFileReader(final BAMInputStream inputStream, - final File indexFile, - final boolean eagerDecode, - final String source, - final ValidationStringency validationStringency, - final SAMRecordFactory factory) - throws IOException { - mIndexFile = indexFile; - mIsSeekable = true; - mInputStream = inputStream; - mStream = new BinaryCodec(new DataInputStream((InputStream)inputStream)); - this.eagerDecode = eagerDecode; - this.mValidationStringency = validationStringency; - this.samRecordFactory = factory; - readHeader(source); - mFirstRecordPointer = inputStream.getFilePointer(); - } - - /** - * If true, writes the source of every read into the source SAMRecords. - * @param enabled true to write source information into each SAMRecord. - */ - void enableFileSource(final SAMFileReader reader, final boolean enabled) { - this.mFileReader = enabled ? reader : null; - } - - /** - * If true, uses the caching version of the index reader. - * @param enabled true to write source information into each SAMRecord. - */ - public void enableIndexCaching(final boolean enabled) { - if(mIndex != null) - throw new SAMException("Unable to turn on index caching; index file has already been loaded."); - this.mEnableIndexCaching = enabled; - } - - /** - * If false, disable the use of memory mapping for accessing index files (default behavior is to use memory mapping). - * This is slower but more scalable when accessing large numbers of BAM files sequentially. - * @param enabled True to use memory mapping, false to use regular I/O. - */ - public void enableIndexMemoryMapping(final boolean enabled) { - if (mIndex != null) { - throw new SAMException("Unable to change index memory mapping; index file has already been loaded."); - } - this.mEnableIndexMemoryMapping = enabled; - } - - @Override void enableCrcChecking(final boolean enabled) { - this.mInputStream.setCheckCrcs(enabled); - } - - @Override void setSAMRecordFactory(final SAMRecordFactory factory) { this.samRecordFactory = factory; } - - /** - * @return true if ths is a BAM file, and has an index - */ - public boolean hasIndex() { - return (mIndexFile != null); - } - - /** - * Retrieves the index for the given file type. Ensure that the index is of the specified type. - * @return An index of the given type. - */ - public BAMIndex getIndex() { - if(mIndexFile == null) - throw new SAMException("No index is available for this BAM file."); - if(mIndex == null) - mIndex = mEnableIndexCaching ? new CachingBAMFileIndex(mIndexFile, getFileHeader().getSequenceDictionary(), mEnableIndexMemoryMapping) - : new DiskBasedBAMFileIndex(mIndexFile, getFileHeader().getSequenceDictionary(), mEnableIndexMemoryMapping); - return mIndex; - } - - void close() { - if (mStream != null) { - mStream.close(); - } - if (mIndex != null) { - mIndex.close(); - } - mStream = null; - mFileHeader = null; - mIndex = null; - } - - SAMFileHeader getFileHeader() { - return mFileHeader; - } - - /** - * Set error-checking level for subsequent SAMRecord reads. - */ - void setValidationStringency(final SAMFileReader.ValidationStringency validationStringency) { - this.mValidationStringency = validationStringency; - } - - SAMFileReader.ValidationStringency getValidationStringency() { - return this.mValidationStringency; - } - - /** - * Prepare to iterate through the SAMRecords in file order. - * Only a single iterator on a BAM file can be extant at a time. If getIterator() or a query method has been called once, - * that iterator must be closed before getIterator() can be called again. - * A somewhat peculiar aspect of this method is that if the file is not seekable, a second call to - * getIterator() begins its iteration where the last one left off. That is the best that can be - * done in that situation. - */ - CloseableIterator getIterator() { - if (mStream == null) { - throw new IllegalStateException("File reader is closed"); - } - if (mCurrentIterator != null) { - throw new IllegalStateException("Iteration in progress"); - } - if (mIsSeekable) { - try { - mInputStream.seek(mFirstRecordPointer); - } catch (IOException exc) { - throw new RuntimeException(exc.getMessage(), exc); - } - } - mCurrentIterator = new BAMFileIterator(); - return mCurrentIterator; - } - - @Override - CloseableIterator getIterator(final SAMFileSpan chunks) { - if (mStream == null) { - throw new IllegalStateException("File reader is closed"); - } - if (mCurrentIterator != null) { - throw new IllegalStateException("Iteration in progress"); - } - if (!(chunks instanceof BAMFileSpan)) { - throw new IllegalStateException("BAMFileReader cannot handle this type of file span."); - } - - // Create an iterator over the given chunk boundaries. - mCurrentIterator = new BAMFileIndexIterator(((BAMFileSpan)chunks).toCoordinateArray()); - return mCurrentIterator; - } - - /** - * Gets an unbounded pointer to the first record in the BAM file. Because the reader doesn't necessarily know - * when the file ends, the rightmost bound of the file pointer will not end exactly where the file ends. However, - * the rightmost bound is guaranteed to be after the last read in the file. - * @return An unbounded pointer to the first record in the BAM file. - */ - @Override - SAMFileSpan getFilePointerSpanningReads() { - return new BAMFileSpan(new Chunk(mFirstRecordPointer,Long.MAX_VALUE)); - } - - /** - * Prepare to iterate through the SAMRecords that match the given interval. - * Only a single iterator on a BAMFile can be extant at a time. The previous one must be closed - * before calling any of the methods that return an iterator. - * - * Note that an unmapped SAMRecord may still have a reference name and an alignment start for sorting - * purposes (typically this is the coordinate of its mate), and will be found by this method if the coordinate - * matches the specified interval. - * - * Note that this method is not necessarily efficient in terms of disk I/O. The index does not have perfect - * resolution, so some SAMRecords may be read and then discarded because they do not match the specified interval. - * - * @param sequence Reference sequence sought. - * @param start Desired SAMRecords must overlap or be contained in the interval specified by start and end. - * A value of zero implies the start of the reference sequence. - * @param end A value of zero implies the end of the reference sequence. - * @param contained If true, the alignments for the SAMRecords must be completely contained in the interval - * specified by start and end. If false, the SAMRecords need only overlap the interval. - * @return Iterator for the matching SAMRecords - */ - CloseableIterator query(final String sequence, final int start, final int end, final boolean contained) { - if (mStream == null) { - throw new IllegalStateException("File reader is closed"); - } - if (mCurrentIterator != null) { - throw new IllegalStateException("Iteration in progress"); - } - if (!mIsSeekable) { - throw new UnsupportedOperationException("Cannot query stream-based BAM file"); - } - mCurrentIterator = createIndexIterator(sequence, start, end, contained? QueryType.CONTAINED: QueryType.OVERLAPPING); - return mCurrentIterator; - } - - /** - * Prepare to iterate through the SAMRecords with the given alignment start. - * Only a single iterator on a BAMFile can be extant at a time. The previous one must be closed - * before calling any of the methods that return an iterator. - * - * Note that an unmapped SAMRecord may still have a reference name and an alignment start for sorting - * purposes (typically this is the coordinate of its mate), and will be found by this method if the coordinate - * matches the specified interval. - * - * Note that this method is not necessarily efficient in terms of disk I/O. The index does not have perfect - * resolution, so some SAMRecords may be read and then discarded because they do not match the specified interval. - * - * @param sequence Reference sequence sought. - * @param start Alignment start sought. - * @return Iterator for the matching SAMRecords. - */ - CloseableIterator queryAlignmentStart(final String sequence, final int start) { - if (mStream == null) { - throw new IllegalStateException("File reader is closed"); - } - if (mCurrentIterator != null) { - throw new IllegalStateException("Iteration in progress"); - } - if (!mIsSeekable) { - throw new UnsupportedOperationException("Cannot query stream-based BAM file"); - } - mCurrentIterator = createIndexIterator(sequence, start, -1, QueryType.STARTING_AT); - return mCurrentIterator; - } - - public CloseableIterator queryUnmapped() { - if (mStream == null) { - throw new IllegalStateException("File reader is closed"); - } - if (mCurrentIterator != null) { - throw new IllegalStateException("Iteration in progress"); - } - if (!mIsSeekable) { - throw new UnsupportedOperationException("Cannot query stream-based BAM file"); - } - try { - final long startOfLastLinearBin = getIndex().getStartOfLastLinearBin(); - if (startOfLastLinearBin != -1) { - mInputStream.seek(startOfLastLinearBin); - } else { - // No mapped reads in file, just start at the first read in file. - mInputStream.seek(mFirstRecordPointer); - } - mCurrentIterator = new BAMFileIndexUnmappedIterator(); - return mCurrentIterator; - } catch (IOException e) { - throw new RuntimeException("IOException seeking to unmapped reads", e); - } - } - - /** - * Reads the header from the file or stream - * @param source Note that this is used only for reporting errors. - */ - private void readHeader(final String source) - throws IOException { - - final byte[] buffer = new byte[4]; - mStream.readBytes(buffer); - if (!Arrays.equals(buffer, BAMFileConstants.BAM_MAGIC)) { - throw new IOException("Invalid BAM file header"); - } - - final int headerTextLength = mStream.readInt(); - final String textHeader = mStream.readString(headerTextLength); - final SAMTextHeaderCodec headerCodec = new SAMTextHeaderCodec(); - headerCodec.setValidationStringency(mValidationStringency); - mFileHeader = headerCodec.decode(new StringLineReader(textHeader), - source); - - final int sequenceCount = mStream.readInt(); - if (mFileHeader.getSequenceDictionary().size() > 0) { - // It is allowed to have binary sequences but no text sequences, so only validate if both are present - if (sequenceCount != mFileHeader.getSequenceDictionary().size()) { - throw new SAMFormatException("Number of sequences in text header (" + - mFileHeader.getSequenceDictionary().size() + - ") != number of sequences in binary header (" + sequenceCount + ") for file " + source); - } - for (int i = 0; i < sequenceCount; i++) { - final SAMSequenceRecord binarySequenceRecord = readSequenceRecord(source); - final SAMSequenceRecord sequenceRecord = mFileHeader.getSequence(i); - if (!sequenceRecord.getSequenceName().equals(binarySequenceRecord.getSequenceName())) { - throw new SAMFormatException("For sequence " + i + ", text and binary have different names in file " + - source); - } - if (sequenceRecord.getSequenceLength() != binarySequenceRecord.getSequenceLength()) { - throw new SAMFormatException("For sequence " + i + ", text and binary have different lengths in file " + - source); - } - } - } else { - // If only binary sequences are present, copy them into mFileHeader - final List sequences = new ArrayList(sequenceCount); - for (int i = 0; i < sequenceCount; i++) { - sequences.add(readSequenceRecord(source)); - } - mFileHeader.setSequenceDictionary(new SAMSequenceDictionary(sequences)); - } - } - - /** - * Reads a single binary sequence record from the file or stream - * @param source Note that this is used only for reporting errors. - */ - private SAMSequenceRecord readSequenceRecord(final String source) { - final int nameLength = mStream.readInt(); - if (nameLength <= 1) { - throw new SAMFormatException("Invalid BAM file header: missing sequence name in file " + source); - } - final String sequenceName = mStream.readString(nameLength - 1); - // Skip the null terminator - mStream.readByte(); - final int sequenceLength = mStream.readInt(); - return new SAMSequenceRecord(SAMSequenceRecord.truncateSequenceName(sequenceName), sequenceLength); - } - - /** - * Iterator for non-indexed sequential iteration through all SAMRecords in file. - * Starting point of iteration is wherever current file position is when the iterator is constructed. - */ - private class BAMFileIterator implements CloseableIterator { - private SAMRecord mNextRecord = null; - private final BAMRecordCodec bamRecordCodec; - private long samRecordIndex = 0; // Records at what position (counted in records) we are at in the file - - BAMFileIterator() { - this(true); - } - - /** - * @param advance Trick to enable subclass to do more setup before advancing - */ - BAMFileIterator(final boolean advance) { - this.bamRecordCodec = new BAMRecordCodec(getFileHeader(), samRecordFactory); - this.bamRecordCodec.setInputStream(BAMFileReader.this.mStream.getInputStream()); - - if (advance) { - advance(); - } - } - - public void close() { - if (mCurrentIterator != null && this != mCurrentIterator) { - throw new IllegalStateException("Attempt to close non-current iterator"); - } - mCurrentIterator = null; - } - - public boolean hasNext() { - return (mNextRecord != null); - } - - public SAMRecord next() { - final SAMRecord result = mNextRecord; - advance(); - return result; - } - - public void remove() { - throw new UnsupportedOperationException("Not supported: remove"); - } - - void advance() { - try { - mNextRecord = getNextRecord(); - - if (mNextRecord != null) { - ++this.samRecordIndex; - // Because some decoding is done lazily, the record needs to remember the validation stringency. - mNextRecord.setValidationStringency(mValidationStringency); - - if (mValidationStringency != ValidationStringency.SILENT) { - final List validationErrors = mNextRecord.isValid(); - SAMUtils.processValidationErrors(validationErrors, - this.samRecordIndex, BAMFileReader.this.getValidationStringency()); - } - } - if (eagerDecode && mNextRecord != null) { - mNextRecord.eagerDecode(); - } - } catch (IOException exc) { - throw new RuntimeException(exc.getMessage(), exc); - } - } - - /** - * Read the next record from the input stream. - */ - SAMRecord getNextRecord() throws IOException { - final long startCoordinate = mInputStream.getFilePointer(); - final SAMRecord next = bamRecordCodec.decode(); - final long stopCoordinate = mInputStream.getFilePointer(); - - if(mFileReader != null && next != null) - next.setFileSource(new SAMFileSource(mFileReader,new BAMFileSpan(new Chunk(startCoordinate,stopCoordinate)))); - - return next; - } - - /** - * @return The record that will be return by the next call to next() - */ - protected SAMRecord peek() { - return mNextRecord; - } - } - - /** - * Prepare to iterate through SAMRecords matching the target interval. - * @param sequence Desired reference sequence. - * @param start 1-based start of target interval, inclusive. - * @param end 1-based end of target interval, inclusive. - * @param queryType contained, overlapping, or starting-at query. - */ - private CloseableIterator createIndexIterator(final String sequence, - final int start, - final int end, - final QueryType queryType) { - long[] filePointers = null; - - // Hit the index to determine the chunk boundaries for the required data. - final SAMFileHeader fileHeader = getFileHeader(); - final int referenceIndex = fileHeader.getSequenceIndex(sequence); - if (referenceIndex != -1) { - final BAMIndex fileIndex = getIndex(); - final BAMFileSpan fileSpan = fileIndex.getSpanOverlapping(referenceIndex, start, end); - filePointers = fileSpan != null ? fileSpan.toCoordinateArray() : null; - } - - // Create an iterator over the above chunk boundaries. - final BAMFileIndexIterator iterator = new BAMFileIndexIterator(filePointers); - - // Add some preprocessing filters for edge-case reads that don't fit into this - // query type. - return new BAMQueryFilteringIterator(iterator,sequence,start,end,queryType); - } - - enum QueryType {CONTAINED, OVERLAPPING, STARTING_AT} - - /** - * Look for BAM index file according to standard naming convention. - * - * @param dataFile BAM file name. - * @return Index file name, or null if not found. - */ - private static File findIndexFile(final File dataFile) { - // If input is foo.bam, look for foo.bai - final String bamExtension = ".bam"; - File indexFile; - final String fileName = dataFile.getName(); - if (fileName.endsWith(bamExtension)) { - final String bai = fileName.substring(0, fileName.length() - bamExtension.length()) + BAMIndex.BAMIndexSuffix; - indexFile = new File(dataFile.getParent(), bai); - if (indexFile.exists()) { - return indexFile; - } - } - - // If foo.bai doesn't exist look for foo.bam.bai - indexFile = new File(dataFile.getParent(), dataFile.getName() + ".bai"); - if (indexFile.exists()) { - return indexFile; - } else { - return null; - } - } - - private class BAMFileIndexIterator extends BAMFileIterator { - - private long[] mFilePointers = null; - private int mFilePointerIndex = 0; - private long mFilePointerLimit = -1; - - /** - * Prepare to iterate through SAMRecords stored in the specified compressed blocks at the given offset. - * @param filePointers the block / offset combination, stored in chunk format. - */ - BAMFileIndexIterator(final long[] filePointers) { - super(false); // delay advance() until after construction - mFilePointers = filePointers; - advance(); - } - - SAMRecord getNextRecord() - throws IOException { - // Advance to next file block if necessary - while (mInputStream.getFilePointer() >= mFilePointerLimit) { - if (mFilePointers == null || - mFilePointerIndex >= mFilePointers.length) { - return null; - } - final long startOffset = mFilePointers[mFilePointerIndex++]; - final long endOffset = mFilePointers[mFilePointerIndex++]; - mInputStream.seek(startOffset); - mFilePointerLimit = endOffset; - } - // Pull next record from stream - return super.getNextRecord(); - } - } - - /** - * A decorating iterator that filters out records that are outside the bounds of the - * given query parameters. - */ - private class BAMQueryFilteringIterator implements CloseableIterator { - /** - * The wrapped iterator. - */ - private final CloseableIterator wrappedIterator; - - /** - * The next record to be returned. Will be null if no such record exists. - */ - private SAMRecord mNextRecord; - - private final int mReferenceIndex; - private final int mRegionStart; - private final int mRegionEnd; - private final QueryType mQueryType; - - public BAMQueryFilteringIterator(final CloseableIterator iterator,final String sequence, final int start, final int end, final QueryType queryType) { - this.wrappedIterator = iterator; - final SAMFileHeader fileHeader = getFileHeader(); - mReferenceIndex = fileHeader.getSequenceIndex(sequence); - mRegionStart = start; - if (queryType == QueryType.STARTING_AT) { - mRegionEnd = mRegionStart; - } else { - mRegionEnd = (end <= 0) ? Integer.MAX_VALUE : end; - } - mQueryType = queryType; - mNextRecord = advance(); - } - - /** - * Returns true if a next element exists; false otherwise. - */ - public boolean hasNext() { - return mNextRecord != null; - } - - /** - * Gets the next record from the given iterator. - * @return The next SAM record in the iterator. - */ - public SAMRecord next() { - if(!hasNext()) - throw new NoSuchElementException("BAMQueryFilteringIterator: no next element available"); - final SAMRecord currentRead = mNextRecord; - mNextRecord = advance(); - return currentRead; - } - - /** - * Closes down the existing iterator. - */ - public void close() { - if (this != mCurrentIterator) { - throw new IllegalStateException("Attempt to close non-current iterator"); - } - mCurrentIterator = null; - } - - /** - * @throws UnsupportedOperationException always. - */ - public void remove() { - throw new UnsupportedOperationException("Not supported: remove"); - } - - SAMRecord advance() { - while (true) { - // Pull next record from stream - if(!wrappedIterator.hasNext()) - return null; - - final SAMRecord record = wrappedIterator.next(); - // If beyond the end of this reference sequence, end iteration - final int referenceIndex = record.getReferenceIndex(); - if (referenceIndex != mReferenceIndex) { - if (referenceIndex < 0 || - referenceIndex > mReferenceIndex) { - return null; - } - // If before this reference sequence, continue - continue; - } - if (mRegionStart == 0 && mRegionEnd == Integer.MAX_VALUE) { - // Quick exit to avoid expensive alignment end calculation - return record; - } - final int alignmentStart = record.getAlignmentStart(); - // If read is unmapped but has a coordinate, return it if the coordinate is within - // the query region, regardless of whether the mapped mate will be returned. - final int alignmentEnd; - if (mQueryType == QueryType.STARTING_AT) { - alignmentEnd = -1; - } else { - alignmentEnd = (record.getAlignmentEnd() != SAMRecord.NO_ALIGNMENT_START? - record.getAlignmentEnd(): alignmentStart); - } - - if (alignmentStart > mRegionEnd) { - // If scanned beyond target region, end iteration - return null; - } - // Filter for overlap with region - if (mQueryType == QueryType.CONTAINED) { - if (alignmentStart >= mRegionStart && alignmentEnd <= mRegionEnd) { - return record; - } - } else if (mQueryType == QueryType.OVERLAPPING) { - if (alignmentEnd >= mRegionStart && alignmentStart <= mRegionEnd) { - return record; - } - } else { - if (alignmentStart == mRegionStart) { - return record; - } - } - } - } - } - - private class BAMFileIndexUnmappedIterator extends BAMFileIterator { - private BAMFileIndexUnmappedIterator() { - while (this.hasNext() && peek().getReferenceIndex() != SAMRecord.NO_ALIGNMENT_REFERENCE_INDEX) { - advance(); - } - } - } - -} diff --git a/public/java/src/net/sf/samtools/GATKChunk.java b/public/java/src/net/sf/samtools/GATKChunk.java index 5d349e72e6..e9335a86d2 100644 --- a/public/java/src/net/sf/samtools/GATKChunk.java +++ b/public/java/src/net/sf/samtools/GATKChunk.java @@ -40,6 +40,10 @@ public GATKChunk(final long start, final long stop) { super(start,stop); } + public GATKChunk(final long blockStart, final int blockOffsetStart, final long blockEnd, final int blockOffsetEnd) { + super(blockStart << 16 | blockOffsetStart,blockEnd << 16 | blockOffsetEnd); + } + public GATKChunk(final Chunk chunk) { super(chunk.getChunkStart(),chunk.getChunkEnd()); } diff --git a/public/java/src/net/sf/samtools/PicardNamespaceUtils.java b/public/java/src/net/sf/samtools/PicardNamespaceUtils.java new file mode 100644 index 0000000000..b645f8fdce --- /dev/null +++ b/public/java/src/net/sf/samtools/PicardNamespaceUtils.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package net.sf.samtools; + +/** + * Utils that insist on being in the same package as Picard. + */ +public class PicardNamespaceUtils { + /** + * Private constructor only. Do not instantiate. + */ + private PicardNamespaceUtils() {} + + public static void setFileSource(final SAMRecord read, final SAMFileSource fileSource) { + read.setFileSource(fileSource); + } +} diff --git a/public/java/src/net/sf/samtools/util/BAMInputStream.java b/public/java/src/net/sf/samtools/util/BAMInputStream.java deleted file mode 100644 index d825c23d51..0000000000 --- a/public/java/src/net/sf/samtools/util/BAMInputStream.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2011, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package net.sf.samtools.util; - -import java.io.IOException; - -/** - * An input stream formulated for use reading BAM files. Supports - */ -public interface BAMInputStream { - /** - * Seek to the given position in the file. Note that pos is a special virtual file pointer, - * not an actual byte offset. - * - * @param pos virtual file pointer - */ - public void seek(final long pos) throws IOException; - - /** - * @return virtual file pointer that can be passed to seek() to return to the current position. This is - * not an actual byte offset, so arithmetic on file pointers cannot be done to determine the distance between - * the two. - */ - public long getFilePointer(); - - /** - * Determines whether or not the inflater will re-calculated the CRC on the decompressed data - * and check it against the value stored in the GZIP header. CRC checking is an expensive - * operation and should be used accordingly. - */ - public void setCheckCrcs(final boolean check); - - public int read() throws java.io.IOException; - - public int read(byte[] bytes) throws java.io.IOException; - - public int read(byte[] bytes, int i, int i1) throws java.io.IOException; - - public long skip(long l) throws java.io.IOException; - - public int available() throws java.io.IOException; - - public void close() throws java.io.IOException; - - public void mark(int i); - - public void reset() throws java.io.IOException; - - public boolean markSupported(); -} diff --git a/public/java/src/net/sf/samtools/util/BlockCompressedInputStream.java b/public/java/src/net/sf/samtools/util/BlockCompressedInputStream.java deleted file mode 100755 index fae2fc89b4..0000000000 --- a/public/java/src/net/sf/samtools/util/BlockCompressedInputStream.java +++ /dev/null @@ -1,483 +0,0 @@ -/* - * The MIT License - * - * Copyright (c) 2009 The Broad Institute - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -package net.sf.samtools.util; - - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.RandomAccessFile; -import java.net.URL; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.Arrays; - -import net.sf.samtools.FileTruncatedException; - -/* - * Utility class for reading BGZF block compressed files. The caller can treat this file like any other InputStream. - * It probably is not necessary to wrap this stream in a buffering stream, because there is internal buffering. - * The advantage of BGZF over conventional GZip format is that BGZF allows for seeking without having to read the - * entire file up to the location being sought. Note that seeking is only possible if the ctor(File) is used. - * - * c.f. http://samtools.sourceforge.net/SAM1.pdf for details of BGZF format - */ -public class BlockCompressedInputStream extends InputStream implements BAMInputStream { - private InputStream mStream = null; - private SeekableStream mFile = null; - private byte[] mFileBuffer = null; - private byte[] mCurrentBlock = null; - private int mCurrentOffset = 0; - private long mBlockAddress = 0; - private int mLastBlockLength = 0; - private final BlockGunzipper blockGunzipper = new BlockGunzipper(); - - - /** - * Note that seek() is not supported if this ctor is used. - */ - public BlockCompressedInputStream(final InputStream stream) { - mStream = IOUtil.toBufferedStream(stream); - mFile = null; - } - - /** - * Use this ctor if you wish to call seek() - */ - public BlockCompressedInputStream(final File file) - throws IOException { - mFile = new SeekableFileStream(file); - mStream = null; - - } - - public BlockCompressedInputStream(final URL url) { - mFile = new SeekableBufferedStream(new SeekableHTTPStream(url)); - mStream = null; - } - - /** - * For providing some arbitrary data source. No additional buffering is - * provided, so if the underlying source is not buffered, wrap it in a - * SeekableBufferedStream before passing to this ctor. - */ - public BlockCompressedInputStream(final SeekableStream strm) { - mFile = strm; - mStream = null; - } - - /** - * Determines whether or not the inflater will re-calculated the CRC on the decompressed data - * and check it against the value stored in the GZIP header. CRC checking is an expensive - * operation and should be used accordingly. - */ - public void setCheckCrcs(final boolean check) { - this.blockGunzipper.setCheckCrcs(check); - } - - /** - * @return the number of bytes that can be read (or skipped over) from this input stream without blocking by the - * next caller of a method for this input stream. The next caller might be the same thread or another thread. - * Note that although the next caller can read this many bytes without blocking, the available() method call itself - * may block in order to fill an internal buffer if it has been exhausted. - */ - public int available() - throws IOException { - if (mCurrentBlock == null || mCurrentOffset == mCurrentBlock.length) { - readBlock(); - } - if (mCurrentBlock == null) { - return 0; - } - return mCurrentBlock.length - mCurrentOffset; - } - - /** - * Closes the underlying InputStream or RandomAccessFile - */ - public void close() - throws IOException { - if (mFile != null) { - mFile.close(); - mFile = null; - } else if (mStream != null) { - mStream.close(); - mStream = null; - } - // Encourage garbage collection - mFileBuffer = null; - mCurrentBlock = null; - } - - /** - * Reads the next byte of data from the input stream. The value byte is returned as an int in the range 0 to 255. - * If no byte is available because the end of the stream has been reached, the value -1 is returned. - * This method blocks until input data is available, the end of the stream is detected, or an exception is thrown. - - * @return the next byte of data, or -1 if the end of the stream is reached. - */ - public int read() - throws IOException { - return (available() > 0) ? mCurrentBlock[mCurrentOffset++] : -1; - } - - /** - * Reads some number of bytes from the input stream and stores them into the buffer array b. The number of bytes - * actually read is returned as an integer. This method blocks until input data is available, end of file is detected, - * or an exception is thrown. - * - * read(buf) has the same effect as read(buf, 0, buf.length). - * - * @param buffer the buffer into which the data is read. - * @return the total number of bytes read into the buffer, or -1 is there is no more data because the end of - * the stream has been reached. - */ - public int read(final byte[] buffer) - throws IOException { - return read(buffer, 0, buffer.length); - } - - private volatile ByteArrayOutputStream buf = null; - private static final byte eol = '\n'; - private static final byte eolCr = '\r'; - - /** - * Reads a whole line. A line is considered to be terminated by either a line feed ('\n'), - * carriage return ('\r') or carriage return followed by a line feed ("\r\n"). - * - * @return A String containing the contents of the line, excluding the line terminating - * character, or null if the end of the stream has been reached - * - * @exception IOException If an I/O error occurs - * @ - */ - public String readLine() throws IOException { - int available = available(); - if (available == 0) { - return null; - } - if(null == buf){ // lazy initialisation - buf = new ByteArrayOutputStream(8192); - } - buf.reset(); - boolean done = false; - boolean foundCr = false; // \r found flag - while (!done) { - int linetmpPos = mCurrentOffset; - int bCnt = 0; - while((available-- > 0)){ - final byte c = mCurrentBlock[linetmpPos++]; - if(c == eol){ // found \n - done = true; - break; - } else if(foundCr){ // previous char was \r - --linetmpPos; // current char is not \n so put it back - done = true; - break; - } else if(c == eolCr){ // found \r - foundCr = true; - continue; // no ++bCnt - } - ++bCnt; - } - if(mCurrentOffset < linetmpPos){ - buf.write(mCurrentBlock, mCurrentOffset, bCnt); - mCurrentOffset = linetmpPos; - } - available = available(); - if(available == 0){ - // EOF - done = true; - } - } - return buf.toString(); - } - - /** - * Reads up to len bytes of data from the input stream into an array of bytes. An attempt is made to read - * as many as len bytes, but a smaller number may be read. The number of bytes actually read is returned as an integer. - * - * This method blocks until input data is available, end of file is detected, or an exception is thrown. - * - * @param buffer buffer into which data is read. - * @param offset the start offset in array b at which the data is written. - * @param length the maximum number of bytes to read. - * @return the total number of bytes read into the buffer, or -1 if there is no more data because the end of - * the stream has been reached. - */ - public int read(final byte[] buffer, int offset, int length) - throws IOException { - final int originalLength = length; - while (length > 0) { - final int available = available(); - if (available == 0) { - // Signal EOF to caller - if (originalLength == length) { - return -1; - } - break; - } - final int copyLength = Math.min(length, available); - System.arraycopy(mCurrentBlock, mCurrentOffset, buffer, offset, copyLength); - mCurrentOffset += copyLength; - offset += copyLength; - length -= copyLength; - } - return originalLength - length; - } - - /** - * Seek to the given position in the file. Note that pos is a special virtual file pointer, - * not an actual byte offset. - * - * @param pos virtual file pointer - */ - public void seek(final long pos) - throws IOException { - if (mFile == null) { - throw new IOException("Cannot seek on stream based file"); - } - // Decode virtual file pointer - // Upper 48 bits is the byte offset into the compressed stream of a block. - // Lower 16 bits is the byte offset into the uncompressed stream inside the block. - final long compressedOffset = BlockCompressedFilePointerUtil.getBlockAddress(pos); - final int uncompressedOffset = BlockCompressedFilePointerUtil.getBlockOffset(pos); - final int available; - if (mBlockAddress == compressedOffset && mCurrentBlock != null) { - available = mCurrentBlock.length; - } else { - mFile.seek(compressedOffset); - mBlockAddress = compressedOffset; - mLastBlockLength = 0; - readBlock(); - available = available(); - } - if (uncompressedOffset > available || - (uncompressedOffset == available && !eof())) { - throw new IOException("Invalid file pointer: " + pos); - } - mCurrentOffset = uncompressedOffset; - } - - private boolean eof() throws IOException { - if (mFile.eof()) { - return true; - } - // If the last remaining block is the size of the EMPTY_GZIP_BLOCK, this is the same as being at EOF. - return (mFile.length() - (mBlockAddress + mLastBlockLength) == BlockCompressedStreamConstants.EMPTY_GZIP_BLOCK.length); - } - - /** - * @return virtual file pointer that can be passed to seek() to return to the current position. This is - * not an actual byte offset, so arithmetic on file pointers cannot be done to determine the distance between - * the two. - */ - public long getFilePointer() { - if (mCurrentOffset == mCurrentBlock.length) { - // If current offset is at the end of the current block, file pointer should point - // to the beginning of the next block. - return BlockCompressedFilePointerUtil.makeFilePointer(mBlockAddress + mLastBlockLength, 0); - } - return BlockCompressedFilePointerUtil.makeFilePointer(mBlockAddress, mCurrentOffset); - } - - public static long getFileBlock(final long bgzfOffset) { - return BlockCompressedFilePointerUtil.getBlockAddress(bgzfOffset); - } - - /** - * @param stream Must be at start of file. Throws RuntimeException if !stream.markSupported(). - * @return true if the given file looks like a valid BGZF file. - */ - public static boolean isValidFile(final InputStream stream) - throws IOException { - if (!stream.markSupported()) { - throw new RuntimeException("Cannot test non-buffered stream"); - } - stream.mark(BlockCompressedStreamConstants.BLOCK_HEADER_LENGTH); - final byte[] buffer = new byte[BlockCompressedStreamConstants.BLOCK_HEADER_LENGTH]; - final int count = readBytes(stream, buffer, 0, BlockCompressedStreamConstants.BLOCK_HEADER_LENGTH); - stream.reset(); - return count == BlockCompressedStreamConstants.BLOCK_HEADER_LENGTH && isValidBlockHeader(buffer); - } - - private static boolean isValidBlockHeader(final byte[] buffer) { - return (buffer[0] == BlockCompressedStreamConstants.GZIP_ID1 && - (buffer[1] & 0xFF) == BlockCompressedStreamConstants.GZIP_ID2 && - (buffer[3] & BlockCompressedStreamConstants.GZIP_FLG) != 0 && - buffer[10] == BlockCompressedStreamConstants.GZIP_XLEN && - buffer[12] == BlockCompressedStreamConstants.BGZF_ID1 && - buffer[13] == BlockCompressedStreamConstants.BGZF_ID2); - } - - private void readBlock() - throws IOException { - - if (mFileBuffer == null) { - mFileBuffer = new byte[BlockCompressedStreamConstants.MAX_COMPRESSED_BLOCK_SIZE]; - } - int count = readBytes(mFileBuffer, 0, BlockCompressedStreamConstants.BLOCK_HEADER_LENGTH); - if (count == 0) { - // Handle case where there is no empty gzip block at end. - mCurrentOffset = 0; - mBlockAddress += mLastBlockLength; - mCurrentBlock = new byte[0]; - return; - } - if (count != BlockCompressedStreamConstants.BLOCK_HEADER_LENGTH) { - throw new IOException("Premature end of file"); - } - final int blockLength = unpackInt16(mFileBuffer, BlockCompressedStreamConstants.BLOCK_LENGTH_OFFSET) + 1; - if (blockLength < BlockCompressedStreamConstants.BLOCK_HEADER_LENGTH || blockLength > mFileBuffer.length) { - throw new IOException("Unexpected compressed block length: " + blockLength); - } - final int remaining = blockLength - BlockCompressedStreamConstants.BLOCK_HEADER_LENGTH; - count = readBytes(mFileBuffer, BlockCompressedStreamConstants.BLOCK_HEADER_LENGTH, remaining); - if (count != remaining) { - throw new FileTruncatedException("Premature end of file"); - } - inflateBlock(mFileBuffer, blockLength); - mCurrentOffset = 0; - mBlockAddress += mLastBlockLength; - mLastBlockLength = blockLength; - } - - private void inflateBlock(final byte[] compressedBlock, final int compressedLength) - throws IOException { - final int uncompressedLength = unpackInt32(compressedBlock, compressedLength-4); - byte[] buffer = mCurrentBlock; - mCurrentBlock = null; - if (buffer == null || buffer.length != uncompressedLength) { - try { - buffer = new byte[uncompressedLength]; - } catch (NegativeArraySizeException e) { - throw new RuntimeException("BGZF file has invalid uncompressedLength: " + uncompressedLength, e); - } - } - blockGunzipper.unzipBlock(buffer, compressedBlock, compressedLength); - mCurrentBlock = buffer; - } - - private int readBytes(final byte[] buffer, final int offset, final int length) - throws IOException { - if (mFile != null) { - return readBytes(mFile, buffer, offset, length); - } else if (mStream != null) { - return readBytes(mStream, buffer, offset, length); - } else { - return 0; - } - } - - private static int readBytes(final SeekableStream file, final byte[] buffer, final int offset, final int length) - throws IOException { - int bytesRead = 0; - while (bytesRead < length) { - final int count = file.read(buffer, offset + bytesRead, length - bytesRead); - if (count <= 0) { - break; - } - bytesRead += count; - } - return bytesRead; - } - - private static int readBytes(final InputStream stream, final byte[] buffer, final int offset, final int length) - throws IOException { - int bytesRead = 0; - while (bytesRead < length) { - final int count = stream.read(buffer, offset + bytesRead, length - bytesRead); - if (count <= 0) { - break; - } - bytesRead += count; - } - return bytesRead; - } - - private int unpackInt16(final byte[] buffer, final int offset) { - return ((buffer[offset] & 0xFF) | - ((buffer[offset+1] & 0xFF) << 8)); - } - - private int unpackInt32(final byte[] buffer, final int offset) { - return ((buffer[offset] & 0xFF) | - ((buffer[offset+1] & 0xFF) << 8) | - ((buffer[offset+2] & 0xFF) << 16) | - ((buffer[offset+3] & 0xFF) << 24)); - } - - public enum FileTermination {HAS_TERMINATOR_BLOCK, HAS_HEALTHY_LAST_BLOCK, DEFECTIVE} - - public static FileTermination checkTermination(final File file) - throws IOException { - final long fileSize = file.length(); - if (fileSize < BlockCompressedStreamConstants.EMPTY_GZIP_BLOCK.length) { - return FileTermination.DEFECTIVE; - } - final RandomAccessFile raFile = new RandomAccessFile(file, "r"); - try { - raFile.seek(fileSize - BlockCompressedStreamConstants.EMPTY_GZIP_BLOCK.length); - byte[] buf = new byte[BlockCompressedStreamConstants.EMPTY_GZIP_BLOCK.length]; - raFile.readFully(buf); - if (Arrays.equals(buf, BlockCompressedStreamConstants.EMPTY_GZIP_BLOCK)) { - return FileTermination.HAS_TERMINATOR_BLOCK; - } - final int bufsize = (int)Math.min(fileSize, BlockCompressedStreamConstants.MAX_COMPRESSED_BLOCK_SIZE); - buf = new byte[bufsize]; - raFile.seek(fileSize - bufsize); - raFile.read(buf); - for (int i = buf.length - BlockCompressedStreamConstants.EMPTY_GZIP_BLOCK.length; - i >= 0; --i) { - if (!preambleEqual(BlockCompressedStreamConstants.GZIP_BLOCK_PREAMBLE, - buf, i, BlockCompressedStreamConstants.GZIP_BLOCK_PREAMBLE.length)) { - continue; - } - final ByteBuffer byteBuffer = ByteBuffer.wrap(buf, i + BlockCompressedStreamConstants.GZIP_BLOCK_PREAMBLE.length, 4); - byteBuffer.order(ByteOrder.LITTLE_ENDIAN); - final int totalBlockSizeMinusOne = byteBuffer.getShort() & 0xFFFF; - if (buf.length - i == totalBlockSizeMinusOne + 1) { - return FileTermination.HAS_HEALTHY_LAST_BLOCK; - } else { - return FileTermination.DEFECTIVE; - } - } - return FileTermination.DEFECTIVE; - } finally { - raFile.close(); - } - } - - private static boolean preambleEqual(final byte[] preamble, final byte[] buf, final int startOffset, final int length) { - for (int i = 0; i < length; ++i) { - if (preamble[i] != buf[i + startOffset]) { - return false; - } - } - return true; - } -} - - diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMReaderPosition.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BAMAccessPlan.java similarity index 58% rename from public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMReaderPosition.java rename to public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BAMAccessPlan.java index 0a6173c1e4..1649713658 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMReaderPosition.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BAMAccessPlan.java @@ -27,8 +27,10 @@ import net.sf.picard.util.PeekableIterator; import net.sf.samtools.GATKBAMFileSpan; import net.sf.samtools.GATKChunk; +import net.sf.samtools.util.BlockCompressedFilePointerUtil; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import java.util.LinkedList; import java.util.List; /** @@ -38,7 +40,7 @@ * Time: 10:47 PM * To change this template use File | Settings | File Templates. */ -class SAMReaderPosition { +class BAMAccessPlan { private final SAMReaderID reader; private final BlockInputStream inputStream; @@ -51,7 +53,7 @@ class SAMReaderPosition { private long nextBlockAddress; - SAMReaderPosition(final SAMReaderID reader, final BlockInputStream inputStream, GATKBAMFileSpan fileSpan) { + BAMAccessPlan(final SAMReaderID reader, final BlockInputStream inputStream, GATKBAMFileSpan fileSpan) { this.reader = reader; this.inputStream = inputStream; @@ -84,11 +86,45 @@ public int getFirstOffsetInBlock() { } /** - * Retrieves the last offset of interest in the block returned by getBlockAddress(). - * @return First block of interest in this segment. + * Gets the spans overlapping the given block; used to copy the contents of the block into the circular buffer. + * @param blockAddress Block address for which to search. + * @param filePosition Block address at which to terminate the last chunk if the last chunk goes beyond this span. + * @return list of chunks containing that block. */ - public int getLastOffsetInBlock() { - return (nextBlockAddress == positionIterator.peek().getBlockEnd()) ? positionIterator.peek().getBlockOffsetEnd() : 65536; + public List getSpansOverlappingBlock(long blockAddress, long filePosition) { + List spansOverlapping = new LinkedList(); + // While the position iterator overlaps the given block, pull out spans to report. + while(positionIterator.hasNext() && positionIterator.peek().getBlockStart() <= blockAddress) { + // Create a span over as much of the block as is covered by this chunk. + int blockOffsetStart = (blockAddress == positionIterator.peek().getBlockStart()) ? positionIterator.peek().getBlockOffsetStart() : 0; + + // Calculate the end of this span. If the span extends past this block, cap it using the current file position. + long blockEnd; + int blockOffsetEnd; + if(blockAddress < positionIterator.peek().getBlockEnd()) { + blockEnd = filePosition; + blockOffsetEnd = 0; + } + else { + blockEnd = positionIterator.peek().getBlockEnd(); + blockOffsetEnd = positionIterator.peek().getBlockOffsetEnd(); + } + + GATKChunk newChunk = new GATKChunk(blockAddress,blockOffsetStart,blockEnd,blockOffsetEnd); + + if(newChunk.getChunkStart() <= newChunk.getChunkEnd()) + spansOverlapping.add(new GATKChunk(blockAddress,blockOffsetStart,blockEnd,blockOffsetEnd)); + + // If the value currently stored in the position iterator ends past the current block, we must be done. Abort. + if(!positionIterator.hasNext() || positionIterator.peek().getBlockEnd() > blockAddress) + break; + + // If the position iterator ends before the block ends, pull the position iterator forward. + if(positionIterator.peek().getBlockEnd() <= blockAddress) + positionIterator.next(); + } + + return spansOverlapping; } public void reset() { @@ -111,20 +147,16 @@ private void initialize() { * @param filePosition The current position within the file. */ void advancePosition(final long filePosition) { - nextBlockAddress = filePosition >> 16; + nextBlockAddress = BlockCompressedFilePointerUtil.getBlockAddress(filePosition); // Check the current file position against the iterator; if the iterator is before the current file position, // draw the iterator forward. Remember when performing the check that coordinates are half-open! - while(positionIterator.hasNext() && isFilePositionPastEndOfChunk(filePosition,positionIterator.peek())) { + while(positionIterator.hasNext() && isFilePositionPastEndOfChunk(filePosition,positionIterator.peek())) positionIterator.next(); - // If the block iterator has shot past the file pointer, bring the file pointer flush with the start of the current block. - if(positionIterator.hasNext() && filePosition < positionIterator.peek().getChunkStart()) { - nextBlockAddress = positionIterator.peek().getBlockStart(); - //System.out.printf("SAMReaderPosition: next block address advanced to %d%n",nextBlockAddress); - break; - } - } + // If the block iterator has shot past the file pointer, bring the file pointer flush with the start of the current block. + if(positionIterator.hasNext() && filePosition < positionIterator.peek().getChunkStart()) + nextBlockAddress = positionIterator.peek().getBlockStart(); // If we've shot off the end of the block pointer, notify consumers that iteration is complete. if(!positionIterator.hasNext()) diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BGZFBlockLoadingDispatcher.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BGZFBlockLoadingDispatcher.java index f468d20204..d75e91bf3a 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BGZFBlockLoadingDispatcher.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BGZFBlockLoadingDispatcher.java @@ -44,12 +44,12 @@ public class BGZFBlockLoadingDispatcher { private final ExecutorService threadPool; - private final Queue inputQueue; + private final Queue inputQueue; public BGZFBlockLoadingDispatcher(final int numThreads, final int numFileHandles) { threadPool = Executors.newFixedThreadPool(numThreads); fileHandleCache = new FileHandleCache(numFileHandles); - inputQueue = new LinkedList(); + inputQueue = new LinkedList(); threadPool.execute(new BlockLoader(this,fileHandleCache,true)); } @@ -58,7 +58,7 @@ public BGZFBlockLoadingDispatcher(final int numThreads, final int numFileHandles * Initiates a request for a new block load. * @param readerPosition Position at which to load. */ - void queueBlockLoad(final SAMReaderPosition readerPosition) { + void queueBlockLoad(final BAMAccessPlan readerPosition) { synchronized(inputQueue) { inputQueue.add(readerPosition); inputQueue.notify(); @@ -69,7 +69,7 @@ void queueBlockLoad(final SAMReaderPosition readerPosition) { * Claims the next work request from the queue. * @return The next work request, or null if none is available. */ - SAMReaderPosition claimNextWorkRequest() { + BAMAccessPlan claimNextWorkRequest() { synchronized(inputQueue) { while(inputQueue.isEmpty()) { try { diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BlockInputStream.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BlockInputStream.java index cb37bad312..fda5d818c6 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BlockInputStream.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BlockInputStream.java @@ -26,24 +26,21 @@ import net.sf.samtools.GATKBAMFileSpan; import net.sf.samtools.GATKChunk; -import net.sf.samtools.util.BAMInputStream; -import net.sf.samtools.util.BlockCompressedFilePointerUtil; import net.sf.samtools.util.BlockCompressedInputStream; -import net.sf.samtools.util.RuntimeEOFException; -import net.sf.samtools.util.SeekableStream; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Arrays; -import java.util.Iterator; import java.util.LinkedList; +import java.util.List; /** * Presents decompressed blocks to the SAMFileReader. */ -public class BlockInputStream extends SeekableStream implements BAMInputStream { +public class BlockInputStream extends InputStream { /** * Mechanism for triggering block loads. */ @@ -65,9 +62,9 @@ public class BlockInputStream extends SeekableStream implements BAMInputStream { private Throwable error; /** - * Current position. + * Current accessPlan. */ - private SAMReaderPosition position; + private BAMAccessPlan accessPlan; /** * A stream of compressed data blocks. @@ -94,11 +91,6 @@ public class BlockInputStream extends SeekableStream implements BAMInputStream { */ private final BlockCompressedInputStream validatingInputStream; - /** - * Has the buffer been filled since last request? - */ - private boolean bufferFilled = false; - /** * Create a new block presenting input stream with a dedicated buffer. * @param dispatcher the block loading messenger. @@ -118,7 +110,7 @@ public class BlockInputStream extends SeekableStream implements BAMInputStream { this.dispatcher = dispatcher; // TODO: Kill the region when all we want to do is start at the beginning of the stream and run to the end of the stream. - this.position = new SAMReaderPosition(reader,this,new GATKBAMFileSpan(new GATKChunk(0,Long.MAX_VALUE))); + this.accessPlan = new BAMAccessPlan(reader,this,new GATKBAMFileSpan(new GATKChunk(0,Long.MAX_VALUE))); // The block offsets / block positions guarantee that the ending offset/position in the data structure maps to // the point in the file just following the last read. These two arrays should never be empty; initializing @@ -151,7 +143,7 @@ public long getFilePointer() { synchronized(lock) { // Find the current block within the input stream. int blockIndex; - for(blockIndex = 0; blockIndex+1 < blockOffsets.size() && buffer.position() >= blockOffsets.get(blockIndex + 1); blockIndex++) + for(blockIndex = 0; blockIndex+1 < blockOffsets.size() && buffer.position() > blockOffsets.get(blockIndex+1); blockIndex++) ; filePointer = blockPositions.get(blockIndex) + (buffer.position()-blockOffsets.get(blockIndex)); } @@ -164,51 +156,8 @@ public long getFilePointer() { return filePointer; } - public void seek(long target) { - //System.out.printf("Thread %s, BlockInputStream %s: seeking to block %d, offset %d%n",Thread.currentThread().getId(),this,BlockCompressedFilePointerUtil.getBlockAddress(target),BlockCompressedFilePointerUtil.getBlockOffset(target)); - synchronized(lock) { - clearBuffers(); - - // Ensure that the position filled in by submitAccessPlan() is in sync with the seek target just specified. - position.advancePosition(target); - - // If the position advances past the end of the target, that must mean that we seeked to a point at the end - // of one of the chunk list's subregions. Make a note of our current position and punt on loading any data. - if(target < position.getBlockAddress() << 16) { - blockOffsets.clear(); - blockOffsets.add(0); - blockPositions.clear(); - blockPositions.add(target); - } - else { - waitForBufferFill(); - // A buffer fill will load the relevant data from the shard, but the buffer position still needs to be - // advanced as appropriate. - Iterator blockOffsetIterator = blockOffsets.descendingIterator(); - Iterator blockPositionIterator = blockPositions.descendingIterator(); - while(blockOffsetIterator.hasNext() && blockPositionIterator.hasNext()) { - final int blockOffset = blockOffsetIterator.next(); - final long blockPosition = blockPositionIterator.next(); - if((blockPosition >> 16) == (target >> 16) && (blockPosition&0xFFFF) < (target&0xFFFF)) { - buffer.position(blockOffset + (int)(target&0xFFFF)-(int)(blockPosition&0xFFFF)); - break; - } - } - } - - if(validatingInputStream != null) { - try { - validatingInputStream.seek(target); - } - catch(IOException ex) { - throw new ReviewedStingException("Unable to validate against Picard input stream",ex); - } - } - } - } - private void clearBuffers() { - this.position.reset(); + this.accessPlan.reset(); // Buffer semantics say that outside of a lock, buffer should always be prepared for reading. // Indicate no data to be read. @@ -225,29 +174,41 @@ private void clearBuffers() { public boolean eof() { synchronized(lock) { // TODO: Handle multiple empty BGZF blocks at end of the file. - return position != null && (position.getBlockAddress() < 0 || position.getBlockAddress() >= length); + return accessPlan != null && (accessPlan.getBlockAddress() < 0 || accessPlan.getBlockAddress() >= length); } } - public void setCheckCrcs(final boolean check) { - // TODO: Implement - } - /** - * Submits a new access plan for the given dataset. - * @param position The next seek point for BAM data in this reader. + * Submits a new access plan for the given dataset and seeks to the given point. + * @param accessPlan The next seek point for BAM data in this reader. */ - public void submitAccessPlan(final SAMReaderPosition position) { + public void submitAccessPlan(final BAMAccessPlan accessPlan) { //System.out.printf("Thread %s: submitting access plan for block at position: %d%n",Thread.currentThread().getId(),position.getBlockAddress()); - synchronized(lock) { - // Assume that the access plan is going to tell us to start where we are and move forward. - // If this isn't the case, we'll soon receive a seek request and the buffer will be forced to reset. - if(this.position != null && position.getBlockAddress() < this.position.getBlockAddress()) - position.advancePosition(this.position.getBlockAddress() << 16); + this.accessPlan = accessPlan; + accessPlan.reset(); + + clearBuffers(); + + // Pull the iterator past any oddball chunks at the beginning of the shard (chunkEnd < chunkStart, empty chunks, etc). + // TODO: Don't pass these empty chunks in. + accessPlan.advancePosition(makeFilePointer(accessPlan.getBlockAddress(),0)); + + if(accessPlan.getBlockAddress() >= 0) { + waitForBufferFill(); + } + + if(validatingInputStream != null) { + try { + validatingInputStream.seek(makeFilePointer(accessPlan.getBlockAddress(),0)); + } + catch(IOException ex) { + throw new ReviewedStingException("Unable to validate against Picard input stream",ex); + } } - this.position = position; + } + private void compactBuffer() { // Compact buffer to maximize storage space. int bytesToRemove = 0; @@ -286,27 +247,14 @@ private void compactBuffer() { * Push contents of incomingBuffer into the end of this buffer. * MUST be called from a thread that is NOT the reader thread. * @param incomingBuffer The data being pushed into this input stream. - * @param position target position for the data. + * @param accessPlan target access plan for the data. * @param filePosition the current position of the file pointer */ - public void copyIntoBuffer(final ByteBuffer incomingBuffer, final SAMReaderPosition position, final long filePosition) { + public void copyIntoBuffer(final ByteBuffer incomingBuffer, final BAMAccessPlan accessPlan, final long filePosition) { synchronized(lock) { try { - compactBuffer(); - // Open up the buffer for more reading. - buffer.limit(buffer.capacity()); - - // Advance the position to take the most recent read into account. - final long lastBlockAddress = position.getBlockAddress(); - final int blockOffsetStart = position.getFirstOffsetInBlock(); - final int blockOffsetEnd = position.getLastOffsetInBlock(); - - // Where did this read end? It either ended in the middle of a block (for a bounding chunk) or it ended at the start of the next block. - final long endOfRead = (blockOffsetEnd < incomingBuffer.remaining()) ? (lastBlockAddress << 16) | blockOffsetEnd : filePosition << 16; - - byte[] validBytes = null; if(validatingInputStream != null) { - validBytes = new byte[incomingBuffer.remaining()]; + byte[] validBytes = new byte[incomingBuffer.remaining()]; byte[] currentBytes = new byte[incomingBuffer.remaining()]; int pos = incomingBuffer.position(); @@ -317,7 +265,7 @@ public void copyIntoBuffer(final ByteBuffer incomingBuffer, final SAMReaderPosit incomingBuffer.position(pos); long currentFilePointer = validatingInputStream.getFilePointer(); - validatingInputStream.seek(lastBlockAddress << 16); + validatingInputStream.seek(makeFilePointer(accessPlan.getBlockAddress(), 0)); validatingInputStream.read(validBytes); validatingInputStream.seek(currentFilePointer); @@ -325,33 +273,41 @@ public void copyIntoBuffer(final ByteBuffer incomingBuffer, final SAMReaderPosit throw new ReviewedStingException(String.format("Bytes being inserted into BlockInputStream %s are incorrect",this)); } - this.position = position; - position.advancePosition(filePosition << 16); + compactBuffer(); + // Open up the buffer for more reading. + buffer.limit(buffer.capacity()); + + // Get the spans overlapping this particular block... + List spansOverlapping = accessPlan.getSpansOverlappingBlock(accessPlan.getBlockAddress(),filePosition); + + // ...and advance the block + this.accessPlan = accessPlan; + accessPlan.advancePosition(makeFilePointer(filePosition, 0)); - if(buffer.remaining() < incomingBuffer.remaining()) { - //System.out.printf("Thread %s: waiting for available space in buffer; buffer remaining = %d, incoming buffer remaining = %d%n",Thread.currentThread().getId(),buffer.remaining(),incomingBuffer.remaining()); + if(buffer.remaining() < incomingBuffer.remaining()) lock.wait(); - //System.out.printf("Thread %s: waited for available space in buffer; buffer remaining = %d, incoming buffer remaining = %d%n", Thread.currentThread().getId(), buffer.remaining(), incomingBuffer.remaining()); - } - // Remove the last position in the list and add in the last read position, in case the two are different. - blockOffsets.removeLast(); - blockOffsets.add(buffer.position()); - blockPositions.removeLast(); - blockPositions.add(lastBlockAddress << 16 | blockOffsetStart); + final int bytesInIncomingBuffer = incomingBuffer.limit(); + + for(GATKChunk spanOverlapping: spansOverlapping) { + // Clear out the endcap tracking state and add in the starting position for this transfer. + blockOffsets.removeLast(); + blockOffsets.add(buffer.position()); + blockPositions.removeLast(); + blockPositions.add(spanOverlapping.getChunkStart()); - // Stream the buffer into the data stream. - incomingBuffer.position(blockOffsetStart); - incomingBuffer.limit(Math.min(incomingBuffer.limit(),blockOffsetEnd)); - buffer.put(incomingBuffer); + // Stream the buffer into the data stream. + incomingBuffer.limit((spanOverlapping.getBlockEnd() > spanOverlapping.getBlockStart()) ? bytesInIncomingBuffer : spanOverlapping.getBlockOffsetEnd()); + incomingBuffer.position(spanOverlapping.getBlockOffsetStart()); + buffer.put(incomingBuffer); - // Then, add the last position read to the very end of the list, just past the end of the last buffer. - blockOffsets.add(buffer.position()); - blockPositions.add(endOfRead); + // Add the endcap for this transfer. + blockOffsets.add(buffer.position()); + blockPositions.add(spanOverlapping.getChunkEnd()); + } // Set up the buffer for reading. buffer.flip(); - bufferFilled = true; lock.notify(); } @@ -447,12 +403,8 @@ public int read(byte[] bytes, final int offset, final int length) { if(remaining < length) return length - remaining; - // Otherwise, if at eof(), return -1. - else if(eof()) - return -1; - - // Otherwise, we must've hit a bug in the system. - throw new ReviewedStingException("BUG: read returned no data, but eof() reports false."); + // Otherwise, return -1. + return -1; } public void close() { @@ -472,20 +424,26 @@ public String getSource() { private void waitForBufferFill() { synchronized(lock) { - bufferFilled = false; if(buffer.remaining() == 0 && !eof()) { //System.out.printf("Thread %s is waiting for a buffer fill from position %d to buffer %s%n",Thread.currentThread().getId(),position.getBlockAddress(),this); - dispatcher.queueBlockLoad(position); + dispatcher.queueBlockLoad(accessPlan); try { lock.wait(); } catch(InterruptedException ex) { throw new ReviewedStingException("Interrupt occurred waiting for buffer to fill",ex); } - - if(bufferFilled && buffer.remaining() == 0) - throw new RuntimeEOFException("No more data left in InputStream"); } } } + + /** + * Create an encoded BAM file pointer given the address of a BGZF block and an offset. + * @param blockAddress Physical address on disk of a BGZF block. + * @param blockOffset Offset into the uncompressed data stored in the BGZF block. + * @return 64-bit pointer encoded according to the BAM spec. + */ + public static long makeFilePointer(final long blockAddress, final int blockOffset) { + return blockAddress << 16 | blockOffset; + } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BlockLoader.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BlockLoader.java index ab42998026..81a37e53ca 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BlockLoader.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BlockLoader.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -70,29 +70,29 @@ public BlockLoader(final BGZFBlockLoadingDispatcher dispatcher, final FileHandle public void run() { for(;;) { - SAMReaderPosition readerPosition = null; + BAMAccessPlan accessPlan = null; try { - readerPosition = dispatcher.claimNextWorkRequest(); - FileInputStream inputStream = fileHandleCache.claimFileInputStream(readerPosition.getReader()); + accessPlan = dispatcher.claimNextWorkRequest(); + FileInputStream inputStream = fileHandleCache.claimFileInputStream(accessPlan.getReader()); - long blockAddress = readerPosition.getBlockAddress(); + //long blockAddress = readerPosition.getBlockAddress(); //System.out.printf("Thread %s: BlockLoader: copying bytes from %s at position %d into %s%n",Thread.currentThread().getId(),inputStream,blockAddress,readerPosition.getInputStream()); - ByteBuffer compressedBlock = readBGZFBlock(inputStream,readerPosition.getBlockAddress()); + ByteBuffer compressedBlock = readBGZFBlock(inputStream,accessPlan.getBlockAddress()); long nextBlockAddress = position(inputStream); - fileHandleCache.releaseFileInputStream(readerPosition.getReader(),inputStream); + fileHandleCache.releaseFileInputStream(accessPlan.getReader(),inputStream); ByteBuffer block = decompress ? decompressBGZFBlock(compressedBlock) : compressedBlock; int bytesCopied = block.remaining(); - BlockInputStream bamInputStream = readerPosition.getInputStream(); - bamInputStream.copyIntoBuffer(block,readerPosition,nextBlockAddress); + BlockInputStream bamInputStream = accessPlan.getInputStream(); + bamInputStream.copyIntoBuffer(block,accessPlan,nextBlockAddress); //System.out.printf("Thread %s: BlockLoader: copied %d bytes from %s at position %d into %s%n",Thread.currentThread().getId(),bytesCopied,inputStream,blockAddress,readerPosition.getInputStream()); } catch(Throwable error) { - if(readerPosition != null && readerPosition.getInputStream() != null) - readerPosition.getInputStream().reportException(error); + if(accessPlan != null && accessPlan.getInputStream() != null) + accessPlan.getInputStream().reportException(error); } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/ReadShard.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/ReadShard.java index 8d73b1b158..96b55674ae 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/ReadShard.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/ReadShard.java @@ -36,7 +36,7 @@ */ public class ReadShard extends Shard { /** - * What is the maximum number of reads which should go into a read shard. + * What is the maximum number of reads per BAM file which should go into a read shard. */ public static int MAX_READS = 10000; diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java index c040b53c4e..a4681cffd3 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java @@ -567,9 +567,14 @@ private StingSAMIterator getIterator(SAMReaders readers, Shard shard, boolean en if(threadAllocation.getNumIOThreads() > 0) { BlockInputStream inputStream = readers.getInputStream(id); - inputStream.submitAccessPlan(new SAMReaderPosition(id,inputStream,(GATKBAMFileSpan)shard.getFileSpans().get(id))); + inputStream.submitAccessPlan(new BAMAccessPlan(id, inputStream, (GATKBAMFileSpan) shard.getFileSpans().get(id))); + BAMRecordCodec codec = new BAMRecordCodec(getHeader(id),factory); + codec.setInputStream(inputStream); + iterator = new BAMCodecIterator(inputStream,readers.getReader(id),codec); + } + else { + iterator = readers.getReader(id).iterator(shard.getFileSpans().get(id)); } - iterator = readers.getReader(id).iterator(shard.getFileSpans().get(id)); if(shard.getGenomeLocs().size() > 0) iterator = new IntervalOverlapFilteringIterator(iterator,shard.getGenomeLocs()); iteratorMap.put(readers.getReader(id), iterator); @@ -577,8 +582,6 @@ private StingSAMIterator getIterator(SAMReaders readers, Shard shard, boolean en MergingSamRecordIterator mergingIterator = readers.createMergingIterator(iteratorMap); - - return applyDecoratingIterators(shard.getReadMetrics(), enableVerification, readProperties.useOriginalBaseQualities(), @@ -592,6 +595,49 @@ private StingSAMIterator getIterator(SAMReaders readers, Shard shard, boolean en readProperties.defaultBaseQualities()); } + private class BAMCodecIterator implements CloseableIterator { + private final BlockInputStream inputStream; + private final SAMFileReader reader; + private final BAMRecordCodec codec; + private SAMRecord nextRead; + + private BAMCodecIterator(final BlockInputStream inputStream, final SAMFileReader reader, final BAMRecordCodec codec) { + this.inputStream = inputStream; + this.reader = reader; + this.codec = codec; + advance(); + } + + public boolean hasNext() { + return nextRead != null; + } + + public SAMRecord next() { + if(!hasNext()) + throw new NoSuchElementException("Unable to retrieve next record from BAMCodecIterator; input stream is empty"); + SAMRecord currentRead = nextRead; + advance(); + return currentRead; + } + + public void close() { + // NO-OP. + } + + public void remove() { + throw new UnsupportedOperationException("Unable to remove from BAMCodecIterator"); + } + + private void advance() { + final long startCoordinate = inputStream.getFilePointer(); + nextRead = codec.decode(); + final long stopCoordinate = inputStream.getFilePointer(); + + if(reader != null && nextRead != null) + PicardNamespaceUtils.setFileSource(nextRead,new SAMFileSource(reader,new GATKBAMFileSpan(new GATKChunk(startCoordinate,stopCoordinate)))); + } + } + /** * Filter reads based on user-specified criteria. * @@ -871,12 +917,9 @@ public ReaderInitializer(final SAMReaderID readerID) { public ReaderInitializer call() { final File indexFile = findIndexFile(readerID.samFile); try { - if (threadAllocation.getNumIOThreads() > 0) { + if (threadAllocation.getNumIOThreads() > 0) blockInputStream = new BlockInputStream(dispatcher,readerID,false); - reader = new SAMFileReader(blockInputStream,indexFile,false); - } - else - reader = new SAMFileReader(readerID.samFile,indexFile,false); + reader = new SAMFileReader(readerID.samFile,indexFile,false); } catch ( RuntimeIOException e ) { if ( e.getCause() != null && e.getCause() instanceof FileNotFoundException ) throw new UserException.CouldNotReadInputFile(readerID.samFile, e); From 2f800b078c07c1a6fc1dd5ed97282dc17c177e7e Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Wed, 8 Feb 2012 15:27:16 -0500 Subject: [PATCH 231/356] Changes to default behavior of UG: multi-allelic mode is always on; max number of alternate alleles to genotype is 3; alleles in the SNP model are ranked by their likelihood sum (Guillermo will do this for indels); SB is computed again. --- ...elGenotypeLikelihoodsCalculationModel.java | 5 +- ...NPGenotypeLikelihoodsCalculationModel.java | 85 ++++++++++--------- .../genotyper/UnifiedArgumentCollection.java | 20 ++--- .../walkers/genotyper/UnifiedGenotyper.java | 2 +- .../genotyper/UnifiedGenotyperEngine.java | 2 +- .../UnifiedGenotyperIntegrationTest.java | 27 ++++-- 6 files changed, 72 insertions(+), 69 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java index 0422fbf035..49c131ce29 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java @@ -55,9 +55,8 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood private final boolean getAlleleListFromVCF; private boolean DEBUG = false; - private final boolean doMultiAllelicCalls; + private final boolean doMultiAllelicCalls = true; private boolean ignoreSNPAllelesWhenGenotypingIndels = false; - private final int maxAlternateAlleles; private PairHMMIndelErrorModel pairModel; private static ThreadLocal>> indelLikelihoodMap = @@ -88,8 +87,6 @@ protected IndelGenotypeLikelihoodsCalculationModel(UnifiedArgumentCollection UAC minIndelCountForGenotyping = UAC.MIN_INDEL_COUNT_FOR_GENOTYPING; HAPLOTYPE_SIZE = UAC.INDEL_HAPLOTYPE_SIZE; DEBUG = UAC.OUTPUT_DEBUG_INDEL_INFO; - maxAlternateAlleles = UAC.MAX_ALTERNATE_ALLELES; - doMultiAllelicCalls = UAC.MULTI_ALLELIC; haplotypeMap = new LinkedHashMap(); ignoreSNPAllelesWhenGenotypingIndels = UAC.IGNORE_SNP_ALLELES; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java index ea53c815d1..6f1f86c6d3 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java @@ -43,13 +43,24 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsCalculationModel { - private boolean ALLOW_MULTIPLE_ALLELES; - private final boolean useAlleleFromVCF; + final LikelihoodSum[] likelihoodSums = new LikelihoodSum[4]; + + private final class LikelihoodSum implements Comparable { + public double sum = 0.0; + public Allele base; + + public LikelihoodSum(Allele base) { this.base = base; } + + public int compareTo(LikelihoodSum other) { + final double diff = sum - other.sum; + return ( diff < 0.0 ) ? 1 : (diff > 0.0 ) ? -1 : 0; + } + } + protected SNPGenotypeLikelihoodsCalculationModel(UnifiedArgumentCollection UAC, Logger logger) { super(UAC, logger); - ALLOW_MULTIPLE_ALLELES = UAC.MULTI_ALLELIC; useAlleleFromVCF = UAC.GenotypingMode == GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES; // make sure the PL cache has been initialized with enough alleles @@ -69,7 +80,6 @@ public VariantContext getLikelihoods(final RefMetaDataTracker tracker, if ( !(priors instanceof DiploidSNPGenotypePriors) ) throw new StingException("Only diploid-based SNP priors are supported in the SNP GL model"); - final boolean[] basesToUse = new boolean[4]; final byte refBase = ref.getBase(); final int indexOfRefBase = BaseUtils.simpleBaseToBaseIndex(refBase); @@ -95,46 +105,40 @@ public VariantContext getLikelihoods(final RefMetaDataTracker tracker, // find the alternate allele(s) that we should be using if ( alternateAlleleToUse != null ) { - basesToUse[BaseUtils.simpleBaseToBaseIndex(alternateAlleleToUse.getBases()[0])] = true; + alleles.add(alternateAlleleToUse); } else if ( useAlleleFromVCF ) { final VariantContext vc = UnifiedGenotyperEngine.getVCFromAllelesRod(tracker, ref, ref.getLocus(), true, logger, UAC.alleles); // ignore places where we don't have a SNP if ( vc == null || !vc.isSNP() ) return null; - - for ( Allele allele : vc.getAlternateAlleles() ) - basesToUse[BaseUtils.simpleBaseToBaseIndex(allele.getBases()[0])] = true; + + alleles.addAll(vc.getAlternateAlleles()); } else { - determineAlternateAlleles(basesToUse, refBase, GLs); - - // how many alternate alleles are we using? - int alleleCounter = Utils.countSetBits(basesToUse); + alleles.addAll(determineAlternateAlleles(refBase, GLs)); // if there are no non-ref alleles... - if ( alleleCounter == 0 ) { + if ( alleles.size() == 1 ) { // if we only want variants, then we don't need to calculate genotype likelihoods if ( UAC.OutputMode == UnifiedGenotyperEngine.OUTPUT_MODE.EMIT_VARIANTS_ONLY ) return builder.make(); // otherwise, choose any alternate allele (it doesn't really matter) - basesToUse[indexOfRefBase == 0 ? 1 : 0] = true; + alleles.add(Allele.create(BaseUtils.baseIndexToSimpleBase(indexOfRefBase == 0 ? 1 : 0))); } } // create the alternate alleles and the allele ordering (the ordering is crucial for the GLs) - final int numAltAlleles = Utils.countSetBits(basesToUse); - final int[] alleleOrdering = new int[numAltAlleles + 1]; - alleleOrdering[0] = indexOfRefBase; - int alleleOrderingIndex = 1; - int numLikelihoods = 1; - for ( int i = 0; i < 4; i++ ) { - if ( i != indexOfRefBase && basesToUse[i] ) { - alleles.add(Allele.create(BaseUtils.baseIndexToSimpleBase(i), false)); - alleleOrdering[alleleOrderingIndex++] = i; - numLikelihoods += alleleOrderingIndex; - } + final int numAlleles = alleles.size(); + final int numAltAlleles = numAlleles - 1; + + final int[] alleleOrdering = new int[numAlleles]; + int alleleOrderingIndex = 0; + int numLikelihoods = 0; + for ( Allele allele : alleles ) { + alleleOrdering[alleleOrderingIndex++] = BaseUtils.simpleBaseToBaseIndex(allele.getBases()[0]); + numLikelihoods += alleleOrderingIndex; } builder.alleles(alleles); @@ -165,13 +169,14 @@ public VariantContext getLikelihoods(final RefMetaDataTracker tracker, return builder.genotypes(genotypes).make(); } - - // fills in the allelesToUse array - protected void determineAlternateAlleles(final boolean[] allelesToUse, final byte ref, final List sampleDataList) { + + // determines the alleles to use + protected List determineAlternateAlleles(final byte ref, final List sampleDataList) { final int baseIndexOfRef = BaseUtils.simpleBaseToBaseIndex(ref); final int PLindexOfRef = DiploidGenotype.createDiploidGenotype(ref, ref).ordinal(); - final double[] likelihoodCounts = new double[4]; + for ( int i = 0; i < 4; i++ ) + likelihoodSums[i] = new LikelihoodSum(Allele.create(BaseUtils.baseIndexToSimpleBase(i), false)); // based on the GLs, find the alternate alleles with the most probability for ( SampleGenotypeData sampleData : sampleDataList ) { @@ -180,25 +185,21 @@ protected void determineAlternateAlleles(final boolean[] allelesToUse, final byt if ( PLindexOfBestGL != PLindexOfRef ) { int[] alleles = UnifiedGenotyperEngine.PLIndexToAlleleIndex[3][PLindexOfBestGL]; if ( alleles[0] != baseIndexOfRef ) - likelihoodCounts[alleles[0]] += likelihoods[PLindexOfBestGL] - likelihoods[PLindexOfRef]; + likelihoodSums[alleles[0]].sum += likelihoods[PLindexOfBestGL] - likelihoods[PLindexOfRef]; // don't double-count it if ( alleles[1] != baseIndexOfRef && alleles[1] != alleles[0] ) - likelihoodCounts[alleles[1]] += likelihoods[PLindexOfBestGL] - likelihoods[PLindexOfRef]; + likelihoodSums[alleles[1]].sum += likelihoods[PLindexOfBestGL] - likelihoods[PLindexOfRef]; } } - if ( ALLOW_MULTIPLE_ALLELES ) { - for ( int i = 0; i < 4; i++ ) { - if ( likelihoodCounts[i] > 0.0 ) { - allelesToUse[i] = true; - } - } - } else { - // set the non-ref base which has the maximum sum of non-ref GLs - final int indexOfMax = MathUtils.maxElementIndex(likelihoodCounts); - if ( likelihoodCounts[indexOfMax] > 0.0 ) - allelesToUse[indexOfMax] = true; + Collections.sort(Arrays.asList(likelihoodSums)); + final List allelesToUse = new ArrayList(3); + for ( LikelihoodSum sum : likelihoodSums ) { + if ( sum.sum > 0.0 ) + allelesToUse.add(sum.base); } + + return allelesToUse; } public ReadBackedPileup createBAQedPileup( final ReadBackedPileup pileup ) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java index 16159393f0..82e411c25c 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java @@ -84,8 +84,8 @@ public class UnifiedArgumentCollection { /** * This argument is not enabled by default because it increases the runtime by an appreciable amount. */ - @Argument(fullName = "computeSLOD", shortName = "sl", doc = "If provided, we will calculate the SLOD", required = false) - public boolean COMPUTE_SLOD = false; + @Argument(fullName = "noSLOD", shortName = "nosl", doc = "If provided, we will not calculate the SLOD", required = false) + public boolean NO_SLOD = false; /** * When the UnifiedGenotyper is put into GENOTYPE_GIVEN_ALLELES mode it will genotype the samples using only the alleles provide in this rod binding @@ -103,21 +103,12 @@ public class UnifiedArgumentCollection { @Argument(fullName = "max_deletion_fraction", shortName = "deletions", doc = "Maximum fraction of reads with deletions spanning this locus for it to be callable [to disable, set to < 0 or > 1; default:0.05]", required = false) public Double MAX_DELETION_FRACTION = 0.05; - /** - * The default behavior of the Unified Genotyper is to allow the genotyping of just one alternate allele in discovery mode; using this flag - * will enable the discovery of multiple alternate alleles. Please note that this works for SNPs only and that it is still highly experimental. - * For advanced users only. - */ - @Advanced - @Argument(fullName = "multiallelic", shortName = "multiallelic", doc = "Allow the discovery of multiple alleles", required = false) - public boolean MULTI_ALLELIC = false; - /** * If there are more than this number of alternate alleles presented to the genotyper (either through discovery or GENOTYPE_GIVEN ALLELES), - * then this site will be skipped and a warning printed. Note that genotyping sites with many alternate alleles is both CPU and memory intensive. + * then only this many alleles will be used. Note that genotyping sites with many alternate alleles is both CPU and memory intensive. */ @Argument(fullName = "max_alternate_alleles", shortName = "maxAlleles", doc = "Maximum number of alternate alleles to genotype", required = false) - public int MAX_ALTERNATE_ALLELES = 5; + public int MAX_ALTERNATE_ALLELES = 3; // indel-related arguments /** @@ -168,7 +159,7 @@ public UnifiedArgumentCollection clone() { uac.PCR_error = PCR_error; uac.GenotypingMode = GenotypingMode; uac.OutputMode = OutputMode; - uac.COMPUTE_SLOD = COMPUTE_SLOD; + uac.NO_SLOD = NO_SLOD; uac.STANDARD_CONFIDENCE_FOR_CALLING = STANDARD_CONFIDENCE_FOR_CALLING; uac.STANDARD_CONFIDENCE_FOR_EMITTING = STANDARD_CONFIDENCE_FOR_EMITTING; uac.MIN_BASE_QUALTY_SCORE = MIN_BASE_QUALTY_SCORE; @@ -185,7 +176,6 @@ public UnifiedArgumentCollection clone() { // todo- arguments to remove uac.IGNORE_SNP_ALLELES = IGNORE_SNP_ALLELES; uac.DONT_DO_BANDED_INDEL_COMPUTATION = DONT_DO_BANDED_INDEL_COMPUTATION; - uac.MULTI_ALLELIC = MULTI_ALLELIC; return uac; } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java index b3f0954a2d..1106fcb527 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java @@ -240,7 +240,7 @@ private Set getHeaderInfo() { headerInfo.addAll(annotationEngine.getVCFAnnotationDescriptions()); // annotation (INFO) fields from UnifiedGenotyper - if ( UAC.COMPUTE_SLOD ) + if ( !UAC.NO_SLOD ) headerInfo.add(new VCFInfoHeaderLine(VCFConstants.STRAND_BIAS_KEY, 1, VCFHeaderLineType.Float, "Strand Bias")); headerInfo.add(new VCFInfoHeaderLine(VCFConstants.DOWNSAMPLED_KEY, 0, VCFHeaderLineType.Flag, "Were any of the samples downsampled?")); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java index 2da4b66c48..2eba6d8841 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java @@ -407,7 +407,7 @@ else if ( UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_M if ( !limitedContext && rawContext.hasPileupBeenDownsampled() ) attributes.put(VCFConstants.DOWNSAMPLED_KEY, true); - if ( UAC.COMPUTE_SLOD && !limitedContext && !bestGuessIsRef ) { + if ( !UAC.NO_SLOD && !limitedContext && !bestGuessIsRef ) { //final boolean DEBUG_SLOD = false; // the overall lod diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java index 7285b0fb8a..fd6738123f 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java @@ -15,9 +15,9 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest { - private final static String baseCommand = "-T UnifiedGenotyper -R " + b36KGReference + " -NO_HEADER -glm BOTH --dbsnp " + b36dbSNP129; - private final static String baseCommandIndels = "-T UnifiedGenotyper -R " + b36KGReference + " -NO_HEADER -glm INDEL -mbq 20 --dbsnp " + b36dbSNP129; - private final static String baseCommandIndelsb37 = "-T UnifiedGenotyper -R " + b37KGReference + " -NO_HEADER -glm INDEL -mbq 20 --dbsnp " + b37dbSNP132; + private final static String baseCommand = "-T UnifiedGenotyper -R " + b36KGReference + " -nosl -NO_HEADER -glm BOTH --dbsnp " + b36dbSNP129; + private final static String baseCommandIndels = "-T UnifiedGenotyper -R " + b36KGReference + " -nosl -NO_HEADER -glm INDEL -mbq 20 --dbsnp " + b36dbSNP129; + private final static String baseCommandIndelsb37 = "-T UnifiedGenotyper -R " + b37KGReference + " -nosl -NO_HEADER -glm INDEL -mbq 20 --dbsnp " + b37dbSNP132; // -------------------------------------------------------------------------------------------------------------- // @@ -28,7 +28,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest { public void testMultiSamplePilot1() { WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec( baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000", 1, - Arrays.asList("653172b43b19003d9f7df6dab21f4b09")); + Arrays.asList("9ab4e98ce437a1c5e1eee338de49ee7e")); executeTest("test MultiSample Pilot1", spec); } @@ -56,6 +56,14 @@ public void testSingleSamplePilot2() { executeTest("test SingleSample Pilot2", spec); } + @Test + public void testMultipleSNPAlleles() { + WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec( + "-T UnifiedGenotyper -R " + b37KGReference + " -nosl -NO_HEADER -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + validationDataLocation + "multiallelic.snps.bam -o %s -L " + validationDataLocation + "multiallelic.snps.intervals", 1, + Arrays.asList("aabc4b3a312aba18b78e14750d8c8e62")); + executeTest("test Multiple SNP alleles", spec); + } + // -------------------------------------------------------------------------------------------------------------- // // testing compressed output @@ -114,8 +122,7 @@ public void testParallelization() { @Test public void testCallingParameters() { HashMap e = new HashMap(); - e.put( "--min_base_quality_score 26", "7acb1a5aee5fdadb0cc0ea07a212efc6" ); - e.put( "--computeSLOD", "6172d2f3d370132f4c57a26aa94c256e" ); + e.put( "--min_base_quality_score 26", "258c1b33349eb3b2d395ec4d69302725" ); for ( Map.Entry entry : e.entrySet() ) { WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec( @@ -125,6 +132,14 @@ public void testCallingParameters() { } } + @Test + public void testSLOD() { + WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec( + "-T UnifiedGenotyper -R " + b36KGReference + " -NO_HEADER -glm BOTH --dbsnp " + b36dbSNP129 + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,010,000", 1, + Arrays.asList("6172d2f3d370132f4c57a26aa94c256e")); + executeTest("test SLOD", spec); + } + @Test public void testOutputParameter() { HashMap e = new HashMap(); From 4316437a625ec383f0b48b605b8056ea187a9309 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Wed, 8 Feb 2012 16:00:12 -0500 Subject: [PATCH 232/356] Initial version of R script that will be called by new BQSR to replace the entire AnalyzeCovariates program From 270b160d8759a22b4a2e416e5417b225c6847f04 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Wed, 8 Feb 2012 16:26:32 -0500 Subject: [PATCH 233/356] Incorporating feedback from Mauricio on the plots From d561914d4fb49b169e75487ee4286762cf6c74b7 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Wed, 8 Feb 2012 23:28:55 -0500 Subject: [PATCH 234/356] Revert "First implementation of GATKReportGatherer" premature push from my part. Roger is still working on the new format and we need to update the other tools to operate correctly with the new GATKReport. This reverts commit aea0de314220810c2666055dc75f04f9010436ad. --- .../sting/gatk/report/GATKReport.java | 65 ++--- .../sting/gatk/report/GATKReportGatherer.java | 46 ---- .../sting/gatk/report/GATKReportTable.java | 252 +++++++----------- .../sting/gatk/report/GATKReportUnitTest.java | 128 ++------- 4 files changed, 134 insertions(+), 357 deletions(-) delete mode 100644 public/java/src/org/broadinstitute/sting/gatk/report/GATKReportGatherer.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReport.java b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReport.java index c0abe74500..608b5d1d0a 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReport.java +++ b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReport.java @@ -24,8 +24,7 @@ public GATKReport() { /** * Create a new GATKReport with the contents of a GATKReport on disk. - * - * @param filename the path to the file to load + * @param filename the path to the file to load */ public GATKReport(String filename) { this(new File(filename)); @@ -33,8 +32,7 @@ public GATKReport(String filename) { /** * Create a new GATKReport with the contents of a GATKReport on disk. - * - * @param file the file to load + * @param file the file to load */ public GATKReport(File file) { loadReport(file); @@ -42,8 +40,7 @@ public GATKReport(File file) { /** * Load a GATKReport file from disk - * - * @param file the file to load + * @param file the file to load */ private void loadReport(File file) { try { @@ -51,11 +48,12 @@ private void loadReport(File file) { GATKReportTable table = null; String[] header = null; + int id = 0; GATKReportVersion version = null; List columnStarts = null; String line; - while ((line = reader.readLine()) != null) { + while ( (line = reader.readLine()) != null ) { if (line.startsWith(GATKREPORT_HEADER_PREFIX)) { @@ -73,7 +71,7 @@ private void loadReport(File file) { header = null; columnStarts = null; - } else if (line.trim().isEmpty()) { + } else if ( line.trim().isEmpty() ) { // do nothing } else { if (table != null) { @@ -99,22 +97,19 @@ private void loadReport(File file) { if (header == null) { header = splitLine; - // Set the first column as the primary key - table.addPrimaryKey(header[0]); - // Set every other column as column - for (int i = 1; i < header.length; i++) { - table.addColumn(header[i], ""); + table.addPrimaryKey("id", false); + + for ( String columnName : header ) { + table.addColumn(columnName, ""); } + id = 0; } else { - //Get primary key Value from the current line array - String primaryKey = splitLine[0]; - //Input all the remaining values - for (int columnIndex = 1; columnIndex < header.length; columnIndex++) { - table.set(primaryKey, header[columnIndex], splitLine[columnIndex]); + for (int columnIndex = 0; columnIndex < header.length; columnIndex++) { + table.set(id, header[columnIndex], splitLine[columnIndex]); } - + id++; } } } @@ -129,8 +124,8 @@ private void loadReport(File file) { /** * Add a new table to the collection * - * @param tableName the name of the table - * @param tableDescription the description of the table + * @param tableName the name of the table + * @param tableDescription the description of the table */ public void addTable(String tableName, String tableDescription) { addTable(tableName, tableDescription, true); @@ -144,7 +139,7 @@ public void addTable(String tableName, String tableDescription, boolean sortByPr /** * Return true if table with a given name exists * - * @param tableName the name of the table + * @param tableName the name of the table * @return true if the table exists, false otherwise */ public boolean hasTable(String tableName) { @@ -154,8 +149,8 @@ public boolean hasTable(String tableName) { /** * Return a table with a given name * - * @param tableName the name of the table - * @return the table object + * @param tableName the name of the table + * @return the table object */ public GATKReportTable getTable(String tableName) { GATKReportTable table = tables.get(tableName); @@ -167,7 +162,7 @@ public GATKReportTable getTable(String tableName) { /** * Print all tables contained within this container to a PrintStream * - * @param out the PrintStream to which the tables should be written + * @param out the PrintStream to which the tables should be written */ public void print(PrintStream out) { for (GATKReportTable table : tables.values()) { @@ -180,24 +175,4 @@ public void print(PrintStream out) { public Collection getTables() { return tables.values(); } - - public void combineWith(GATKReport input) { - - // For every input table, add values - System.out.println("This.tables: keySet"); - for (String s : tables.keySet()) - System.out.println(s); - - // todo test tables exist - - - for (String tableName : input.tables.keySet()) { - System.out.println("Input table key: " + tableName); - if (tables.containsKey(tableName)) - tables.get(tableName).mergeRows(input.getTable(tableName)); - else - throw new ReviewedStingException("Failed to combine GATKReport, tables don't match!"); - } - - } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportGatherer.java b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportGatherer.java deleted file mode 100644 index 0d15971ae1..0000000000 --- a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportGatherer.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.broadinstitute.sting.gatk.report; - -import org.broadinstitute.sting.commandline.Gatherer; -import org.broadinstitute.sting.utils.exceptions.UserException; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.PrintStream; -import java.util.List; - -/** - * Created by IntelliJ IDEA. - * User: roger - * Date: 1/9/12 - * Time: 11:17 PM - * To change this template use File | Settings | File Templates. - */ -public class GATKReportGatherer extends Gatherer { - @Override - public void gather(List inputs, File output) { - //Combines inputs GATKReport to one output - - PrintStream o; - try { - o = new PrintStream(output); - } catch (FileNotFoundException e) { - throw new UserException("File to be output by CoverageByRG Gather function was not found"); - } - - GATKReport current = new GATKReport(); - boolean isFirst = true; - for (File input : inputs) { - - // If the table is empty - if (isFirst) { - current = new GATKReport(input); - isFirst = false; - } else { - GATKReport toAdd = new GATKReport(input); - current.combineWith(toAdd); - } - } - - current.print(o); - } -} diff --git a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java index ac18891d78..b72b20e0b7 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java +++ b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java @@ -4,10 +4,7 @@ import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import java.io.PrintStream; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.TreeSet; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -15,12 +12,12 @@ * A data structure that allows data to be collected over the course of a walker's computation, then have that data * written to a PrintStream such that it's human-readable, AWK-able, and R-friendly (given that you load it using the * GATKReport loader module). - *

+ * * The goal of this object is to use the same data structure for both accumulating data during a walker's computation * and emitting that data to a file for easy analysis in R (or any other program/language that can take in a table of * results). Thus, all of the infrastructure below is designed simply to make printing the following as easy as * possible: - *

+ * * ##:GATKReport.v0.1 ErrorRatePerCycle : The error rate per sequenced position in the reads * cycle errorrate.61PA8.7 qualavg.61PA8.7 * 0 0.007451835696110506 25.474613284804366 @@ -32,60 +29,60 @@ * 6 5.452562704471102E-4 36.1217248908297 * 7 5.452562704471102E-4 36.1910480349345 * 8 5.452562704471102E-4 36.00345705967977 - *

+ * * Here, we have a GATKReport table - a well-formatted, easy to read representation of some tabular data. Every single * table has this same GATKReport.v0.1 header, which permits multiple files from different sources to be cat-ed * together, which makes it very easy to pull tables from different programs into R via a single file. - *

+ * * ------------ * Definitions: - *

+ * * Table info: - * The first line, structured as - * ##:

:
- *

+ * The first line, structured as + * ##:

:
+ * * Table header: - * The second line, specifying a unique name for each column in the table. - *

- * The first column mentioned in the table header is the "primary key" column - a column that provides the unique - * identifier for each row in the table. Once this column is created, any element in the table can be referenced by - * the row-column coordinate, i.e. "primary key"-"column name" coordinate. - *

- * When a column is added to a table, a default value must be specified (usually 0). This is the initial value for - * an element in a column. This permits operations like increment() and decrement() to work properly on columns that - * are effectively counters for a particular event. - *

- * Finally, the display property for each column can be set during column creation. This is useful when a given - * column stores an intermediate result that will be used later on, perhaps to calculate the value of another column. - * In these cases, it's obviously necessary to store the value required for further computation, but it's not - * necessary to actually print the intermediate column. - *

+ * The second line, specifying a unique name for each column in the table. + * + * The first column mentioned in the table header is the "primary key" column - a column that provides the unique + * identifier for each row in the table. Once this column is created, any element in the table can be referenced by + * the row-column coordinate, i.e. "primary key"-"column name" coordinate. + * + * When a column is added to a table, a default value must be specified (usually 0). This is the initial value for + * an element in a column. This permits operations like increment() and decrement() to work properly on columns that + * are effectively counters for a particular event. + * + * Finally, the display property for each column can be set during column creation. This is useful when a given + * column stores an intermediate result that will be used later on, perhaps to calculate the value of another column. + * In these cases, it's obviously necessary to store the value required for further computation, but it's not + * necessary to actually print the intermediate column. + * * Table body: - * The values of the table itself. - *

+ * The values of the table itself. + * * --------------- * Implementation: - *

+ * * The implementation of this table has two components: - * 1. A TreeSet that stores all the values ever specified for the primary key. Any get() operation that - * refers to an element where the primary key object does not exist will result in its implicit creation. I - * haven't yet decided if this is a good idea... - *

- * 2. A HashMap that stores a mapping from column name to column contents. Each - * GATKReportColumn is effectively a map (in fact, GATKReportColumn extends TreeMap) between - * primary key and the column value. This means that, given N columns, the primary key information is stored - * N+1 times. This is obviously wasteful and can likely be handled much more elegantly in future implementations. - *

+ * 1. A TreeSet that stores all the values ever specified for the primary key. Any get() operation that + * refers to an element where the primary key object does not exist will result in its implicit creation. I + * haven't yet decided if this is a good idea... + * + * 2. A HashMap that stores a mapping from column name to column contents. Each + * GATKReportColumn is effectively a map (in fact, GATKReportColumn extends TreeMap) between + * primary key and the column value. This means that, given N columns, the primary key information is stored + * N+1 times. This is obviously wasteful and can likely be handled much more elegantly in future implementations. + * * ------------------------------ * Element and column operations: - *

+ * * In addition to simply getting and setting values, this object also permits some simple operations to be applied to * individual elements or to whole columns. For instance, an element can be easily incremented without the hassle of * calling get(), incrementing the obtained value by 1, and then calling set() with the new value. Also, some vector * operations are supported. For instance, two whole columns can be divided and have the result be set to a third * column. This is especially useful when aggregating counts in two intermediate columns that will eventually need to * be manipulated row-by-row to compute the final column. - *

+ * * Note: I've made no attempt whatsoever to make these operations efficient. Right now, some of the methods check the * type of the stored object using an instanceof call and attempt to do the right thing. Others cast the contents of * the cell to a Number, call the Number.toDouble() method and compute a result. This is clearly not the ideal design, @@ -95,9 +92,7 @@ * @author Khalid Shakir */ public class GATKReportTable { - /** - * REGEX that matches any table with an invalid name - */ + /** REGEX that matches any table with an invalid name */ public final static String INVALID_TABLE_NAME_REGEX = "[^a-zA-Z0-9_\\-\\.]"; private static final GATKReportVersion LATEST_REPORT_VERSION = GATKReportVersion.V0_2; private String tableName; @@ -114,8 +109,8 @@ public class GATKReportTable { /** * Verifies that a table or column name has only alphanumeric characters - no spaces or special characters allowed * - * @param name the name of the table or column - * @return true if the name is valid, false if otherwise + * @param name the name of the table or column + * @return true if the name is valid, false if otherwise */ private boolean isValidName(String name) { Pattern p = Pattern.compile(INVALID_TABLE_NAME_REGEX); @@ -127,8 +122,8 @@ private boolean isValidName(String name) { /** * Verifies that a table or column name has only alphanumeric characters - no spaces or special characters allowed * - * @param description the name of the table or column - * @return true if the name is valid, false if otherwise + * @param description the name of the table or column + * @return true if the name is valid, false if otherwise */ private boolean isValidDescription(String description) { Pattern p = Pattern.compile("\\r|\\n"); @@ -140,15 +135,15 @@ private boolean isValidDescription(String description) { /** * Construct a new GATK report table with the specified name and description * - * @param tableName the name of the table - * @param tableDescription the description of the table + * @param tableName the name of the table + * @param tableDescription the description of the table */ public GATKReportTable(String tableName, String tableDescription) { this(tableName, tableDescription, true); } public GATKReportTable(String tableName, String tableDescription, boolean sortByPrimaryKey) { - if (!isValidName(tableName)) { + if (!isValidName(tableName)) { throw new ReviewedStingException("Attempted to set a GATKReportTable name of '" + tableName + "'. GATKReportTable names must be purely alphanumeric - no spaces or special characters are allowed."); } @@ -174,7 +169,7 @@ protected void setVersion(GATKReportVersion version) { /** * Add a primary key column. This becomes the unique identifier for every column in the table. * - * @param primaryKeyName the name of the primary key column + * @param primaryKeyName the name of the primary key column */ public void addPrimaryKey(String primaryKeyName) { addPrimaryKey(primaryKeyName, true); @@ -183,8 +178,8 @@ public void addPrimaryKey(String primaryKeyName) { /** * Add an optionally visible primary key column. This becomes the unique identifier for every column in the table, and will always be printed as the first column. * - * @param primaryKeyName the name of the primary key column - * @param display should this primary key be displayed? + * @param primaryKeyName the name of the primary key column + * @param display should this primary key be displayed? */ public void addPrimaryKey(String primaryKeyName, boolean display) { if (!isValidName(primaryKeyName)) { @@ -200,7 +195,6 @@ public void addPrimaryKey(String primaryKeyName, boolean display) { /** * Returns the first primary key matching the dotted column values. * Ex: dbsnp.eval.called.all.novel.all - * * @param dottedColumnValues Period concatenated values. * @return The first primary key matching the column values or throws an exception. */ @@ -214,7 +208,6 @@ public Object getPrimaryKey(String dottedColumnValues) { /** * Returns true if there is at least on row with the dotted column values. * Ex: dbsnp.eval.called.all.novel.all - * * @param dottedColumnValues Period concatenated values. * @return true if there is at least one row matching the columns. */ @@ -225,7 +218,6 @@ public boolean containsPrimaryKey(String dottedColumnValues) { /** * Returns the first primary key matching the dotted column values. * Ex: dbsnp.eval.called.all.novel.all - * * @param dottedColumnValues Period concatenated values. * @return The first primary key matching the column values or null. */ @@ -236,7 +228,6 @@ private Object findPrimaryKey(String dottedColumnValues) { /** * Returns the first primary key matching the column values. * Ex: new String[] { "dbsnp", "eval", "called", "all", "novel", "all" } - * * @param columnValues column values. * @return The first primary key matching the column values. */ @@ -244,7 +235,7 @@ private Object findPrimaryKey(Object[] columnValues) { for (Object primaryKey : primaryKeyColumn) { boolean matching = true; for (int i = 0; matching && i < columnValues.length; i++) { - matching = ObjectUtils.equals(columnValues[i], get(primaryKey, i + 1)); + matching = ObjectUtils.equals(columnValues[i], get(primaryKey, i+1)); } if (matching) return primaryKey; @@ -255,8 +246,8 @@ private Object findPrimaryKey(Object[] columnValues) { /** * Add a column to the report and specify the default value that should be supplied if a given position in the table is never explicitly set. * - * @param columnName the name of the column - * @param defaultValue the default value for the column + * @param columnName the name of the column + * @param defaultValue the default value for the column */ public void addColumn(String columnName, Object defaultValue) { addColumn(columnName, defaultValue, null); @@ -265,13 +256,12 @@ public void addColumn(String columnName, Object defaultValue) { public void addColumn(String columnName, Object defaultValue, String format) { addColumn(columnName, defaultValue, true, format); } - /** * Add a column to the report, specify the default column value, and specify whether the column should be displayed in the final output (useful when intermediate columns are necessary for later calculations, but are not required to be in the output file. * - * @param columnName the name of the column - * @param defaultValue the default value of the column - * @param display if true - the column will be displayed; if false - the column will be hidden + * @param columnName the name of the column + * @param defaultValue the default value of the column + * @param display if true - the column will be displayed; if false - the column will be hidden */ public void addColumn(String columnName, Object defaultValue, boolean display) { addColumn(columnName, defaultValue, display, null); @@ -287,8 +277,8 @@ public void addColumn(String columnName, Object defaultValue, boolean display, S /** * Check if the requested element exists, and if not, create it. * - * @param primaryKey the primary key value - * @param columnName the name of the column + * @param primaryKey the primary key value + * @param columnName the name of the column */ private void verifyEntry(Object primaryKey, String columnName) { if (!columns.containsKey(columnName)) { @@ -309,9 +299,9 @@ public boolean containsKey(Object primaryKey) { /** * Set the value for a given position in the table * - * @param primaryKey the primary key value - * @param columnName the name of the column - * @param value the value to set + * @param primaryKey the primary key value + * @param columnName the name of the column + * @param value the value to set */ public void set(Object primaryKey, String columnName, Object value) { verifyEntry(primaryKey, columnName); @@ -322,13 +312,13 @@ public void set(Object primaryKey, String columnName, Object value) { /** * Get a value from the given position in the table * - * @param primaryKey the primary key value - * @param columnName the name of the column - * @return the value stored at the specified position in the table + * @param primaryKey the primary key value + * @param columnName the name of the column + * @return the value stored at the specified position in the table */ public Object get(Object primaryKey, String columnName) { verifyEntry(primaryKey, columnName); - + return columns.get(columnName).get(primaryKey); } @@ -337,7 +327,7 @@ public Object get(Object primaryKey, String columnName) { * * @param primaryKey the primary key value * @param columnIndex the index of the column - * @return the value stored at the specified position in the table + * @return the value stored at the specified position in the table */ private Object get(Object primaryKey, int columnIndex) { return columns.getByIndex(columnIndex).get(primaryKey); @@ -346,8 +336,8 @@ private Object get(Object primaryKey, int columnIndex) { /** * Increment an element in the table. This implementation is awful - a functor would probably be better. * - * @param primaryKey the primary key value - * @param columnName the name of the column + * @param primaryKey the primary key value + * @param columnName the name of the column */ public void increment(Object primaryKey, String columnName) { Object oldValue = get(primaryKey, columnName); @@ -375,8 +365,8 @@ public void increment(Object primaryKey, String columnName) { /** * Decrement an element in the table. This implementation is awful - a functor would probably be better. * - * @param primaryKey the primary key value - * @param columnName the name of the column + * @param primaryKey the primary key value + * @param columnName the name of the column */ public void decrement(Object primaryKey, String columnName) { Object oldValue = get(primaryKey, columnName); @@ -404,9 +394,9 @@ public void decrement(Object primaryKey, String columnName) { /** * Add the specified value to an element in the table * - * @param primaryKey the primary key value - * @param columnName the name of the column - * @param valueToAdd the value to add + * @param primaryKey the primary key value + * @param columnName the name of the column + * @param valueToAdd the value to add */ public void add(Object primaryKey, String columnName, Object valueToAdd) { Object oldValue = get(primaryKey, columnName); @@ -434,8 +424,8 @@ public void add(Object primaryKey, String columnName, Object valueToAdd) { /** * Subtract the specified value from an element in the table * - * @param primaryKey the primary key value - * @param columnName the name of the column + * @param primaryKey the primary key value + * @param columnName the name of the column * @param valueToSubtract the value to subtract */ public void subtract(Object primaryKey, String columnName, Object valueToSubtract) { @@ -464,9 +454,9 @@ public void subtract(Object primaryKey, String columnName, Object valueToSubtrac /** * Multiply the specified value to an element in the table * - * @param primaryKey the primary key value - * @param columnName the name of the column - * @param valueToMultiply the value to multiply by + * @param primaryKey the primary key value + * @param columnName the name of the column + * @param valueToMultiply the value to multiply by */ public void multiply(Object primaryKey, String columnName, Object valueToMultiply) { Object oldValue = get(primaryKey, columnName); @@ -494,9 +484,9 @@ public void multiply(Object primaryKey, String columnName, Object valueToMultipl /** * Divide the specified value from an element in the table * - * @param primaryKey the primary key value - * @param columnName the name of the column - * @param valueToDivide the value to divide by + * @param primaryKey the primary key value + * @param columnName the name of the column + * @param valueToDivide the value to divide by */ public void divide(Object primaryKey, String columnName, Object valueToDivide) { Object oldValue = get(primaryKey, columnName); @@ -524,9 +514,9 @@ public void divide(Object primaryKey, String columnName, Object valueToDivide) { /** * Add two columns to each other and set the results to a third column * - * @param columnToSet the column that should hold the results - * @param augend the column that shall be the augend - * @param addend the column that shall be the addend + * @param columnToSet the column that should hold the results + * @param augend the column that shall be the augend + * @param addend the column that shall be the addend */ public void addColumns(String columnToSet, String augend, String addend) { for (Object primaryKey : primaryKeyColumn) { @@ -542,8 +532,8 @@ public void addColumns(String columnToSet, String augend, String addend) { /** * Subtract one column from another and set the results to a third column * - * @param columnToSet the column that should hold the results - * @param minuend the column that shall be the minuend (the a in a - b) + * @param columnToSet the column that should hold the results + * @param minuend the column that shall be the minuend (the a in a - b) * @param subtrahend the column that shall be the subtrahend (the b in a - b) */ public void subtractColumns(String columnToSet, String minuend, String subtrahend) { @@ -561,8 +551,8 @@ public void subtractColumns(String columnToSet, String minuend, String subtrahen * Multiply two columns by each other and set the results to a third column * * @param columnToSet the column that should hold the results - * @param multiplier the column that shall be the multiplier - * @param multiplicand the column that shall be the multiplicand + * @param multiplier the column that shall be the multiplier + * @param multiplicand the column that shall be the multiplicand */ public void multiplyColumns(String columnToSet, String multiplier, String multiplicand) { for (Object primaryKey : primaryKeyColumn) { @@ -578,9 +568,9 @@ public void multiplyColumns(String columnToSet, String multiplier, String multip /** * Divide two columns by each other and set the results to a third column * - * @param columnToSet the column that should hold the results - * @param numeratorColumn the column that shall be the numerator - * @param denominatorColumn the column that shall be the denominator + * @param columnToSet the column that should hold the results + * @param numeratorColumn the column that shall be the numerator + * @param denominatorColumn the column that shall be the denominator */ public void divideColumns(String columnToSet, String numeratorColumn, String denominatorColumn) { for (Object primaryKey : primaryKeyColumn) { @@ -595,11 +585,10 @@ public void divideColumns(String columnToSet, String numeratorColumn, String den /** * Return the print width of the primary key column - * - * @return the width of the primary key column + * @return the width of the primary key column */ public int getPrimaryKeyColumnWidth() { - int maxWidth = getPrimaryKeyName().length(); + int maxWidth = primaryKeyName.length(); for (Object primaryKey : primaryKeyColumn) { int width = primaryKey.toString().length(); @@ -615,7 +604,7 @@ public int getPrimaryKeyColumnWidth() { /** * Write the table to the PrintStream, formatted nicely to be human-readable, AWK-able, and R-friendly. * - * @param out the PrintStream to which the table should be written + * @param out the PrintStream to which the table should be written */ public void write(PrintStream out) { // Get the column widths for everything @@ -631,15 +620,13 @@ public void write(PrintStream out) { // Emit the table header, taking into account the padding requirement if the primary key is a hidden column boolean needsPadding = false; if (primaryKeyDisplay) { - out.printf(primaryKeyFormat, getPrimaryKeyName()); + out.printf(primaryKeyFormat, primaryKeyName); needsPadding = true; } for (String columnName : columns.keySet()) { if (columns.get(columnName).isDisplayable()) { - if (needsPadding) { - out.printf(" "); - } + if (needsPadding) { out.printf(" "); } out.printf(columnFormats.get(columnName).getNameFormat(), columnName); needsPadding = true; @@ -658,9 +645,7 @@ public void write(PrintStream out) { for (String columnName : columns.keySet()) { if (columns.get(columnName).isDisplayable()) { - if (needsPadding) { - out.printf(" "); - } + if (needsPadding) { out.printf(" "); } String value = columns.get(columnName).getStringValue(primaryKey); out.printf(columnFormats.get(columnName).getValueFormat(), value); @@ -690,49 +675,4 @@ public String getTableDescription() { public GATKReportColumns getColumns() { return columns; } - - public void mergeRows(GATKReportTable input) { - /* - * This function is different from addRowsFrom because we will add the ability to sum,average, etc rows - * TODO: Add other combining algorithms - */ - - // Make sure the columns match AND the Primary Key - if (input.getColumns().keySet().equals(this.getColumns().keySet()) && - input.getPrimaryKeyName().equals(this.getPrimaryKeyName())) { - this.addRowsFrom(input); - } else - throw new ReviewedStingException("Failed to combine GATKReportTable, columns don't match!"); - } - - public void addRowsFrom(GATKReportTable input) { - // add column by column - - // For every column - for (String columnKey : input.getColumns().keySet()) { - GATKReportColumn current = this.getColumns().get(columnKey); - GATKReportColumn toAdd = input.getColumns().get(columnKey); - // We want to take the current column and add all the values from input - - // The column is a map of values - for (Object rowKey : toAdd.keySet()) { - // We add every value from toAdd to the current - if (!current.containsKey(rowKey)) { - this.set(rowKey, columnKey, toAdd.get(rowKey)); - System.out.printf("Putting row with PK: %s \n", rowKey); - } else { - - // TODO we should be able to handle combining data by adding, averaging, etc. - this.set(rowKey, columnKey, toAdd.get(rowKey)); - - System.out.printf("OVERWRITING Row with PK: %s \n", rowKey); - } - } - } - - } - - public String getPrimaryKeyName() { - return primaryKeyName; - } } diff --git a/public/java/test/org/broadinstitute/sting/gatk/report/GATKReportUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/report/GATKReportUnitTest.java index b9a89fcfe7..b3b9ab555c 100644 --- a/public/java/test/org/broadinstitute/sting/gatk/report/GATKReportUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/report/GATKReportUnitTest.java @@ -49,23 +49,23 @@ public void testParse() throws Exception { @DataProvider(name = "rightAlignValues") public Object[][] getRightAlignValues() { - return new Object[][]{ - new Object[]{null, true}, - new Object[]{"null", true}, - new Object[]{"NA", true}, - new Object[]{"0", true}, - new Object[]{"0.0", true}, - new Object[]{"-0", true}, - new Object[]{"-0.0", true}, - new Object[]{String.valueOf(Long.MAX_VALUE), true}, - new Object[]{String.valueOf(Long.MIN_VALUE), true}, - new Object[]{String.valueOf(Float.MIN_NORMAL), true}, - new Object[]{String.valueOf(Double.MAX_VALUE), true}, - new Object[]{String.valueOf(Double.MIN_VALUE), true}, - new Object[]{String.valueOf(Double.POSITIVE_INFINITY), true}, - new Object[]{String.valueOf(Double.NEGATIVE_INFINITY), true}, - new Object[]{String.valueOf(Double.NaN), true}, - new Object[]{"hello", false} + return new Object[][] { + new Object[] {null, true}, + new Object[] {"null", true}, + new Object[] {"NA", true}, + new Object[] {"0", true}, + new Object[] {"0.0", true}, + new Object[] {"-0", true}, + new Object[] {"-0.0", true}, + new Object[] {String.valueOf(Long.MAX_VALUE), true}, + new Object[] {String.valueOf(Long.MIN_VALUE), true}, + new Object[] {String.valueOf(Float.MIN_NORMAL), true}, + new Object[] {String.valueOf(Double.MAX_VALUE), true}, + new Object[] {String.valueOf(Double.MIN_VALUE), true}, + new Object[] {String.valueOf(Double.POSITIVE_INFINITY), true}, + new Object[] {String.valueOf(Double.NEGATIVE_INFINITY), true}, + new Object[] {String.valueOf(Double.NaN), true}, + new Object[] {"hello", false} }; } @@ -73,96 +73,4 @@ public Object[][] getRightAlignValues() { public void testIsRightAlign(String value, boolean expected) { Assert.assertEquals(GATKReportColumn.isRightAlign(value), expected, "right align of '" + value + "'"); } - - @Test - public void testGATKReportGatherer() { - - /* - GATKReportTable actual1 = new GATKReportTable("TableName", "Description"); - actual1.addPrimaryKey("key"); - actual1.addColumn("colA", 0); - actual1.addColumn("colB", 0); - actual1.set("row1", "colA", 1); - actual1.set("row1", "colB", 2); - - GATKReportTable actual2 = new GATKReportTable("TableName", "Description"); - actual2.addPrimaryKey("key"); - actual2.addColumn("colA", 0); - actual2.addColumn("colB", 0); - actual2.set("row2", "colA", 3); - actual2.set("row2", "colB", 4); - - GATKReportTable actual3 = new GATKReportTable("TableName", "Description"); - actual3.addPrimaryKey("key"); - actual3.addColumn("colA", 0); - actual3.addColumn("colB", 0); - actual3.set("row3", "colA", 5); - actual3.set("row3", "colB", 6); - - actual1.mergeRows(actual2); - actual1.mergeRows(actual3); - actual1.write(System.out); - */ - - GATKReportTable expected = new GATKReportTable("TableName", "Description"); - expected.addPrimaryKey("key"); - expected.addColumn("colA", 0); - expected.addColumn("colB", 0); - expected.set("row1", "colA", 1); - expected.set("row1", "colB", 2); - expected.set("row2", "colA", 3); - expected.set("row2", "colB", 4); - expected.set("row3", "colA", 5); - expected.set("row3", "colB", 6); - expected.write(System.out); - - GATKReport report1, report2, report3; - report1 = new GATKReport(); - report1.addTable("TableName", "Description"); - report1.getTable("TableName").addPrimaryKey("key"); - report1.getTable("TableName").addColumn("colA", 0); - report1.getTable("TableName").addColumn("colB", 0); - report1.getTable("TableName").set("row1", "colA", 1); - report1.getTable("TableName").set("row1", "colB", 2); - - report2 = new GATKReport(); - report2.addTable("TableName", "Description"); - report2.getTable("TableName").addPrimaryKey("key"); - report2.getTable("TableName").addColumn("colA", 0); - report2.getTable("TableName").addColumn("colB", 0); - report2.getTable("TableName").set("row2", "colA", 3); - report2.getTable("TableName").set("row2", "colB", 4); - - report3 = new GATKReport(); - report3.addTable("TableName", "Description"); - report3.getTable("TableName").addPrimaryKey("key"); - report3.getTable("TableName").addColumn("colA", 0); - report3.getTable("TableName").addColumn("colB", 0); - report3.getTable("TableName").set("row3", "colA", 5); - report3.getTable("TableName").set("row3", "colB", 6); - - report1.combineWith(report2); - report1.combineWith(report3); - - report1.print(System.out); - /* - File a = new File("/home/roger/tbls/a.tbl"); - File b = new File("/home/roger/tbls/b.tbl"); - File c = new File("/home/roger/tbls/c.tbl"); - File out = new File("/home/roger/tbls/out.tbl"); - - - List FileList = new ArrayList(); - FileList.add(a); - FileList.add(b); - FileList.add(c); - - GATKReportGatherer gatherer = new GATKReportGatherer(); - gatherer.gather(FileList, out); - System.out.print(out); - */ - - //Assert.assertEquals(1,1); - - } -} \ No newline at end of file +} From 5b3d875833992fcea1e59ab3f4b00bf9d73d21fd Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Thu, 9 Feb 2012 09:05:08 -0500 Subject: [PATCH 235/356] Incorporating Mark's suggestions on the AnalyzeCovariates plots. From b57d4250bfae1ea8c6cdb5da8fca83e5affa01ea Mon Sep 17 00:00:00 2001 From: Matt Hanna Date: Thu, 9 Feb 2012 11:24:52 -0500 Subject: [PATCH 236/356] Documentation request by Eric. At each stage of the GATK where filtering occurs, added documentation suggesting the goal of the filtering along with examples of suggested inputs and outputs. --- .../gatk/datasources/providers/LocusView.java | 9 +- .../IntervalOverlapFilteringIterator.java | 203 ++++++++++++++++++ .../gatk/datasources/reads/SAMDataSource.java | 162 -------------- .../sting/gatk/executive/WindowMaker.java | 18 +- .../sting/gatk/traversals/TraverseLoci.java | 4 +- 5 files changed, 228 insertions(+), 168 deletions(-) create mode 100644 public/java/src/org/broadinstitute/sting/gatk/datasources/reads/IntervalOverlapFilteringIterator.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/providers/LocusView.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/providers/LocusView.java index f9ed0cb747..a3ce6dd278 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/providers/LocusView.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/providers/LocusView.java @@ -25,9 +25,14 @@ */ /** - * A queue of locus context entries. + * The two goals of the LocusView are as follows: + * 1) To provide a 'trigger track' iteration interface so that TraverseLoci can easily switch + * between iterating over all bases in a region, only covered bases in a region covered by + * reads, only bases in a region covered by RODs, or any other sort of trigger track + * implementation one can think of. + * 2) To manage the copious number of iterators that have to be jointly pulled through the + * genome to make a locus traversal function. */ - public abstract class LocusView extends LocusIterator implements View { /** * The locus bounding this view. diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/IntervalOverlapFilteringIterator.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/IntervalOverlapFilteringIterator.java new file mode 100644 index 0000000000..4005f1c321 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/IntervalOverlapFilteringIterator.java @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.datasources.reads; + +import net.sf.samtools.SAMRecord; +import net.sf.samtools.util.CloseableIterator; +import org.broadinstitute.sting.utils.GenomeLoc; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; + +import java.util.List; +import java.util.NoSuchElementException; + +/** + * High efficiency filtering iterator designed to filter out reads only included + * in the query results due to the granularity of the BAM index. + * + * Built into the BAM index is a notion of 16kbase granularity -- an index query for + * two regions contained within a 16kbase chunk (say, chr1:5-10 and chr1:11-20) will + * return exactly the same regions within the BAM file. This iterator is optimized + * to subtract out reads which do not at all overlap the interval list passed to the + * constructor. + * + * Example: + * interval list: chr20:6-10 + * Reads that would pass through the filter: chr20:6-10, chr20:1-15, chr20:1-7, chr20:8-15. + * Reads that would be discarded by the filter: chr20:1-5, chr20:11-15. + */ +class IntervalOverlapFilteringIterator implements CloseableIterator { + /** + * The wrapped iterator. + */ + private CloseableIterator iterator; + + /** + * The next read, queued up and ready to go. + */ + private SAMRecord nextRead; + + /** + * Rather than using the straight genomic bounds, use filter out only mapped reads. + */ + private boolean keepOnlyUnmappedReads; + + /** + * Custom representation of interval bounds. + * Makes it simpler to track current position. + */ + private int[] intervalContigIndices; + private int[] intervalStarts; + private int[] intervalEnds; + + /** + * Position within the interval list. + */ + private int currentBound = 0; + + public IntervalOverlapFilteringIterator(CloseableIterator iterator, List intervals) { + this.iterator = iterator; + + // Look at the interval list to detect whether we should worry about unmapped reads. + // If we find a mix of mapped/unmapped intervals, throw an exception. + boolean foundMappedIntervals = false; + for(GenomeLoc location: intervals) { + if(! GenomeLoc.isUnmapped(location)) + foundMappedIntervals = true; + keepOnlyUnmappedReads |= GenomeLoc.isUnmapped(location); + } + + + if(foundMappedIntervals) { + if(keepOnlyUnmappedReads) + throw new ReviewedStingException("Tried to apply IntervalOverlapFilteringIterator to a mixed of mapped and unmapped intervals. Please apply this filter to only mapped or only unmapped reads"); + this.intervalContigIndices = new int[intervals.size()]; + this.intervalStarts = new int[intervals.size()]; + this.intervalEnds = new int[intervals.size()]; + int i = 0; + for(GenomeLoc interval: intervals) { + intervalContigIndices[i] = interval.getContigIndex(); + intervalStarts[i] = interval.getStart(); + intervalEnds[i] = interval.getStop(); + i++; + } + } + + advance(); + } + + public boolean hasNext() { + return nextRead != null; + } + + public SAMRecord next() { + if(nextRead == null) + throw new NoSuchElementException("No more reads left in this iterator."); + SAMRecord currentRead = nextRead; + advance(); + return currentRead; + } + + public void remove() { + throw new UnsupportedOperationException("Cannot remove from an IntervalOverlapFilteringIterator"); + } + + + public void close() { + iterator.close(); + } + + private void advance() { + nextRead = null; + + if(!iterator.hasNext()) + return; + + SAMRecord candidateRead = iterator.next(); + while(nextRead == null && (keepOnlyUnmappedReads || currentBound < intervalStarts.length)) { + if(!keepOnlyUnmappedReads) { + // Mapped read filter; check against GenomeLoc-derived bounds. + if(readEndsOnOrAfterStartingBound(candidateRead)) { + // This read ends after the current interval begins. + // Promising, but this read must be checked against the ending bound. + if(readStartsOnOrBeforeEndingBound(candidateRead)) { + // Yes, this read is within both bounds. This must be our next read. + nextRead = candidateRead; + break; + } + else { + // Oops, we're past the end bound. Increment the current bound and try again. + currentBound++; + continue; + } + } + } + else { + // Found an unmapped read. We're done. + if(candidateRead.getReadUnmappedFlag()) { + nextRead = candidateRead; + break; + } + } + + // No more reads available. Stop the search. + if(!iterator.hasNext()) + break; + + // No reasonable read found; advance the iterator. + candidateRead = iterator.next(); + } + } + + /** + * Check whether the read lies after the start of the current bound. If the read is unmapped but placed, its + * end will be distorted, so rely only on the alignment start. + * @param read The read to position-check. + * @return True if the read starts after the current bounds. False otherwise. + */ + private boolean readEndsOnOrAfterStartingBound(final SAMRecord read) { + return + // Read ends on a later contig, or... + read.getReferenceIndex() > intervalContigIndices[currentBound] || + // Read ends of this contig... + (read.getReferenceIndex() == intervalContigIndices[currentBound] && + // either after this location, or... + (read.getAlignmentEnd() >= intervalStarts[currentBound] || + // read is unmapped but positioned and alignment start is on or after this start point. + (read.getReadUnmappedFlag() && read.getAlignmentStart() >= intervalStarts[currentBound]))); + } + + /** + * Check whether the read lies before the end of the current bound. + * @param read The read to position-check. + * @return True if the read starts after the current bounds. False otherwise. + */ + private boolean readStartsOnOrBeforeEndingBound(final SAMRecord read) { + return + // Read starts on a prior contig, or... + read.getReferenceIndex() < intervalContigIndices[currentBound] || + // Read starts on this contig and the alignment start is registered before this end point. + (read.getReferenceIndex() == intervalContigIndices[currentBound] && read.getAlignmentStart() <= intervalEnds[currentBound]); + } +} diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java index a4681cffd3..27b9e7f778 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java @@ -39,7 +39,6 @@ import org.broadinstitute.sting.gatk.filters.ReadFilter; import org.broadinstitute.sting.gatk.iterators.*; import org.broadinstitute.sting.gatk.resourcemanagement.ThreadAllocation; -import org.broadinstitute.sting.utils.GenomeLoc; import org.broadinstitute.sting.utils.GenomeLocParser; import org.broadinstitute.sting.utils.GenomeLocSortedSet; import org.broadinstitute.sting.utils.SimpleTimer; @@ -976,167 +975,6 @@ public SAMRecord next() { */ private class ReadGroupMapping extends HashMap {} - /** - * Filters out reads that do not overlap the current GenomeLoc. - * Note the custom implementation: BAM index querying returns all reads that could - * possibly overlap the given region (and quite a few extras). In order not to drag - * down performance, this implementation is highly customized to its task. - */ - private class IntervalOverlapFilteringIterator implements CloseableIterator { - /** - * The wrapped iterator. - */ - private CloseableIterator iterator; - - /** - * The next read, queued up and ready to go. - */ - private SAMRecord nextRead; - - /** - * Rather than using the straight genomic bounds, use filter out only mapped reads. - */ - private boolean keepOnlyUnmappedReads; - - /** - * Custom representation of interval bounds. - * Makes it simpler to track current position. - */ - private int[] intervalContigIndices; - private int[] intervalStarts; - private int[] intervalEnds; - - /** - * Position within the interval list. - */ - private int currentBound = 0; - - public IntervalOverlapFilteringIterator(CloseableIterator iterator, List intervals) { - this.iterator = iterator; - - // Look at the interval list to detect whether we should worry about unmapped reads. - // If we find a mix of mapped/unmapped intervals, throw an exception. - boolean foundMappedIntervals = false; - for(GenomeLoc location: intervals) { - if(! GenomeLoc.isUnmapped(location)) - foundMappedIntervals = true; - keepOnlyUnmappedReads |= GenomeLoc.isUnmapped(location); - } - - - if(foundMappedIntervals) { - if(keepOnlyUnmappedReads) - throw new ReviewedStingException("Tried to apply IntervalOverlapFilteringIterator to a mixed of mapped and unmapped intervals. Please apply this filter to only mapped or only unmapped reads"); - this.intervalContigIndices = new int[intervals.size()]; - this.intervalStarts = new int[intervals.size()]; - this.intervalEnds = new int[intervals.size()]; - int i = 0; - for(GenomeLoc interval: intervals) { - intervalContigIndices[i] = interval.getContigIndex(); - intervalStarts[i] = interval.getStart(); - intervalEnds[i] = interval.getStop(); - i++; - } - } - - advance(); - } - - public boolean hasNext() { - return nextRead != null; - } - - public SAMRecord next() { - if(nextRead == null) - throw new NoSuchElementException("No more reads left in this iterator."); - SAMRecord currentRead = nextRead; - advance(); - return currentRead; - } - - public void remove() { - throw new UnsupportedOperationException("Cannot remove from an IntervalOverlapFilteringIterator"); - } - - - public void close() { - iterator.close(); - } - - private void advance() { - nextRead = null; - - if(!iterator.hasNext()) - return; - - SAMRecord candidateRead = iterator.next(); - while(nextRead == null && (keepOnlyUnmappedReads || currentBound < intervalStarts.length)) { - if(!keepOnlyUnmappedReads) { - // Mapped read filter; check against GenomeLoc-derived bounds. - if(readEndsOnOrAfterStartingBound(candidateRead)) { - // This read ends after the current interval begins. - // Promising, but this read must be checked against the ending bound. - if(readStartsOnOrBeforeEndingBound(candidateRead)) { - // Yes, this read is within both bounds. This must be our next read. - nextRead = candidateRead; - break; - } - else { - // Oops, we're past the end bound. Increment the current bound and try again. - currentBound++; - continue; - } - } - } - else { - // Found an unmapped read. We're done. - if(candidateRead.getReadUnmappedFlag()) { - nextRead = candidateRead; - break; - } - } - - // No more reads available. Stop the search. - if(!iterator.hasNext()) - break; - - // No reasonable read found; advance the iterator. - candidateRead = iterator.next(); - } - } - - /** - * Check whether the read lies after the start of the current bound. If the read is unmapped but placed, its - * end will be distorted, so rely only on the alignment start. - * @param read The read to position-check. - * @return True if the read starts after the current bounds. False otherwise. - */ - private boolean readEndsOnOrAfterStartingBound(final SAMRecord read) { - return - // Read ends on a later contig, or... - read.getReferenceIndex() > intervalContigIndices[currentBound] || - // Read ends of this contig... - (read.getReferenceIndex() == intervalContigIndices[currentBound] && - // either after this location, or... - (read.getAlignmentEnd() >= intervalStarts[currentBound] || - // read is unmapped but positioned and alignment start is on or after this start point. - (read.getReadUnmappedFlag() && read.getAlignmentStart() >= intervalStarts[currentBound]))); - } - - /** - * Check whether the read lies before the end of the current bound. - * @param read The read to position-check. - * @return True if the read starts after the current bounds. False otherwise. - */ - private boolean readStartsOnOrBeforeEndingBound(final SAMRecord read) { - return - // Read starts on a prior contig, or... - read.getReferenceIndex() < intervalContigIndices[currentBound] || - // Read starts on this contig and the alignment start is registered before this end point. - (read.getReferenceIndex() == intervalContigIndices[currentBound] && read.getAlignmentStart() <= intervalEnds[currentBound]); - } - } - /** * Locates the index file alongside the given BAM, if present. * TODO: This is currently a hachetjob that reaches into Picard and pulls out its index file locator. Replace with something more permanent. diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/WindowMaker.java b/public/java/src/org/broadinstitute/sting/gatk/executive/WindowMaker.java index d1f5d80daf..da11d36ddd 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/executive/WindowMaker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/executive/WindowMaker.java @@ -17,9 +17,21 @@ import java.util.NoSuchElementException; /** - * Buffer shards of data which may or may not contain multiple loci into - * iterators of all data which cover an interval. Its existence is an homage - * to Mark's stillborn WindowMaker, RIP 2009. + * Transforms an iterator of reads which overlap the given interval list into an iterator of covered single-base loci + * completely contained within the interval list. To do this, it creates a LocusIteratorByState which will emit a single-bp + * locus for every base covered by the read iterator, then uses the WindowMakerIterator.advance() to filter down that stream of + * loci to only those covered by the given interval list. + * + * Example: + * Incoming stream of reads: A:chr20:1-5, B:chr20:2-6, C:chr20:2-7, D:chr20:3-8, E:chr20:5-10 + * Incoming intervals: chr20:3-7 + * + * Locus iterator by state will produce the following stream of data: + * chr1:1 {A}, chr1:2 {A,B,C}, chr1:3 {A,B,C,D}, chr1:4 {A,B,C,D}, chr1:5 {A,B,C,D,E}, + * chr1:6 {B,C,D,E}, chr1:7 {C,D,E}, chr1:8 {D,E}, chr1:9 {E}, chr1:10 {E} + * + * WindowMakerIterator will then filter the incoming stream, emitting the following stream: + * chr1:3 {A,B,C,D}, chr1:4 {A,B,C,D}, chr1:5 {A,B,C,D,E}, chr1:6 {B,C,D,E}, chr1:7 {C,D,E} * * @author mhanna * @version 0.1 diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseLoci.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseLoci.java index d99e7c3539..1d14a7f35d 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseLoci.java +++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseLoci.java @@ -102,7 +102,9 @@ public T traverse( LocusWalker walker, } /** - * Gets the best view of loci for this walker given the available data. + * Gets the best view of loci for this walker given the available data. The view will function as a 'trigger track' + * of sorts, providing a consistent interface so that TraverseLoci doesn't need to be reimplemented for any new datatype + * that comes along. * @param walker walker to interrogate. * @param dataProvider Data which which to drive the locus view. * @return A view of the locus data, where one iteration of the locus view maps to one iteration of the traversal. From 0f728a0604c7c531b0419361673e8791b403f48b Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Thu, 9 Feb 2012 14:02:34 -0500 Subject: [PATCH 237/356] The Exact model now subsets the VC to the first N alleles when the VC contains more than the maximum number of alleles (instead of throwing it out completely as it did previously). [Perhaps the culling should be done by the UG engine? But theoretically the Exact model can be called outside of the UG and we'd still want the context subsetted.] --- .../AlleleFrequencyCalculationModel.java | 18 ++- .../genotyper/ExactAFCalculationModel.java | 26 ++- .../genotyper/UnifiedGenotyperEngine.java | 149 +++++++++--------- 3 files changed, 107 insertions(+), 86 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/AlleleFrequencyCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/AlleleFrequencyCalculationModel.java index 681cc1fa68..9f2403bbf7 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/AlleleFrequencyCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/AlleleFrequencyCalculationModel.java @@ -27,7 +27,7 @@ import org.apache.log4j.Logger; import org.broadinstitute.sting.utils.variantcontext.Allele; -import org.broadinstitute.sting.utils.variantcontext.GenotypesContext; +import org.broadinstitute.sting.utils.variantcontext.VariantContext; import java.io.PrintStream; import java.util.List; @@ -41,10 +41,11 @@ public abstract class AlleleFrequencyCalculationModel implements Cloneable { public enum Model { /** The default model with the best performance in all cases */ - EXACT, + EXACT } protected int N; + protected int MAX_ALTERNATE_ALLELES_TO_GENOTYPE; protected Logger logger; protected PrintStream verboseWriter; @@ -53,20 +54,21 @@ protected enum GenotypeType { AA, AB, BB } protected static final double VALUE_NOT_CALCULATED = Double.NEGATIVE_INFINITY; - protected AlleleFrequencyCalculationModel(UnifiedArgumentCollection UAC, int N, Logger logger, PrintStream verboseWriter) { + protected AlleleFrequencyCalculationModel(final UnifiedArgumentCollection UAC, final int N, final Logger logger, final PrintStream verboseWriter) { this.N = N; + this.MAX_ALTERNATE_ALLELES_TO_GENOTYPE = UAC.MAX_ALTERNATE_ALLELES; this.logger = logger; this.verboseWriter = verboseWriter; } /** * Must be overridden by concrete subclasses - * @param GLs genotype likelihoods - * @param Alleles Alleles corresponding to GLs + * @param vc variant context with alleles and genotype likelihoods * @param log10AlleleFrequencyPriors priors * @param result (pre-allocated) object to store likelihoods results + * @return the alleles used for genotyping */ - protected abstract void getLog10PNonRef(GenotypesContext GLs, List Alleles, - double[][] log10AlleleFrequencyPriors, - AlleleFrequencyCalculationResult result); + protected abstract List getLog10PNonRef(final VariantContext vc, + final double[][] log10AlleleFrequencyPriors, + final AlleleFrequencyCalculationResult result); } \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java index d604e8d62c..f9518a35c3 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java @@ -43,14 +43,28 @@ protected ExactAFCalculationModel(UnifiedArgumentCollection UAC, int N, Logger l super(UAC, N, logger, verboseWriter); } - public void getLog10PNonRef(final GenotypesContext GLs, - final List alleles, - final double[][] log10AlleleFrequencyPriors, - final AlleleFrequencyCalculationResult result) { - final int numAlleles = alleles.size(); + public List getLog10PNonRef(final VariantContext vc, + final double[][] log10AlleleFrequencyPriors, + final AlleleFrequencyCalculationResult result) { + + final GenotypesContext GLs = vc.getGenotypes(); + List alleles = vc.getAlleles(); + + // don't try to genotype too many alternate alleles + if ( vc.getAlternateAlleles().size() > MAX_ALTERNATE_ALLELES_TO_GENOTYPE ) { + logger.warn("this tool is currently set to genotype at most " + MAX_ALTERNATE_ALLELES_TO_GENOTYPE + " alternate alleles in a given context, but the context at " + vc.getChr() + ":" + vc.getStart() + " has " + (vc.getAlternateAlleles().size()) + " alternate alleles so only the top alleles will be used; see the --max_alternate_alleles argument"); + + alleles = new ArrayList(MAX_ALTERNATE_ALLELES_TO_GENOTYPE + 1); + alleles.add(vc.getReference()); + for ( int i = 0; i < MAX_ALTERNATE_ALLELES_TO_GENOTYPE; i++ ) + alleles.add(vc.getAlternateAllele(i)); + UnifiedGenotyperEngine.subsetAlleles(vc, alleles, false); + } //linearExact(GLs, log10AlleleFrequencyPriors[0], log10AlleleFrequencyLikelihoods, log10AlleleFrequencyPosteriors); - linearExactMultiAllelic(GLs, numAlleles - 1, log10AlleleFrequencyPriors, result, false); + linearExactMultiAllelic(GLs, alleles.size() - 1, log10AlleleFrequencyPriors, result, false); + + return alleles; } private static final ArrayList getGLs(GenotypesContext GLs) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java index 2eba6d8841..aa5776007e 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java @@ -295,12 +295,6 @@ public VariantCallContext calculateGenotypes(RefMetaDataTracker tracker, Referen } AlleleFrequencyCalculationResult AFresult = alleleFrequencyCalculationResult.get(); - // don't try to genotype too many alternate alleles - if ( vc.getAlternateAlleles().size() > UAC.MAX_ALTERNATE_ALLELES ) { - logger.warn("the Unified Genotyper is currently set to genotype at most " + UAC.MAX_ALTERNATE_ALLELES + " alternate alleles in a given context, but the context at " + vc.getChr() + ":" + vc.getStart() + " has " + vc.getAlternateAlleles().size() + " alternate alleles; see the --max_alternate_alleles argument"); - return null; - } - // estimate our confidence in a reference call and return if ( vc.getNSamples() == 0 ) { if ( limitedContext ) @@ -313,25 +307,32 @@ public VariantCallContext calculateGenotypes(RefMetaDataTracker tracker, Referen // 'zero' out the AFs (so that we don't have to worry if not all samples have reads at this position) clearAFarray(AFresult.log10AlleleFrequencyLikelihoods); clearAFarray(AFresult.log10AlleleFrequencyPosteriors); - afcm.get().getLog10PNonRef(vc.getGenotypes(), vc.getAlleles(), getAlleleFrequencyPriors(model), AFresult); + List allelesUsedInGenotyping = afcm.get().getLog10PNonRef(vc, getAlleleFrequencyPriors(model), AFresult); // is the most likely frequency conformation AC=0 for all alternate alleles? boolean bestGuessIsRef = true; // determine which alternate alleles have AF>0 - boolean[] altAllelesToUse = new boolean[vc.getAlternateAlleles().size()]; + final List myAlleles = new ArrayList(vc.getAlleles().size()); + myAlleles.add(vc.getReference()); for ( int i = 0; i < vc.getAlternateAlleles().size(); i++ ) { - int indexOfBestAC = MathUtils.maxElementIndex(AFresult.log10AlleleFrequencyPosteriors[i]); + final Allele alternateAllele = vc.getAlternateAllele(i); + final int indexOfAllele = allelesUsedInGenotyping.indexOf(alternateAllele); + // the genotyping model may have stripped it out + if ( indexOfAllele == -1 ) + continue; + + int indexOfBestAC = MathUtils.maxElementIndex(AFresult.log10AlleleFrequencyPosteriors[indexOfAllele-1]); // if the most likely AC is not 0, then this is a good alternate allele to use; // make sure to test against log10PosteriorOfAFzero since that no longer is an entry in the array - if ( indexOfBestAC != 0 && AFresult.log10AlleleFrequencyPosteriors[i][indexOfBestAC] > AFresult.log10PosteriorOfAFzero ) { - altAllelesToUse[i] = true; + if ( indexOfBestAC != 0 && AFresult.log10AlleleFrequencyPosteriors[indexOfAllele-1][indexOfBestAC] > AFresult.log10PosteriorOfAFzero ) { + myAlleles.add(alternateAllele); bestGuessIsRef = false; } // if in GENOTYPE_GIVEN_ALLELES mode, we still want to allow the use of a poor allele else if ( UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES ) { - altAllelesToUse[i] = true; + myAlleles.add(alternateAllele); } } @@ -367,20 +368,6 @@ else if ( UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_M return limitedContext ? null : estimateReferenceConfidence(vc, stratifiedContexts, getGenotypePriors(model).getHeterozygosity(), true, 1.0 - PofF); } - // strip out any alleles that aren't going to be used in the VariantContext - final List myAlleles; - if ( UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.DISCOVERY ) { - myAlleles = new ArrayList(vc.getAlleles().size()); - myAlleles.add(vc.getReference()); - for ( int i = 0; i < vc.getAlternateAlleles().size(); i++ ) { - if ( altAllelesToUse[i] ) - myAlleles.add(vc.getAlternateAllele(i)); - } - } else { - // use all of the alleles if we are given them by the user - myAlleles = vc.getAlleles(); - } - // start constructing the resulting VC final GenomeLoc loc = genomeLocParser.createGenomeLoc(vc); final VariantContextBuilder builder = new VariantContextBuilder("UG_call", loc.getContig(), loc.getStart(), loc.getStop(), myAlleles); @@ -394,7 +381,7 @@ else if ( UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_M } // create the genotypes - final GenotypesContext genotypes = assignGenotypes(vc, altAllelesToUse); + final GenotypesContext genotypes = subsetAlleles(vc, myAlleles, true); // print out stats if we have a writer if ( verboseWriter != null && !limitedContext ) @@ -414,7 +401,7 @@ else if ( UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_M VariantContext vcOverall = calculateLikelihoods(tracker, refContext, stratifiedContexts, AlignmentContextUtils.ReadOrientation.COMPLETE, vc.getAlternateAllele(0), false, model); clearAFarray(AFresult.log10AlleleFrequencyLikelihoods); clearAFarray(AFresult.log10AlleleFrequencyPosteriors); - afcm.get().getLog10PNonRef(vcOverall.getGenotypes(), vc.getAlleles(), getAlleleFrequencyPriors(model), AFresult); + afcm.get().getLog10PNonRef(vcOverall, getAlleleFrequencyPriors(model), AFresult); //double overallLog10PofNull = AFresult.log10AlleleFrequencyPosteriors[0]; double overallLog10PofF = MathUtils.log10sumLog10(AFresult.log10AlleleFrequencyPosteriors[0], 0); //if ( DEBUG_SLOD ) System.out.println("overallLog10PofF=" + overallLog10PofF); @@ -423,7 +410,7 @@ else if ( UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_M VariantContext vcForward = calculateLikelihoods(tracker, refContext, stratifiedContexts, AlignmentContextUtils.ReadOrientation.FORWARD, vc.getAlternateAllele(0), false, model); clearAFarray(AFresult.log10AlleleFrequencyLikelihoods); clearAFarray(AFresult.log10AlleleFrequencyPosteriors); - afcm.get().getLog10PNonRef(vcForward.getGenotypes(), vc.getAlleles(), getAlleleFrequencyPriors(model), AFresult); + afcm.get().getLog10PNonRef(vcForward, getAlleleFrequencyPriors(model), AFresult); //double[] normalizedLog10Posteriors = MathUtils.normalizeFromLog10(AFresult.log10AlleleFrequencyPosteriors, true); double forwardLog10PofNull = AFresult.log10PosteriorOfAFzero; double forwardLog10PofF = MathUtils.log10sumLog10(AFresult.log10AlleleFrequencyPosteriors[0], 0); @@ -433,7 +420,7 @@ else if ( UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_M VariantContext vcReverse = calculateLikelihoods(tracker, refContext, stratifiedContexts, AlignmentContextUtils.ReadOrientation.REVERSE, vc.getAlternateAllele(0), false, model); clearAFarray(AFresult.log10AlleleFrequencyLikelihoods); clearAFarray(AFresult.log10AlleleFrequencyPosteriors); - afcm.get().getLog10PNonRef(vcReverse.getGenotypes(), vc.getAlleles(), getAlleleFrequencyPriors(model), AFresult); + afcm.get().getLog10PNonRef(vcReverse, getAlleleFrequencyPriors(model), AFresult); //normalizedLog10Posteriors = MathUtils.normalizeFromLog10(AFresult.log10AlleleFrequencyPosteriors, true); double reverseLog10PofNull = AFresult.log10PosteriorOfAFzero; double reverseLog10PofF = MathUtils.log10sumLog10(AFresult.log10AlleleFrequencyPosteriors[0], 0); @@ -772,30 +759,36 @@ public static VariantContext getVCFromAllelesRod(RefMetaDataTracker tracker, Ref /** * @param vc variant context with genotype likelihoods - * @param allelesToUse bit vector describing which alternate alleles from the vc are okay to use * @return genotypes */ - public static GenotypesContext assignGenotypes(final VariantContext vc, - final boolean[] allelesToUse) { + public static GenotypesContext assignGenotypes(final VariantContext vc) { + return subsetAlleles(vc, vc.getAlleles(), true); + } + + /** + * @param vc variant context with genotype likelihoods + * @param allelesToUse which alleles from the vc are okay to use + * @param assignGenotypes true if we should change the genotypes based on the (subsetted) PLs + * @return genotypes + */ + public static GenotypesContext subsetAlleles(final VariantContext vc, + final List allelesToUse, + final boolean assignGenotypes) { - // the no-called genotypes - final GenotypesContext GLs = vc.getGenotypes(); + // the genotypes with PLs + final GenotypesContext oldGTs = vc.getGenotypes(); // samples - final List sampleIndices = GLs.getSampleNamesOrderedByName(); + final List sampleIndices = oldGTs.getSampleNamesOrderedByName(); - // the new called genotypes to create - final GenotypesContext calls = GenotypesContext.create(); + // the new genotypes to create + final GenotypesContext newGTs = GenotypesContext.create(); // we need to determine which of the alternate alleles (and hence the likelihoods) to use and carry forward - final int numOriginalAltAlleles = allelesToUse.length; - final List newAlleles = new ArrayList(numOriginalAltAlleles+1); - newAlleles.add(vc.getReference()); - for ( int i = 0; i < numOriginalAltAlleles; i++ ) { - if ( allelesToUse[i] ) - newAlleles.add(vc.getAlternateAllele(i)); - } - final int numNewAltAlleles = newAlleles.size() - 1; + final int numOriginalAltAlleles = vc.getAlternateAlleles().size(); + final int numNewAltAlleles = allelesToUse.size() - 1; + + // which PLs should be carried forward? ArrayList likelihoodIndexesToUse = null; // an optimization: if we are supposed to use all (or none in the case of a ref call) of the alleles, @@ -804,20 +797,27 @@ public static GenotypesContext assignGenotypes(final VariantContext vc, likelihoodIndexesToUse = new ArrayList(30); final int[][] PLcache = PLIndexToAlleleIndex[numOriginalAltAlleles]; + final boolean[] altAlleleIndexToUse = new boolean[numOriginalAltAlleles]; + for ( int i = 0; i < numOriginalAltAlleles; i++ ) { + if ( allelesToUse.contains(vc.getAlternateAllele(i)) ) + altAlleleIndexToUse[i] = true; + } + for ( int PLindex = 0; PLindex < PLcache.length; PLindex++ ) { - int[] alleles = PLcache[PLindex]; + final int[] alleles = PLcache[PLindex]; // consider this entry only if both of the alleles are good - if ( (alleles[0] == 0 || allelesToUse[alleles[0] - 1]) && (alleles[1] == 0 || allelesToUse[alleles[1] - 1]) ) + if ( (alleles[0] == 0 || altAlleleIndexToUse[alleles[0] - 1]) && (alleles[1] == 0 || altAlleleIndexToUse[alleles[1] - 1]) ) likelihoodIndexesToUse.add(PLindex); } } // create the new genotypes - for ( int k = GLs.size() - 1; k >= 0; k-- ) { - final String sample = sampleIndices.get(k); - final Genotype g = GLs.get(sample); - if ( !g.hasLikelihoods() ) + for ( int k = 0; k < oldGTs.size(); k++ ) { + final Genotype g = oldGTs.get(sampleIndices.get(k)); + if ( !g.hasLikelihoods() ) { + newGTs.add(new Genotype(g.getSampleName(), NO_CALL_ALLELES, Genotype.NO_LOG10_PERROR, null, null, false)); continue; + } // create the new likelihoods array from the alleles we are allowed to use final double[] originalLikelihoods = g.getLikelihoods().getAsVector(); @@ -834,29 +834,34 @@ public static GenotypesContext assignGenotypes(final VariantContext vc, newLikelihoods = MathUtils.normalizeFromLog10(newLikelihoods, false, true); } - // if there is no mass on the (new) likelihoods and we actually have alternate alleles, then just no-call the sample - if ( MathUtils.sum(newLikelihoods) > SUM_GL_THRESH_NOCALL ) { - calls.add(new Genotype(g.getSampleName(), NO_CALL_ALLELES, Genotype.NO_LOG10_PERROR, null, null, false)); + // if there is no mass on the (new) likelihoods or we weren't asked to assign a genotype, then just no-call the sample + if ( !assignGenotypes || MathUtils.sum(newLikelihoods) > SUM_GL_THRESH_NOCALL ) { + newGTs.add(new Genotype(g.getSampleName(), NO_CALL_ALLELES, Genotype.NO_LOG10_PERROR, null, null, false)); continue; } - // find the genotype with maximum likelihoods - int PLindex = numNewAltAlleles == 0 ? 0 : MathUtils.maxElementIndex(newLikelihoods); - int[] alleles = PLIndexToAlleleIndex[numNewAltAlleles][PLindex]; - - ArrayList myAlleles = new ArrayList(); - myAlleles.add(newAlleles.get(alleles[0])); - myAlleles.add(newAlleles.get(alleles[1])); - - final double qual = numNewAltAlleles == 0 ? Genotype.NO_LOG10_PERROR : GenotypeLikelihoods.getQualFromLikelihoods(PLindex, newLikelihoods); - Map attrs = new HashMap(g.getAttributes()); - if ( numNewAltAlleles == 0 ) - attrs.remove(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY); - else - attrs.put(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY, GenotypeLikelihoods.fromLog10Likelihoods(newLikelihoods)); - calls.add(new Genotype(sample, myAlleles, qual, null, attrs, false)); + final Genotype newGT = assignGenotype(g, newLikelihoods, allelesToUse, numNewAltAlleles); + newGTs.add(newGT); } - - return calls; + + return newGTs; + } + + protected static Genotype assignGenotype(Genotype originalGT, double[] newLikelihoods, List allelesToUse, int numNewAltAlleles) { + // find the genotype with maximum likelihoods + int PLindex = numNewAltAlleles == 0 ? 0 : MathUtils.maxElementIndex(newLikelihoods); + int[] alleles = PLIndexToAlleleIndex[numNewAltAlleles][PLindex]; + + ArrayList myAlleles = new ArrayList(); + myAlleles.add(allelesToUse.get(alleles[0])); + myAlleles.add(allelesToUse.get(alleles[1])); + + final double qual = numNewAltAlleles == 0 ? Genotype.NO_LOG10_PERROR : GenotypeLikelihoods.getQualFromLikelihoods(PLindex, newLikelihoods); + Map attrs = new HashMap(originalGT.getAttributes()); + if ( numNewAltAlleles == 0 ) + attrs.remove(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY); + else + attrs.put(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY, GenotypeLikelihoods.fromLog10Likelihoods(newLikelihoods)); + return new Genotype(originalGT.getSampleName(), myAlleles, qual, null, attrs, false); } } From 7a937dd1ebc5186f864bde7736fe00a8f2325f88 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Thu, 9 Feb 2012 16:14:22 -0500 Subject: [PATCH 238/356] Several bug fixes to new genotyping strategy. Update integration tests for multi-allelic indels accordingly. --- .../genotyper/ExactAFCalculationModel.java | 4 +-- .../genotyper/UnifiedGenotyperEngine.java | 32 ++++++++++++------- .../UnifiedGenotyperIntegrationTest.java | 4 +-- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java index f9518a35c3..d833e9f8e8 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java @@ -47,7 +47,7 @@ public List getLog10PNonRef(final VariantContext vc, final double[][] log10AlleleFrequencyPriors, final AlleleFrequencyCalculationResult result) { - final GenotypesContext GLs = vc.getGenotypes(); + GenotypesContext GLs = vc.getGenotypes(); List alleles = vc.getAlleles(); // don't try to genotype too many alternate alleles @@ -58,7 +58,7 @@ public List getLog10PNonRef(final VariantContext vc, alleles.add(vc.getReference()); for ( int i = 0; i < MAX_ALTERNATE_ALLELES_TO_GENOTYPE; i++ ) alleles.add(vc.getAlternateAllele(i)); - UnifiedGenotyperEngine.subsetAlleles(vc, alleles, false); + GLs = UnifiedGenotyperEngine.subsetAlleles(vc, alleles, false); } //linearExact(GLs, log10AlleleFrequencyPriors[0], log10AlleleFrequencyLikelihoods, log10AlleleFrequencyPosteriors); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java index aa5776007e..c84c944b84 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java @@ -795,6 +795,10 @@ public static GenotypesContext subsetAlleles(final VariantContext vc, // then we can keep the PLs as is; otherwise, we determine which ones to keep if ( numNewAltAlleles != numOriginalAltAlleles && numNewAltAlleles > 0 ) { likelihoodIndexesToUse = new ArrayList(30); + + // make sure that we've cached enough data + if ( numOriginalAltAlleles > PLIndexToAlleleIndex.length - 1 ) + calculatePLcache(numOriginalAltAlleles); final int[][] PLcache = PLIndexToAlleleIndex[numOriginalAltAlleles]; final boolean[] altAlleleIndexToUse = new boolean[numOriginalAltAlleles]; @@ -834,20 +838,29 @@ public static GenotypesContext subsetAlleles(final VariantContext vc, newLikelihoods = MathUtils.normalizeFromLog10(newLikelihoods, false, true); } - // if there is no mass on the (new) likelihoods or we weren't asked to assign a genotype, then just no-call the sample - if ( !assignGenotypes || MathUtils.sum(newLikelihoods) > SUM_GL_THRESH_NOCALL ) { + // if there is no mass on the (new) likelihoods, then just no-call the sample + if ( MathUtils.sum(newLikelihoods) > SUM_GL_THRESH_NOCALL ) { newGTs.add(new Genotype(g.getSampleName(), NO_CALL_ALLELES, Genotype.NO_LOG10_PERROR, null, null, false)); - continue; } - - final Genotype newGT = assignGenotype(g, newLikelihoods, allelesToUse, numNewAltAlleles); - newGTs.add(newGT); + else { + Map attrs = new HashMap(g.getAttributes()); + if ( numNewAltAlleles == 0 ) + attrs.remove(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY); + else + attrs.put(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY, GenotypeLikelihoods.fromLog10Likelihoods(newLikelihoods)); + + // if we weren't asked to assign a genotype, then just no-call the sample + if ( !assignGenotypes || MathUtils.sum(newLikelihoods) > SUM_GL_THRESH_NOCALL ) + newGTs.add(new Genotype(g.getSampleName(), NO_CALL_ALLELES, Genotype.NO_LOG10_PERROR, null, attrs, false)); + else + newGTs.add(assignGenotype(g, newLikelihoods, allelesToUse, numNewAltAlleles, attrs)); + } } return newGTs; } - protected static Genotype assignGenotype(Genotype originalGT, double[] newLikelihoods, List allelesToUse, int numNewAltAlleles) { + protected static Genotype assignGenotype(Genotype originalGT, double[] newLikelihoods, List allelesToUse, int numNewAltAlleles, Map attrs) { // find the genotype with maximum likelihoods int PLindex = numNewAltAlleles == 0 ? 0 : MathUtils.maxElementIndex(newLikelihoods); int[] alleles = PLIndexToAlleleIndex[numNewAltAlleles][PLindex]; @@ -857,11 +870,6 @@ protected static Genotype assignGenotype(Genotype originalGT, double[] newLikeli myAlleles.add(allelesToUse.get(alleles[1])); final double qual = numNewAltAlleles == 0 ? Genotype.NO_LOG10_PERROR : GenotypeLikelihoods.getQualFromLikelihoods(PLindex, newLikelihoods); - Map attrs = new HashMap(originalGT.getAttributes()); - if ( numNewAltAlleles == 0 ) - attrs.remove(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY); - else - attrs.put(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY, GenotypeLikelihoods.fromLog10Likelihoods(newLikelihoods)); return new Genotype(originalGT.getSampleName(), myAlleles, qual, null, attrs, false); } } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java index fd6738123f..125242a2f0 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java @@ -300,7 +300,7 @@ public void testWithIndelAllelesPassedIn3() { WalkerTest.WalkerTestSpec spec3 = new WalkerTest.WalkerTestSpec( baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "ALL.wgs.union_v2.20101123.indels.sites.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,080,000", 1, - Arrays.asList("d356cbaf240d7025d1aecdabaff3a3e0")); + Arrays.asList("e4d2904b406f37d99fbe8f52ae75254f")); executeTest("test MultiSample Pilot2 indels with complicated records", spec3); } @@ -309,7 +309,7 @@ public void testWithIndelAllelesPassedIn4() { WalkerTest.WalkerTestSpec spec4 = new WalkerTest.WalkerTestSpec( baseCommandIndelsb37 + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "ALL.wgs.union_v2_chr20_100_110K.20101123.indels.sites.vcf -I " + validationDataLocation + "phase1_GBR_realigned.chr20.100K-110K.bam -o %s -L 20:100,000-110,000", 1, - Arrays.asList("1d1956fd7b0f0d30935674b2f5019860")); + Arrays.asList("21f7b6c8b7eaccad1754a832bac79a65")); executeTest("test MultiSample Phase1 indels with complicated records", spec4); } From 5af373a3a13a3558d758fb280e8cdced5c4bc949 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Wed, 8 Feb 2012 14:39:55 -0500 Subject: [PATCH 239/356] BQSR with indels integrated! * added support to base before deletion in the pileup * refactored covariates to operate on mismatches, insertions and deletions at the same time * all code is in private so original BQSR is still working as usual in public * outputs a molten CSV with mismatches, insertions and deletions, time to play! * barely tested, passes my very simple tests... haven't tested edge cases. --- .../sting/gatk/iterators/LocusIteratorByState.java | 4 ++-- .../SNPGenotypeLikelihoodsCalculationModel.java | 7 +++++-- .../org/broadinstitute/sting/utils/NGSPlatform.java | 12 +++++++++++- .../sting/utils/pileup/AbstractReadBackedPileup.java | 6 +++--- .../utils/pileup/ExtendedEventPileupElement.java | 2 +- .../sting/utils/pileup/PileupElement.java | 9 ++++++++- .../pileup/ReadBackedExtendedEventPileupImpl.java | 2 +- .../sting/utils/pileup/ReadBackedPileupImpl.java | 4 ++-- .../sting/utils/sam/ArtificialSAMUtils.java | 4 ++-- .../sting/utils/sam/GATKSAMRecordUnitTest.java | 4 ++-- 10 files changed, 37 insertions(+), 17 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java b/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java index 316a20a704..6edae38161 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java +++ b/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java @@ -470,7 +470,7 @@ else if (op != CigarOperator.D || readInfo.includeReadsWithDeletionAtLoci()) { if (op == CigarOperator.D) { if (readInfo.includeReadsWithDeletionAtLoci()) { // only add deletions to the pileup if we are authorized to do so - pile.add(new PileupElement(read, readOffset, true, nextOp == CigarOperator.I, nextOp == CigarOperator.S || (state.getGenomeOffset() == 0 && read.getSoftStart() != read.getAlignmentStart()))); + pile.add(new PileupElement(read, readOffset, true, nextOp == CigarOperator.D, nextOp == CigarOperator.I, nextOp == CigarOperator.S || (state.getGenomeOffset() == 0 && read.getSoftStart() != read.getAlignmentStart()))); size++; nDeletions++; if (read.getMappingQuality() == 0) @@ -479,7 +479,7 @@ else if (op != CigarOperator.D || readInfo.includeReadsWithDeletionAtLoci()) { } else { if (!filterBaseInRead(read, location.getStart())) { - pile.add(new PileupElement(read, readOffset, false, nextOp == CigarOperator.I, nextOp == CigarOperator.S || (state.getGenomeOffset() == 0 && read.getSoftStart() != read.getAlignmentStart()))); + pile.add(new PileupElement(read, readOffset, false, nextOp == CigarOperator.D, nextOp == CigarOperator.I, nextOp == CigarOperator.S || (state.getGenomeOffset() == 0 && read.getSoftStart() != read.getAlignmentStart()))); size++; if (read.getMappingQuality() == 0) nMQ0Reads++; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java index ea53c815d1..6171b01eb5 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java @@ -39,7 +39,10 @@ import org.broadinstitute.sting.utils.pileup.ReadBackedPileupImpl; import org.broadinstitute.sting.utils.variantcontext.*; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsCalculationModel { @@ -212,7 +215,7 @@ public ReadBackedPileup createBAQedPileup( final ReadBackedPileup pileup ) { public class BAQedPileupElement extends PileupElement { public BAQedPileupElement( final PileupElement PE ) { - super(PE.getRead(), PE.getOffset(), PE.isDeletion(), PE.isBeforeInsertion(), PE.isNextToSoftClip()); + super(PE.getRead(), PE.getOffset(), PE.isDeletion(), PE.isBeforeDeletion(), PE.isBeforeInsertion(), PE.isNextToSoftClip()); } @Override diff --git a/public/java/src/org/broadinstitute/sting/utils/NGSPlatform.java b/public/java/src/org/broadinstitute/sting/utils/NGSPlatform.java index 4f01f2b7aa..597dc48034 100644 --- a/public/java/src/org/broadinstitute/sting/utils/NGSPlatform.java +++ b/public/java/src/org/broadinstitute/sting/utils/NGSPlatform.java @@ -87,7 +87,7 @@ public static final NGSPlatform fromReadGroup(SAMReadGroupRecord rg) { /** * Returns the NGSPlatform corresponding to the PL tag in the read group * @param plFromRG -- the PL field (or equivalent) in a ReadGroup object - * @return an NGSPlatform object matching the PL field of the header, of UNKNOWN if there was no match + * @return an NGSPlatform object matching the PL field of the header, or UNKNOWN if there was no match */ public static final NGSPlatform fromReadGroupPL(final String plFromRG) { if ( plFromRG == null ) return UNKNOWN; @@ -105,4 +105,14 @@ public static final NGSPlatform fromReadGroupPL(final String plFromRG) { return UNKNOWN; } + + /** + * checks whether or not the requested platform is listed in the set (and is not unknown) + * + * @param platform the read group string that describes the platform used + * @return true if the platform is known (i.e. it's in the list and is not UNKNOWN) + */ + public static final boolean isKnown (final String platform) { + return fromReadGroupPL(platform) != UNKNOWN; + } } diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java b/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java index 82e4038421..70ad70f43b 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java @@ -177,7 +177,7 @@ private PileupElementTracker readsOffsets2Pileup(List reads, for (int i = 0; i < reads.size(); i++) { GATKSAMRecord read = reads.get(i); int offset = offsets.get(i); - pileup.add(createNewPileupElement(read, offset, false, false, false)); // only used to create fake pileups for testing so ancillary information is not important + pileup.add(createNewPileupElement(read, offset, false, false, false, false)); // only used to create fake pileups for testing so ancillary information is not important } return pileup; @@ -196,7 +196,7 @@ private PileupElementTracker readsOffsets2Pileup(List reads, UnifiedPileupElementTracker pileup = new UnifiedPileupElementTracker(); for (GATKSAMRecord read : reads) { - pileup.add(createNewPileupElement(read, offset, false, false, false)); // only used to create fake pileups for testing so ancillary information is not important + pileup.add(createNewPileupElement(read, offset, false, false, false, false)); // only used to create fake pileups for testing so ancillary information is not important } return pileup; @@ -204,7 +204,7 @@ private PileupElementTracker readsOffsets2Pileup(List reads, protected abstract AbstractReadBackedPileup createNewPileup(GenomeLoc loc, PileupElementTracker pileupElementTracker); - protected abstract PE createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeInsertion, boolean isNextToSoftClip); + protected abstract PE createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeDeletion, boolean isBeforeInsertion, boolean isNextToSoftClip); // -------------------------------------------------------- // diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/ExtendedEventPileupElement.java b/public/java/src/org/broadinstitute/sting/utils/pileup/ExtendedEventPileupElement.java index 921da2a1f1..506442d03c 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/ExtendedEventPileupElement.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/ExtendedEventPileupElement.java @@ -48,7 +48,7 @@ public enum Type { public ExtendedEventPileupElement(GATKSAMRecord read, int offset, int eventLength, String eventBases, Type type) { - super(read, offset, type == Type.DELETION, false, false); // extended events are slated for removal + super(read, offset, type == Type.DELETION, false, false, false); // extended events are slated for removal this.read = read; this.offset = offset; this.eventLength = eventLength; diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java b/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java index a4830223e2..9df22700e5 100755 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java @@ -24,6 +24,7 @@ public class PileupElement implements Comparable { protected final GATKSAMRecord read; protected final int offset; protected final boolean isDeletion; + protected final boolean isBeforeDeletion; protected final boolean isBeforeInsertion; protected final boolean isNextToSoftClip; @@ -33,6 +34,7 @@ public class PileupElement implements Comparable { * @param read the read we are adding to the pileup * @param offset the position in the read for this base. All deletions must be left aligned! (-1 is only allowed for reads starting with insertions) * @param isDeletion whether or not this base is a deletion + * @param isBeforeDeletion whether or not this base is before a deletion * @param isBeforeInsertion whether or not this base is before an insertion * @param isNextToSoftClip whether or not this base is next to a soft clipped base */ @@ -40,13 +42,14 @@ public class PileupElement implements Comparable { "read != null", "offset >= -1", "offset <= read.getReadLength()"}) - public PileupElement(final GATKSAMRecord read, final int offset, final boolean isDeletion, final boolean isBeforeInsertion, final boolean isNextToSoftClip) { + public PileupElement(final GATKSAMRecord read, final int offset, final boolean isDeletion, final boolean isBeforeDeletion, final boolean isBeforeInsertion, final boolean isNextToSoftClip) { if (offset < 0 && isDeletion) throw new ReviewedStingException("Pileup Element cannot create a deletion with a negative offset"); this.read = read; this.offset = offset; this.isDeletion = isDeletion; + this.isBeforeDeletion = isBeforeDeletion; this.isBeforeInsertion = isBeforeInsertion; this.isNextToSoftClip = isNextToSoftClip; } @@ -55,6 +58,10 @@ public boolean isDeletion() { return isDeletion; } + public boolean isBeforeDeletion() { + return isBeforeDeletion; + } + public boolean isBeforeInsertion() { return isBeforeInsertion; } diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java index df334f557f..357195daa7 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java @@ -96,7 +96,7 @@ protected ReadBackedExtendedEventPileupImpl createNewPileup(GenomeLoc loc, Pileu } @Override - protected ExtendedEventPileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeInsertion, boolean isNextToSoftClip) { + protected ExtendedEventPileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeDeletion, boolean isBeforeInsertion, boolean isNextToSoftClip) { throw new UnsupportedOperationException("Not enough information provided to create a new pileup element"); } diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java index 20b1000017..7a6ebef218 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java @@ -71,7 +71,7 @@ protected ReadBackedPileupImpl createNewPileup(GenomeLoc loc, PileupElementTrack } @Override - protected PileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeInsertion, boolean isNextToSoftClip) { - return new PileupElement(read, offset, isDeletion, isBeforeInsertion, isNextToSoftClip); + protected PileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeDeletion, boolean isBeforeInsertion, boolean isNextToSoftClip) { + return new PileupElement(read, offset, isDeletion, isBeforeDeletion, isBeforeInsertion, isNextToSoftClip); } } diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/ArtificialSAMUtils.java b/public/java/src/org/broadinstitute/sting/utils/sam/ArtificialSAMUtils.java index 1175a038f0..b17e325fca 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/ArtificialSAMUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/ArtificialSAMUtils.java @@ -361,10 +361,10 @@ public static ReadBackedPileup createReadBackedPileup(final SAMFileHeader header final GATKSAMRecord left = pair.get(0); final GATKSAMRecord right = pair.get(1); - pileupElements.add(new PileupElement(left, pos - leftStart, false, false, false)); + pileupElements.add(new PileupElement(left, pos - leftStart, false, false, false, false)); if (pos >= right.getAlignmentStart() && pos <= right.getAlignmentEnd()) { - pileupElements.add(new PileupElement(right, pos - rightStart, false, false, false)); + pileupElements.add(new PileupElement(right, pos - rightStart, false, false, false, false)); } } diff --git a/public/java/test/org/broadinstitute/sting/utils/sam/GATKSAMRecordUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/sam/GATKSAMRecordUnitTest.java index 729503f843..520fb70405 100755 --- a/public/java/test/org/broadinstitute/sting/utils/sam/GATKSAMRecordUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/sam/GATKSAMRecordUnitTest.java @@ -42,8 +42,8 @@ public void testReducedReads() { @Test public void testReducedReadPileupElement() { - PileupElement readp = new PileupElement(read, 0, false, false, false); - PileupElement reducedreadp = new PileupElement(reducedRead, 0, false, false, false); + PileupElement readp = new PileupElement(read, 0, false, false, false, false); + PileupElement reducedreadp = new PileupElement(reducedRead, 0, false, false, false, false); Assert.assertFalse(readp.getRead().isReducedRead()); From f53cd3de1b9b92fe9e4048a3201c54512fb5c6ce Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Fri, 10 Feb 2012 11:07:32 -0500 Subject: [PATCH 240/356] Based on Ryan's suggestion, there's a new contract for genotyping multiple alleles. Now the requester submits alleles in any arbitrary order - rankings aren't needed. If the Exact model decides that it needs to subset the alleles because too many were requested, it does so based on PL mass (in other words, I moved this code from the SNPGenotypeLikelihoodsCalculationModel to the Exact model). Now subsetting alleles is consistent. --- .../genotyper/ExactAFCalculationModel.java | 55 ++++++++++++++++++- ...NPGenotypeLikelihoodsCalculationModel.java | 33 ++++------- .../genotyper/UnifiedGenotyperEngine.java | 4 +- .../UnifiedGenotyperIntegrationTest.java | 8 +-- 4 files changed, 69 insertions(+), 31 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java index d833e9f8e8..ed737064db 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java @@ -56,8 +56,7 @@ public List getLog10PNonRef(final VariantContext vc, alleles = new ArrayList(MAX_ALTERNATE_ALLELES_TO_GENOTYPE + 1); alleles.add(vc.getReference()); - for ( int i = 0; i < MAX_ALTERNATE_ALLELES_TO_GENOTYPE; i++ ) - alleles.add(vc.getAlternateAllele(i)); + alleles.addAll(chooseMostLikelyAlternateAlleles(vc, MAX_ALTERNATE_ALLELES_TO_GENOTYPE)); GLs = UnifiedGenotyperEngine.subsetAlleles(vc, alleles, false); } @@ -67,6 +66,58 @@ public List getLog10PNonRef(final VariantContext vc, return alleles; } + private static final class LikelihoodSum implements Comparable { + public double sum = 0.0; + public Allele allele; + + public LikelihoodSum(Allele allele) { this.allele = allele; } + + public int compareTo(LikelihoodSum other) { + final double diff = sum - other.sum; + return ( diff < 0.0 ) ? 1 : (diff > 0.0 ) ? -1 : 0; + } + } + + private static final int PL_INDEX_OF_HOM_REF = 0; + private static final List chooseMostLikelyAlternateAlleles(VariantContext vc, int numAllelesToChoose) { + final int numOriginalAltAlleles = vc.getAlternateAlleles().size(); + final LikelihoodSum[] likelihoodSums = new LikelihoodSum[numOriginalAltAlleles]; + for ( int i = 0; i < numOriginalAltAlleles; i++ ) + likelihoodSums[i] = new LikelihoodSum(vc.getAlternateAllele(i)); + + // make sure that we've cached enough data + if ( numOriginalAltAlleles > UnifiedGenotyperEngine.PLIndexToAlleleIndex.length - 1 ) + UnifiedGenotyperEngine.calculatePLcache(numOriginalAltAlleles); + + // based on the GLs, find the alternate alleles with the most probability; sum the GLs for the most likely genotype + final ArrayList GLs = getGLs(vc.getGenotypes()); + for ( final double[] likelihoods : GLs ) { + final int PLindexOfBestGL = MathUtils.maxElementIndex(likelihoods); + if ( PLindexOfBestGL != PL_INDEX_OF_HOM_REF ) { + int[] alleles = UnifiedGenotyperEngine.PLIndexToAlleleIndex[numOriginalAltAlleles][PLindexOfBestGL]; + if ( alleles[0] != 0 ) + likelihoodSums[alleles[0]-1].sum += likelihoods[PLindexOfBestGL] - likelihoods[PL_INDEX_OF_HOM_REF]; + // don't double-count it + if ( alleles[1] != 0 && alleles[1] != alleles[0] ) + likelihoodSums[alleles[1]-1].sum += likelihoods[PLindexOfBestGL] - likelihoods[PL_INDEX_OF_HOM_REF]; + } + } + + // sort them by probability mass and choose the best ones + Collections.sort(Arrays.asList(likelihoodSums)); + final ArrayList bestAlleles = new ArrayList(numAllelesToChoose); + for ( int i = 0; i < numAllelesToChoose; i++ ) + bestAlleles.add(likelihoodSums[i].allele); + + final ArrayList orderedBestAlleles = new ArrayList(numAllelesToChoose); + for ( Allele allele : vc.getAlternateAlleles() ) { + if ( bestAlleles.contains(allele) ) + orderedBestAlleles.add(allele); + } + + return orderedBestAlleles; + } + private static final ArrayList getGLs(GenotypesContext GLs) { ArrayList genotypeLikelihoods = new ArrayList(GLs.size()); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java index 6f1f86c6d3..c078be2f2b 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java @@ -45,20 +45,8 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsC private final boolean useAlleleFromVCF; - final LikelihoodSum[] likelihoodSums = new LikelihoodSum[4]; - - private final class LikelihoodSum implements Comparable { - public double sum = 0.0; - public Allele base; - - public LikelihoodSum(Allele base) { this.base = base; } - - public int compareTo(LikelihoodSum other) { - final double diff = sum - other.sum; - return ( diff < 0.0 ) ? 1 : (diff > 0.0 ) ? -1 : 0; - } - } - + private final double[] likelihoodSums = new double[4]; + protected SNPGenotypeLikelihoodsCalculationModel(UnifiedArgumentCollection UAC, Logger logger) { super(UAC, logger); useAlleleFromVCF = UAC.GenotypingMode == GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES; @@ -176,27 +164,26 @@ protected List determineAlternateAlleles(final byte ref, final List allelesToUse = new ArrayList(3); - for ( LikelihoodSum sum : likelihoodSums ) { - if ( sum.sum > 0.0 ) - allelesToUse.add(sum.base); + for ( int i = 0; i < 4; i++ ) { + if ( likelihoodSums[i] > 0.0 ) + allelesToUse.add(Allele.create(BaseUtils.baseIndexToSimpleBase(i), false)); } return allelesToUse; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java index c84c944b84..0156890ac8 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java @@ -767,7 +767,7 @@ public static GenotypesContext assignGenotypes(final VariantContext vc) { /** * @param vc variant context with genotype likelihoods - * @param allelesToUse which alleles from the vc are okay to use + * @param allelesToUse which alleles from the vc are okay to use; *** must be in the same relative order as those in the original VC *** * @param assignGenotypes true if we should change the genotypes based on the (subsetted) PLs * @return genotypes */ @@ -860,7 +860,7 @@ public static GenotypesContext subsetAlleles(final VariantContext vc, return newGTs; } - protected static Genotype assignGenotype(Genotype originalGT, double[] newLikelihoods, List allelesToUse, int numNewAltAlleles, Map attrs) { + protected static Genotype assignGenotype(final Genotype originalGT, final double[] newLikelihoods, final List allelesToUse, final int numNewAltAlleles, final Map attrs) { // find the genotype with maximum likelihoods int PLindex = numNewAltAlleles == 0 ? 0 : MathUtils.maxElementIndex(newLikelihoods); int[] alleles = PLIndexToAlleleIndex[numNewAltAlleles][PLindex]; diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java index 125242a2f0..fc4f0f46bd 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java @@ -28,7 +28,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest { public void testMultiSamplePilot1() { WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec( baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000", 1, - Arrays.asList("9ab4e98ce437a1c5e1eee338de49ee7e")); + Arrays.asList("202b337ebbea3def1be8495eb363dfa8")); executeTest("test MultiSample Pilot1", spec); } @@ -60,7 +60,7 @@ public void testSingleSamplePilot2() { public void testMultipleSNPAlleles() { WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec( "-T UnifiedGenotyper -R " + b37KGReference + " -nosl -NO_HEADER -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + validationDataLocation + "multiallelic.snps.bam -o %s -L " + validationDataLocation + "multiallelic.snps.intervals", 1, - Arrays.asList("aabc4b3a312aba18b78e14750d8c8e62")); + Arrays.asList("b53cb55a5f868663068812b13578af57")); executeTest("test Multiple SNP alleles", spec); } @@ -300,7 +300,7 @@ public void testWithIndelAllelesPassedIn3() { WalkerTest.WalkerTestSpec spec3 = new WalkerTest.WalkerTestSpec( baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "ALL.wgs.union_v2.20101123.indels.sites.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,080,000", 1, - Arrays.asList("e4d2904b406f37d99fbe8f52ae75254f")); + Arrays.asList("c9897b80615c53a4ea10a4b193d56d9c")); executeTest("test MultiSample Pilot2 indels with complicated records", spec3); } @@ -309,7 +309,7 @@ public void testWithIndelAllelesPassedIn4() { WalkerTest.WalkerTestSpec spec4 = new WalkerTest.WalkerTestSpec( baseCommandIndelsb37 + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "ALL.wgs.union_v2_chr20_100_110K.20101123.indels.sites.vcf -I " + validationDataLocation + "phase1_GBR_realigned.chr20.100K-110K.bam -o %s -L 20:100,000-110,000", 1, - Arrays.asList("21f7b6c8b7eaccad1754a832bac79a65")); + Arrays.asList("5282fdb1711a532d726c13507bf80a21")); executeTest("test MultiSample Phase1 indels with complicated records", spec4); } From 0722df46dbd8f48f8bb1bbf55cd5df2074186cff Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 10 Feb 2012 11:11:14 -0500 Subject: [PATCH 241/356] gsafolkLSFLogs creates a subset of LSF MySQL db related to gsafolk only Creates a SQL table in the MySQL server calcium at the Broad that contains only key information about the LSF usage of members of the gsafolk fairshare group Does this by first building a list of gsafolk uids, selecting lsf info from matter's table, and inserts this information into the gsafolk_lsf queue as part of the GATK schema. The standard way to run this is with incremental refreshes enabled, so that the program only fetches new raw lsf records with timestamps beyond the max timestamp present in the GATK LSF table. The default way to run this program is via cron with 'python private/python/gsafolkLSFLogs.py' From 48cc4b913a7c432135ab14d40e4c1420435cd2e4 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 10 Feb 2012 11:30:36 -0500 Subject: [PATCH 242/356] bugfix for incremental refresh in gsafolkLSFlogs From 1fb19a0f98001d4ae71f2eb9c775051079bcb2ab Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Fri, 10 Feb 2012 11:43:48 -0500 Subject: [PATCH 243/356] Moving the covariates and shared functionality to public so Ryan can work on the recalibration on the fly without breaking the build. Supposedly all the secret sauce is in the BQSR walker, which sits in private. --- .../gatk/walkers/bqsr/ContextCovariate.java | 101 +++ .../sting/gatk/walkers/bqsr/Covariate.java | 62 ++ .../gatk/walkers/bqsr/CovariateKeySet.java | 63 ++ .../gatk/walkers/bqsr/CovariateValues.java | 37 + .../gatk/walkers/bqsr/CycleCovariate.java | 199 +++++ .../walkers/bqsr/QualityScoreCovariate.java | 77 ++ .../gatk/walkers/bqsr/ReadGroupCovariate.java | 57 ++ .../gatk/walkers/bqsr/RecalDataManager.java | 698 ++++++++++++++++++ .../sting/gatk/walkers/bqsr/RecalDatum.java | 112 +++ .../walkers/bqsr/RecalDatumOptimized.java | 115 +++ .../bqsr/RecalibrationArgumentCollection.java | 102 +++ 11 files changed, 1623 insertions(+) create mode 100644 public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java create mode 100755 public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/Covariate.java create mode 100644 public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/CovariateKeySet.java create mode 100644 public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/CovariateValues.java create mode 100755 public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/CycleCovariate.java create mode 100755 public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/QualityScoreCovariate.java create mode 100755 public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ReadGroupCovariate.java create mode 100644 public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalDataManager.java create mode 100755 public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalDatum.java create mode 100755 public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalDatumOptimized.java create mode 100755 public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalibrationArgumentCollection.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java new file mode 100644 index 0000000000..a46543f671 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2011 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR + * THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.walkers.bqsr; + +import org.broadinstitute.sting.utils.exceptions.UserException; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; + +import java.util.Arrays; + +/** + * Created by IntelliJ IDEA. + * User: rpoplin + * Date: 9/26/11 + */ + +public class ContextCovariate implements StandardCovariate { + + private int mismatchesContextSize; + private int insertionsContextSize; + private int deletionsContextSize; + + private String mismatchesNoContext = ""; + private String insertionsNoContext = ""; + private String deletionsNoContext = ""; + + // Initialize any member variables using the command-line arguments passed to the walkers + @Override + public void initialize(final RecalibrationArgumentCollection RAC) { + mismatchesContextSize = RAC.MISMATCHES_CONTEXT_SIZE; + insertionsContextSize = RAC.INSERTIONS_CONTEXT_SIZE; + deletionsContextSize = RAC.DELETIONS_CONTEXT_SIZE; + + if (mismatchesContextSize <= 0 || insertionsContextSize <= 0 || deletionsContextSize <= 0) + throw new UserException(String.format("Context Size must be positive, if you don't want to use the context covariate, just turn it off instead. Mismatches: %d Insertions: %d Deletions:%d", mismatchesContextSize, insertionsContextSize, deletionsContextSize)); + + // initialize no context strings given the size of the context for each covariate type + mismatchesNoContext = makeAllNStringWithLength(mismatchesContextSize); + insertionsNoContext = makeAllNStringWithLength(insertionsContextSize); + deletionsNoContext = makeAllNStringWithLength( deletionsContextSize); + } + + @Override + public CovariateValues getValues(final GATKSAMRecord read) { + int l = read.getReadLength(); + String[] mismatches = new String [l]; + String[] insertions = new String [l]; + String[] deletions = new String [l]; + + byte[] bases = read.getReadBases(); + for (int i = 0; i < read.getReadLength(); i++) { + mismatches[i] = contextWith(bases, i, mismatchesContextSize, mismatchesNoContext); + insertions[i] = contextWith(bases, i, insertionsContextSize, insertionsNoContext); + deletions[i] = contextWith(bases, i, deletionsContextSize, deletionsNoContext); + } + return new CovariateValues(mismatches, insertions, deletions); + } + + /** + * calculates the context of a base indenpendent of the covariate mode + * + * @param bases the bases in the read to build the context from + * @param offset the position in the read to calculate the context for + * @param contextSize context size to use building the context + * @param noContextString string to return if the position is not far enough in the read to have a full context before. + * @return + */ + private String contextWith(byte [] bases, int offset, int contextSize, String noContextString) { + return (offset < contextSize) ? noContextString : new String(Arrays.copyOfRange(bases, offset - contextSize, offset)); + } + + private String makeAllNStringWithLength(int length) { + String s = ""; + for (int i=0; i DISCRETE_CYCLE_PLATFORMS = EnumSet.of(NGSPlatform.ILLUMINA, NGSPlatform.SOLID, NGSPlatform.PACBIO, NGSPlatform.COMPLETE_GENOMICS); + private final static EnumSet FLOW_CYCLE_PLATFORMS = EnumSet.of(NGSPlatform.LS454, NGSPlatform.ION_TORRENT); + + // Initialize any member variables using the command-line arguments passed to the walkers + @Override + public void initialize(final RecalibrationArgumentCollection RAC) { + if (RAC.DEFAULT_PLATFORM != null && !NGSPlatform.isKnown(RAC.DEFAULT_PLATFORM)) + throw new UserException.CommandLineException("The requested default platform (" + RAC.DEFAULT_PLATFORM + ") is not a recognized platform."); + } + + // Used to pick out the covariate's value from attributes of the read + @Override + public CovariateValues getValues(final GATKSAMRecord read) { + Integer [] cycles = new Integer[read.getReadLength()]; + final NGSPlatform ngsPlatform = read.getNGSPlatform(); + + // Discrete cycle platforms + if (DISCRETE_CYCLE_PLATFORMS.contains(ngsPlatform)) { + final int init; + final int increment; + if (!read.getReadNegativeStrandFlag()) { + // Differentiate between first and second of pair. + // The sequencing machine cycle keeps incrementing for the second read in a pair. So it is possible for a read group + // to have an error affecting quality at a particular cycle on the first of pair which carries over to the second of pair. + // Therefore the cycle covariate must differentiate between first and second of pair reads. + // This effect can not be corrected by pulling out the first of pair and second of pair flags into a separate covariate because + // the current sequential model would consider the effects independently instead of jointly. + if (read.getReadPairedFlag() && read.getSecondOfPairFlag()) { + //second of pair, positive strand + init = -1; + increment = -1; + } + else { + //first of pair, positive strand + init = 1; + increment = 1; + } + + } + else { + if (read.getReadPairedFlag() && read.getSecondOfPairFlag()) { + //second of pair, negative strand + init = -read.getReadLength(); + increment = 1; + } + else { + //first of pair, negative strand + init = read.getReadLength(); + increment = -1; + } + } + + int cycle = init; + for (int i = 0; i < read.getReadLength(); i++) { + cycles[i] = cycle; + cycle += increment; + } + } + + // Flow cycle platforms + else if (FLOW_CYCLE_PLATFORMS.contains(ngsPlatform)) { + + final int readLength = read.getReadLength(); + final byte[] bases = read.getReadBases(); + + // Differentiate between first and second of pair. + // The sequencing machine cycle keeps incrementing for the second read in a pair. So it is possible for a read group + // to have an error affecting quality at a particular cycle on the first of pair which carries over to the second of pair. + // Therefore the cycle covariate must differentiate between first and second of pair reads. + // This effect can not be corrected by pulling out the first of pair and second of pair flags into a separate covariate because + // the current sequential model would consider the effects independently instead of jointly. + final boolean multiplyByNegative1 = read.getReadPairedFlag() && read.getSecondOfPairFlag(); + + int cycle = multiplyByNegative1 ? -1 : 1; + + // BUGBUG: Consider looking at degradation of base quality scores in homopolymer runs to detect when the cycle incremented even though the nucleotide didn't change + // For example, AAAAAAA was probably read in two flow cycles but here we count it as one + if (!read.getReadNegativeStrandFlag()) { // Forward direction + int iii = 0; + while (iii < readLength) { + while (iii < readLength && bases[iii] == (byte) 'T') { + cycles[iii] = cycle; + iii++; + } + while (iii < readLength && bases[iii] == (byte) 'A') { + cycles[iii] = cycle; + iii++; + } + while (iii < readLength && bases[iii] == (byte) 'C') { + cycles[iii] = cycle; + iii++; + } + while (iii < readLength && bases[iii] == (byte) 'G') { + cycles[iii] = cycle; + iii++; + } + if (iii < readLength) { + if (multiplyByNegative1) + cycle--; + else + cycle++; + } + if (iii < readLength && !BaseUtils.isRegularBase(bases[iii])) { + cycles[iii] = cycle; + iii++; + } + + } + } + else { // Negative direction + int iii = readLength - 1; + while (iii >= 0) { + while (iii >= 0 && bases[iii] == (byte) 'T') { + cycles[iii] = cycle; + iii--; + } + while (iii >= 0 && bases[iii] == (byte) 'A') { + cycles[iii] = cycle; + iii--; + } + while (iii >= 0 && bases[iii] == (byte) 'C') { + cycles[iii] = cycle; + iii--; + } + while (iii >= 0 && bases[iii] == (byte) 'G') { + cycles[iii] = cycle; + iii--; + } + if (iii >= 0) { + if (multiplyByNegative1) + cycle--; + else + cycle++; + } + if (iii >= 0 && !BaseUtils.isRegularBase(bases[iii])) { + cycles[iii] = cycle; + iii--; + } + } + } + } + + // Unknown platforms + else { + throw new UserException("The platform (" + read.getReadGroup().getPlatform() + ") associated with read group " + read.getReadGroup() + " is not a recognized platform. Implemented options are e.g. illumina, 454, and solid"); + } + + return new CovariateValues(cycles, cycles, cycles); + } + +} \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/QualityScoreCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/QualityScoreCovariate.java new file mode 100755 index 0000000000..0d36f3ff4b --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/QualityScoreCovariate.java @@ -0,0 +1,77 @@ +package org.broadinstitute.sting.gatk.walkers.bqsr; + +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; + +import java.util.Arrays; + +/* + * Copyright (c) 2009 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/** + * Created by IntelliJ IDEA. + * User: rpoplin + * Date: Nov 3, 2009 + * + * The Reported Quality Score covariate. + */ + +public class QualityScoreCovariate implements RequiredCovariate { + + private byte defaultMismatchesQuality; // walker parameter. Must be > 0 to be used, otherwise we use the quality from the read. + private byte defaultInsertionsQuality; // walker parameter. Must be > 0 to be used, otherwise we use the quality from the read. + private byte defaultDeletionsQuality; // walker parameter. Must be > 0 to be used, otherwise we use the quality from the read. + + // Initialize any member variables using the command-line arguments passed to the walkers + @Override + public void initialize(final RecalibrationArgumentCollection RAC) { + defaultMismatchesQuality = RAC.MISMATCHES_DEFAULT_QUALITY; + defaultInsertionsQuality = RAC.INSERTIONS_DEFAULT_QUALITY; + defaultDeletionsQuality = RAC.DELETIONS_DEFAULT_QUALITY; + } + + @Override + public CovariateValues getValues(final GATKSAMRecord read) { + int readLength = read.getReadLength(); + + Byte [] mismatches = new Byte[readLength]; + Byte [] insertions = new Byte[readLength]; + Byte [] deletions = new Byte[readLength]; + + byte [] baseQualities = read.getBaseQualities(); + + if (defaultMismatchesQuality >= 0) + Arrays.fill(mismatches, defaultMismatchesQuality); // if the user decides to override the base qualities in the read, use the flat value + else { + for (int i=0; i dataCollapsedByCovariate; // Tables where everything except read group, quality score, and given covariate has been collapsed + + public final static String ORIGINAL_QUAL_ATTRIBUTE_TAG = "OQ"; // The tag that holds the original quality scores + public final static String COLOR_SPACE_QUAL_ATTRIBUTE_TAG = "CQ"; // The tag that holds the color space quality scores for SOLID bams + public final static String COLOR_SPACE_ATTRIBUTE_TAG = "CS"; // The tag that holds the color space for SOLID bams + public final static String COLOR_SPACE_INCONSISTENCY_TAG = "ZC"; // A new tag made up for the recalibrator which will hold an array of ints which say if this base is inconsistent with its color + private static boolean warnUserNullReadGroup = false; + private static boolean warnUserNullPlatform = false; + + private static final String COVARS_ATTRIBUTE = "COVARS"; // used to store covariates array as a temporary attribute inside GATKSAMRecord.\ + + + public enum SOLID_RECAL_MODE { + /** + * Treat reference inserted bases as reference matching bases. Very unsafe! + */ + DO_NOTHING, + /** + * Set reference inserted bases and the previous base (because of color space alignment details) to Q0. This is the default option. + */ + SET_Q_ZERO, + /** + * In addition to setting the quality scores to zero, also set the base itself to 'N'. This is useful to visualize in IGV. + */ + SET_Q_ZERO_BASE_N, + /** + * Look at the color quality scores and probabilistically decide to change the reference inserted base to be the base which is implied by the original color space instead of the reference. + */ + REMOVE_REF_BIAS + } + + public enum SOLID_NOCALL_STRATEGY { + /** + * When a no call is detected throw an exception to alert the user that recalibrating this SOLiD data is unsafe. This is the default option. + */ + THROW_EXCEPTION, + /** + * Leave the read in the output bam completely untouched. This mode is only okay if the no calls are very rare. + */ + LEAVE_READ_UNRECALIBRATED, + /** + * Mark these reads as failing vendor quality checks so they can be filtered out by downstream analyses. + */ + PURGE_READ + } + + public RecalDataManager() { + nestedHashMap = new NestedHashMap(); + dataCollapsedReadGroup = null; + dataCollapsedQualityScore = null; + dataCollapsedByCovariate = null; + } + + public RecalDataManager(final boolean createCollapsedTables, final int numCovariates) { + if (createCollapsedTables) { // Initialize all the collapsed tables, only used by TableRecalibrationWalker + nestedHashMap = null; + dataCollapsedReadGroup = new NestedHashMap(); + dataCollapsedQualityScore = new NestedHashMap(); + dataCollapsedByCovariate = new ArrayList(); + for (int iii = 0; iii < numCovariates - 2; iii++) { // readGroup and QualityScore aren't counted here, their tables are separate + dataCollapsedByCovariate.add(new NestedHashMap()); + } + } + else { + nestedHashMap = new NestedHashMap(); + dataCollapsedReadGroup = null; + dataCollapsedQualityScore = null; + dataCollapsedByCovariate = null; + } + } + + public static CovariateKeySet getAllCovariateValuesFor(GATKSAMRecord read) { + return (CovariateKeySet) read.getTemporaryAttribute(COVARS_ATTRIBUTE); + } + + /** + * Add the given mapping to all of the collapsed hash tables + * + * @param key The list of comparables that is the key for this mapping + * @param fullDatum The RecalDatum which is the data for this mapping + * @param PRESERVE_QSCORES_LESS_THAN The threshold in report quality for adding to the aggregate collapsed table + */ + public final void addToAllTables(final Object[] key, final RecalDatum fullDatum, final int PRESERVE_QSCORES_LESS_THAN) { + + // The full dataset isn't actually ever used for anything because of the sequential calculation so no need to keep the full data HashMap around + //data.put(key, thisDatum); // add the mapping to the main table + + final int qualityScore = Integer.parseInt(key[1].toString()); + final Object[] readGroupCollapsedKey = new Object[1]; + final Object[] qualityScoreCollapsedKey = new Object[2]; + final Object[] covariateCollapsedKey = new Object[3]; + RecalDatum collapsedDatum; + + // Create dataCollapsedReadGroup, the table where everything except read group has been collapsed + if (qualityScore >= PRESERVE_QSCORES_LESS_THAN) { + readGroupCollapsedKey[0] = key[0]; // Make a new key with just the read group + collapsedDatum = (RecalDatum) dataCollapsedReadGroup.get(readGroupCollapsedKey); + if (collapsedDatum == null) { + dataCollapsedReadGroup.put(new RecalDatum(fullDatum), readGroupCollapsedKey); + } + else { + collapsedDatum.combine(fullDatum); // using combine instead of increment in order to calculate overall aggregateQReported + } + } + + // Create dataCollapsedQuality, the table where everything except read group and quality score has been collapsed + qualityScoreCollapsedKey[0] = key[0]; // Make a new key with the read group ... + qualityScoreCollapsedKey[1] = key[1]; // and quality score + collapsedDatum = (RecalDatum) dataCollapsedQualityScore.get(qualityScoreCollapsedKey); + if (collapsedDatum == null) { + dataCollapsedQualityScore.put(new RecalDatum(fullDatum), qualityScoreCollapsedKey); + } + else { + collapsedDatum.increment(fullDatum); + } + + // Create dataCollapsedByCovariate's, the tables where everything except read group, quality score, and given covariate has been collapsed + for (int iii = 0; iii < dataCollapsedByCovariate.size(); iii++) { + covariateCollapsedKey[0] = key[0]; // Make a new key with the read group ... + covariateCollapsedKey[1] = key[1]; // and quality score ... + final Object theCovariateElement = key[iii + 2]; // and the given covariate + if (theCovariateElement != null) { + covariateCollapsedKey[2] = theCovariateElement; + collapsedDatum = (RecalDatum) dataCollapsedByCovariate.get(iii).get(covariateCollapsedKey); + if (collapsedDatum == null) { + dataCollapsedByCovariate.get(iii).put(new RecalDatum(fullDatum), covariateCollapsedKey); + } + else { + collapsedDatum.increment(fullDatum); + } + } + } + } + + /** + * Loop over all the collapsed tables and turn the recalDatums found there into an empirical quality score + * that will be used in the sequential calculation in TableRecalibrationWalker + * + * @param smoothing The smoothing parameter that goes into empirical quality score calculation + * @param maxQual At which value to cap the quality scores + */ + public final void generateEmpiricalQualities(final int smoothing, final int maxQual) { + + recursivelyGenerateEmpiricalQualities(dataCollapsedReadGroup.data, smoothing, maxQual); + recursivelyGenerateEmpiricalQualities(dataCollapsedQualityScore.data, smoothing, maxQual); + for (NestedHashMap map : dataCollapsedByCovariate) { + recursivelyGenerateEmpiricalQualities(map.data, smoothing, maxQual); + checkForSingletons(map.data); + } + } + + private void recursivelyGenerateEmpiricalQualities(final Map data, final int smoothing, final int maxQual) { + + for (Object comp : data.keySet()) { + final Object val = data.get(comp); + if (val instanceof RecalDatum) { // We are at the end of the nested hash maps + ((RecalDatum) val).calcCombinedEmpiricalQuality(smoothing, maxQual); + } + else { // Another layer in the nested hash map + recursivelyGenerateEmpiricalQualities((Map) val, smoothing, maxQual); + } + } + } + + private void checkForSingletons(final Map data) { + // todo -- this looks like it's better just as a data.valueSet() call? + for (Object comp : data.keySet()) { + final Object val = data.get(comp); + if (val instanceof RecalDatum) { // We are at the end of the nested hash maps + if (data.keySet().size() == 1) { + data.clear(); // don't TableRecalibrate a non-required covariate if it only has one element because that correction has already been done ... + // in a previous step of the sequential calculation model + } + } + else { // Another layer in the nested hash map + checkForSingletons((Map) val); + } + } + } + + /** + * Get the appropriate collapsed table out of the set of all the tables held by this Object + * + * @param covariate Which covariate indexes the desired collapsed HashMap + * @return The desired collapsed HashMap + */ + public final NestedHashMap getCollapsedTable(final int covariate) { + if (covariate == 0) { + return dataCollapsedReadGroup; // Table where everything except read group has been collapsed + } + else if (covariate == 1) { + return dataCollapsedQualityScore; // Table where everything except read group and quality score has been collapsed + } + else { + return dataCollapsedByCovariate.get(covariate - 2); // Table where everything except read group, quality score, and given covariate has been collapsed + } + } + + /** + * Section of code shared between the two recalibration walkers which uses the command line arguments to adjust attributes of the read such as quals or platform string + * + * @param read The read to adjust + * @param RAC The list of shared command line arguments + */ + public static void parseSAMRecord(final GATKSAMRecord read, final RecalibrationArgumentCollection RAC) { + GATKSAMReadGroupRecord readGroup = ((GATKSAMRecord) read).getReadGroup(); + + if (RAC.FORCE_PLATFORM != null && (readGroup.getPlatform() == null || !readGroup.getPlatform().equals(RAC.FORCE_PLATFORM))) { + readGroup.setPlatform(RAC.FORCE_PLATFORM); + } + + if (readGroup.getPlatform() == null) { + if (RAC.DEFAULT_PLATFORM != null) { + if (!warnUserNullPlatform) { + Utils.warnUser("The input .bam file contains reads with no platform information. " + + "Defaulting to platform = " + RAC.DEFAULT_PLATFORM + ". " + + "First observed at read with name = " + read.getReadName()); + warnUserNullPlatform = true; + } + readGroup.setPlatform(RAC.DEFAULT_PLATFORM); + } + else { + throw new UserException.MalformedBAM(read, "The input .bam file contains reads with no platform information. First observed at read with name = " + read.getReadName()); + } + } + } + + /** + * Parse through the color space of the read and add a new tag to the SAMRecord that says which bases are inconsistent with the color space + * + * @param read The SAMRecord to parse + */ + public static void parseColorSpace(final GATKSAMRecord read) { + + // If this is a SOLID read then we have to check if the color space is inconsistent. This is our only sign that SOLID has inserted the reference base + if (ReadUtils.isSOLiDRead(read)) { + if (read.getAttribute(RecalDataManager.COLOR_SPACE_INCONSISTENCY_TAG) == null) { // Haven't calculated the inconsistency array yet for this read + final Object attr = read.getAttribute(RecalDataManager.COLOR_SPACE_ATTRIBUTE_TAG); + if (attr != null) { + byte[] colorSpace; + if (attr instanceof String) { + colorSpace = ((String) attr).getBytes(); + } + else { + throw new UserException.MalformedBAM(read, String.format("Value encoded by %s in %s isn't a string!", RecalDataManager.COLOR_SPACE_ATTRIBUTE_TAG, read.getReadName())); + } + + // Loop over the read and calculate first the inferred bases from the color and then check if it is consistent with the read + byte[] readBases = read.getReadBases(); + if (read.getReadNegativeStrandFlag()) { + readBases = BaseUtils.simpleReverseComplement(read.getReadBases()); + } + final byte[] inconsistency = new byte[readBases.length]; + int iii; + byte prevBase = colorSpace[0]; // The sentinel + for (iii = 0; iii < readBases.length; iii++) { + final byte thisBase = getNextBaseFromColor(read, prevBase, colorSpace[iii + 1]); + inconsistency[iii] = (byte) (thisBase == readBases[iii] ? 0 : 1); + prevBase = readBases[iii]; + } + read.setAttribute(RecalDataManager.COLOR_SPACE_INCONSISTENCY_TAG, inconsistency); + + } + else { + throw new UserException.MalformedBAM(read, "Unable to find color space information in SOLiD read. First observed at read with name = " + read.getReadName() + + " Unfortunately this .bam file can not be recalibrated without color space information because of potential reference bias."); + } + } + } + } + + /** + * Parse through the color space of the read and apply the desired --solid_recal_mode correction to the bases + * This method doesn't add the inconsistent tag to the read like parseColorSpace does + * + * @param read The SAMRecord to parse + * @param originalQualScores The array of original quality scores to modify during the correction + * @param solidRecalMode Which mode of solid recalibration to apply + * @param refBases The reference for this read + * @return A new array of quality scores that have been ref bias corrected + */ + public static byte[] calcColorSpace(final GATKSAMRecord read, byte[] originalQualScores, final SOLID_RECAL_MODE solidRecalMode, final byte[] refBases) { + + final Object attr = read.getAttribute(RecalDataManager.COLOR_SPACE_ATTRIBUTE_TAG); + if (attr != null) { + byte[] colorSpace; + if (attr instanceof String) { + colorSpace = ((String) attr).getBytes(); + } + else { + throw new ReviewedStingException(String.format("Value encoded by %s in %s isn't a string!", RecalDataManager.COLOR_SPACE_ATTRIBUTE_TAG, read.getReadName())); + } + + // Loop over the read and calculate first the inferred bases from the color and then check if it is consistent with the read + byte[] readBases = read.getReadBases(); + final byte[] colorImpliedBases = readBases.clone(); + byte[] refBasesDirRead = AlignmentUtils.alignmentToByteArray(read.getCigar(), read.getReadBases(), refBases); //BUGBUG: This needs to change when read walkers are changed to give the aligned refBases + if (read.getReadNegativeStrandFlag()) { + readBases = BaseUtils.simpleReverseComplement(read.getReadBases()); + refBasesDirRead = BaseUtils.simpleReverseComplement(refBasesDirRead.clone()); + } + final int[] inconsistency = new int[readBases.length]; + byte prevBase = colorSpace[0]; // The sentinel + for (int iii = 0; iii < readBases.length; iii++) { + final byte thisBase = getNextBaseFromColor(read, prevBase, colorSpace[iii + 1]); + colorImpliedBases[iii] = thisBase; + inconsistency[iii] = (thisBase == readBases[iii] ? 0 : 1); + prevBase = readBases[iii]; + } + + // Now that we have the inconsistency array apply the desired correction to the inconsistent bases + if (solidRecalMode == SOLID_RECAL_MODE.SET_Q_ZERO) { // Set inconsistent bases and the one before it to Q0 + final boolean setBaseN = false; + originalQualScores = solidRecalSetToQZero(read, readBases, inconsistency, originalQualScores, refBasesDirRead, setBaseN); + } + else if (solidRecalMode == SOLID_RECAL_MODE.SET_Q_ZERO_BASE_N) { + final boolean setBaseN = true; + originalQualScores = solidRecalSetToQZero(read, readBases, inconsistency, originalQualScores, refBasesDirRead, setBaseN); + } + else if (solidRecalMode == SOLID_RECAL_MODE.REMOVE_REF_BIAS) { // Use the color space quality to probabilistically remove ref bases at inconsistent color space bases + solidRecalRemoveRefBias(read, readBases, inconsistency, colorImpliedBases, refBasesDirRead); + } + + } + else { + throw new UserException.MalformedBAM(read, "Unable to find color space information in SOLiD read. First observed at read with name = " + read.getReadName() + + " Unfortunately this .bam file can not be recalibrated without color space information because of potential reference bias."); + } + + return originalQualScores; + } + + public static boolean checkNoCallColorSpace(final GATKSAMRecord read) { + if (ReadUtils.isSOLiDRead(read)) { + final Object attr = read.getAttribute(RecalDataManager.COLOR_SPACE_ATTRIBUTE_TAG); + if (attr != null) { + byte[] colorSpace; + if (attr instanceof String) { + colorSpace = ((String) attr).substring(1).getBytes(); // trim off the Sentinel + } + else { + throw new ReviewedStingException(String.format("Value encoded by %s in %s isn't a string!", RecalDataManager.COLOR_SPACE_ATTRIBUTE_TAG, read.getReadName())); + } + + for (byte color : colorSpace) { + if (color != (byte) '0' && color != (byte) '1' && color != (byte) '2' && color != (byte) '3') { + return true; // There is a bad color in this SOLiD read and the user wants to skip over it + } + } + + } + else { + throw new UserException.MalformedBAM(read, "Unable to find color space information in SOLiD read. First observed at read with name = " + read.getReadName() + + " Unfortunately this .bam file can not be recalibrated without color space information because of potential reference bias."); + } + } + + return false; // There aren't any color no calls in this SOLiD read + } + + /** + * Perform the SET_Q_ZERO solid recalibration. Inconsistent color space bases and their previous base are set to quality zero + * + * @param read The SAMRecord to recalibrate + * @param readBases The bases in the read which have been RC'd if necessary + * @param inconsistency The array of 1/0 that says if this base is inconsistent with its color + * @param originalQualScores The array of original quality scores to set to zero if needed + * @param refBases The reference which has been RC'd if necessary + * @param setBaseN Should we also set the base to N as well as quality zero in order to visualize in IGV or something similar + * @return The byte array of original quality scores some of which might have been set to zero + */ + private static byte[] solidRecalSetToQZero(final GATKSAMRecord read, byte[] readBases, final int[] inconsistency, final byte[] originalQualScores, final byte[] refBases, final boolean setBaseN) { + + final boolean negStrand = read.getReadNegativeStrandFlag(); + for (int iii = 1; iii < originalQualScores.length; iii++) { + if (inconsistency[iii] == 1) { + if (readBases[iii] == refBases[iii]) { + if (negStrand) { + originalQualScores[originalQualScores.length - (iii + 1)] = (byte) 0; + } + else { + originalQualScores[iii] = (byte) 0; + } + if (setBaseN) { + readBases[iii] = (byte) 'N'; + } + } + // Set the prev base to Q0 as well + if (readBases[iii - 1] == refBases[iii - 1]) { + if (negStrand) { + originalQualScores[originalQualScores.length - iii] = (byte) 0; + } + else { + originalQualScores[iii - 1] = (byte) 0; + } + if (setBaseN) { + readBases[iii - 1] = (byte) 'N'; + } + } + } + } + if (negStrand) { + readBases = BaseUtils.simpleReverseComplement(readBases.clone()); // Put the bases back in reverse order to stuff them back in the read + } + read.setReadBases(readBases); + + return originalQualScores; + } + + /** + * Peform the REMOVE_REF_BIAS solid recalibration. Look at the color space qualities and probabilistically decide if the base should be change to match the color or left as reference + * + * @param read The SAMRecord to recalibrate + * @param readBases The bases in the read which have been RC'd if necessary + * @param inconsistency The array of 1/0 that says if this base is inconsistent with its color + * @param colorImpliedBases The bases implied by the color space, RC'd if necessary + * @param refBases The reference which has been RC'd if necessary + */ + private static void solidRecalRemoveRefBias(final GATKSAMRecord read, byte[] readBases, final int[] inconsistency, final byte[] colorImpliedBases, final byte[] refBases) { + + final Object attr = read.getAttribute(RecalDataManager.COLOR_SPACE_QUAL_ATTRIBUTE_TAG); + if (attr != null) { + byte[] colorSpaceQuals; + if (attr instanceof String) { + String x = (String) attr; + colorSpaceQuals = x.getBytes(); + SAMUtils.fastqToPhred(colorSpaceQuals); + } + else { + throw new ReviewedStingException(String.format("Value encoded by %s in %s isn't a string!", RecalDataManager.COLOR_SPACE_QUAL_ATTRIBUTE_TAG, read.getReadName())); + } + + for (int iii = 1; iii < inconsistency.length - 1; iii++) { + if (inconsistency[iii] == 1) { + for (int jjj = iii - 1; jjj <= iii; jjj++) { // Correct this base and the one before it along the direction of the read + if (jjj == iii || inconsistency[jjj] == 0) { // Don't want to correct the previous base a second time if it was already corrected in the previous step + if (readBases[jjj] == refBases[jjj]) { + if (colorSpaceQuals[jjj] == colorSpaceQuals[jjj + 1]) { // Equal evidence for the color implied base and the reference base, so flip a coin + final int rand = GenomeAnalysisEngine.getRandomGenerator().nextInt(2); + if (rand == 0) { // The color implied base won the coin flip + readBases[jjj] = colorImpliedBases[jjj]; + } + } + else { + final int maxQuality = Math.max((int) colorSpaceQuals[jjj], (int) colorSpaceQuals[jjj + 1]); + final int minQuality = Math.min((int) colorSpaceQuals[jjj], (int) colorSpaceQuals[jjj + 1]); + int diffInQuality = maxQuality - minQuality; + int numLow = minQuality; + if (numLow == 0) { + numLow++; + diffInQuality++; + } + final int numHigh = Math.round(numLow * (float) Math.pow(10.0f, (float) diffInQuality / 10.0f)); // The color with higher quality is exponentially more likely + final int rand = GenomeAnalysisEngine.getRandomGenerator().nextInt(numLow + numHigh); + if (rand >= numLow) { // higher q score won + if (maxQuality == (int) colorSpaceQuals[jjj]) { + readBases[jjj] = colorImpliedBases[jjj]; + } // else ref color had higher q score, and won out, so nothing to do here + } + else { // lower q score won + if (minQuality == (int) colorSpaceQuals[jjj]) { + readBases[jjj] = colorImpliedBases[jjj]; + } // else ref color had lower q score, and won out, so nothing to do here + } + } + } + } + } + } + } + + if (read.getReadNegativeStrandFlag()) { + readBases = BaseUtils.simpleReverseComplement(readBases.clone()); // Put the bases back in reverse order to stuff them back in the read + } + read.setReadBases(readBases); + } + else { // No color space quality tag in file + throw new UserException.MalformedBAM(read, "REMOVE_REF_BIAS recal mode requires color space qualities but they can't be found for read: " + read.getReadName()); + } + } + + /** + * Given the base and the color calculate the next base in the sequence + * + * @param prevBase The base + * @param color The color + * @return The next base in the sequence + */ + private static byte getNextBaseFromColor(GATKSAMRecord read, final byte prevBase, final byte color) { + switch (color) { + case '0': + return prevBase; + case '1': + return performColorOne(prevBase); + case '2': + return performColorTwo(prevBase); + case '3': + return performColorThree(prevBase); + default: + throw new UserException.MalformedBAM(read, "Unrecognized color space in SOLID read, color = " + (char) color + + " Unfortunately this bam file can not be recalibrated without full color space information because of potential reference bias."); + } + } + + /** + * Check if this base is inconsistent with its color space. If it is then SOLID inserted the reference here and we should reduce the quality + * + * @param read The read which contains the color space to check against + * @param offset The offset in the read at which to check + * @return Returns true if the base was inconsistent with the color space + */ + public static boolean isInconsistentColorSpace(final GATKSAMRecord read, final int offset) { + final Object attr = read.getAttribute(RecalDataManager.COLOR_SPACE_INCONSISTENCY_TAG); + if (attr != null) { + final byte[] inconsistency = (byte[]) attr; + // NOTE: The inconsistency array is in the direction of the read, not aligned to the reference! + if (read.getReadNegativeStrandFlag()) { // Negative direction + return inconsistency[inconsistency.length - offset - 1] != (byte) 0; + } + else { // Forward direction + return inconsistency[offset] != (byte) 0; + } + + // This block of code is for if you want to check both the offset and the next base for color space inconsistency + //if( read.getReadNegativeStrandFlag() ) { // Negative direction + // if( offset == 0 ) { + // return inconsistency[0] != 0; + // } else { + // return (inconsistency[inconsistency.length - offset - 1] != 0) || (inconsistency[inconsistency.length - offset] != 0); + // } + //} else { // Forward direction + // if( offset == inconsistency.length - 1 ) { + // return inconsistency[inconsistency.length - 1] != 0; + // } else { + // return (inconsistency[offset] != 0) || (inconsistency[offset + 1] != 0); + // } + //} + + } + else { // No inconsistency array, so nothing is inconsistent + return false; + } + } + + /** + * Computes all requested covariates for every offset in the given read + * by calling covariate.getValues(..). + * + * @param read The read for which to compute covariate values. + * @param requestedCovariates The list of requested covariates. + * @return An array of covariate values where result[i][j] is the covariate + * value for the ith position in the read and the jth covariate in + * reqeustedCovariates list. + */ + public static void computeCovariates(final GATKSAMRecord read, final List requestedCovariates) { + final int numRequestedCovariates = requestedCovariates.size(); + final int readLength = read.getReadLength(); + final CovariateKeySet covariateKeySet = new CovariateKeySet(readLength, numRequestedCovariates); + + // Loop through the list of requested covariates and compute the values of each covariate for all positions in this read + for (Covariate covariate : requestedCovariates) + covariateKeySet.addCovariate(covariate.getValues(read)); + + read.setTemporaryAttribute(COVARS_ATTRIBUTE, covariateKeySet); + } + + /** + * Perform a certain transversion (A <-> C or G <-> T) on the base. + * + * @param base the base [AaCcGgTt] + * @return the transversion of the base, or the input base if it's not one of the understood ones + */ + private static byte performColorOne(byte base) { + switch (base) { + case 'A': + case 'a': + return 'C'; + case 'C': + case 'c': + return 'A'; + case 'G': + case 'g': + return 'T'; + case 'T': + case 't': + return 'G'; + default: + return base; + } + } + + /** + * Perform a transition (A <-> G or C <-> T) on the base. + * + * @param base the base [AaCcGgTt] + * @return the transition of the base, or the input base if it's not one of the understood ones + */ + private static byte performColorTwo(byte base) { + switch (base) { + case 'A': + case 'a': + return 'G'; + case 'C': + case 'c': + return 'T'; + case 'G': + case 'g': + return 'A'; + case 'T': + case 't': + return 'C'; + default: + return base; + } + } + + /** + * Return the complement (A <-> T or C <-> G) of a base. + * + * @param base the base [AaCcGgTt] + * @return the complementary base, or the input base if it's not one of the understood ones + */ + private static byte performColorThree(byte base) { + switch (base) { + case 'A': + case 'a': + return 'T'; + case 'C': + case 'c': + return 'G'; + case 'G': + case 'g': + return 'C'; + case 'T': + case 't': + return 'A'; + default: + return base; + } + } +} diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalDatum.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalDatum.java new file mode 100755 index 0000000000..91f865180a --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalDatum.java @@ -0,0 +1,112 @@ +package org.broadinstitute.sting.gatk.walkers.bqsr; + +/* + * Copyright (c) 2009 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/** + * Created by IntelliJ IDEA. + * User: rpoplin + * Date: Nov 3, 2009 + * + * An individual piece of recalibration data. Each bin counts up the number of observations and the number of reference mismatches seen for that combination of covariates. + */ + +public class RecalDatum extends RecalDatumOptimized { + + private double estimatedQReported; // estimated reported quality score based on combined data's individual q-reporteds and number of observations + private double empiricalQuality; // the empirical quality for datums that have been collapsed together (by read group and reported quality, for example) + + //--------------------------------------------------------------------------------------------------------------- + // + // constructors + // + //--------------------------------------------------------------------------------------------------------------- + + public RecalDatum() { + numObservations = 0L; + numMismatches = 0L; + estimatedQReported = 0.0; + empiricalQuality = 0.0; + } + + public RecalDatum(final long _numObservations, final long _numMismatches, final double _estimatedQReported, final double _empiricalQuality) { + numObservations = _numObservations; + numMismatches = _numMismatches; + estimatedQReported = _estimatedQReported; + empiricalQuality = _empiricalQuality; + } + + public RecalDatum(final RecalDatum copy) { + this.numObservations = copy.numObservations; + this.numMismatches = copy.numMismatches; + this.estimatedQReported = copy.estimatedQReported; + this.empiricalQuality = copy.empiricalQuality; + } + + //--------------------------------------------------------------------------------------------------------------- + // + // increment methods + // + //--------------------------------------------------------------------------------------------------------------- + + public final void combine(final RecalDatum other) { + final double sumErrors = this.calcExpectedErrors() + other.calcExpectedErrors(); + this.increment(other.numObservations, other.numMismatches); + this.estimatedQReported = -10 * Math.log10(sumErrors / (double) this.numObservations); + //if( this.estimatedQReported > QualityUtils.MAX_REASONABLE_Q_SCORE ) { this.estimatedQReported = QualityUtils.MAX_REASONABLE_Q_SCORE; } + } + + //--------------------------------------------------------------------------------------------------------------- + // + // methods to derive empirical quality score + // + //--------------------------------------------------------------------------------------------------------------- + + public final void calcCombinedEmpiricalQuality(final int smoothing, final int maxQual) { + this.empiricalQuality = empiricalQualDouble(smoothing, maxQual); // cache the value so we don't call log over and over again + } + + //--------------------------------------------------------------------------------------------------------------- + // + // misc. methods + // + //--------------------------------------------------------------------------------------------------------------- + + public final double getEstimatedQReported() { + return estimatedQReported; + } + + public final double getEmpiricalQuality() { + return empiricalQuality; + } + + private double calcExpectedErrors() { + return (double) this.numObservations * qualToErrorProb(estimatedQReported); + } + + private double qualToErrorProb(final double qual) { + return Math.pow(10.0, qual / -10.0); + } +} diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalDatumOptimized.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalDatumOptimized.java new file mode 100755 index 0000000000..2333808206 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalDatumOptimized.java @@ -0,0 +1,115 @@ +package org.broadinstitute.sting.gatk.walkers.bqsr; + +import org.broadinstitute.sting.utils.QualityUtils; + +import java.util.List; + +/* + * Copyright (c) 2010 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/** + * Created by IntelliJ IDEA. + * User: rpoplin + * Date: Jan 6, 2010 + * + * An individual piece of recalibration data. Optimized for CountCovariates. Extras added to make TableRecalibration fast have been removed. + * Each bin counts up the number of observations and the number of reference mismatches seen for that combination of covariates. + */ + +public class RecalDatumOptimized { + + protected long numObservations; // number of bases seen in total + protected long numMismatches; // number of bases seen that didn't match the reference + + //--------------------------------------------------------------------------------------------------------------- + // + // constructors + // + //--------------------------------------------------------------------------------------------------------------- + + public RecalDatumOptimized() { + numObservations = 0L; + numMismatches = 0L; + } + + public RecalDatumOptimized(final long _numObservations, final long _numMismatches) { + numObservations = _numObservations; + numMismatches = _numMismatches; + } + + public RecalDatumOptimized(final RecalDatumOptimized copy) { + this.numObservations = copy.numObservations; + this.numMismatches = copy.numMismatches; + } + + //--------------------------------------------------------------------------------------------------------------- + // + // increment methods + // + //--------------------------------------------------------------------------------------------------------------- + + public synchronized final void increment(final long incObservations, final long incMismatches) { + numObservations += incObservations; + numMismatches += incMismatches; + } + + public synchronized final void increment(final RecalDatumOptimized other) { + increment(other.numObservations, other.numMismatches); + } + + public synchronized final void increment(final List data) { + for (RecalDatumOptimized other : data) { + this.increment(other); + } + } + + //--------------------------------------------------------------------------------------------------------------- + // + // methods to derive empirical quality score + // + //--------------------------------------------------------------------------------------------------------------- + + public final double empiricalQualDouble(final int smoothing, final double maxQual) { + final double doubleMismatches = (double) (numMismatches + smoothing); + final double doubleObservations = (double) (numObservations + smoothing); + double empiricalQual = -10 * Math.log10(doubleMismatches / doubleObservations); + return Math.min(empiricalQual, maxQual); + } + + public final byte empiricalQualByte(final int smoothing) { + final double doubleMismatches = (double) (numMismatches + smoothing); + final double doubleObservations = (double) (numObservations + smoothing); + return QualityUtils.probToQual(1.0 - doubleMismatches / doubleObservations); // This is capped at Q40 + } + + public final byte empiricalQualByte() { + return empiricalQualByte(0); // 'default' behavior is to use smoothing value of zero + } + + public final String outputToCSV() { + return String.format("%d,%d,%d", numObservations, numMismatches, (int) empiricalQualByte()); + } + +} diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalibrationArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalibrationArgumentCollection.java new file mode 100755 index 0000000000..38e7051e48 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalibrationArgumentCollection.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2010 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR + * THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.walkers.bqsr; + +import org.broadinstitute.sting.commandline.Argument; +import org.broadinstitute.sting.commandline.Hidden; + +/** + * Created by IntelliJ IDEA. + * User: rpoplin + * Date: Nov 27, 2009 + * + * A collection of the arguments that are common to both CovariateCounterWalker and TableRecalibrationWalker. + * This set of arguments will also be passed to the constructor of every Covariate when it is instantiated. + */ + +public class RecalibrationArgumentCollection { + + /** + * CountCovariates and TableRecalibration accept a --solid_recal_mode flag which governs how the recalibrator handles the + * reads which have had the reference inserted because of color space inconsistencies. + */ + @Argument(fullName = "solid_recal_mode", shortName = "sMode", required = false, doc = "How should we recalibrate solid bases in which the reference was inserted? Options = DO_NOTHING, SET_Q_ZERO, SET_Q_ZERO_BASE_N, or REMOVE_REF_BIAS") + public RecalDataManager.SOLID_RECAL_MODE SOLID_RECAL_MODE = RecalDataManager.SOLID_RECAL_MODE.SET_Q_ZERO; + + /** + * CountCovariates and TableRecalibration accept a --solid_nocall_strategy flag which governs how the recalibrator handles + * no calls in the color space tag. Unfortunately because of the reference inserted bases mentioned above, reads with no calls in + * their color space tag can not be recalibrated. + */ + @Argument(fullName = "solid_nocall_strategy", shortName = "solid_nocall_strategy", doc = "Defines the behavior of the recalibrator when it encounters no calls in the color space. Options = THROW_EXCEPTION, LEAVE_READ_UNRECALIBRATED, or PURGE_READ", required = false) + public RecalDataManager.SOLID_NOCALL_STRATEGY SOLID_NOCALL_STRATEGY = RecalDataManager.SOLID_NOCALL_STRATEGY.THROW_EXCEPTION; + + /** + * The context covariate will use a context of this size to calculate it's covariate value for base mismatches + */ + @Argument(fullName = "mismatches_context_size", shortName = "mcs", doc = "size of the k-mer context to be used for base mismatches", required = false) + public int MISMATCHES_CONTEXT_SIZE = 2; + + /** + * The context covariate will use a context of this size to calculate it's covariate value for base insertions + */ + @Argument(fullName = "insertions_context_size", shortName = "ics", doc = "size of the k-mer context to be used for base insertions", required = false) + public int INSERTIONS_CONTEXT_SIZE = 8; + + /** + * The context covariate will use a context of this size to calculate it's covariate value for base deletions + */ + @Argument(fullName = "deletions_context_size", shortName = "dcs", doc = "size of the k-mer context to be used for base deletions", required = false) + public int DELETIONS_CONTEXT_SIZE = 8; + + /** + * A default base qualities to use as a prior (reported quality) in the mismatch covariate model. This value will replace all base qualities in the read for this default value. Negative value turns it off (default is off) + */ + @Argument(fullName = "mismatches_default_quality", shortName = "mdq", doc = "default quality for the base mismatches covariate", required = false) + public byte MISMATCHES_DEFAULT_QUALITY = -1; + + /** + * A default base qualities to use as a prior (reported quality) in the insertion covariate model. This parameter is used for all reads without insertion quality scores for each base. (default is on) + */ + @Argument(fullName = "insertions_default_quality", shortName = "idq", doc = "default quality for the base insertions covariate", required = false) + public byte INSERTIONS_DEFAULT_QUALITY = 45; + + /** + * A default base qualities to use as a prior (reported quality) in the mismatch covariate model. This value will replace all base qualities in the read for this default value. Negative value turns it off (default is off) + */ + @Argument(fullName = "deletions_default_quality", shortName = "ddq", doc = "default quality for the base deletions covariate", required = false) + public byte DELETIONS_DEFAULT_QUALITY = 45; + + + @Hidden + @Argument(fullName = "default_platform", shortName = "dP", required = false, doc = "If a read has no platform then default to the provided String. Valid options are illumina, 454, and solid.") + public String DEFAULT_PLATFORM = null; + @Hidden + @Argument(fullName = "force_platform", shortName = "fP", required = false, doc = "If provided, the platform of EVERY read will be forced to be the provided String. Valid options are illumina, 454, and solid.") + public String FORCE_PLATFORM = null; + + +} From a7c6f255e9dcf53caf83d1ab24605e2f554da671 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Fri, 10 Feb 2012 13:33:57 -0500 Subject: [PATCH 244/356] Adding the old gatherer to BQSR for now, the old gatherer will still work for us to scatter/gather our tests. From f1990981fcf715a4d3762b11bd8d4f79dd948701 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Fri, 10 Feb 2012 14:00:53 -0500 Subject: [PATCH 245/356] A little BQSR scala script to use with scatter/gather From f52f1f659f9c1d9a0eccc18830dd8e9c7e037cdc Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Fri, 10 Feb 2012 14:15:59 -0500 Subject: [PATCH 246/356] Multiallelic implementation of the TDT should be a pairwise list of values as per Mark Daly. Integration tests change because the count in the header is now A instead of 1. --- .../TransmissionDisequilibriumTest.java | 49 ++++++++----------- .../VariantAnnotatorIntegrationTest.java | 2 +- 2 files changed, 22 insertions(+), 29 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java index d84ba44bc1..1f8ccf6525 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TransmissionDisequilibriumTest.java @@ -9,6 +9,7 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation; import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.RodRequiringAnnotation; import org.broadinstitute.sting.utils.MathUtils; +import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineCount; import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType; import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine; import org.broadinstitute.sting.utils.exceptions.UserException; @@ -58,41 +59,33 @@ public Map annotate(RefMetaDataTracker tracker, AnnotatorCompati // return the descriptions used for the VCF INFO meta field public List getKeyNames() { return Arrays.asList("TDT"); } - public List getDescriptions() { return Arrays.asList(new VCFInfoHeaderLine("TDT", 1, VCFHeaderLineType.Float, "Test statistic from Wittkowski transmission disequilibrium test.")); } + public List getDescriptions() { return Arrays.asList(new VCFInfoHeaderLine("TDT", VCFHeaderLineCount.A, VCFHeaderLineType.Float, "Test statistic from Wittkowski transmission disequilibrium test.")); } // Following derivation in http://en.wikipedia.org/wiki/Transmission_disequilibrium_test#A_modified_version_of_the_TDT - private double calculateTDT( final VariantContext vc, final Set triosToTest ) { + private List calculateTDT( final VariantContext vc, final Set triosToTest ) { - double nABGivenABandBB = 0.0; - double nBBGivenABandBB = 0.0; - double nAAGivenABandAB = 0.0; - double nBBGivenABandAB = 0.0; - double nAAGivenAAandAB = 0.0; - double nABGivenAAandAB = 0.0; + List pairwiseTDTs = new ArrayList(10); + final int HomRefIndex = 0; // for each pair of alleles, add the likelihoods - int numAlleles = vc.getNAlleles(); - for ( int allele1 = 0; allele1 < numAlleles; allele1++ ) { - final int HOM1index = determineHomIndex(allele1, numAlleles); - - for ( int allele2 = allele1 + 1; allele2 < numAlleles; allele2++ ) { - - // TODO -- cache these for better performance - final int HETindex = HOM1index + (allele2 - allele1); - final int HOM2index = determineHomIndex(allele2, numAlleles); - - nABGivenABandBB += calculateNChildren(vc, triosToTest, HETindex, HETindex, HOM2index) + calculateNChildren(vc, triosToTest, HETindex, HOM2index, HETindex); - nBBGivenABandBB += calculateNChildren(vc, triosToTest, HOM2index, HETindex, HOM2index) + calculateNChildren(vc, triosToTest, HOM2index, HOM2index, HETindex); - nAAGivenABandAB += calculateNChildren(vc, triosToTest, HOM1index, HETindex, HETindex); - nBBGivenABandAB += calculateNChildren(vc, triosToTest, HOM2index, HETindex, HETindex); - nAAGivenAAandAB += calculateNChildren(vc, triosToTest, HOM1index, HOM1index, HETindex) + calculateNChildren(vc, triosToTest, HOM1index, HETindex, HOM1index); - nABGivenAAandAB += calculateNChildren(vc, triosToTest, HETindex, HOM1index, HETindex) + calculateNChildren(vc, triosToTest, HETindex, HETindex, HOM1index); - } + int numAltAlleles = vc.getAlternateAlleles().size(); + for ( int alt = 1; alt <= numAltAlleles; alt++ ) { + final int HetIndex = alt; + final int HomVarIndex = determineHomIndex(alt, numAltAlleles+1); + + final double nABGivenABandBB = calculateNChildren(vc, triosToTest, HetIndex, HetIndex, HomVarIndex) + calculateNChildren(vc, triosToTest, HetIndex, HomVarIndex, HetIndex); + final double nBBGivenABandBB = calculateNChildren(vc, triosToTest, HomVarIndex, HetIndex, HomVarIndex) + calculateNChildren(vc, triosToTest, HomVarIndex, HomVarIndex, HetIndex); + final double nAAGivenABandAB = calculateNChildren(vc, triosToTest, HomRefIndex, HetIndex, HetIndex); + final double nBBGivenABandAB = calculateNChildren(vc, triosToTest, HomVarIndex, HetIndex, HetIndex); + final double nAAGivenAAandAB = calculateNChildren(vc, triosToTest, HomRefIndex, HomRefIndex, HetIndex) + calculateNChildren(vc, triosToTest, HomRefIndex, HetIndex, HomRefIndex); + final double nABGivenAAandAB = calculateNChildren(vc, triosToTest, HetIndex, HomRefIndex, HetIndex) + calculateNChildren(vc, triosToTest, HetIndex, HetIndex, HomRefIndex); + + final double numer = (nABGivenABandBB - nBBGivenABandBB) + 2.0 * (nAAGivenABandAB - nBBGivenABandAB) + (nAAGivenAAandAB - nABGivenAAandAB); + final double denom = (nABGivenABandBB + nBBGivenABandBB) + 4.0 * (nAAGivenABandAB + nBBGivenABandAB) + (nAAGivenAAandAB + nABGivenAAandAB); + pairwiseTDTs.add((numer * numer) / denom); } - final double numer = (nABGivenABandBB - nBBGivenABandBB) + 2.0 * (nAAGivenABandAB - nBBGivenABandAB) + (nAAGivenAAandAB - nABGivenAAandAB); - final double denom = (nABGivenABandBB + nBBGivenABandBB) + 4.0 * (nAAGivenABandAB + nBBGivenABandAB) + (nAAGivenAAandAB + nABGivenAAandAB); - return (numer * numer) / denom; + return pairwiseTDTs; } private double calculateNChildren( final VariantContext vc, final Set triosToTest, final int childIdx, final int momIdx, final int dadIdx ) { diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java index 0d9d9bcd89..7984a00c0a 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java @@ -179,7 +179,7 @@ public void testSnpEffAnnotationsUnsupportedVersion() { @Test public void testTDTAnnotation() { - final String MD5 = "0aedd760e8099f0b95d53a41bdcd793e"; + final String MD5 = "a78c1e950740d3c13c0258960c5fa8e1"; WalkerTestSpec spec = new WalkerTestSpec( "-T VariantAnnotator -R " + b37KGReference + " -A TransmissionDisequilibriumTest --variant:vcf " + validationDataLocation + "ug.random50000.subset300bp.chr1.family.vcf" + " -L " + validationDataLocation + "ug.random50000.subset300bp.chr1.family.vcf -NO_HEADER -ped " + validationDataLocation + "ug.random50000.family.ped -o %s", 1, From 9b8fd4c2ff6c63bc60fbab973590ca495cf99cfd Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Sat, 11 Feb 2012 10:57:20 -0500 Subject: [PATCH 247/356] Updating the half of the code that makes use of the recalibration information to work with the new refactoring of the bqsr. Reverting the covariate interface change in the original bqsr because the error model enum was moved to a different class and didn't make sense any more. --- .../traversals/TraverseActiveRegions.java | 6 +- .../gatk/walkers/ActiveRegionWalker.java | 4 +- .../walkers/annotator/MVLikelihoodRatio.java | 2 +- .../gatk/walkers/bqsr/ContextCovariate.java | 5 + .../sting/gatk/walkers/bqsr/Covariate.java | 1 + .../gatk/walkers/bqsr/CovariateKeySet.java | 12 +- .../gatk/walkers/bqsr/CycleCovariate.java | 5 + .../walkers/bqsr/QualityScoreCovariate.java | 36 +++--- .../gatk/walkers/bqsr/ReadGroupCovariate.java | 6 + .../gatk/walkers/bqsr/RecalDataManager.java | 69 ++++++----- .../recalibration/ContextCovariate.java | 2 +- .../recalibration/CountCovariatesWalker.java | 2 +- .../gatk/walkers/recalibration/Covariate.java | 2 +- .../walkers/recalibration/CycleCovariate.java | 2 +- .../walkers/recalibration/DinucCovariate.java | 2 +- .../recalibration/GCContentCovariate.java | 2 +- .../recalibration/HomopolymerCovariate.java | 2 +- .../MappingQualityCovariate.java | 2 +- .../recalibration/MinimumNQSCovariate.java | 2 +- .../recalibration/PositionCovariate.java | 2 +- .../recalibration/PrimerRoundCovariate.java | 2 +- .../recalibration/QualityScoreCovariate.java | 14 +-- .../recalibration/ReadGroupCovariate.java | 2 +- .../recalibration/RecalDataManager.java | 5 +- .../TableRecalibrationWalker.java | 2 +- .../recalibration/BaseRecalibration.java | 111 +++++++++--------- .../sting/utils/sam/GATKSAMRecord.java | 40 +------ 27 files changed, 164 insertions(+), 178 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java index 58c2df877e..70fe437555 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java +++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java @@ -107,7 +107,7 @@ public T traverse( final ActiveRegionWalker walker, } // If this is the last pileup for this shard calculate the minimum alignment start so that we know - // which active regions in the work queue are now safe to process + // which active regions in the work queue are now safe to process if( !locusView.hasNext() ) { for( final PileupElement p : locus.getBasePileup() ) { final GATKSAMRecord read = p.getRead(); @@ -135,7 +135,7 @@ public T traverse( final ActiveRegionWalker walker, } } - // Since we've sufficiently past this point (or this contig!) in the workQueue we can unload those regions and process them + // Since we've traversed sufficiently past this point (or this contig!) in the workQueue we can unload those regions and process them while( workQueue.peek() != null && (workQueue.peek().getExtendedLoc().getStop() < minStart || !workQueue.peek().getExtendedLoc().getContig().equals(dataProvider.getLocus().getContig())) ) { final ActiveRegion activeRegion = workQueue.remove(); sum = processActiveRegion( activeRegion, myReads, workQueue, sum, walker ); @@ -190,7 +190,7 @@ private T processActiveRegion( final ActiveRegion activeRegion, final LinkedHash reads.removeAll( placedReads ); // remove all the reads which have been placed into their active region logger.debug(">> Map call with " + activeRegion.getReads().size() + " " + (activeRegion.isActive ? "active" : "inactive") + " reads @ " + activeRegion.getLocation() + " with full extent: " + activeRegion.getReferenceLoc()); - final M x = walker.map( activeRegion, null ); // BUGBUG: tracker needs to be filled in and passed to the walker + final M x = walker.map( activeRegion, null ); return walker.reduce( x, sum ); } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java index 244870c78a..6403f15a2f 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/ActiveRegionWalker.java @@ -5,14 +5,12 @@ import org.broadinstitute.sting.commandline.Input; import org.broadinstitute.sting.commandline.IntervalBinding; import org.broadinstitute.sting.commandline.Output; -import org.broadinstitute.sting.commandline.RodBinding; import org.broadinstitute.sting.gatk.contexts.AlignmentContext; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; import org.broadinstitute.sting.gatk.filters.DuplicateReadFilter; import org.broadinstitute.sting.gatk.filters.FailsVendorQualityCheckFilter; import org.broadinstitute.sting.gatk.filters.NotPrimaryAlignmentFilter; import org.broadinstitute.sting.gatk.filters.UnmappedReadFilter; -import org.broadinstitute.sting.gatk.refdata.ReadMetaDataTracker; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; import org.broadinstitute.sting.utils.GenomeLoc; import org.broadinstitute.sting.utils.GenomeLocParser; @@ -77,7 +75,7 @@ public boolean wantsNonPrimaryReads() { public abstract double isActive(final RefMetaDataTracker tracker, final ReferenceContext ref, final AlignmentContext context); // Map over the ActiveRegion - public abstract MapType map(final ActiveRegion activeRegion, final ReadMetaDataTracker metaDataTracker); + public abstract MapType map(final ActiveRegion activeRegion, final RefMetaDataTracker metaDataTracker); public final GenomeLocSortedSet extendIntervals( final GenomeLocSortedSet intervals, final GenomeLocParser genomeLocParser, IndexedFastaSequenceFile reference ) { final int activeRegionExtension = this.getClass().getAnnotation(ActiveRegionExtension.class).extension(); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MVLikelihoodRatio.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MVLikelihoodRatio.java index 889cc634c3..e38d7d1424 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MVLikelihoodRatio.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MVLikelihoodRatio.java @@ -22,8 +22,8 @@ * User: chartl * Date: 9/14/11 * Time: 12:24 PM - * To change this template use File | Settings | File Templates. */ + public class MVLikelihoodRatio extends InfoFieldAnnotation implements ExperimentalAnnotation, RodRequiringAnnotation { private MendelianViolation mendelianViolation = null; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java index a46543f671..c7b90606c6 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java @@ -98,4 +98,9 @@ private String makeAllNStringWithLength(int length) { return s; } + // Used to get the covariate's value from input csv file during on-the-fly recalibration + @Override + public final Comparable getValue(final String str) { + return str; + } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/Covariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/Covariate.java index d1726dd130..b99cd3c3c3 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/Covariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/Covariate.java @@ -53,6 +53,7 @@ public interface Covariate { */ public CovariateValues getValues(GATKSAMRecord read); + public Comparable getValue(String str); // Used to get the covariate's value from input csv file during on-the-fly recalibration } interface RequiredCovariate extends Covariate {} diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/CovariateKeySet.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/CovariateKeySet.java index 04a0684b66..f71bb03e57 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/CovariateKeySet.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/CovariateKeySet.java @@ -15,18 +15,18 @@ public class CovariateKeySet { private int nextCovariateIndex; - private final String mismatchesCovariateName = "M"; - private final String insertionsCovariateName = "I"; - private final String deletionsCovariateName = "D"; + public final static String mismatchesCovariateName = "M"; + public final static String insertionsCovariateName = "I"; + public final static String deletionsCovariateName = "D"; public CovariateKeySet(int readLength, int numberOfCovariates) { numberOfCovariates++; // +1 because we are adding the mismatch covariate (to comply with the molten table format) this.mismatchesKeySet = new Object[readLength][numberOfCovariates]; this.insertionsKeySet = new Object[readLength][numberOfCovariates]; this.deletionsKeySet = new Object[readLength][numberOfCovariates]; - initializeCovariateKeySet(this.mismatchesKeySet, this.mismatchesCovariateName); - initializeCovariateKeySet(this.insertionsKeySet, this.insertionsCovariateName); - initializeCovariateKeySet(this.deletionsKeySet, this.deletionsCovariateName); + initializeCovariateKeySet(this.mismatchesKeySet, mismatchesCovariateName); + initializeCovariateKeySet(this.insertionsKeySet, insertionsCovariateName); + initializeCovariateKeySet(this.deletionsKeySet, deletionsCovariateName); this.nextCovariateIndex = 0; } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/CycleCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/CycleCovariate.java index f996de50ea..a5795c018c 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/CycleCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/CycleCovariate.java @@ -196,4 +196,9 @@ else if (FLOW_CYCLE_PLATFORMS.contains(ngsPlatform)) { return new CovariateValues(cycles, cycles, cycles); } + // Used to get the covariate's value from input csv file during on-the-fly recalibration + @Override + public final Comparable getValue(final String str) { + return Integer.parseInt(str); + } } \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/QualityScoreCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/QualityScoreCovariate.java index 0d36f3ff4b..b48e486acd 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/QualityScoreCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/QualityScoreCovariate.java @@ -39,39 +39,35 @@ public class QualityScoreCovariate implements RequiredCovariate { - private byte defaultMismatchesQuality; // walker parameter. Must be > 0 to be used, otherwise we use the quality from the read. - private byte defaultInsertionsQuality; // walker parameter. Must be > 0 to be used, otherwise we use the quality from the read. - private byte defaultDeletionsQuality; // walker parameter. Must be > 0 to be used, otherwise we use the quality from the read. - // Initialize any member variables using the command-line arguments passed to the walkers @Override public void initialize(final RecalibrationArgumentCollection RAC) { - defaultMismatchesQuality = RAC.MISMATCHES_DEFAULT_QUALITY; - defaultInsertionsQuality = RAC.INSERTIONS_DEFAULT_QUALITY; - defaultDeletionsQuality = RAC.DELETIONS_DEFAULT_QUALITY; } @Override public CovariateValues getValues(final GATKSAMRecord read) { int readLength = read.getReadLength(); - - Byte [] mismatches = new Byte[readLength]; - Byte [] insertions = new Byte[readLength]; - Byte [] deletions = new Byte[readLength]; - + + Integer [] mismatches = new Integer[readLength]; + Integer [] insertions = new Integer[readLength]; + Integer [] deletions = new Integer[readLength]; + byte [] baseQualities = read.getBaseQualities(); + byte [] baseInsertionQualities = read.getBaseInsertionQualities(); + byte [] baseDeletionQualities = read.getBaseDeletionQualities(); - if (defaultMismatchesQuality >= 0) - Arrays.fill(mismatches, defaultMismatchesQuality); // if the user decides to override the base qualities in the read, use the flat value - else { - for (int i=0; i dataCollapsedByCovariate; // Tables where everything except read group, quality score, and given covariate has been collapsed + public final NestedHashMap nestedHashMap; // The full dataset + private final HashMap dataCollapsedReadGroup; // Table where everything except read group has been collapsed + private final HashMap dataCollapsedQualityScore; // Table where everything except read group and quality score has been collapsed + private final HashMap> dataCollapsedByCovariate; // Tables where everything except read group, quality score, and given covariate has been collapsed - public final static String ORIGINAL_QUAL_ATTRIBUTE_TAG = "OQ"; // The tag that holds the original quality scores public final static String COLOR_SPACE_QUAL_ATTRIBUTE_TAG = "CQ"; // The tag that holds the color space quality scores for SOLID bams public final static String COLOR_SPACE_ATTRIBUTE_TAG = "CS"; // The tag that holds the color space for SOLID bams public final static String COLOR_SPACE_INCONSISTENCY_TAG = "ZC"; // A new tag made up for the recalibrator which will hold an array of ints which say if this base is inconsistent with its color - private static boolean warnUserNullReadGroup = false; private static boolean warnUserNullPlatform = false; private static final String COVARS_ATTRIBUTE = "COVARS"; // used to store covariates array as a temporary attribute inside GATKSAMRecord.\ + public enum BaseRecalibrationType { + BASE_SUBSTITUTION, + BASE_INSERTION, + BASE_DELETION + } public enum SOLID_RECAL_MODE { /** @@ -109,13 +113,18 @@ public RecalDataManager() { } public RecalDataManager(final boolean createCollapsedTables, final int numCovariates) { - if (createCollapsedTables) { // Initialize all the collapsed tables, only used by TableRecalibrationWalker + if (createCollapsedTables) { // Initialize all the collapsed tables, only used by on-the-fly recalibration nestedHashMap = null; - dataCollapsedReadGroup = new NestedHashMap(); - dataCollapsedQualityScore = new NestedHashMap(); - dataCollapsedByCovariate = new ArrayList(); - for (int iii = 0; iii < numCovariates - 2; iii++) { // readGroup and QualityScore aren't counted here, their tables are separate - dataCollapsedByCovariate.add(new NestedHashMap()); + dataCollapsedReadGroup = new HashMap(); + dataCollapsedQualityScore = new HashMap(); + dataCollapsedByCovariate = new HashMap>(); + for ( final BaseRecalibrationType errorModel : BaseRecalibrationType.values() ) { + dataCollapsedReadGroup.put(errorModel, new NestedHashMap()); + dataCollapsedQualityScore.put(errorModel, new NestedHashMap()); + dataCollapsedByCovariate.put(errorModel, new ArrayList()); + for (int iii = 0; iii < numCovariates - 2; iii++) { // readGroup and QualityScore aren't counted here, their tables are separate + dataCollapsedByCovariate.get(errorModel).add(new NestedHashMap()); + } } } else { @@ -137,7 +146,7 @@ public static CovariateKeySet getAllCovariateValuesFor(GATKSAMRecord read) { * @param fullDatum The RecalDatum which is the data for this mapping * @param PRESERVE_QSCORES_LESS_THAN The threshold in report quality for adding to the aggregate collapsed table */ - public final void addToAllTables(final Object[] key, final RecalDatum fullDatum, final int PRESERVE_QSCORES_LESS_THAN) { + public final void addToAllTables(final Object[] key, final RecalDatum fullDatum, final int PRESERVE_QSCORES_LESS_THAN, final BaseRecalibrationType errorModel ) { // The full dataset isn't actually ever used for anything because of the sequential calculation so no need to keep the full data HashMap around //data.put(key, thisDatum); // add the mapping to the main table @@ -151,9 +160,9 @@ public final void addToAllTables(final Object[] key, final RecalDatum fullDatum, // Create dataCollapsedReadGroup, the table where everything except read group has been collapsed if (qualityScore >= PRESERVE_QSCORES_LESS_THAN) { readGroupCollapsedKey[0] = key[0]; // Make a new key with just the read group - collapsedDatum = (RecalDatum) dataCollapsedReadGroup.get(readGroupCollapsedKey); + collapsedDatum = (RecalDatum) dataCollapsedReadGroup.get(errorModel).get(readGroupCollapsedKey); if (collapsedDatum == null) { - dataCollapsedReadGroup.put(new RecalDatum(fullDatum), readGroupCollapsedKey); + dataCollapsedReadGroup.get(errorModel).put(new RecalDatum(fullDatum), readGroupCollapsedKey); } else { collapsedDatum.combine(fullDatum); // using combine instead of increment in order to calculate overall aggregateQReported @@ -163,9 +172,9 @@ public final void addToAllTables(final Object[] key, final RecalDatum fullDatum, // Create dataCollapsedQuality, the table where everything except read group and quality score has been collapsed qualityScoreCollapsedKey[0] = key[0]; // Make a new key with the read group ... qualityScoreCollapsedKey[1] = key[1]; // and quality score - collapsedDatum = (RecalDatum) dataCollapsedQualityScore.get(qualityScoreCollapsedKey); + collapsedDatum = (RecalDatum) dataCollapsedQualityScore.get(errorModel).get(qualityScoreCollapsedKey); if (collapsedDatum == null) { - dataCollapsedQualityScore.put(new RecalDatum(fullDatum), qualityScoreCollapsedKey); + dataCollapsedQualityScore.get(errorModel).put(new RecalDatum(fullDatum), qualityScoreCollapsedKey); } else { collapsedDatum.increment(fullDatum); @@ -178,9 +187,9 @@ public final void addToAllTables(final Object[] key, final RecalDatum fullDatum, final Object theCovariateElement = key[iii + 2]; // and the given covariate if (theCovariateElement != null) { covariateCollapsedKey[2] = theCovariateElement; - collapsedDatum = (RecalDatum) dataCollapsedByCovariate.get(iii).get(covariateCollapsedKey); + collapsedDatum = (RecalDatum) dataCollapsedByCovariate.get(errorModel).get(iii).get(covariateCollapsedKey); if (collapsedDatum == null) { - dataCollapsedByCovariate.get(iii).put(new RecalDatum(fullDatum), covariateCollapsedKey); + dataCollapsedByCovariate.get(errorModel).get(iii).put(new RecalDatum(fullDatum), covariateCollapsedKey); } else { collapsedDatum.increment(fullDatum); @@ -198,11 +207,13 @@ public final void addToAllTables(final Object[] key, final RecalDatum fullDatum, */ public final void generateEmpiricalQualities(final int smoothing, final int maxQual) { - recursivelyGenerateEmpiricalQualities(dataCollapsedReadGroup.data, smoothing, maxQual); - recursivelyGenerateEmpiricalQualities(dataCollapsedQualityScore.data, smoothing, maxQual); - for (NestedHashMap map : dataCollapsedByCovariate) { - recursivelyGenerateEmpiricalQualities(map.data, smoothing, maxQual); - checkForSingletons(map.data); + for( final BaseRecalibrationType errorModel : BaseRecalibrationType.values() ) { + recursivelyGenerateEmpiricalQualities(dataCollapsedReadGroup.get(errorModel).data, smoothing, maxQual); + recursivelyGenerateEmpiricalQualities(dataCollapsedQualityScore.get(errorModel).data, smoothing, maxQual); + for (NestedHashMap map : dataCollapsedByCovariate.get(errorModel)) { + recursivelyGenerateEmpiricalQualities(map.data, smoothing, maxQual); + checkForSingletons(map.data); + } } } @@ -241,15 +252,15 @@ private void checkForSingletons(final Map data) { * @param covariate Which covariate indexes the desired collapsed HashMap * @return The desired collapsed HashMap */ - public final NestedHashMap getCollapsedTable(final int covariate) { + public final NestedHashMap getCollapsedTable(final int covariate, final BaseRecalibrationType errorModel) { if (covariate == 0) { - return dataCollapsedReadGroup; // Table where everything except read group has been collapsed + return dataCollapsedReadGroup.get(errorModel); // Table where everything except read group has been collapsed } else if (covariate == 1) { - return dataCollapsedQualityScore; // Table where everything except read group and quality score has been collapsed + return dataCollapsedQualityScore.get(errorModel); // Table where everything except read group and quality score has been collapsed } else { - return dataCollapsedByCovariate.get(covariate - 2); // Table where everything except read group, quality score, and given covariate has been collapsed + return dataCollapsedByCovariate.get(errorModel).get(covariate - 2); // Table where everything except read group, quality score, and given covariate has been collapsed } } @@ -260,7 +271,7 @@ else if (covariate == 1) { * @param RAC The list of shared command line arguments */ public static void parseSAMRecord(final GATKSAMRecord read, final RecalibrationArgumentCollection RAC) { - GATKSAMReadGroupRecord readGroup = ((GATKSAMRecord) read).getReadGroup(); + GATKSAMReadGroupRecord readGroup = read.getReadGroup(); if (RAC.FORCE_PLATFORM != null && (readGroup.getPlatform() == null || !readGroup.getPlatform().equals(RAC.FORCE_PLATFORM))) { readGroup.setPlatform(RAC.FORCE_PLATFORM); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java index 875782fdc0..e1a7772dbc 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ContextCovariate.java @@ -56,7 +56,7 @@ public void initialize(final RecalibrationArgumentCollection RAC) { } @Override - public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable) { byte[] bases = read.getReadBases(); for (int i = 0; i < read.getReadLength(); i++) comparable[i] = (i < CONTEXT_SIZE) ? allN : new String(Arrays.copyOfRange(bases, i - CONTEXT_SIZE, i)); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java index 626460be6c..a99f35f458 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java @@ -378,7 +378,7 @@ public CountedData map(RefMetaDataTracker tracker, ReferenceContext ref, Alignme } RecalDataManager.parseColorSpace(gatkRead); - gatkRead.setTemporaryAttribute(COVARS_ATTRIBUTE, RecalDataManager.computeCovariates(gatkRead, requestedCovariates, BaseRecalibration.BaseRecalibrationType.BASE_SUBSTITUTION)); + gatkRead.setTemporaryAttribute(COVARS_ATTRIBUTE, RecalDataManager.computeCovariates(gatkRead, requestedCovariates)); } // Skip this position if base quality is zero diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/Covariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/Covariate.java index e4edb8ca68..9d5747023f 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/Covariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/Covariate.java @@ -43,7 +43,7 @@ public interface Covariate { public Comparable getValue(String str); // Used to get the covariate's value from input csv file in TableRecalibrationWalker - public void getValues(GATKSAMRecord read, Comparable[] comparable, BaseRecalibration.BaseRecalibrationType modelType); + public void getValues(GATKSAMRecord read, Comparable[] comparable); //Takes an array of size (at least) read.getReadLength() and fills it with covariate //values for each position in the read. This method was created as an optimization over calling getValue( read, offset ) for each offset and allows //read-specific calculations to be done just once rather than for each offset. diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java index 4244af7d11..b8d13ca10b 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CycleCovariate.java @@ -66,7 +66,7 @@ public void initialize(final RecalibrationArgumentCollection RAC) { // Used to pick out the covariate's value from attributes of the read @Override - public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable) { //----------------------------- // Illumina, Solid, PacBio, and Complete Genomics diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/DinucCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/DinucCovariate.java index 2fa1b33cab..9a401d09f2 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/DinucCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/DinucCovariate.java @@ -66,7 +66,7 @@ public void initialize(final RecalibrationArgumentCollection RAC) { * Takes an array of size (at least) read.getReadLength() and fills it with the covariate values for each position in the read. */ @Override - public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable) { final HashMap dinucHashMapRef = this.dinucHashMap; //optimize access to dinucHashMap final int readLength = read.getReadLength(); final boolean negativeStrand = read.getReadNegativeStrandFlag(); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/GCContentCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/GCContentCovariate.java index 7b209ae5cf..14ffd35a46 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/GCContentCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/GCContentCovariate.java @@ -82,7 +82,7 @@ private Comparable getValue(final SAMRecord read, final int offset) { } @Override - public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable) { for (int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/HomopolymerCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/HomopolymerCovariate.java index fd67edc3b3..004fb0bdb0 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/HomopolymerCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/HomopolymerCovariate.java @@ -95,7 +95,7 @@ private Comparable getValue(final SAMRecord read, final int offset) { } @Override - public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable) { for (int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MappingQualityCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MappingQualityCovariate.java index e22049890c..54fa18106e 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MappingQualityCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MappingQualityCovariate.java @@ -55,7 +55,7 @@ public final Comparable getValue(final String str) { } @Override - public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable) { for (int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read); // BUGBUG: this can be optimized } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MinimumNQSCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MinimumNQSCovariate.java index 1dfb915b9a..ecaa550060 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MinimumNQSCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/MinimumNQSCovariate.java @@ -65,7 +65,7 @@ private Comparable getValue(final SAMRecord read, final int offset) { } @Override - public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable) { for (int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PositionCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PositionCovariate.java index fbd1efc47b..fd720697f5 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PositionCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PositionCovariate.java @@ -55,7 +55,7 @@ private Comparable getValue(final SAMRecord read, final int offset) { } @Override - public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable) { for (int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PrimerRoundCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PrimerRoundCovariate.java index 8dfa118849..d6bdea5bfc 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PrimerRoundCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/PrimerRoundCovariate.java @@ -62,7 +62,7 @@ private Comparable getValue(final SAMRecord read, final int offset) { } @Override - public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable) { for (int iii = 0; iii < read.getReadLength(); iii++) { comparable[iii] = getValue(read, iii); // BUGBUG: this can be optimized } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/QualityScoreCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/QualityScoreCovariate.java index 1ed4a6fe85..a29a0530c9 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/QualityScoreCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/QualityScoreCovariate.java @@ -46,16 +46,10 @@ public void initialize(final RecalibrationArgumentCollection RAC) { } @Override - public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { - if (modelType == BaseRecalibration.BaseRecalibrationType.BASE_SUBSTITUTION) { - byte[] baseQualities = read.getBaseQualities(); - for (int i = 0; i < read.getReadLength(); i++) { - comparable[i] = (int) baseQualities[i]; - } - } - else { // model == BASE_INSERTION || model == BASE_DELETION - Arrays.fill(comparable, 45); // Some day in the future when base insertion and base deletion quals exist the samtools API will - // be updated and the original quals will be pulled here, but for now we assume the original quality is a flat Q45 + public void getValues(final GATKSAMRecord read, final Comparable[] comparable) { + byte[] baseQualities = read.getBaseQualities(); + for (int i = 0; i < read.getReadLength(); i++) { + comparable[i] = (int) baseQualities[i]; } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ReadGroupCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ReadGroupCovariate.java index 27e1d82635..33adf44172 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ReadGroupCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/ReadGroupCovariate.java @@ -44,7 +44,7 @@ public void initialize(final RecalibrationArgumentCollection RAC) { } @Override - public void getValues(final GATKSAMRecord read, final Comparable[] comparable, final BaseRecalibration.BaseRecalibrationType modelType) { + public void getValues(final GATKSAMRecord read, final Comparable[] comparable) { final String readGroupId = read.getReadGroup().getReadGroupId(); for (int i = 0; i < read.getReadLength(); i++) { comparable[i] = readGroupId; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java index 311e33f8a5..1a6b8cfcb8 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/RecalDataManager.java @@ -63,7 +63,6 @@ public class RecalDataManager { public final static String COLOR_SPACE_QUAL_ATTRIBUTE_TAG = "CQ"; // The tag that holds the color space quality scores for SOLID bams public final static String COLOR_SPACE_ATTRIBUTE_TAG = "CS"; // The tag that holds the color space for SOLID bams public final static String COLOR_SPACE_INCONSISTENCY_TAG = "ZC"; // A new tag made up for the recalibrator which will hold an array of ints which say if this base is inconsistent with its color - private static boolean warnUserNullReadGroup = false; private static boolean warnUserNullPlatform = false; public enum SOLID_RECAL_MODE { @@ -604,7 +603,7 @@ public static boolean isInconsistentColorSpace(final GATKSAMRecord read, final i * value for the ith position in the read and the jth covariate in * reqeustedCovariates list. */ - public static Comparable[][] computeCovariates(final GATKSAMRecord gatkRead, final List requestedCovariates, final BaseRecalibration.BaseRecalibrationType modelType) { + public static Comparable[][] computeCovariates(final GATKSAMRecord gatkRead, final List requestedCovariates) { //compute all covariates for this read final int numRequestedCovariates = requestedCovariates.size(); final int readLength = gatkRead.getReadLength(); @@ -613,7 +612,7 @@ public static Comparable[][] computeCovariates(final GATKSAMRecord gatkRead, fin final Comparable[] tempCovariateValuesHolder = new Comparable[readLength]; for (int i = 0; i < numRequestedCovariates; i++) { // Loop through the list of requested covariates and compute the values of each covariate for all positions in this read - requestedCovariates.get(i).getValues(gatkRead, tempCovariateValuesHolder, modelType); + requestedCovariates.get(i).getValues(gatkRead, tempCovariateValuesHolder); for (int j = 0; j < readLength; j++) covariateValues_offset_x_covar[j][i] = tempCovariateValuesHolder[j]; // copy values into a 2D array that allows all covar types to be extracted at once for an offset j by doing covariateValues_offset_x_covar[j]. This avoids the need to later iterate over covar types. } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java index cd848cd9e0..08151321fa 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java @@ -405,7 +405,7 @@ else if (RAC.SOLID_NOCALL_STRATEGY == RecalDataManager.SOLID_NOCALL_STRATEGY.PUR } //compute all covariate values for this read - final Comparable[][] covariateValues_offset_x_covar = RecalDataManager.computeCovariates(read, requestedCovariates, BaseRecalibration.BaseRecalibrationType.BASE_SUBSTITUTION); + final Comparable[][] covariateValues_offset_x_covar = RecalDataManager.computeCovariates(read, requestedCovariates); // For each base in the read for (int offset = 0; offset < read.getReadLength(); offset++) { diff --git a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java index 75d4b1e170..2c1bc494a8 100644 --- a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java +++ b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java @@ -25,14 +25,12 @@ package org.broadinstitute.sting.utils.recalibration; -import org.broadinstitute.sting.gatk.walkers.recalibration.Covariate; -import org.broadinstitute.sting.gatk.walkers.recalibration.RecalDataManager; -import org.broadinstitute.sting.gatk.walkers.recalibration.RecalDatum; -import org.broadinstitute.sting.gatk.walkers.recalibration.RecalibrationArgumentCollection; +import org.broadinstitute.sting.gatk.walkers.bqsr.*; import org.broadinstitute.sting.utils.QualityUtils; import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.collections.NestedHashMap; import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import org.broadinstitute.sting.utils.text.XReadLines; @@ -52,19 +50,13 @@ public class BaseRecalibration { - public enum BaseRecalibrationType { - BASE_SUBSTITUTION, - BASE_INSERTION, - BASE_DELETION - } - private RecalDataManager dataManager; // Holds the data HashMap, mostly used by TableRecalibrationWalker to create collapsed data hashmaps private final ArrayList requestedCovariates = new ArrayList(); // List of covariates to be used in this calculation public static final Pattern COMMENT_PATTERN = Pattern.compile("^#.*"); public static final Pattern COVARIATE_PATTERN = Pattern.compile("^ReadGroup,QualityScore,.*"); public static final String EOF_MARKER = "EOF"; private static final int MAX_QUALITY_SCORE = 65; //BUGBUG: what value to use here? - private NestedHashMap qualityScoreByFullCovariateKey = new NestedHashMap(); // Caches the result of performSequentialQualityCalculation(..) for all sets of covariate values. + private NestedHashMap qualityScoreByFullCovariateKey = new NestedHashMap(); // Caches the result of performSequentialQualityCalculation(...) for all sets of covariate values. public BaseRecalibration( final File RECAL_FILE ) { // Get a list of all available covariates @@ -89,7 +81,7 @@ else if( COVARIATE_PATTERN.matcher(line).matches() ) { // The line string is eit throw new UserException.MalformedFile( RECAL_FILE, "Malformed input recalibration file. Found covariate names intermingled with data in file: " + RECAL_FILE ); } else { // Found the covariate list in input file, loop through all of them and instantiate them String[] vals = line.split(","); - for( int iii = 0; iii < vals.length - 3; iii++ ) { // There are n-3 covariates. The last three items are nObservations, nMismatch, and Qempirical + for( int iii = 0; iii < vals.length - 4; iii++ ) { // There are n-4 covariates. The last four items are ErrorModel, nObservations, nMismatch, and Qempirical boolean foundClass = false; for( Class covClass : classes ) { if( (vals[iii] + "Covariate").equalsIgnoreCase( covClass.getSimpleName() ) ) { @@ -160,7 +152,7 @@ private void addCSVData(final File file, final String line) { final String[] vals = line.split(","); // Check if the data line is malformed, for example if the read group string contains a comma then it won't be parsed correctly - if( vals.length != requestedCovariates.size() + 3 ) { // +3 because of nObservations, nMismatch, and Qempirical + if( vals.length != requestedCovariates.size() + 4 ) { // +4 because of ErrorModel, nObservations, nMismatch, and Qempirical throw new UserException.MalformedFile(file, "Malformed input recalibration file. Found data line with too many fields: " + line + " --Perhaps the read group string contains a comma and isn't being parsed correctly."); } @@ -172,39 +164,63 @@ private void addCSVData(final File file, final String line) { cov = requestedCovariates.get( iii ); key[iii] = cov.getValue( vals[iii] ); } - + final String modelString = vals[iii++]; + final RecalDataManager.BaseRecalibrationType errorModel = ( modelString.equals(CovariateKeySet.mismatchesCovariateName) ? RecalDataManager.BaseRecalibrationType.BASE_SUBSTITUTION : + ( modelString.equals(CovariateKeySet.insertionsCovariateName) ? RecalDataManager.BaseRecalibrationType.BASE_INSERTION : + ( modelString.equals(CovariateKeySet.deletionsCovariateName) ? RecalDataManager.BaseRecalibrationType.BASE_DELETION : null ) ) ); + // Create a new datum using the number of observations, number of mismatches, and reported quality score final RecalDatum datum = new RecalDatum( Long.parseLong( vals[iii] ), Long.parseLong( vals[iii + 1] ), Double.parseDouble( vals[1] ), 0.0 ); // Add that datum to all the collapsed tables which will be used in the sequential calculation - dataManager.addToAllTables( key, datum, QualityUtils.MIN_USABLE_Q_SCORE ); //BUGBUG: used to be Q5 now is Q6, probably doesn't matter + + dataManager.addToAllTables( key, datum, QualityUtils.MIN_USABLE_Q_SCORE, errorModel ); //BUGBUG: used to be Q5 now is Q6, probably doesn't matter } - public byte[] recalibrateRead( final GATKSAMRecord read, final byte[] originalQuals, final BaseRecalibrationType modelType ) { + public void recalibrateRead( final GATKSAMRecord read ) { - final byte[] recalQuals = originalQuals.clone(); - //compute all covariate values for this read - final Comparable[][] covariateValues_offset_x_covar = - RecalDataManager.computeCovariates(read, requestedCovariates, modelType); - - // For each base in the read - for( int offset = 0; offset < read.getReadLength(); offset++ ) { - - final Object[] fullCovariateKey = covariateValues_offset_x_covar[offset]; - - Byte qualityScore = (Byte) qualityScoreByFullCovariateKey.get(fullCovariateKey); - if(qualityScore == null) - { - qualityScore = performSequentialQualityCalculation( fullCovariateKey ); - qualityScoreByFullCovariateKey.put(qualityScore, fullCovariateKey); + RecalDataManager.computeCovariates(read, requestedCovariates); + final CovariateKeySet covariateKeySet = RecalDataManager.getAllCovariateValuesFor( read ); + + for( final RecalDataManager.BaseRecalibrationType errorModel : RecalDataManager.BaseRecalibrationType.values() ) { + final byte[] originalQuals = ( errorModel == RecalDataManager.BaseRecalibrationType.BASE_SUBSTITUTION ? read.getBaseQualities() : + ( errorModel == RecalDataManager.BaseRecalibrationType.BASE_INSERTION ? read.getBaseDeletionQualities() : + ( errorModel == RecalDataManager.BaseRecalibrationType.BASE_DELETION ? read.getBaseDeletionQualities() : null ) ) ); + final byte[] recalQuals = originalQuals.clone(); + + // For each base in the read + for( int offset = 0; offset < read.getReadLength(); offset++ ) { + + final Object[] fullCovariateKey = + ( errorModel == RecalDataManager.BaseRecalibrationType.BASE_SUBSTITUTION ? covariateKeySet.getMismatchesKeySet(offset) : + ( errorModel == RecalDataManager.BaseRecalibrationType.BASE_INSERTION ? covariateKeySet.getInsertionsKeySet(offset) : + ( errorModel == RecalDataManager.BaseRecalibrationType.BASE_DELETION ? covariateKeySet.getDeletionsKeySet(offset) : null ) ) ); + + Byte qualityScore = (Byte) qualityScoreByFullCovariateKey.get(fullCovariateKey); + if( qualityScore == null ) { + qualityScore = performSequentialQualityCalculation( errorModel, fullCovariateKey ); + qualityScoreByFullCovariateKey.put(qualityScore, fullCovariateKey); + } + + recalQuals[offset] = qualityScore; } - - recalQuals[offset] = qualityScore; - } - - preserveQScores( originalQuals, recalQuals ); // Overwrite the work done if original quality score is too low - return recalQuals; + preserveQScores( originalQuals, recalQuals ); // Overwrite the work done if original quality score is too low + switch (errorModel) { + case BASE_SUBSTITUTION: + read.setBaseQualities( recalQuals ); + break; + case BASE_INSERTION: + read.setAttribute( GATKSAMRecord.BQSR_BASE_INSERTION_QUALITIES, recalQuals ); + break; + case BASE_DELETION: + read.setAttribute( GATKSAMRecord.BQSR_BASE_DELETION_QUALITIES, recalQuals ); + break; + default: + throw new ReviewedStingException("Unrecognized Base Recalibration type: " + errorModel ); + } + } + } /** @@ -222,7 +238,7 @@ public byte[] recalibrateRead( final GATKSAMRecord read, final byte[] originalQu * @param key The list of Comparables that were calculated from the covariates * @return A recalibrated quality score as a byte */ - private byte performSequentialQualityCalculation( final Object... key ) { + private byte performSequentialQualityCalculation( final RecalDataManager.BaseRecalibrationType errorModel, final Object... key ) { final byte qualFromRead = (byte)Integer.parseInt(key[1].toString()); final Object[] readGroupCollapsedKey = new Object[1]; @@ -231,7 +247,7 @@ private byte performSequentialQualityCalculation( final Object... key ) { // The global quality shift (over the read group only) readGroupCollapsedKey[0] = key[0]; - final RecalDatum globalRecalDatum = ((RecalDatum)dataManager.getCollapsedTable(0).get( readGroupCollapsedKey )); + final RecalDatum globalRecalDatum = ((RecalDatum)dataManager.getCollapsedTable(0, errorModel).get( readGroupCollapsedKey )); double globalDeltaQ = 0.0; if( globalRecalDatum != null ) { final double globalDeltaQEmpirical = globalRecalDatum.getEmpiricalQuality(); @@ -242,7 +258,7 @@ private byte performSequentialQualityCalculation( final Object... key ) { // The shift in quality between reported and empirical qualityScoreCollapsedKey[0] = key[0]; qualityScoreCollapsedKey[1] = key[1]; - final RecalDatum qReportedRecalDatum = ((RecalDatum)dataManager.getCollapsedTable(1).get( qualityScoreCollapsedKey )); + final RecalDatum qReportedRecalDatum = ((RecalDatum)dataManager.getCollapsedTable(1, errorModel).get( qualityScoreCollapsedKey )); double deltaQReported = 0.0; if( qReportedRecalDatum != null ) { final double deltaQReportedEmpirical = qReportedRecalDatum.getEmpiricalQuality(); @@ -256,7 +272,7 @@ private byte performSequentialQualityCalculation( final Object... key ) { covariateCollapsedKey[1] = key[1]; for( int iii = 2; iii < key.length; iii++ ) { covariateCollapsedKey[2] = key[iii]; // The given covariate - final RecalDatum covariateRecalDatum = ((RecalDatum)dataManager.getCollapsedTable(iii).get( covariateCollapsedKey )); + final RecalDatum covariateRecalDatum = ((RecalDatum)dataManager.getCollapsedTable(iii, errorModel).get( covariateCollapsedKey )); if( covariateRecalDatum != null ) { deltaQCovariateEmpirical = covariateRecalDatum.getEmpiricalQuality(); deltaQCovariates += ( deltaQCovariateEmpirical - qualFromRead - (globalDeltaQ + deltaQReported) ); @@ -265,18 +281,6 @@ private byte performSequentialQualityCalculation( final Object... key ) { final double newQuality = qualFromRead + globalDeltaQ + deltaQReported + deltaQCovariates; return QualityUtils.boundQual( (int)Math.round(newQuality), (byte)MAX_QUALITY_SCORE ); - - // Verbose printouts used to validate with old recalibrator - //if(key.contains(null)) { - // System.out.println( key + String.format(" => %d + %.2f + %.2f + %.2f + %.2f = %d", - // qualFromRead, globalDeltaQ, deltaQReported, deltaQPos, deltaQDinuc, newQualityByte)); - //} - //else { - // System.out.println( String.format("%s %s %s %s => %d + %.2f + %.2f + %.2f + %.2f = %d", - // key.get(0).toString(), key.get(3).toString(), key.get(2).toString(), key.get(1).toString(), qualFromRead, globalDeltaQ, deltaQReported, deltaQPos, deltaQDinuc, newQualityByte) ); - //} - - //return newQualityByte; } /** @@ -291,5 +295,4 @@ private void preserveQScores( final byte[] originalQuals, final byte[] recalQual } } } - } diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java index bdcf2b210b..f6b3d759c6 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java @@ -54,7 +54,6 @@ public class GATKSAMRecord extends BAMRecord { // Base Quality Score Recalibrator specific attribute tags public static final String BQSR_BASE_INSERTION_QUALITIES = "BI"; public static final String BQSR_BASE_DELETION_QUALITIES = "BD"; - public static final String BQSR_BASES_HAVE_BEEN_RECALIBRATED_TAG = "BR"; // the SAMRecord data we're caching private String mReadString = null; @@ -163,27 +162,6 @@ public boolean equals(Object o) { return super.equals(o); } - - @Override - public byte[] getBaseQualities() { - return super.getBaseQualities(); - /* - if( getAttribute( BQSR_BASES_HAVE_BEEN_RECALIBRATED_TAG ) != null ) { - return super.getBaseQualities(); - } else { - // if the recal data was populated in the engine then recalibrate the quality scores on the fly - if( GenomeAnalysisEngine.hasBaseRecalibration() ) { - final byte[] quals = GenomeAnalysisEngine.getBaseRecalibration().recalibrateRead( this, super.getBaseQualities() ); - setBaseQualities(quals); - setAttribute( BQSR_BASES_HAVE_BEEN_RECALIBRATED_TAG, true ); - return quals; - } else { // just use the qualities that are in the read since we don't have the sufficient information to recalibrate on the fly - return super.getBaseQualities(); - } - } - */ - } - /** * Accessors for base insertion and base deletion quality scores */ @@ -191,13 +169,8 @@ public byte[] getBaseInsertionQualities() { byte[] quals = getByteArrayAttribute( BQSR_BASE_INSERTION_QUALITIES ); if( quals == null ) { quals = new byte[getBaseQualities().length]; - Arrays.fill(quals, (byte) 45); // allow for differing default values between BaseInsertions and BaseDeletions - // if the recal data was populated in the engine then recalibrate the quality scores on the fly - // else give default values which are flat Q45 - if( GenomeAnalysisEngine.hasBaseRecalibration() ) { - quals = GenomeAnalysisEngine.getBaseRecalibration().recalibrateRead( this, quals, BaseRecalibration.BaseRecalibrationType.BASE_INSERTION ); // the original quals here are the flat base insertion/deletion quals, NOT the original base qualities - } - // add the qual array to the read so that we don't have to do the recalibration work again + Arrays.fill(quals, (byte) 45); // Some day in the future when base insertion and base deletion quals exist the samtools API will + // be updated and the original quals will be pulled here, but for now we assume the original quality is a flat Q45 setAttribute( BQSR_BASE_INSERTION_QUALITIES, quals ); } return quals; @@ -207,13 +180,8 @@ public byte[] getBaseDeletionQualities() { byte[] quals = getByteArrayAttribute( BQSR_BASE_DELETION_QUALITIES ); if( quals == null ) { quals = new byte[getBaseQualities().length]; - Arrays.fill(quals, (byte) 45); - // if the recal data was populated in the engine then recalibrate the quality scores on the fly - // else give default values which are flat Q45 - if( GenomeAnalysisEngine.hasBaseRecalibration() ) { - quals = GenomeAnalysisEngine.getBaseRecalibration().recalibrateRead( this, quals, BaseRecalibration.BaseRecalibrationType.BASE_DELETION ); // the original quals here are the flat base insertion/deletion quals, NOT the original base qualities - } - // add the qual array to the read so that we don't have to do the recalibration work again + Arrays.fill(quals, (byte) 45); // Some day in the future when base insertion and base deletion quals exist the samtools API will + // be updated and the original quals will be pulled here, but for now we assume the original quality is a flat Q45 setAttribute( BQSR_BASE_DELETION_QUALITIES, quals ); } return quals; From 3caa1b83bb220d1de1f7d5d3bc3b04b86f09c519 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Sat, 11 Feb 2012 11:48:32 -0500 Subject: [PATCH 248/356] Updating HC integration tests --- .../sting/gatk/walkers/bqsr/RecalDataManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalDataManager.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalDataManager.java index a143ff98d2..8a255391f3 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalDataManager.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalDataManager.java @@ -181,7 +181,7 @@ public final void addToAllTables(final Object[] key, final RecalDatum fullDatum, } // Create dataCollapsedByCovariate's, the tables where everything except read group, quality score, and given covariate has been collapsed - for (int iii = 0; iii < dataCollapsedByCovariate.size(); iii++) { + for (int iii = 0; iii < dataCollapsedByCovariate.get(errorModel).size(); iii++) { covariateCollapsedKey[0] = key[0]; // Make a new key with the read group ... covariateCollapsedKey[1] = key[1]; // and quality score ... final Object theCovariateElement = key[iii + 2]; // and the given covariate From ac9250b12b1448ae1b265fde486e661d6ef8b68e Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Sat, 11 Feb 2012 23:02:05 -0500 Subject: [PATCH 249/356] Don't assume chrom20, just pull from the file list From 41ffd08d534cc56701b0257d01922dcef886a199 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Mon, 13 Feb 2012 12:35:09 -0500 Subject: [PATCH 250/356] On the fly base quality score recalibration now happens up front in a SAMIterator on input instead of in a lazy-loading fashion if the BQSR table is provided as an engine argument. On the fly recalibration is now completely hooked up and live. --- .../sting/gatk/GenomeAnalysisEngine.java | 11 ++-- .../sting/gatk/ReadProperties.java | 38 +++++++------- .../gatk/datasources/reads/SAMDataSource.java | 14 +++++- .../traversals/TraverseActiveRegions.java | 4 +- .../gatk/walkers/bqsr/ContextCovariate.java | 2 +- .../gatk/walkers/bqsr/CovariateKeySet.java | 33 ++++++++++-- .../sting/utils/baq/BAQSamIterator.java | 2 +- .../sting/utils/fragments/FragmentUtils.java | 19 ++++++- .../utils/recalibration/BQSRSamIterator.java | 50 +++++++++++++++++++ .../recalibration/BaseRecalibration.java | 35 ++++--------- .../sting/utils/sam/GATKSAMRecord.java | 35 +++++++++++-- .../reads/DownsamplerBenchmark.java | 3 +- .../LocusIteratorByStateUnitTest.java | 1 + 13 files changed, 183 insertions(+), 64 deletions(-) create mode 100644 public/java/src/org/broadinstitute/sting/utils/recalibration/BQSRSamIterator.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java b/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java index c0db75aa9c..50ef4653b9 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java @@ -185,12 +185,12 @@ public void setReferenceMetaDataFiles(Collection referenceMetaDataFi public static void resetRandomGenerator(long seed) { randomGenerator.setSeed(seed); } /** - * Static base quality score recalibration helper object + * Base Quality Score Recalibration helper object */ - private static BaseRecalibration baseRecalibration = null; - public static BaseRecalibration getBaseRecalibration() { return baseRecalibration; } - public static boolean hasBaseRecalibration() { return baseRecalibration != null; } - public static void setBaseRecalibration(File recalFile) { baseRecalibration = new BaseRecalibration(recalFile); } + private BaseRecalibration baseRecalibration = null; + public BaseRecalibration getBaseRecalibration() { return baseRecalibration; } + public boolean hasBaseRecalibration() { return baseRecalibration != null; } + public void setBaseRecalibration(File recalFile) { baseRecalibration = new BaseRecalibration(recalFile); } /** * Actually run the GATK with the specified walker. @@ -770,6 +770,7 @@ private SAMDataSource createReadsDataSource(GATKArgumentCollection argCollection getWalkerBAQApplicationTime() == BAQ.ApplicationTime.ON_INPUT ? argCollection.BAQMode : BAQ.CalculationMode.OFF, getWalkerBAQQualityMode(), refReader, + getBaseRecalibration(), argCollection.defaultBaseQualities); } diff --git a/public/java/src/org/broadinstitute/sting/gatk/ReadProperties.java b/public/java/src/org/broadinstitute/sting/gatk/ReadProperties.java index daa8ff60db..db22886ce1 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/ReadProperties.java +++ b/public/java/src/org/broadinstitute/sting/gatk/ReadProperties.java @@ -7,6 +7,7 @@ import org.broadinstitute.sting.gatk.datasources.reads.SAMReaderID; import org.broadinstitute.sting.gatk.filters.ReadFilter; import org.broadinstitute.sting.utils.baq.BAQ; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; import java.util.Collection; /** @@ -27,23 +28,20 @@ * information about how they should be downsampled, sorted, and filtered. */ public class ReadProperties { - private Collection readers = null; - private SAMFileHeader header = null; - private SAMFileReader.ValidationStringency validationStringency = SAMFileReader.ValidationStringency.STRICT; - private DownsamplingMethod downsamplingMethod = null; - private ValidationExclusion exclusionList = null; - private Collection supplementalFilters = null; - private boolean includeReadsWithDeletionAtLoci = false; - private boolean useOriginalBaseQualities = false; - private boolean generateExtendedEvents = false; - private BAQ.CalculationMode cmode = BAQ.CalculationMode.OFF; - private BAQ.QualityMode qmode = BAQ.QualityMode.DONT_MODIFY; - IndexedFastaSequenceFile refReader = null; // read for BAQ, if desired - private byte defaultBaseQualities; - - // do we want to generate additional piles of "extended" events (indels) -// immediately after the reference base such event is associated with? - + private final Collection readers; + private final SAMFileHeader header; + private final SAMFileReader.ValidationStringency validationStringency; + private final DownsamplingMethod downsamplingMethod; + private final ValidationExclusion exclusionList; + private final Collection supplementalFilters; + private final boolean includeReadsWithDeletionAtLoci; + private final boolean useOriginalBaseQualities; + private final boolean generateExtendedEvents; + private final BAQ.CalculationMode cmode; + private final BAQ.QualityMode qmode; + private final IndexedFastaSequenceFile refReader; // read for BAQ, if desired + private final BaseRecalibration bqsrApplier; + private final byte defaultBaseQualities; /** * Return true if the walker wants to see reads that contain deletions when looking at locus pileups @@ -126,6 +124,8 @@ public IndexedFastaSequenceFile getRefReader() { return refReader; } + public BaseRecalibration getBQSRApplier() { return bqsrApplier; } + /** * @return Default base quality value to fill reads missing base quality information. */ @@ -165,8 +165,9 @@ public ReadProperties( Collection samFiles, boolean includeReadsWithDeletionAtLoci, boolean generateExtendedEvents, BAQ.CalculationMode cmode, - BAQ.QualityMode qmode, + BAQ.QualityMode qmode, IndexedFastaSequenceFile refReader, + BaseRecalibration bqsrApplier, byte defaultBaseQualities) { this.readers = samFiles; this.header = header; @@ -180,6 +181,7 @@ public ReadProperties( Collection samFiles, this.cmode = cmode; this.qmode = qmode; this.refReader = refReader; + this.bqsrApplier = bqsrApplier; this.defaultBaseQualities = defaultBaseQualities; } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java index 27b9e7f778..70284b2a6a 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/SAMDataSource.java @@ -46,6 +46,8 @@ import org.broadinstitute.sting.utils.baq.BAQSamIterator; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; +import org.broadinstitute.sting.utils.recalibration.BQSRSamIterator; +import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; import org.broadinstitute.sting.utils.sam.GATKSamRecordFactory; import java.io.File; @@ -201,6 +203,7 @@ public SAMDataSource( BAQ.CalculationMode.OFF, BAQ.QualityMode.DONT_MODIFY, null, // no BAQ + null, // no BQSR (byte) -1); } @@ -237,6 +240,7 @@ public SAMDataSource( BAQ.CalculationMode cmode, BAQ.QualityMode qmode, IndexedFastaSequenceFile refReader, + BaseRecalibration bqsrApplier, byte defaultBaseQualities) { this.readMetrics = new ReadMetrics(); this.genomeLocParser = genomeLocParser; @@ -309,6 +313,7 @@ public SAMDataSource( cmode, qmode, refReader, + bqsrApplier, defaultBaseQualities); // cache the read group id (original) -> read group id (merged) @@ -591,6 +596,7 @@ private StingSAMIterator getIterator(SAMReaders readers, Shard shard, boolean en readProperties.getBAQCalculationMode(), readProperties.getBAQQualityMode(), readProperties.getRefReader(), + readProperties.getBQSRApplier(), readProperties.defaultBaseQualities()); } @@ -660,9 +666,10 @@ protected StingSAMIterator applyDecoratingIterators(ReadMetrics readMetrics, BAQ.CalculationMode cmode, BAQ.QualityMode qmode, IndexedFastaSequenceFile refReader, + BaseRecalibration bqsrApplier, byte defaultBaseQualities) { - if ( useOriginalBaseQualities || defaultBaseQualities >= 0 ) - // only wrap if we are replacing the original qualitiies or using a default base quality + if (useOriginalBaseQualities || defaultBaseQualities >= 0) + // only wrap if we are replacing the original qualities or using a default base quality wrappedIterator = new ReadFormattingIterator(wrappedIterator, useOriginalBaseQualities, defaultBaseQualities); // NOTE: this (and other filtering) should be done before on-the-fly sorting @@ -675,6 +682,9 @@ protected StingSAMIterator applyDecoratingIterators(ReadMetrics readMetrics, if (!noValidationOfReadOrder && enableVerification) wrappedIterator = new VerifyingSamIterator(genomeLocParser,wrappedIterator); + if (bqsrApplier != null) + wrappedIterator = new BQSRSamIterator(wrappedIterator, bqsrApplier); + if (cmode != BAQ.CalculationMode.OFF) wrappedIterator = new BAQSamIterator(refReader, wrappedIterator, cmode, qmode); diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java index 70fe437555..92c508f854 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java +++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java @@ -68,7 +68,7 @@ public T traverse( final ActiveRegionWalker walker, if(prevLoc != null) { for(int iii = prevLoc.getStart() + 1; iii < location.getStart(); iii++ ) { final GenomeLoc fakeLoc = engine.getGenomeLocParser().createGenomeLoc(prevLoc.getContig(), iii, iii); - if( initialIntervals.overlaps( fakeLoc ) ) { + if( initialIntervals == null || initialIntervals.overlaps( fakeLoc ) ) { final double isActiveProb = ( walker.presetActiveRegions == null ? walker.isActive( null, null, null ) : ( walker.presetActiveRegions.overlaps(fakeLoc) ? 1.0 : 0.0 ) ); isActiveList.add( isActiveProb ); @@ -89,7 +89,7 @@ public T traverse( final ActiveRegionWalker walker, final RefMetaDataTracker tracker = referenceOrderedDataView.getReferenceOrderedDataAtLocus(locus.getLocation(), refContext); // Call the walkers isActive function for this locus and add them to the list to be integrated later - if( initialIntervals.overlaps( location ) ) { + if( initialIntervals == null || initialIntervals.overlaps( location ) ) { final double isActiveProb = ( walker.presetActiveRegions == null ? walker.isActive( tracker, refContext, locus ) : ( walker.presetActiveRegions.overlaps(location) ? 1.0 : 0.0 ) ); isActiveList.add( isActiveProb ); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java index c7b90606c6..64f1d08a87 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java @@ -79,7 +79,7 @@ public CovariateValues getValues(final GATKSAMRecord read) { } /** - * calculates the context of a base indenpendent of the covariate mode + * calculates the context of a base independent of the covariate mode * * @param bases the bases in the read to build the context from * @param offset the position in the read to calculate the context for diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/CovariateKeySet.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/CovariateKeySet.java index f71bb03e57..1b62160a3d 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/CovariateKeySet.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/CovariateKeySet.java @@ -1,5 +1,7 @@ package org.broadinstitute.sting.gatk.walkers.bqsr; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; + /** * The object temporarily held by a read that describes all of it's covariates. * @@ -15,9 +17,9 @@ public class CovariateKeySet { private int nextCovariateIndex; - public final static String mismatchesCovariateName = "M"; - public final static String insertionsCovariateName = "I"; - public final static String deletionsCovariateName = "D"; + private static String mismatchesCovariateName = "M"; + private static String insertionsCovariateName = "I"; + private static String deletionsCovariateName = "D"; public CovariateKeySet(int readLength, int numberOfCovariates) { numberOfCovariates++; // +1 because we are adding the mismatch covariate (to comply with the molten table format) @@ -36,7 +38,30 @@ public void addCovariate(CovariateValues covariate) { transposeCovariateValues(deletionsKeySet, covariate.getDeletions()); nextCovariateIndex++; } - + + public static RecalDataManager.BaseRecalibrationType getErrorModelFromString(final String modelString) { + if (modelString.equals(mismatchesCovariateName)) + return RecalDataManager.BaseRecalibrationType.BASE_SUBSTITUTION; + else if (modelString.equals(insertionsCovariateName)) + return RecalDataManager.BaseRecalibrationType.BASE_INSERTION; + else if (modelString.equals(deletionsCovariateName)) + return RecalDataManager.BaseRecalibrationType.BASE_DELETION; + throw new ReviewedStingException("Unrecognized Base Recalibration model string: " + modelString); + } + + public Object[] getKeySet(final int readPosition, final RecalDataManager.BaseRecalibrationType errorModel) { + switch (errorModel) { + case BASE_SUBSTITUTION: + return getMismatchesKeySet(readPosition); + case BASE_INSERTION: + return getInsertionsKeySet(readPosition); + case BASE_DELETION: + return getDeletionsKeySet(readPosition); + default: + throw new ReviewedStingException("Unrecognized Base Recalibration type: " + errorModel ); + } + } + public Object[] getMismatchesKeySet(int readPosition) { return mismatchesKeySet[readPosition]; } diff --git a/public/java/src/org/broadinstitute/sting/utils/baq/BAQSamIterator.java b/public/java/src/org/broadinstitute/sting/utils/baq/BAQSamIterator.java index 26356a4a4d..adfeef5180 100644 --- a/public/java/src/org/broadinstitute/sting/utils/baq/BAQSamIterator.java +++ b/public/java/src/org/broadinstitute/sting/utils/baq/BAQSamIterator.java @@ -34,7 +34,7 @@ public class BAQSamIterator implements StingSAMIterator { "cmode != null" , "qmode != null"}) public BAQSamIterator(IndexedFastaSequenceFile refReader, StingSAMIterator it, BAQ.CalculationMode cmode, BAQ.QualityMode qmode) { - if ( cmode == BAQ.CalculationMode.OFF) throw new ReviewedStingException("BUG: shouldn't create BAQSamIterator with calculation mode OFF"); + if ( cmode == BAQ.CalculationMode.OFF ) throw new ReviewedStingException("BUG: shouldn't create BAQSamIterator with calculation mode OFF"); if ( qmode == BAQ.QualityMode.DONT_MODIFY ) throw new ReviewedStingException("BUG: shouldn't create BAQSamIterator with quailty mode DONT_MODIFY"); this.refReader = refReader; diff --git a/public/java/src/org/broadinstitute/sting/utils/fragments/FragmentUtils.java b/public/java/src/org/broadinstitute/sting/utils/fragments/FragmentUtils.java index 68bf6dce8e..7104b1eddb 100644 --- a/public/java/src/org/broadinstitute/sting/utils/fragments/FragmentUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/fragments/FragmentUtils.java @@ -4,6 +4,7 @@ import net.sf.samtools.CigarElement; import net.sf.samtools.CigarOperator; import net.sf.samtools.SAMRecord; +import org.broadinstitute.sting.gatk.walkers.bqsr.RecalDataManager; import org.broadinstitute.sting.utils.collections.Pair; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.pileup.PileupElement; @@ -150,13 +151,23 @@ public final static List mergeOverlappingPairedFragments( List MIN_QUAL_BAD_OVERLAP && secondReadQuals[iii-firstReadStop] > MIN_QUAL_BAD_OVERLAP && firstReadBases[iii] != secondReadBases[iii-firstReadStop] ) { @@ -164,16 +175,22 @@ public final static List mergeOverlappingPairedFragments( List secondReadQuals[iii-firstReadStop] ? firstReadBases[iii] : secondReadBases[iii-firstReadStop] ); quals[iii] = ( firstReadQuals[iii] > secondReadQuals[iii-firstReadStop] ? firstReadQuals[iii] : secondReadQuals[iii-firstReadStop] ); + insertionQuals[iii] = ( firstReadQuals[iii] > secondReadQuals[iii-firstReadStop] ? firstReadInsertionQuals[iii] : secondReadInsertionQuals[iii-firstReadStop] ); // Purposefully checking the highest base quality score + deletionQuals[iii] = ( firstReadQuals[iii] > secondReadQuals[iii-firstReadStop] ? firstReadDeletionQuals[iii] : secondReadDeletionQuals[iii-firstReadStop] ); // Purposefully checking the highest base quality score } for(int iii = firstRead.getReadLength(); iii < numBases; iii++) { bases[iii] = secondReadBases[iii-firstReadStop]; quals[iii] = secondReadQuals[iii-firstReadStop]; + insertionQuals[iii] = secondReadInsertionQuals[iii-firstReadStop]; + deletionQuals[iii] = secondReadDeletionQuals[iii-firstReadStop]; } final GATKSAMRecord returnRead = new GATKSAMRecord(firstRead.getHeader()); returnRead.setAlignmentStart(firstRead.getUnclippedStart()); returnRead.setReadBases( bases ); - returnRead.setBaseQualities( quals ); + returnRead.setBaseQualities( quals, RecalDataManager.BaseRecalibrationType.BASE_SUBSTITUTION ); + returnRead.setBaseQualities( insertionQuals, RecalDataManager.BaseRecalibrationType.BASE_INSERTION ); + returnRead.setBaseQualities( deletionQuals, RecalDataManager.BaseRecalibrationType.BASE_DELETION ); returnRead.setReadGroup( firstRead.getReadGroup() ); returnRead.setReferenceName( firstRead.getReferenceName() ); final CigarElement c = new CigarElement(bases.length, CigarOperator.M); diff --git a/public/java/src/org/broadinstitute/sting/utils/recalibration/BQSRSamIterator.java b/public/java/src/org/broadinstitute/sting/utils/recalibration/BQSRSamIterator.java new file mode 100644 index 0000000000..048f8e58ca --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/recalibration/BQSRSamIterator.java @@ -0,0 +1,50 @@ +package org.broadinstitute.sting.utils.recalibration; + +import com.google.java.contract.Ensures; +import com.google.java.contract.Requires; +import net.sf.samtools.SAMRecord; +import org.broadinstitute.sting.gatk.iterators.StingSAMIterator; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; + +import java.util.Iterator; + +/** + * Created by IntelliJ IDEA. + * User: rpoplin + * Date: 2/13/12 + */ + +public class BQSRSamIterator implements StingSAMIterator { + private final StingSAMIterator it; + private final BaseRecalibration bqsr; + + /** + * Creates a new BQSRSamIterator and applies BQSR on the fly to incoming reads. + * + * @param it The incoming SamIterator to wrap + * @param bqsr The object which holds the BQSR table information and knows how to apply it + */ + @Requires({ + "it != null", + "bqsr != null"}) + public BQSRSamIterator(StingSAMIterator it, BaseRecalibration bqsr) { + if ( bqsr == null ) throw new ReviewedStingException("BUG: shouldn't create BQSRSamIterator with null recalibration object"); + + this.it = it; + this.bqsr = bqsr; + } + + @Requires("hasNext()") + @Ensures("result != null") + public SAMRecord next() { + SAMRecord read = it.next(); + bqsr.recalibrateRead((GATKSAMRecord) read); + return read; + } + + public boolean hasNext() { return this.it.hasNext(); } + public void remove() { throw new UnsupportedOperationException("Can not remove records from a SAM file via an iterator!"); } + public void close() { it.close(); } + public Iterator iterator() { return this; } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java index 2c1bc494a8..b08365a788 100644 --- a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java +++ b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java @@ -38,6 +38,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.regex.Pattern; @@ -165,10 +166,8 @@ private void addCSVData(final File file, final String line) { key[iii] = cov.getValue( vals[iii] ); } final String modelString = vals[iii++]; - final RecalDataManager.BaseRecalibrationType errorModel = ( modelString.equals(CovariateKeySet.mismatchesCovariateName) ? RecalDataManager.BaseRecalibrationType.BASE_SUBSTITUTION : - ( modelString.equals(CovariateKeySet.insertionsCovariateName) ? RecalDataManager.BaseRecalibrationType.BASE_INSERTION : - ( modelString.equals(CovariateKeySet.deletionsCovariateName) ? RecalDataManager.BaseRecalibrationType.BASE_DELETION : null ) ) ); - + final RecalDataManager.BaseRecalibrationType errorModel = CovariateKeySet.getErrorModelFromString(modelString); + // Create a new datum using the number of observations, number of mismatches, and reported quality score final RecalDatum datum = new RecalDatum( Long.parseLong( vals[iii] ), Long.parseLong( vals[iii + 1] ), Double.parseDouble( vals[1] ), 0.0 ); // Add that datum to all the collapsed tables which will be used in the sequential calculation @@ -183,19 +182,16 @@ public void recalibrateRead( final GATKSAMRecord read ) { final CovariateKeySet covariateKeySet = RecalDataManager.getAllCovariateValuesFor( read ); for( final RecalDataManager.BaseRecalibrationType errorModel : RecalDataManager.BaseRecalibrationType.values() ) { - final byte[] originalQuals = ( errorModel == RecalDataManager.BaseRecalibrationType.BASE_SUBSTITUTION ? read.getBaseQualities() : - ( errorModel == RecalDataManager.BaseRecalibrationType.BASE_INSERTION ? read.getBaseDeletionQualities() : - ( errorModel == RecalDataManager.BaseRecalibrationType.BASE_DELETION ? read.getBaseDeletionQualities() : null ) ) ); + final byte[] originalQuals = read.getBaseQualities( errorModel ); final byte[] recalQuals = originalQuals.clone(); // For each base in the read for( int offset = 0; offset < read.getReadLength(); offset++ ) { - final Object[] fullCovariateKey = - ( errorModel == RecalDataManager.BaseRecalibrationType.BASE_SUBSTITUTION ? covariateKeySet.getMismatchesKeySet(offset) : - ( errorModel == RecalDataManager.BaseRecalibrationType.BASE_INSERTION ? covariateKeySet.getInsertionsKeySet(offset) : - ( errorModel == RecalDataManager.BaseRecalibrationType.BASE_DELETION ? covariateKeySet.getDeletionsKeySet(offset) : null ) ) ); - + final Object[] fullCovariateKeyWithErrorMode = covariateKeySet.getKeySet(offset, errorModel); + + final Object[] fullCovariateKey = Arrays.copyOfRange(fullCovariateKeyWithErrorMode, 0, fullCovariateKeyWithErrorMode.length-1); // need to strip off the error mode which was appended to the list of covariates + Byte qualityScore = (Byte) qualityScoreByFullCovariateKey.get(fullCovariateKey); if( qualityScore == null ) { qualityScore = performSequentialQualityCalculation( errorModel, fullCovariateKey ); @@ -206,21 +202,8 @@ public void recalibrateRead( final GATKSAMRecord read ) { } preserveQScores( originalQuals, recalQuals ); // Overwrite the work done if original quality score is too low - switch (errorModel) { - case BASE_SUBSTITUTION: - read.setBaseQualities( recalQuals ); - break; - case BASE_INSERTION: - read.setAttribute( GATKSAMRecord.BQSR_BASE_INSERTION_QUALITIES, recalQuals ); - break; - case BASE_DELETION: - read.setAttribute( GATKSAMRecord.BQSR_BASE_DELETION_QUALITIES, recalQuals ); - break; - default: - throw new ReviewedStingException("Unrecognized Base Recalibration type: " + errorModel ); - } + read.setBaseQualities( recalQuals, errorModel ); } - } /** diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java index f6b3d759c6..2172cfb94c 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java @@ -25,9 +25,9 @@ package org.broadinstitute.sting.utils.sam; import net.sf.samtools.*; -import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; +import org.broadinstitute.sting.gatk.walkers.bqsr.RecalDataManager; import org.broadinstitute.sting.utils.NGSPlatform; -import org.broadinstitute.sting.utils.recalibration.BaseRecalibration; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import java.util.Arrays; import java.util.HashMap; @@ -163,8 +163,37 @@ public boolean equals(Object o) { } /** - * Accessors for base insertion and base deletion quality scores + * Setters and Accessors for base insertion and base deletion quality scores */ + public void setBaseQualities( final byte[] quals, final RecalDataManager.BaseRecalibrationType errorModel ) { + switch( errorModel ) { + case BASE_SUBSTITUTION: + setBaseQualities(quals); + break; + case BASE_INSERTION: + setAttribute( GATKSAMRecord.BQSR_BASE_INSERTION_QUALITIES, quals ); + break; + case BASE_DELETION: + setAttribute( GATKSAMRecord.BQSR_BASE_DELETION_QUALITIES, quals ); + break; + default: + throw new ReviewedStingException("Unrecognized Base Recalibration type: " + errorModel ); + } + } + + public byte[] getBaseQualities( final RecalDataManager.BaseRecalibrationType errorModel ) { + switch( errorModel ) { + case BASE_SUBSTITUTION: + return getBaseQualities(); + case BASE_INSERTION: + return getBaseInsertionQualities(); + case BASE_DELETION: + return getBaseDeletionQualities(); + default: + throw new ReviewedStingException("Unrecognized Base Recalibration type: " + errorModel ); + } + } + public byte[] getBaseInsertionQualities() { byte[] quals = getByteArrayAttribute( BQSR_BASE_INSERTION_QUALITIES ); if( quals == null ) { diff --git a/public/java/test/org/broadinstitute/sting/gatk/datasources/reads/DownsamplerBenchmark.java b/public/java/test/org/broadinstitute/sting/gatk/datasources/reads/DownsamplerBenchmark.java index 5da8cebf47..20f3e1e352 100644 --- a/public/java/test/org/broadinstitute/sting/gatk/datasources/reads/DownsamplerBenchmark.java +++ b/public/java/test/org/broadinstitute/sting/gatk/datasources/reads/DownsamplerBenchmark.java @@ -79,7 +79,8 @@ public void timeDownsampling(int reps) { false, BAQ.CalculationMode.OFF, BAQ.QualityMode.DONT_MODIFY, - null, + null, // no BAQ + null, // no BQSR (byte)0); GenomeLocParser genomeLocParser = new GenomeLocParser(reader.getFileHeader().getSequenceDictionary()); diff --git a/public/java/test/org/broadinstitute/sting/gatk/iterators/LocusIteratorByStateUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/iterators/LocusIteratorByStateUnitTest.java index 4011594f32..04e11db541 100644 --- a/public/java/test/org/broadinstitute/sting/gatk/iterators/LocusIteratorByStateUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/iterators/LocusIteratorByStateUnitTest.java @@ -308,6 +308,7 @@ private static ReadProperties createTestReadProperties() { BAQ.CalculationMode.OFF, BAQ.QualityMode.DONT_MODIFY, null, // no BAQ + null, // no BQSR (byte) -1 ); } From e9338e2c2040fd1ff258992af5177d39e87058b7 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Mon, 13 Feb 2012 13:40:41 -0500 Subject: [PATCH 251/356] Context covariate needs to look in the reverse direction for negative stranded reads. --- .../gatk/walkers/bqsr/ContextCovariate.java | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java index 64f1d08a87..89a30e4f5f 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java @@ -25,6 +25,7 @@ package org.broadinstitute.sting.gatk.walkers.bqsr; +import org.broadinstitute.sting.utils.BaseUtils; import org.broadinstitute.sting.utils.exceptions.UserException; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; @@ -68,16 +69,31 @@ public CovariateValues getValues(final GATKSAMRecord read) { String[] mismatches = new String [l]; String[] insertions = new String [l]; String[] deletions = new String [l]; - + + final boolean negativeStrand = read.getReadNegativeStrandFlag(); byte[] bases = read.getReadBases(); + if (negativeStrand) { + bases = BaseUtils.simpleReverseComplement(bases); //this is NOT in-place + } for (int i = 0; i < read.getReadLength(); i++) { mismatches[i] = contextWith(bases, i, mismatchesContextSize, mismatchesNoContext); insertions[i] = contextWith(bases, i, insertionsContextSize, insertionsNoContext); deletions[i] = contextWith(bases, i, deletionsContextSize, deletionsNoContext); } + if (negativeStrand) { + reverse(mismatches); + reverse(insertions); + reverse(deletions); + } return new CovariateValues(mismatches, insertions, deletions); } + // Used to get the covariate's value from input csv file during on-the-fly recalibration + @Override + public final Comparable getValue(final String str) { + return str; + } + /** * calculates the context of a base independent of the covariate mode * @@ -98,9 +114,17 @@ private String makeAllNStringWithLength(int length) { return s; } - // Used to get the covariate's value from input csv file during on-the-fly recalibration - @Override - public final Comparable getValue(final String str) { - return str; + /** + * Reverses the given array in place. + * + * @param array any array + */ + private static void reverse(final Comparable[] array) { + final int arrayLength = array.length; + for (int l = 0, r = arrayLength - 1; l < r; l++, r--) { + final Comparable temp = array[l]; + array[l] = array[r]; + array[r] = temp; + } } } From 14981bed10fe519f60a58305bdf71d59f802b123 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Mon, 13 Feb 2012 14:32:03 -0500 Subject: [PATCH 252/356] Cleaning up VariantsToTable: added docs for supported fields; removed one-off hidden arguments for multi-allelics; default behavior is now to include multi-allelics in one record; added option to split multi-allelics into separate records. --- .../walkers/variantutils/VariantsToTable.java | 115 +++++++++--------- .../VariantsToTableIntegrationTest.java | 4 +- 2 files changed, 58 insertions(+), 61 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java index e43d54e144..9f4718ef22 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java @@ -26,7 +26,6 @@ import org.broadinstitute.sting.commandline.*; import org.broadinstitute.sting.gatk.arguments.StandardVariantContextInputArgumentCollection; -import org.broadinstitute.sting.utils.MathUtils; import org.broadinstitute.sting.utils.variantcontext.Allele; import org.broadinstitute.sting.utils.variantcontext.VariantContext; import org.broadinstitute.sting.gatk.contexts.AlignmentContext; @@ -49,7 +48,13 @@ * fields to print with the -F NAME, each of which appears as a single column in * the output file, with a header named NAME, and the value of this field in the VCF * one per line. NAME can be any standard VCF column (CHROM, ID, QUAL) or any binding - * in the INFO field (AC=10). Note that this tool does not support capturing any + * in the INFO field (AC=10). In addition, there are specially supported values like + * EVENTLENGTH (length of the event), TRANSITION (for SNPs), HET (count of het genotypes), + * HOM-REF (count of homozygous reference genotypes), HOM-VAR (count of homozygous variant + * genotypes), NO-CALL (count of no-call genotypes), TYPE (the type of event), VAR (count of + * non-reference genotypes), NSAMPLES (number of samples), NCALLED (number of called samples), + * GQ (from the genotype field; works only for a file with a single sample), and MULTI-ALLELIC + * (is the record from a multi-allelic site). Note that this tool does not support capturing any * GENOTYPE field values. If a VCF record is missing a value, then the tool by * default throws an error, but the special value NA can be emitted instead with * appropriate tool arguments. @@ -121,18 +126,13 @@ public class VariantsToTable extends RodWalker { int nRecords = 0; /** - * By default, only biallelic (REF=A, ALT=B) sites are including in the output. If this flag is provided, then - * VariantsToTable will emit field values for records with multiple ALT alleles. Note that in general this - * can make your resulting file unreadable and malformated according to tools like R, as the representation of - * multi-allelic INFO field values can be lists of values. + * By default, records with multiple ALT alleles will comprise just one line of output; note that in general this can make your resulting file + * unreadable/malformed for certain tools like R, as the representation of multi-allelic INFO field values are often comma-separated lists + * of values. Using the flag will cause multi-allelic records to be split into multiple lines of output (one for each allele in the ALT field); + * INFO field values that are not lists are copied for each of the output records while only the appropriate entry is used for lists. */ - @Advanced - @Argument(fullName="keepMultiAllelic", shortName="KMA", doc="If provided, we will not require the site to be biallelic", required=false) - public boolean keepMultiAllelic = false; - - @Hidden - @Argument(fullName="logACSum", shortName="logACSum", doc="Log sum of AC instead of max value in case of multiallelic variants", required=false) - public boolean logACSum = false; + @Argument(fullName="splitMultiAllelic", shortName="SMA", doc="If provided, we will split multi-allelic records into multiple lines of output", required=false) + public boolean splitMultiAllelic = false; /** * By default, this tool throws a UserException when it encounters a field without a value in some record. This @@ -144,6 +144,7 @@ public class VariantsToTable extends RodWalker { @Advanced @Argument(fullName="allowMissingData", shortName="AMD", doc="If provided, we will not require every record to contain every field", required=false) public boolean ALLOW_MISSING_DATA = false; + private final static String MISSING_DATA = "NA"; public void initialize() { // print out the header @@ -155,9 +156,9 @@ public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentCo return 0; for ( VariantContext vc : tracker.getValues(variantCollection.variants, context.getLocation())) { - if ( (keepMultiAllelic || vc.isBiallelic()) && ( showFiltered || vc.isNotFiltered() ) ) { - List vals = extractFields(vc, fieldsToTake, ALLOW_MISSING_DATA, keepMultiAllelic, logACSum); - out.println(Utils.join("\t", vals)); + if ( showFiltered || vc.isNotFiltered() ) { + for ( final List record : extractFields(vc, fieldsToTake, ALLOW_MISSING_DATA, splitMultiAllelic) ) + out.println(Utils.join("\t", record)); } } @@ -180,22 +181,23 @@ private static final boolean isWildCard(String s) { * * @param vc the VariantContext whose field values we can to capture * @param fields a non-null list of fields to capture from VC - * @param allowMissingData if false, then throws a UserException if any field isn't found in vc. Otherwise - * provides a value of NA - * @param kma if true, multiallelic variants are to be kept - * @param logsum if true, AF and AC are computed based on sum of allele counts. Otherwise, based on allele with highest count. - * @return + * @param allowMissingData if false, then throws a UserException if any field isn't found in vc. Otherwise provides a value of NA + * @param splitMultiAllelic if true, multiallelic variants are to be split into multiple records + * @return List of lists of field values */ - private static List extractFields(VariantContext vc, List fields, boolean allowMissingData, boolean kma, boolean logsum) { - List vals = new ArrayList(); + private static List> extractFields(VariantContext vc, List fields, boolean allowMissingData, boolean splitMultiAllelic) { + + final int numRecordsToProduce = splitMultiAllelic ? vc.getAlternateAlleles().size() : 1; + final List> records = new ArrayList>(numRecordsToProduce); + for ( int i = 0; i < numRecordsToProduce; i++ ) + records.add(new ArrayList(fields.size())); for ( String field : fields ) { - String val = "NA"; if ( getters.containsKey(field) ) { - val = getters.get(field).get(vc); + addFieldValue(getters.get(field).get(vc), records); } else if ( vc.hasAttribute(field) ) { - val = vc.getAttributeAsString(field, null); + addFieldValue(vc.getAttribute(field, null), records); } else if ( isWildCard(field) ) { Set wildVals = new HashSet(); for ( Map.Entry elt : vc.getAttributes().entrySet()) { @@ -204,51 +206,47 @@ private static List extractFields(VariantContext vc, List fields } } + String val = MISSING_DATA; if ( wildVals.size() > 0 ) { List toVal = new ArrayList(wildVals); Collections.sort(toVal); val = Utils.join(",", toVal); } + + addFieldValue(val, records); } else if ( ! allowMissingData ) { throw new UserException(String.format("Missing field %s in vc %s at %s", field, vc.getSource(), vc)); + } else { + addFieldValue(MISSING_DATA, records); } + } - if (field.equals("AF") || field.equals("AC")) { - String afo = val; - - double af=0; - if (afo.contains(",")) { - String[] afs = afo.split(","); - afs[0] = afs[0].substring(1,afs[0].length()); - afs[afs.length-1] = afs[afs.length-1].substring(0,afs[afs.length-1].length()-1); - - double[] afd = new double[afs.length]; - - for (int k=0; k < afd.length; k++) - afd[k] = Double.valueOf(afs[k]); - - if (kma && logsum) - af = MathUtils.sum(afd); - else - af = MathUtils.arrayMax(afd); - //af = Double.valueOf(afs[0]); - - } - else - if (!afo.equals("NA")) - af = Double.valueOf(afo); - - val = Double.toString(af); + return records; + } - } - vals.add(val); + private static void addFieldValue(Object val, List> result) { + final int numResultRecords = result.size(); + + // if we're trying to create a single output record, add it + if ( numResultRecords == 1 ) { + result.get(0).add(val.toString()); + } + // if this field is a list of the proper size, add the appropriate entry to each record + else if ( (val instanceof List) && ((List)val).size() == numResultRecords ) { + final List list = (List)val; + for ( int i = 0; i < numResultRecords; i++ ) + result.get(i).add(list.get(i).toString()); + } + // otherwise, add the original value to all of the records + else { + final String valStr = val.toString(); + for ( List record : result ) + record.add(valStr); } - - return vals; } - public static List extractFields(VariantContext vc, List fields, boolean allowMissingData) { - return extractFields(vc, fields, allowMissingData, false, false); + public static List> extractFields(VariantContext vc, List fields, boolean allowMissingData) { + return extractFields(vc, fields, allowMissingData, false); } // // default reduce -- doesn't do anything at all @@ -321,6 +319,7 @@ public String get(VariantContext vc) { getters.put("VAR", new Getter() { public String get(VariantContext vc) { return Integer.toString(vc.getHetCount() + vc.getHomVarCount()); } }); getters.put("NSAMPLES", new Getter() { public String get(VariantContext vc) { return Integer.toString(vc.getNSamples()); } }); getters.put("NCALLED", new Getter() { public String get(VariantContext vc) { return Integer.toString(vc.getNSamples() - vc.getNoCallCount()); } }); + getters.put("MULTI-ALLELIC", new Getter() { public String get(VariantContext vc) { return Boolean.toString(vc.getAlternateAlleles().size() > 1); } }); getters.put("GQ", new Getter() { public String get(VariantContext vc) { if ( vc.getNSamples() > 1 ) throw new UserException("Cannot get GQ values for multi-sample VCF"); return String.format("%.2f", -10 * vc.getGenotype(0).getLog10PError()); diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTableIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTableIntegrationTest.java index 19021c1c28..0ab593e7ab 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTableIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTableIntegrationTest.java @@ -27,10 +27,8 @@ import org.broadinstitute.sting.WalkerTest; import org.broadinstitute.sting.utils.exceptions.UserException; import org.testng.annotations.Test; -import org.testng.annotations.DataProvider; import java.util.*; -import java.io.File; public class VariantsToTableIntegrationTest extends WalkerTest { private String variantsToTableCmd(String moreArgs) { @@ -38,7 +36,7 @@ private String variantsToTableCmd(String moreArgs) { " --variant:vcf " + validationDataLocation + "/soap_gatk_annotated.vcf" + " -T VariantsToTable" + " -F CHROM -F POS -F ID -F REF -F ALT -F QUAL -F FILTER -F TRANSITION -F DP -F SB -F set -F RankSumP -F refseq.functionalClass*" + - " -L chr1 -KMA -o %s" + moreArgs; + " -L chr1 -o %s" + moreArgs; } @Test(enabled = true) From 0920a1921eb02d3d6000dd380086732dd635cfc6 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Mon, 13 Feb 2012 15:09:53 -0500 Subject: [PATCH 253/356] Minor fixes to splitting multi-allelic records (as regards printing indel alleles correctly); minor code refactoring; adding integration tests to cover +/- splitting multi-allelics. --- .../walkers/variantutils/VariantsToTable.java | 30 ++++++++++++++----- .../VariantsToTableIntegrationTest.java | 22 ++++++++++++++ 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java index 9f4718ef22..4c8e8df5c3 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java @@ -194,7 +194,9 @@ private static List> extractFields(VariantContext vc, List for ( String field : fields ) { - if ( getters.containsKey(field) ) { + if ( splitMultiAllelic && field.equals("ALT") ) { // we need to special case the ALT field when splitting out multi-allelic records + addFieldValue(splitAltAlleles(vc), records); + } else if ( getters.containsKey(field) ) { addFieldValue(getters.get(field).get(vc), records); } else if ( vc.hasAttribute(field) ) { addFieldValue(vc.getAttribute(field, null), records); @@ -271,9 +273,7 @@ public static abstract class Getter { public abstract String get(VariantContext getters.put("REF", new Getter() { public String get(VariantContext vc) { StringBuilder x = new StringBuilder(); - if ( vc.hasReferenceBaseForIndel() && !vc.isSNP() ) - x.append((char)vc.getReferenceBaseForIndel().byteValue()); - x.append(vc.getReference().getDisplayString()); + x.append(getAlleleDisplayString(vc, vc.getReference())); return x.toString(); } }); @@ -285,9 +285,7 @@ public String get(VariantContext vc) { for ( int i = 0; i < n; i++ ) { if ( i != 0 ) x.append(","); - if ( vc.hasReferenceBaseForIndel() && !vc.isSNP() ) - x.append((char)vc.getReferenceBaseForIndel().byteValue()); - x.append(vc.getAlternateAllele(i).getDisplayString()); + x.append(getAlleleDisplayString(vc, vc.getAlternateAllele(i))); } return x.toString(); } @@ -325,5 +323,23 @@ public String get(VariantContext vc) { return String.format("%.2f", -10 * vc.getGenotype(0).getLog10PError()); }}); } + + private static String getAlleleDisplayString(VariantContext vc, Allele allele) { + StringBuilder sb = new StringBuilder(); + if ( vc.hasReferenceBaseForIndel() && !vc.isSNP() ) + sb.append((char)vc.getReferenceBaseForIndel().byteValue()); + sb.append(allele.getDisplayString()); + return sb.toString(); + } + + private static Object splitAltAlleles(VariantContext vc) { + final int numAltAlleles = vc.getAlternateAlleles().size(); + if ( numAltAlleles == 1 ) + return getAlleleDisplayString(vc, vc.getAlternateAllele(0)); + final List alleles = new ArrayList(numAltAlleles); + for ( Allele allele : vc.getAlternateAlleles() ) + alleles.add(getAlleleDisplayString(vc, allele)); + return alleles; + } } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTableIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTableIntegrationTest.java index 0ab593e7ab..6188f22558 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTableIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTableIntegrationTest.java @@ -39,6 +39,14 @@ private String variantsToTableCmd(String moreArgs) { " -L chr1 -o %s" + moreArgs; } + private String variantsToTableMultiAllelicCmd(String moreArgs) { + return "-R " + b37KGReference + + " --variant " + validationDataLocation + "/multiallelic.vcf" + + " -T VariantsToTable" + + " -F CHROM -F POS -F ID -F REF -F ALT -F QUAL -F MULTI-ALLELIC -F AC -F AF" + + " -o %s" + moreArgs; + } + @Test(enabled = true) public void testComplexVariantsToTable() { WalkerTestSpec spec = new WalkerTestSpec(variantsToTableCmd(" -AMD"), @@ -51,4 +59,18 @@ public void testComplexVariantsToTableFail() { WalkerTestSpec spec = new WalkerTestSpec(variantsToTableCmd(""), 1, UserException.class); executeTest("testComplexVariantsToTable-FAIL", spec); } + + @Test(enabled = true) + public void testMultiAllelicOneRecord() { + WalkerTestSpec spec = new WalkerTestSpec(variantsToTableMultiAllelicCmd(""), + Arrays.asList("13dd36c08be6c800f23988e6000d963e")); + executeTest("testMultiAllelicOneRecord", spec).getFirst(); + } + + @Test(enabled = true) + public void testMultiAllelicSplitRecords() { + WalkerTestSpec spec = new WalkerTestSpec(variantsToTableMultiAllelicCmd(" -SMA"), + Arrays.asList("17a0fc80409d2fc00ad2bbb94b3a346b")); + executeTest("testMultiAllelicSplitRecords", spec).getFirst(); + } } From 8742f5e36c7b9484ca7ec1daa4ad69f2347d168f Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Mon, 13 Feb 2012 15:44:30 -0500 Subject: [PATCH 254/356] Updating BQSR scala script to take any number of known sites files and to use the scatter count input argument. From dfcdf92afae5575d88cdd028b21d8afdfcee7615 Mon Sep 17 00:00:00 2001 From: David Roazen Date: Mon, 13 Feb 2012 16:37:31 -0500 Subject: [PATCH 255/356] Revert "Disable HaplotypeCaller integration tests in Stable" These tests should remain enabled in Unstable. This reverts commit 15c5b7aee1327f9dc012d2168f127a4700fe5064. From 8f7587048c8c13fd19770a9110a8e78123d37bcb Mon Sep 17 00:00:00 2001 From: David Roazen Date: Mon, 13 Feb 2012 20:25:52 -0500 Subject: [PATCH 256/356] Update the expected novel TiTv in the HybridSelectionPipelineTest The expected novel TiTv has changed for this set of variants now that multi-allelic mode is on by default. From ae5b42c88456c8f0b93bfd737ae636f6d40b3537 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Tue, 14 Feb 2012 14:01:04 -0500 Subject: [PATCH 257/356] Put base insertion and base deletions in the SAMRecord as a string of quality scores instead of an array of bytes. Start of a proper genotype given alleles mode in HaplotypeCaller --- .../sting/gatk/walkers/bqsr/RecalDataManager.java | 1 + .../org/broadinstitute/sting/utils/Haplotype.java | 4 ++++ .../utils/recalibration/BaseRecalibration.java | 6 ++---- .../sting/utils/sam/GATKSAMRecord.java | 14 ++++++-------- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalDataManager.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalDataManager.java index 8a255391f3..cc60ac0106 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalDataManager.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalDataManager.java @@ -58,6 +58,7 @@ public class RecalDataManager { private final HashMap dataCollapsedQualityScore; // Table where everything except read group and quality score has been collapsed private final HashMap> dataCollapsedByCovariate; // Tables where everything except read group, quality score, and given covariate has been collapsed + public final static String ORIGINAL_QUAL_ATTRIBUTE_TAG = "OQ"; // The tag that holds the original quality scores public final static String COLOR_SPACE_QUAL_ATTRIBUTE_TAG = "CQ"; // The tag that holds the color space quality scores for SOLID bams public final static String COLOR_SPACE_ATTRIBUTE_TAG = "CS"; // The tag that holds the color space for SOLID bams public final static String COLOR_SPACE_INCONSISTENCY_TAG = "ZC"; // A new tag made up for the recalibrator which will hold an array of ints which say if this base is inconsistent with its color diff --git a/public/java/src/org/broadinstitute/sting/utils/Haplotype.java b/public/java/src/org/broadinstitute/sting/utils/Haplotype.java index df682f215f..e10a810fd0 100755 --- a/public/java/src/org/broadinstitute/sting/utils/Haplotype.java +++ b/public/java/src/org/broadinstitute/sting/utils/Haplotype.java @@ -109,6 +109,10 @@ public boolean isReference() { return isReference; } + public byte[] insertAllele( final Allele a ) { + return getBases(); + } + public static LinkedHashMap makeHaplotypeListFromAlleles(List alleleList, int startPos, ReferenceContext ref, final int haplotypeSize, final int numPrefBases) { diff --git a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java index b08365a788..4a366bc024 100644 --- a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java +++ b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java @@ -30,7 +30,6 @@ import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.collections.NestedHashMap; import org.broadinstitute.sting.utils.exceptions.DynamicClassResolutionException; -import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import org.broadinstitute.sting.utils.text.XReadLines; @@ -189,13 +188,12 @@ public void recalibrateRead( final GATKSAMRecord read ) { for( int offset = 0; offset < read.getReadLength(); offset++ ) { final Object[] fullCovariateKeyWithErrorMode = covariateKeySet.getKeySet(offset, errorModel); - final Object[] fullCovariateKey = Arrays.copyOfRange(fullCovariateKeyWithErrorMode, 0, fullCovariateKeyWithErrorMode.length-1); // need to strip off the error mode which was appended to the list of covariates - Byte qualityScore = (Byte) qualityScoreByFullCovariateKey.get(fullCovariateKey); + Byte qualityScore = (Byte) qualityScoreByFullCovariateKey.get(fullCovariateKeyWithErrorMode); if( qualityScore == null ) { qualityScore = performSequentialQualityCalculation( errorModel, fullCovariateKey ); - qualityScoreByFullCovariateKey.put(qualityScore, fullCovariateKey); + qualityScoreByFullCovariateKey.put(qualityScore, fullCovariateKeyWithErrorMode); } recalQuals[offset] = qualityScore; diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java index 2172cfb94c..f5a9b2f456 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java @@ -171,10 +171,10 @@ public void setBaseQualities( final byte[] quals, final RecalDataManager.BaseRec setBaseQualities(quals); break; case BASE_INSERTION: - setAttribute( GATKSAMRecord.BQSR_BASE_INSERTION_QUALITIES, quals ); + setAttribute( GATKSAMRecord.BQSR_BASE_INSERTION_QUALITIES, SAMUtils.phredToFastq(quals) ); break; case BASE_DELETION: - setAttribute( GATKSAMRecord.BQSR_BASE_DELETION_QUALITIES, quals ); + setAttribute( GATKSAMRecord.BQSR_BASE_DELETION_QUALITIES, SAMUtils.phredToFastq(quals) ); break; default: throw new ReviewedStingException("Unrecognized Base Recalibration type: " + errorModel ); @@ -195,23 +195,23 @@ public byte[] getBaseQualities( final RecalDataManager.BaseRecalibrationType err } public byte[] getBaseInsertionQualities() { - byte[] quals = getByteArrayAttribute( BQSR_BASE_INSERTION_QUALITIES ); + byte[] quals = SAMUtils.fastqToPhred( getStringAttribute( BQSR_BASE_INSERTION_QUALITIES ) ); if( quals == null ) { quals = new byte[getBaseQualities().length]; Arrays.fill(quals, (byte) 45); // Some day in the future when base insertion and base deletion quals exist the samtools API will // be updated and the original quals will be pulled here, but for now we assume the original quality is a flat Q45 - setAttribute( BQSR_BASE_INSERTION_QUALITIES, quals ); + setBaseQualities(quals, RecalDataManager.BaseRecalibrationType.BASE_INSERTION); } return quals; } public byte[] getBaseDeletionQualities() { - byte[] quals = getByteArrayAttribute( BQSR_BASE_DELETION_QUALITIES ); + byte[] quals = SAMUtils.fastqToPhred( getStringAttribute( BQSR_BASE_DELETION_QUALITIES ) ); if( quals == null ) { quals = new byte[getBaseQualities().length]; Arrays.fill(quals, (byte) 45); // Some day in the future when base insertion and base deletion quals exist the samtools API will // be updated and the original quals will be pulled here, but for now we assume the original quality is a flat Q45 - setAttribute( BQSR_BASE_DELETION_QUALITIES, quals ); + setBaseQualities(quals, RecalDataManager.BaseRecalibrationType.BASE_DELETION); } return quals; } @@ -259,12 +259,10 @@ public final byte getReducedCount(final int i) { return (i==0) ? firstCount : (byte) Math.min(firstCount + offsetCount, Byte.MAX_VALUE); } - /////////////////////////////////////////////////////////////////////////////// // *** GATKSAMRecord specific methods ***// /////////////////////////////////////////////////////////////////////////////// - /** * Checks whether an attribute has been set for the given key. * From 2f33c5706076f58897eee8fc41e8aaf267bc68bb Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Thu, 16 Feb 2012 13:58:00 -0500 Subject: [PATCH 258/356] No reason to restrict HaplotypeScore to bi-allelic SNPs when the plumbing for multi-allelic events is already present. --- .../sting/gatk/walkers/annotator/HaplotypeScore.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java index 40b5aa4d5a..f323a7be2d 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java @@ -52,8 +52,7 @@ /** * Consistency of the site with two (and only two) segregating haplotypes. Higher scores * are indicative of regions with bad alignments, often leading to artifactual SNP and indel calls. - * Note that the Haplotype Score is only calculated for sites with read coverage; also, for SNPs, the - * site must be bi-allelic. + * Note that the Haplotype Score is only calculated for sites with read coverage. */ public class HaplotypeScore extends InfoFieldAnnotation implements StandardAnnotation { private final static boolean DEBUG = false; @@ -65,12 +64,9 @@ public Map annotate(RefMetaDataTracker tracker, AnnotatorCompati if (stratifiedContexts.size() == 0) // size 0 means that call was made by someone else and we have no data here return null; - if (vc.isSNP() && !vc.isBiallelic()) - return null; - final AlignmentContext context = AlignmentContextUtils.joinContexts(stratifiedContexts.values()); - final int contextWingSize = Math.min(((int) ref.getWindow().size() - 1) / 2, MIN_CONTEXT_WING_SIZE); + final int contextWingSize = Math.min((ref.getWindow().size() - 1) / 2, MIN_CONTEXT_WING_SIZE); final int contextSize = contextWingSize * 2 + 1; final int locus = ref.getLocus().getStart() + (ref.getLocus().getStop() - ref.getLocus().getStart()) / 2; From 78718b8d6a88de69a7537769ca5b4c63bcac8169 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Sat, 18 Feb 2012 10:31:26 -0500 Subject: [PATCH 259/356] Adding Genotype Given Alleles mode to the HaplotypeCaller. It constructs the possible haplotypes via assembly and then injects the desired allele to be genotyped. --- .../traversals/TraverseActiveRegions.java | 3 +- .../broadinstitute/sting/utils/Haplotype.java | 134 +++++++++++++++++- 2 files changed, 133 insertions(+), 4 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java index 92c508f854..3f24e65852 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java +++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraverseActiveRegions.java @@ -69,8 +69,7 @@ public T traverse( final ActiveRegionWalker walker, for(int iii = prevLoc.getStart() + 1; iii < location.getStart(); iii++ ) { final GenomeLoc fakeLoc = engine.getGenomeLocParser().createGenomeLoc(prevLoc.getContig(), iii, iii); if( initialIntervals == null || initialIntervals.overlaps( fakeLoc ) ) { - final double isActiveProb = ( walker.presetActiveRegions == null ? walker.isActive( null, null, null ) - : ( walker.presetActiveRegions.overlaps(fakeLoc) ? 1.0 : 0.0 ) ); + final double isActiveProb = ( walker.presetActiveRegions == null ? 0.0 : ( walker.presetActiveRegions.overlaps(fakeLoc) ? 1.0 : 0.0 ) ); isActiveList.add( isActiveProb ); if( firstIsActiveStart == null ) { firstIsActiveStart = fakeLoc; diff --git a/public/java/src/org/broadinstitute/sting/utils/Haplotype.java b/public/java/src/org/broadinstitute/sting/utils/Haplotype.java index e10a810fd0..d48deab1b6 100755 --- a/public/java/src/org/broadinstitute/sting/utils/Haplotype.java +++ b/public/java/src/org/broadinstitute/sting/utils/Haplotype.java @@ -24,11 +24,17 @@ package org.broadinstitute.sting.utils; +import net.sf.samtools.Cigar; +import net.sf.samtools.CigarElement; +import net.sf.samtools.CigarOperator; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.utils.collections.Pair; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import org.broadinstitute.sting.utils.variantcontext.Allele; import java.util.Arrays; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; @@ -109,8 +115,52 @@ public boolean isReference() { return isReference; } - public byte[] insertAllele( final Allele a ) { - return getBases(); + public byte[] insertAllele( final Allele refAllele, final Allele altAllele, int refInsertLocation, final byte[] paddedRef, final int refStart, + final Cigar haplotypeCigar, final int numBasesAddedToStartOfHaplotype, final int refHaplotypeLength ) { + + if( refAllele.length() != altAllele.length() ) { refInsertLocation++; } + int haplotypeInsertLocation = getHaplotypeCoordinateForReferenceCoordinate(refStart + numBasesAddedToStartOfHaplotype, haplotypeCigar, refInsertLocation); + if( haplotypeInsertLocation == -1 ) { // desired change falls inside deletion so don't bother creating a new haplotype + return getBases().clone(); + } + haplotypeInsertLocation += numBasesAddedToStartOfHaplotype; + final byte[] newHaplotype = getBases().clone(); + + try { + if( refAllele.length() == altAllele.length() ) { // SNP or MNP + for( int iii = 0; iii < altAllele.length(); iii++ ) { + newHaplotype[haplotypeInsertLocation+iii] = altAllele.getBases()[iii]; + } + } else if( refAllele.length() < altAllele.length() ) { // insertion + final int altAlleleLength = altAllele.length(); + for( int iii = newHaplotype.length -1; iii > haplotypeInsertLocation + altAlleleLength; iii-- ) { + newHaplotype[iii] = newHaplotype[iii-altAlleleLength]; + } + for( int iii = 0; iii < altAlleleLength; iii++ ) { + newHaplotype[haplotypeInsertLocation+iii] = altAllele.getBases()[iii]; + } + } else { // deletion + int refHaplotypeOffset = 0; + for( final CigarElement ce : haplotypeCigar.getCigarElements()) { + if(ce.getOperator() == CigarOperator.D) { refHaplotypeOffset += ce.getLength(); } + else if(ce.getOperator() == CigarOperator.I) { refHaplotypeOffset -= ce.getLength(); } + } + for( int iii = 0; iii < altAllele.length(); iii++ ) { + newHaplotype[haplotypeInsertLocation+iii] = altAllele.getBases()[iii]; + } + final int shift = refAllele.length() - altAllele.length(); + for( int iii = haplotypeInsertLocation + altAllele.length(); iii < newHaplotype.length - shift; iii++ ) { + newHaplotype[iii] = newHaplotype[iii+shift]; + } + for( int iii = 0; iii < shift; iii++ ) { + newHaplotype[iii+newHaplotype.length-shift] = paddedRef[refStart+refHaplotypeLength+refHaplotypeOffset+iii]; + } + } + } catch (Exception e) { // event already on haplotype is too large/complex to insert another allele, most likely because of not enough reference padding + return getBases().clone(); + } + + return newHaplotype; } public static LinkedHashMap makeHaplotypeListFromAlleles(List alleleList, int startPos, ReferenceContext ref, @@ -169,4 +219,84 @@ public static LinkedHashMap makeHaplotypeListFromAlleles(List< return haplotypeMap; } + private static Integer getHaplotypeCoordinateForReferenceCoordinate( final int haplotypeStart, final Cigar haplotypeCigar, final int refCoord ) { + int readBases = 0; + int refBases = 0; + boolean fallsInsideDeletion = false; + + int goal = refCoord - haplotypeStart; // The goal is to move this many reference bases + boolean goalReached = refBases == goal; + + Iterator cigarElementIterator = haplotypeCigar.getCigarElements().iterator(); + while (!goalReached && cigarElementIterator.hasNext()) { + CigarElement cigarElement = cigarElementIterator.next(); + int shift = 0; + + if (cigarElement.getOperator().consumesReferenceBases() || cigarElement.getOperator() == CigarOperator.SOFT_CLIP) { + if (refBases + cigarElement.getLength() < goal) + shift = cigarElement.getLength(); + else + shift = goal - refBases; + + refBases += shift; + } + goalReached = refBases == goal; + + if (!goalReached && cigarElement.getOperator().consumesReadBases()) + readBases += cigarElement.getLength(); + + if (goalReached) { + // Is this base's reference position within this cigar element? Or did we use it all? + boolean endsWithinCigar = shift < cigarElement.getLength(); + + // If it isn't, we need to check the next one. There should *ALWAYS* be a next one + // since we checked if the goal coordinate is within the read length, so this is just a sanity check. + if (!endsWithinCigar && !cigarElementIterator.hasNext()) + return -1; + + CigarElement nextCigarElement; + + // if we end inside the current cigar element, we just have to check if it is a deletion + if (endsWithinCigar) + fallsInsideDeletion = cigarElement.getOperator() == CigarOperator.DELETION; + + // if we end outside the current cigar element, we need to check if the next element is an insertion or deletion. + else { + nextCigarElement = cigarElementIterator.next(); + + // if it's an insertion, we need to clip the whole insertion before looking at the next element + if (nextCigarElement.getOperator() == CigarOperator.INSERTION) { + readBases += nextCigarElement.getLength(); + if (!cigarElementIterator.hasNext()) + return -1; + + nextCigarElement = cigarElementIterator.next(); + } + + // if it's a deletion, we will pass the information on to be handled downstream. + fallsInsideDeletion = nextCigarElement.getOperator() == CigarOperator.DELETION; + } + + // If we reached our goal outside a deletion, add the shift + if (!fallsInsideDeletion && cigarElement.getOperator().consumesReadBases()) + readBases += shift; + + // If we reached our goal inside a deletion, but the deletion is the next cigar element then we need + // to add the shift of the current cigar element but go back to it's last element to return the last + // base before the deletion (see warning in function contracts) + else if (fallsInsideDeletion && !endsWithinCigar) + readBases += shift - 1; + + // If we reached our goal inside a deletion then we must backtrack to the last base before the deletion + else if (fallsInsideDeletion && endsWithinCigar) + readBases--; + } + } + + if (!goalReached) + return -1; + + return (fallsInsideDeletion ? -1 : readBases); + } + } From a8be96f63dbc5545c516ada8acd96bf05b59904d Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Sat, 18 Feb 2012 10:54:39 -0500 Subject: [PATCH 260/356] This caching in the BQSR seems to be too slow now that there are so many keys --- .../sting/utils/recalibration/BaseRecalibration.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java index 4a366bc024..74083ced26 100644 --- a/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java +++ b/public/java/src/org/broadinstitute/sting/utils/recalibration/BaseRecalibration.java @@ -190,11 +190,12 @@ public void recalibrateRead( final GATKSAMRecord read ) { final Object[] fullCovariateKeyWithErrorMode = covariateKeySet.getKeySet(offset, errorModel); final Object[] fullCovariateKey = Arrays.copyOfRange(fullCovariateKeyWithErrorMode, 0, fullCovariateKeyWithErrorMode.length-1); // need to strip off the error mode which was appended to the list of covariates - Byte qualityScore = (Byte) qualityScoreByFullCovariateKey.get(fullCovariateKeyWithErrorMode); - if( qualityScore == null ) { - qualityScore = performSequentialQualityCalculation( errorModel, fullCovariateKey ); - qualityScoreByFullCovariateKey.put(qualityScore, fullCovariateKeyWithErrorMode); - } + // BUGBUG: This caching seems to put the entire key set into memory which negates the benefits of storing the delta delta tables? + //Byte qualityScore = (Byte) qualityScoreByFullCovariateKey.get(fullCovariateKeyWithErrorMode); + //if( qualityScore == null ) { + final byte qualityScore = performSequentialQualityCalculation( errorModel, fullCovariateKey ); + // qualityScoreByFullCovariateKey.put(qualityScore, fullCovariateKeyWithErrorMode); + //} recalQuals[offset] = qualityScore; } From fe102a5d4739c2ac9eee1afd8d265c7d56b4951b Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Sat, 18 Feb 2012 11:13:20 -0500 Subject: [PATCH 261/356] Fix for my renaming of the BQSR walker From 0f5674b95e9dd18df55cf09dbdd79f8bc9ec46fa Mon Sep 17 00:00:00 2001 From: Guillermo del Angel Date: Mon, 20 Feb 2012 09:12:51 -0500 Subject: [PATCH 262/356] Redid fix for corner case when forming consensus with reads that start/end with insertions and that don't agree with each other in inserted bases: since I can't iterate over the elements of a HashMap because keys might change during iteration, and since I can't use ConcurrentHashMaps, the code now copies structure of (bases, number of times seen) into ArrayList, which can be addressed by element index in order to iterate on it. --- ...elGenotypeLikelihoodsCalculationModel.java | 77 +++++++++++-------- 1 file changed, 46 insertions(+), 31 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java index 49c131ce29..7ee7b0752c 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java @@ -37,6 +37,7 @@ import org.broadinstitute.sting.utils.Haplotype; import org.broadinstitute.sting.utils.clipping.ReadClipper; import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants; +import org.broadinstitute.sting.utils.collections.Pair; import org.broadinstitute.sting.utils.exceptions.StingException; import org.broadinstitute.sting.utils.pileup.ExtendedEventPileupElement; import org.broadinstitute.sting.utils.pileup.PileupElement; @@ -141,62 +142,76 @@ private ArrayList computeConsensusAlleles(ReferenceContext ref, String indelString = p.getEventBases(); if (p.isInsertion()) { boolean foundKey = false; + // copy of hashmap into temp arrayList + ArrayList> cList = new ArrayList>(); + for (String s : consensusIndelStrings.keySet()) { + cList.add(new Pair(s,consensusIndelStrings.get(s))); + } + if (read.getAlignmentEnd() == loc.getStart()) { // first corner condition: a read has an insertion at the end, and we're right at the insertion. // In this case, the read could have any of the inserted bases and we need to build a consensus - for (String s : consensusIndelStrings.keySet()) { - int cnt = consensusIndelStrings.get(s); + + for (int k=0; k < cList.size(); k++) { + String s = cList.get(k).getFirst(); + int cnt = cList.get(k).getSecond(); + // case 1: current insertion is prefix of indel in hash map if (s.startsWith(indelString)) { - // case 1: current insertion is prefix of indel in hash map - consensusIndelStrings.put(s, cnt + 1); + cList.set(k,new Pair(s,cnt+1)); foundKey = true; - break; - } else if (indelString.startsWith(s)) { + } + else if (indelString.startsWith(s)) { // case 2: indel stored in hash table is prefix of current insertion // In this case, new bases are new key. - consensusIndelStrings.remove(s); - consensusIndelStrings.put(indelString, cnt + 1); foundKey = true; - break; + cList.set(k,new Pair(indelString,cnt+1)); } } if (!foundKey) // none of the above: event bases not supported by previous table, so add new key - consensusIndelStrings.put(indelString, 1); + cList.add(new Pair(indelString,1)); - } else if (read.getAlignmentStart() == loc.getStart() + 1) { + } + else if (read.getAlignmentStart() == loc.getStart()+1) { // opposite corner condition: read will start at current locus with an insertion - for (String s : consensusIndelStrings.keySet()) { - int cnt = consensusIndelStrings.get(s); + for (int k=0; k < cList.size(); k++) { + String s = cList.get(k).getFirst(); + int cnt = cList.get(k).getSecond(); if (s.endsWith(indelString)) { - // case 1: current insertion is suffix of indel in hash map - consensusIndelStrings.put(s, cnt + 1); + // case 1: current insertion (indelString) is suffix of indel in hash map (s) + cList.set(k,new Pair(s,cnt+1)); foundKey = true; - break; - } else if (indelString.endsWith(s)) { - // case 2: indel stored in hash table is suffix of current insertion + } + else if (indelString.endsWith(s)) { + // case 2: indel stored in hash table is prefix of current insertion // In this case, new bases are new key. - - consensusIndelStrings.remove(s); - consensusIndelStrings.put(indelString, cnt + 1); foundKey = true; - break; + cList.set(k,new Pair(indelString,cnt+1)); } } if (!foundKey) // none of the above: event bases not supported by previous table, so add new key - consensusIndelStrings.put(indelString, 1); + cList.add(new Pair(indelString,1)); + + + } + else { + // normal case: insertion somewhere in the middle of a read: add count to arrayList + int cnt = consensusIndelStrings.containsKey(indelString)? consensusIndelStrings.get(indelString):0; + cList.add(new Pair(indelString,cnt+1)); + } - } else { - // normal case: insertion somewhere in the middle of a read: add count to hash map - int cnt = consensusIndelStrings.containsKey(indelString) ? consensusIndelStrings.get(indelString) : 0; - consensusIndelStrings.put(indelString, cnt + 1); + // copy back arrayList into hashMap + consensusIndelStrings.clear(); + for (Pair pair : cList) { + consensusIndelStrings.put(pair.getFirst(),pair.getSecond()); } - } else if (p.isDeletion()) { - indelString = String.format("D%d", p.getEventLength()); - int cnt = consensusIndelStrings.containsKey(indelString) ? consensusIndelStrings.get(indelString) : 0; - consensusIndelStrings.put(indelString, cnt + 1); + } + else if (p.isDeletion()) { + indelString = String.format("D%d",p.getEventLength()); + int cnt = consensusIndelStrings.containsKey(indelString)? consensusIndelStrings.get(indelString):0; + consensusIndelStrings.put(indelString,cnt+1); } } From 75783af6fc1cca8da25d9a919537a8cb28615448 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Tue, 21 Feb 2012 14:10:36 -0500 Subject: [PATCH 263/356] int <-> BitSet conversion utils for MathUtils * added unit tests. --- .../broadinstitute/sting/utils/MathUtils.java | 32 +++++++++++++++++++ .../sting/utils/MathUtilsUnitTest.java | 12 ++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java index a4e9fc7ed0..c9ab3b58e5 100644 --- a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java @@ -1613,4 +1613,36 @@ public static Double[] vectorLog10(Double v1[]) { } + /** + * Creates an integer out of a bitset + * + * @param bitSet the bitset + * @return an integer with the bitset representation + */ + public static int intFrom(final BitSet bitSet) { + int integer = 0; + for (int bitIndex = bitSet.nextSetBit(0); bitIndex >= 0; bitIndex = bitSet.nextSetBit(bitIndex+1)) + integer |= 1 << bitIndex; + + return integer; + } + + /** + * Creates a BitSet representation of a given integer + * + * @param integer the number to turn into a bitset + * @return a bitset representation of the integer + */ + public static BitSet bitSetFrom(int integer) { + BitSet bitSet = new BitSet((int) Math.ceil(Math.sqrt(integer))); + int bitIndex = 0; + while (integer > 0) { + if (integer%2 > 0) + bitSet.set(bitIndex); + bitIndex++; + integer /= 2; + } + return bitSet; + } + } diff --git a/public/java/test/org/broadinstitute/sting/utils/MathUtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/MathUtilsUnitTest.java index 049bdce3e0..5b50c91a60 100755 --- a/public/java/test/org/broadinstitute/sting/utils/MathUtilsUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/MathUtilsUnitTest.java @@ -205,6 +205,16 @@ public void testArrayShuffle() { } } + @Test(enabled = true) + public void testIntAndBitSetConversion() { + Assert.assertEquals(428, MathUtils.intFrom(MathUtils.bitSetFrom(428))); + Assert.assertEquals(239847, MathUtils.intFrom(MathUtils.bitSetFrom(239847))); + Assert.assertEquals(12726, MathUtils.intFrom(MathUtils.bitSetFrom(12726))); + Assert.assertEquals(0, MathUtils.intFrom(MathUtils.bitSetFrom(0))); + Assert.assertEquals(1, MathUtils.intFrom(MathUtils.bitSetFrom(1))); + Assert.assertEquals(65536, MathUtils.intFrom(MathUtils.bitSetFrom(65536))); + } + private boolean hasUniqueElements(Object[] x) { for (int i = 0; i < x.length; i++) for (int j = i + 1; j < x.length; j++) @@ -220,10 +230,10 @@ private boolean hasAllElements(final Object[] expected, final Object[] actual) { return set.isEmpty(); } - private void p (Object []x) { for (Object v: x) System.out.print((Integer) v + " "); System.out.println(); } + } From a611f865584d1970be4871237c2b3e4d8f20217d Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Wed, 22 Feb 2012 12:23:45 -0500 Subject: [PATCH 264/356] CalibrateGenotypeLikelihoods now accepts any number of external likelihood VCFs. We decided in the dev group to have the assigned name be a combination of the sample name provided in the VCF and the name provided to the rod binding. From e39638323b36a52e3fab3cb1574b1eb13c38d8be Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Wed, 22 Feb 2012 12:24:43 -0500 Subject: [PATCH 265/356] Misc cleanup in HaplotypeCaller's HMM code now that we have separate GOP for insertions and deletions From ca7b5e068fac15e94a1ac1d69fe8722b10405d66 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Wed, 22 Feb 2012 15:23:24 -0500 Subject: [PATCH 266/356] updating HaplotypeCaller integration tests after change to separate insertion and deletion GOP. From 2c1b14d35e2db27d274a8bbe1336efd122034c2b Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Wed, 22 Feb 2012 17:20:04 -0500 Subject: [PATCH 267/356] Mostly small changes to my own scala scripts: .vcf.gz compatibility for output files, smarter beagle generation, simple script to scatter-gather combine variants. Whole genome indel calling now uses the gold standard indel set. --- .../gatk/walkers/variantutils/VariantsToPed.java | 10 ++++++++-- .../sting/queue/qscripts/lib/VcfToPed.scala | 14 +++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToPed.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToPed.java index aab230b69e..d8b01e91d8 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToPed.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToPed.java @@ -7,6 +7,7 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; import org.broadinstitute.sting.gatk.walkers.RodWalker; +import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants; import org.broadinstitute.sting.utils.codecs.vcf.VCFHeader; import org.broadinstitute.sting.utils.codecs.vcf.VCFUtils; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; @@ -45,6 +46,9 @@ public class VariantsToPed extends RodWalker { @Output(shortName="fam",fullName="fam",required=true,doc="output fam file") PrintStream outFam; + @Argument(shortName="mgq",fullName="minGenotypeQuality",required=true,doc="If genotype quality is lower than this value, output NO_CALL") + int minGenotypeQuality = 0; + private ValidateVariants vv = new ValidateVariants(); private static double APPROX_CM_PER_BP = 1000000.0/750000.0; @@ -173,9 +177,11 @@ public Integer reduceInit() { return 0; } - private static byte getEncoding(Genotype g, int offset) { + private byte getEncoding(Genotype g, int offset) { byte b; - if ( g.isHomRef() ) { + if ( g.hasAttribute(VCFConstants.GENOTYPE_QUALITY_KEY) && ((Integer) g.getAttribute(VCFConstants.GENOTYPE_QUALITY_KEY)) < minGenotypeQuality ) { + b = NO_CALL; + } else if ( g.isHomRef() ) { b = HOM_REF; } else if ( g.isHomVar() ) { b = HOM_VAR; diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala index 913a62e260..cad8af51d3 100644 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/lib/VcfToPed.scala @@ -47,9 +47,14 @@ class VcfToPed extends QScript { val extract : VCFExtractIntervals = new VCFExtractIntervals(variants,ivals,false) add(extract) } else { + val IS_GZ : Boolean = variants.getName.endsWith(".vcf.gz") var iXRL = new XReadLines(intervals) var chunk = 1; - var subListFile = swapExt(tmpdir,variants,".vcf",".chunk%d.list".format(chunk)) + var subListFile : File = null + if ( IS_GZ ) + subListFile = swapExt(tmpdir,variants,".vcf.gz",".chunk%d.list".format(chunk)) + else + subListFile = swapExt(tmpdir,variants,".vcf",".chunk%d.list".format(chunk)) var subList = new PrintStream(subListFile) var nL = 0; var bedOuts : List[File] = Nil; @@ -58,7 +63,7 @@ class VcfToPed extends QScript { while ( iXRL.hasNext ) { subList.printf("%s%n",iXRL.next()) nL = nL + 1 - if ( nL > 100000 ) { + if ( nL > 10000 ) { val toPed : VariantsToPed = new VariantsToPed toPed.memoryLimit = 2 toPed.reference_sequence = ref @@ -89,7 +94,10 @@ class VcfToPed extends QScript { add(toPed) subList.close() chunk = chunk + 1 - subListFile = swapExt(tmpdir,variants,".vcf",".chunk%d.list".format(chunk)) + if ( IS_GZ ) + subListFile = swapExt(tmpdir,variants,".vcf.gz",".chunk%d.list".format(chunk)) + else + subListFile = swapExt(tmpdir,variants,".vcf",".chunk%d.list".format(chunk)) subList = new PrintStream(subListFile) bedOuts :+= tBed bimOuts :+= bim From 8695738400252c3a1a28b70f56bade32ef34098c Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Wed, 22 Feb 2012 19:00:04 -0500 Subject: [PATCH 268/356] Bug fix in HaplotypeCaller's GENOTYPE_GIVEN_ALLELES mode for insertions greater than length 1. The allele being genotyped was off by one base pair. --- public/java/src/org/broadinstitute/sting/utils/Haplotype.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/Haplotype.java b/public/java/src/org/broadinstitute/sting/utils/Haplotype.java index d48deab1b6..def2fc349d 100755 --- a/public/java/src/org/broadinstitute/sting/utils/Haplotype.java +++ b/public/java/src/org/broadinstitute/sting/utils/Haplotype.java @@ -28,9 +28,7 @@ import net.sf.samtools.CigarElement; import net.sf.samtools.CigarOperator; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; -import org.broadinstitute.sting.utils.collections.Pair; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import org.broadinstitute.sting.utils.variantcontext.Allele; import java.util.Arrays; @@ -133,7 +131,7 @@ public byte[] insertAllele( final Allele refAllele, final Allele altAllele, int } } else if( refAllele.length() < altAllele.length() ) { // insertion final int altAlleleLength = altAllele.length(); - for( int iii = newHaplotype.length -1; iii > haplotypeInsertLocation + altAlleleLength; iii-- ) { + for( int iii = newHaplotype.length - 1; iii > haplotypeInsertLocation + altAlleleLength - 1; iii-- ) { newHaplotype[iii] = newHaplotype[iii-altAlleleLength]; } for( int iii = 0; iii < altAlleleLength; iii++ ) { From 6866a41914c808e9b1d63edbf4860349143cbd14 Mon Sep 17 00:00:00 2001 From: Guillermo del Angel Date: Thu, 23 Feb 2012 09:45:47 -0500 Subject: [PATCH 269/356] Added functionality in pileups to not only determine whether there's an insertion or deletion following the current position, but to also get the indel length and involved bases - definitely needed for extended event removal, and needed for pool caller indel functionality. --- .../gatk/iterators/LocusIteratorByState.java | 15 ++++++--- .../pileup/AbstractReadBackedPileup.java | 1 + .../pileup/ExtendedEventPileupElement.java | 2 +- .../sting/utils/pileup/PileupElement.java | 31 ++++++++++++++++++- .../utils/pileup/ReadBackedPileupImpl.java | 10 ++++-- 5 files changed, 50 insertions(+), 9 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java b/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java index 6edae38161..b8dd033177 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java +++ b/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java @@ -175,8 +175,8 @@ public String toString() { return String.format("%s ro=%d go=%d co=%d cec=%d %s", read.getReadName(), readOffset, genomeOffset, cigarOffset, cigarElementCounter, curElement); } - public CigarOperator peekForwardOnGenome() { - return ( cigarElementCounter + 1 > curElement.getLength() && cigarOffset + 1 < nCigarElements ? cigar.getCigarElement(cigarOffset + 1) : curElement ).getOperator(); + public CigarElement peekForwardOnGenome() { + return ( cigarElementCounter + 1 > curElement.getLength() && cigarOffset + 1 < nCigarElements ? cigar.getCigarElement(cigarOffset + 1) : curElement ); } public CigarOperator stepForwardOnGenome() { @@ -462,15 +462,19 @@ else if (op != CigarOperator.D || readInfo.includeReadsWithDeletionAtLoci()) { final SAMRecordState state = iterator.next(); // state object with the read/offset information final GATKSAMRecord read = (GATKSAMRecord) state.getRead(); // the actual read final CigarOperator op = state.getCurrentCigarOperator(); // current cigar operator - final CigarOperator nextOp = state.peekForwardOnGenome(); // next cigar operator + final CigarElement nextElement = state.peekForwardOnGenome(); // next cigar element + final CigarOperator nextOp = nextElement.getOperator(); final int readOffset = state.getReadOffset(); // the base offset on this read + byte[] insertedBases = Arrays.copyOfRange(read.getReadBases(), readOffset + 1, readOffset + 1 + nextElement.getLength()); + int nextElementLength = nextElement.getLength(); if (op == CigarOperator.N) // N's are never added to any pileup continue; if (op == CigarOperator.D) { if (readInfo.includeReadsWithDeletionAtLoci()) { // only add deletions to the pileup if we are authorized to do so - pile.add(new PileupElement(read, readOffset, true, nextOp == CigarOperator.D, nextOp == CigarOperator.I, nextOp == CigarOperator.S || (state.getGenomeOffset() == 0 && read.getSoftStart() != read.getAlignmentStart()))); + pile.add(new PileupElement(read, readOffset, true, nextOp == CigarOperator.D, nextOp == CigarOperator.I, nextOp == CigarOperator.S || (state.getGenomeOffset() == 0 && read.getSoftStart() != read.getAlignmentStart()), + null,nextOp == CigarOperator.D? nextElementLength:-1)); size++; nDeletions++; if (read.getMappingQuality() == 0) @@ -479,7 +483,8 @@ else if (op != CigarOperator.D || readInfo.includeReadsWithDeletionAtLoci()) { } else { if (!filterBaseInRead(read, location.getStart())) { - pile.add(new PileupElement(read, readOffset, false, nextOp == CigarOperator.D, nextOp == CigarOperator.I, nextOp == CigarOperator.S || (state.getGenomeOffset() == 0 && read.getSoftStart() != read.getAlignmentStart()))); + pile.add(new PileupElement(read, readOffset, false, nextOp == CigarOperator.D, nextOp == CigarOperator.I, nextOp == CigarOperator.S || (state.getGenomeOffset() == 0 && read.getSoftStart() != read.getAlignmentStart()), + new String(insertedBases),nextElementLength)); size++; if (read.getMappingQuality() == 0) nMQ0Reads++; diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java b/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java index 70ad70f43b..7c2a67aba0 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/AbstractReadBackedPileup.java @@ -205,6 +205,7 @@ private PileupElementTracker readsOffsets2Pileup(List reads, protected abstract AbstractReadBackedPileup createNewPileup(GenomeLoc loc, PileupElementTracker pileupElementTracker); protected abstract PE createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeDeletion, boolean isBeforeInsertion, boolean isNextToSoftClip); + protected abstract PE createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeDeletion, boolean isBeforeInsertion, boolean isNextToSoftClip, String nextEventBases, int nextEventLength ); // -------------------------------------------------------- // diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/ExtendedEventPileupElement.java b/public/java/src/org/broadinstitute/sting/utils/pileup/ExtendedEventPileupElement.java index 506442d03c..8df0aa0b8c 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/ExtendedEventPileupElement.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/ExtendedEventPileupElement.java @@ -48,7 +48,7 @@ public enum Type { public ExtendedEventPileupElement(GATKSAMRecord read, int offset, int eventLength, String eventBases, Type type) { - super(read, offset, type == Type.DELETION, false, false, false); // extended events are slated for removal + super(read, offset, type == Type.DELETION, false, false, false,null,-1); // extended events are slated for removal this.read = read; this.offset = offset; this.eventLength = eventLength; diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java b/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java index 9df22700e5..022cadbbe0 100755 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java @@ -27,6 +27,10 @@ public class PileupElement implements Comparable { protected final boolean isBeforeDeletion; protected final boolean isBeforeInsertion; protected final boolean isNextToSoftClip; + protected final int eventLength; + protected final String eventBases; // if it is a deletion, we do not have information about the actual deleted bases + // in the read itself, so we fill the string with D's; for insertions we keep actual inserted bases + /** * Creates a new pileup element. @@ -37,12 +41,15 @@ public class PileupElement implements Comparable { * @param isBeforeDeletion whether or not this base is before a deletion * @param isBeforeInsertion whether or not this base is before an insertion * @param isNextToSoftClip whether or not this base is next to a soft clipped base + * @param nextEventBases bases in event in case element comes before insertion or deletion + * @param nextEventLength length of next event in case it's insertion or deletion */ @Requires({ "read != null", "offset >= -1", "offset <= read.getReadLength()"}) - public PileupElement(final GATKSAMRecord read, final int offset, final boolean isDeletion, final boolean isBeforeDeletion, final boolean isBeforeInsertion, final boolean isNextToSoftClip) { + public PileupElement(final GATKSAMRecord read, final int offset, final boolean isDeletion, final boolean isBeforeDeletion, final boolean isBeforeInsertion, final boolean isNextToSoftClip, + final String nextEventBases, final int nextEventLength) { if (offset < 0 && isDeletion) throw new ReviewedStingException("Pileup Element cannot create a deletion with a negative offset"); @@ -52,6 +59,14 @@ public PileupElement(final GATKSAMRecord read, final int offset, final boolean i this.isBeforeDeletion = isBeforeDeletion; this.isBeforeInsertion = isBeforeInsertion; this.isNextToSoftClip = isNextToSoftClip; + if (isBeforeInsertion) + eventBases = nextEventBases; + else + eventBases = null; // ignore argument in any other case + if (isBeforeDeletion || isBeforeInsertion) + eventLength = nextEventLength; + else + eventLength = -1; } public boolean isDeletion() { @@ -104,6 +119,20 @@ public byte getBaseDeletionQual() { return getBaseDeletionQual(offset); } + /** + * Returns length of the event (number of inserted or deleted bases + */ + public int getEventLength() { + return eventLength; + } + + /** + * Returns actual sequence of inserted bases, or a null if the event is a deletion or if there is no event in the associated read. + */ + public String getEventBases() { + return eventBases; + } + public int getMappingQual() { return read.getMappingQuality(); } diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java index 7a6ebef218..759d64b2fe 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedPileupImpl.java @@ -71,7 +71,13 @@ protected ReadBackedPileupImpl createNewPileup(GenomeLoc loc, PileupElementTrack } @Override - protected PileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeDeletion, boolean isBeforeInsertion, boolean isNextToSoftClip) { - return new PileupElement(read, offset, isDeletion, isBeforeDeletion, isBeforeInsertion, isNextToSoftClip); + protected PileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeDeletion, boolean isBeforeInsertion, + boolean isNextToSoftClip) { + return new PileupElement(read, offset, isDeletion, isBeforeDeletion, isBeforeInsertion, isNextToSoftClip, null,0); + } + + protected PileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeDeletion, boolean isBeforeInsertion, + boolean isNextToSoftClip,String nextEventBases, final int nextEventLength) { + return new PileupElement(read, offset, isDeletion, isBeforeDeletion, isBeforeInsertion, isNextToSoftClip, nextEventBases,nextEventLength); } } From 522ace6d57d84a22143b0991b499a303c38bf21a Mon Sep 17 00:00:00 2001 From: Menachem Fromer Date: Thu, 23 Feb 2012 11:28:22 -0500 Subject: [PATCH 270/356] CNV discovery is also a long-running job (depending on the number of samples) From e0c189909f8c403c48fe8873591c9e4a9e0be380 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Thu, 23 Feb 2012 12:14:48 -0500 Subject: [PATCH 271/356] Added support for breakpoint alleles -- See https://getsatisfaction.com/gsa/topics/support_vcf_4_1_structural_variation_breakend_alleles?utm_content=topic_link&utm_medium=email&utm_source=new_topic -- Added integrationtest to ensure that we can parse and write out breakpoint example --- .../sting/utils/codecs/vcf/AbstractVCFCodec.java | 7 +++++-- .../sting/utils/variantcontext/Allele.java | 8 +++++++- .../utils/codecs/vcf/VCFIntegrationTest.java | 16 +++++++++++++++- public/testdata/breakpoint-example.vcf | 6 ++++++ 4 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 public/testdata/breakpoint-example.vcf diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java index 1bdee802b4..3c2ed18e45 100755 --- a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java +++ b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java @@ -544,12 +544,15 @@ private static void checkAllele(String allele, boolean isRef, int lineNo) { } /** - * return true if this is a symbolic allele (e.g. ) otherwise false + * return true if this is a symbolic allele (e.g. ) or + * structural variation breakend (with [ or ]), otherwise false * @param allele the allele to check * @return true if the allele is a symbolic allele, otherwise false */ private static boolean isSymbolicAllele(String allele) { - return (allele != null && allele.startsWith("<") && allele.endsWith(">") && allele.length() > 2); + return (allele != null && allele.length() > 2 && + ((allele.startsWith("<") && allele.endsWith(">")) || + (allele.contains("[") || allele.contains("]")))); } /** diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/Allele.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/Allele.java index c3f437f11b..52b4109fef 100755 --- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/Allele.java +++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/Allele.java @@ -212,7 +212,13 @@ public static boolean wouldBeNoCallAllele(byte[] bases) { * @return true if the bases represent a symbolic allele */ public static boolean wouldBeSymbolicAllele(byte[] bases) { - return bases.length > 2 && bases[0] == '<' && bases[bases.length-1] == '>'; + if ( bases.length <= 2 ) + return false; + else { + final String strBases = new String(bases); + return (bases[0] == '<' && bases[bases.length-1] == '>') || + (strBases.contains("[") || strBases.contains("]")); + } } /** diff --git a/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/VCFIntegrationTest.java b/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/VCFIntegrationTest.java index c8a0c0ed66..5de6f1417f 100644 --- a/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/VCFIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/VCFIntegrationTest.java @@ -9,7 +9,7 @@ public class VCFIntegrationTest extends WalkerTest { - @Test + @Test(enabled = false) public void testReadingAndWritingWitHNoChanges() { String md5ofInputVCF = "a990ba187a69ca44cb9bc2bb44d00447"; @@ -25,4 +25,18 @@ public void testReadingAndWritingWitHNoChanges() { WalkerTestSpec spec2 = new WalkerTestSpec(test2, 1, Arrays.asList(md5ofInputVCF)); executeTest("Test Variants To VCF from new output", spec2); } + + @Test + // See https://getsatisfaction.com/gsa/topics/support_vcf_4_1_structural_variation_breakend_alleles?utm_content=topic_link&utm_medium=email&utm_source=new_topic + public void testReadingAndWritingBreakpointAlleles() { + String testVCF = testDir + "breakpoint-example.vcf"; + //String testVCF = validationDataLocation + "multiallelic.vcf"; + + String baseCommand = "-R " + b37KGReference + " -NO_HEADER -o %s "; + + String test1 = baseCommand + "-T SelectVariants -V " + testVCF; + WalkerTestSpec spec1 = new WalkerTestSpec(test1, 1, Arrays.asList("")); + executeTest("Test reading and writing breakpoint VCF", spec1); + } + } diff --git a/public/testdata/breakpoint-example.vcf b/public/testdata/breakpoint-example.vcf new file mode 100644 index 0000000000..f015e1721b --- /dev/null +++ b/public/testdata/breakpoint-example.vcf @@ -0,0 +1,6 @@ +##fileformat=VCFv4.1 +#CHROM POS ID REF ALT QUAL FILTER INFO +22 50 bnd_W G G]22:6000] 6 PASS SVTYPE=BND;MATEID=bnd_Y +22 51 bnd_V T ]22:55]T 6 PASS SVTYPE=BND;MATEID=bnd_U +22 55 bnd_U C C[22:51[ 6 PASS SVTYPE=BND;MATEID=bnd_V +22 6000 bnd_Y A A]22:50] 6 PASS SVTYPE=BND;MATEID=bnd_W \ No newline at end of file From ee9a56ad27add21961d7567627a0994ebbf53229 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Thu, 23 Feb 2012 18:25:01 -0500 Subject: [PATCH 272/356] Fix subtle bug in the ReduceReads stash reported by Adam * The tailSet generated every time we flush the reads stash is still being affected by subsequent clears because it is just a pointer to the parent element in the original TreeSet. This is dangerous, and there is a weird condition where the clear will affects it. * Fix by creating a new set, given the tailSet instead of trying to do magic with just the pointer. --- .../sting/utils/sam/AlignmentStartWithNoTiesComparator.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/AlignmentStartWithNoTiesComparator.java b/public/java/src/org/broadinstitute/sting/utils/sam/AlignmentStartWithNoTiesComparator.java index 02512c8dc9..682c766170 100644 --- a/public/java/src/org/broadinstitute/sting/utils/sam/AlignmentStartWithNoTiesComparator.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/AlignmentStartWithNoTiesComparator.java @@ -36,8 +36,10 @@ else if (r2.getReadUnmappedFlag()) result = cmpContig; else { - if (r1.getAlignmentStart() < r2.getAlignmentStart()) result = -1; - else result = 1; + if (r1.getAlignmentStart() < r2.getAlignmentStart()) + result = -1; + else + result = 1; } } From 470375db58fcbdc1d7e5ad34c93bd438f2ba38b6 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Thu, 23 Feb 2012 18:58:46 -0500 Subject: [PATCH 273/356] added integration test for the ReduceReadsStash bug reported by Adam From c9a4c74f7af270e0040ddfb40d0dcb8a350c175d Mon Sep 17 00:00:00 2001 From: Guillermo del Angel Date: Fri, 24 Feb 2012 10:27:59 -0500 Subject: [PATCH 274/356] a) Bug fixes for last commit related to PileupElements (unit tests are forthcoming). b) Changes needed to make pool caller work in GENOTYPE_GIVEN_ALLELES mode c) Bug fix (yet again) for UG when GENOTYPE_GIVEN_ALLELES and EMIT_ALL_SITES are on, when there's no coverage at site and when input vcf has genotypes: output vcf would still inherit genotypes from input vcf. Now, we just build vc from scratch instead of initializing from input vc. We just take location and alleles from vc --- .../sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java | 2 +- .../org/broadinstitute/sting/utils/pileup/PileupElement.java | 3 +++ .../utils/pileup/ReadBackedExtendedEventPileupImpl.java | 5 +++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java index 0156890ac8..a60cc64f7b 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java @@ -253,7 +253,7 @@ private VariantCallContext generateEmptyContext(RefMetaDataTracker tracker, Refe VariantContext vcInput = UnifiedGenotyperEngine.getVCFromAllelesRod(tracker, ref, rawContext.getLocation(), false, logger, UAC.alleles); if ( vcInput == null ) return null; - vc = new VariantContextBuilder(vcInput).source("UG_call").noID().referenceBaseForIndel(ref.getBase()).attributes(new HashMap()).filters(new HashSet()).make(); + vc = new VariantContextBuilder("UG_call", ref.getLocus().getContig(), ref.getLocus().getStart(), ref.getLocus().getStart(), vcInput.getAlleles()).make(); } else { // deal with bad/non-standard reference bases if ( !Allele.acceptableAlleleBases(new byte[]{ref.getBase()}) ) diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java b/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java index 022cadbbe0..9dbfc52f30 100755 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java @@ -69,6 +69,9 @@ public PileupElement(final GATKSAMRecord read, final int offset, final boolean i eventLength = -1; } + public PileupElement(final GATKSAMRecord read, final int offset, final boolean isDeletion, final boolean isBeforeDeletion, final boolean isBeforeInsertion, final boolean isNextToSoftClip) { + this(read,offset, isDeletion, isBeforeDeletion, isBeforeInsertion, isNextToSoftClip, null, -1); + } public boolean isDeletion() { return isDeletion; } diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java index 357195daa7..e547534dd6 100644 --- a/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java +++ b/public/java/src/org/broadinstitute/sting/utils/pileup/ReadBackedExtendedEventPileupImpl.java @@ -99,6 +99,11 @@ protected ReadBackedExtendedEventPileupImpl createNewPileup(GenomeLoc loc, Pileu protected ExtendedEventPileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeDeletion, boolean isBeforeInsertion, boolean isNextToSoftClip) { throw new UnsupportedOperationException("Not enough information provided to create a new pileup element"); } + @Override + protected ExtendedEventPileupElement createNewPileupElement(GATKSAMRecord read, int offset, boolean isDeletion, boolean isBeforeDeletion, boolean isBeforeInsertion, + boolean isNextToSoftClip,String nextEventBases, int nextEventLength) { + throw new UnsupportedOperationException("Not enough information provided to create a new pileup element"); + } /** From 253bb46bcdf23030c9b503a5547e066b8104a8f4 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 24 Feb 2012 09:24:49 -0500 Subject: [PATCH 275/356] Add support to analyzeRunReports to tag xml logs with git version numbers From e94a5340767b2a0ff89de3c96f50e7b9b7c0f415 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 24 Feb 2012 10:00:04 -0500 Subject: [PATCH 276/356] Added dry run and verbose options to gsafolkLSFLogs From 747e1a728fe92e83421a441624b6026fd31856da Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 24 Feb 2012 10:22:38 -0500 Subject: [PATCH 277/356] Script to recreate entire GATKLog db from scratch Useful primarily as a reference. Sometimes necessary when low-level changes are made to the scripts, requiring all of the data to be reprocessed From 80b5c7ad21b6fe85b1946622679e03fc91d077a2 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 24 Feb 2012 15:51:45 -0500 Subject: [PATCH 278/356] Fix gitVersionNumbers script to not print git status messages to our file From 9bad51877e981a8b58e2da406d1e594a7b92a6b8 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 24 Feb 2012 15:52:45 -0500 Subject: [PATCH 279/356] Generalized gsafolkLSFLogs.py to gsafolkLogsForTableau.py -- Now updates both LSF logs and filesystem sizes -- New Tableau emails will include both LSF and FS info! From 50de1a3eabffecb07f4219396a81d617cb4d56e3 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sat, 25 Feb 2012 11:26:36 -0500 Subject: [PATCH 280/356] Fixing bad VCFIntegration tests -- Left disabled a test that should have been enabled -- Didn't add the md5 to the test I actually added -- Now VCFIntegrationTests should be working! --- .../sting/utils/codecs/vcf/VCFIntegrationTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/VCFIntegrationTest.java b/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/VCFIntegrationTest.java index 5de6f1417f..ca5fcf4195 100644 --- a/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/VCFIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/VCFIntegrationTest.java @@ -9,7 +9,7 @@ public class VCFIntegrationTest extends WalkerTest { - @Test(enabled = false) + @Test(enabled = true) public void testReadingAndWritingWitHNoChanges() { String md5ofInputVCF = "a990ba187a69ca44cb9bc2bb44d00447"; @@ -35,7 +35,7 @@ public void testReadingAndWritingBreakpointAlleles() { String baseCommand = "-R " + b37KGReference + " -NO_HEADER -o %s "; String test1 = baseCommand + "-T SelectVariants -V " + testVCF; - WalkerTestSpec spec1 = new WalkerTestSpec(test1, 1, Arrays.asList("")); + WalkerTestSpec spec1 = new WalkerTestSpec(test1, 1, Arrays.asList("76075307afd26b4db6234795d9fb3c2f")); executeTest("Test reading and writing breakpoint VCF", spec1); } From c8a06e53c17cb4a4c8d2e76f2ae24aa32ccbd6ae Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sat, 25 Feb 2012 11:32:50 -0500 Subject: [PATCH 281/356] DoC now properly handles reference N bases + misc. additional cleanups -- DoC now by default ignores bases with reference Ns, so these are not included in the coverage calculations at any stage. -- Added option --includeRefNSites that will include them in the calculation -- Added integration tests that ensures the per base tables (and so all subsequent calculations) work with and without reference N bases included -- Reorganized command line options, tagging advanced options with @Advanced --- .../coverage/DepthOfCoverageWalker.java | 107 ++++++++++++------ .../DepthOfCoverageIntegrationTest.java | 10 ++ 2 files changed, 80 insertions(+), 37 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageWalker.java index cbbb3d43f6..7d1858a634 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageWalker.java @@ -26,6 +26,7 @@ package org.broadinstitute.sting.gatk.walkers.coverage; import net.sf.samtools.SAMReadGroupRecord; +import org.broadinstitute.sting.commandline.Advanced; import org.broadinstitute.sting.commandline.Argument; import org.broadinstitute.sting.commandline.Output; import org.broadinstitute.sting.gatk.contexts.AlignmentContext; @@ -119,21 +120,6 @@ public class DepthOfCoverageWalker extends LocusWalker out; - /** - * Sets the low-coverage cutoff for granular binning. All loci with depth < START are counted in the first bin. - */ - @Argument(fullName = "start", doc = "Starting (left endpoint) for granular binning", required = false) - int start = 1; - /** - * Sets the high-coverage cutoff for granular binning. All loci with depth > END are counted in the last bin. - */ - @Argument(fullName = "stop", doc = "Ending (right endpoint) for granular binning", required = false) - int stop = 500; - /** - * Sets the number of bins for granular binning - */ - @Argument(fullName = "nBins", doc = "Number of bins to use for granular binning", required = false) - int nBins = 499; @Argument(fullName = "minMappingQuality", shortName = "mmq", doc = "Minimum mapping quality of reads to count towards depth. Defaults to -1.", required = false) int minMappingQuality = -1; @Argument(fullName = "maxMappingQuality", doc = "Maximum mapping quality of reads to count towards depth. Defaults to 2^31-1 (Integer.MAX_VALUE).", required = false) @@ -142,16 +128,19 @@ public class DepthOfCoverageWalker extends LocusWalker END are counted in the last bin. + */ + @Advanced + @Argument(fullName = "stop", doc = "Ending (right endpoint) for granular binning", required = false) + int stop = 500; + /** + * Sets the number of bins for granular binning + */ + @Advanced + @Argument(fullName = "nBins", doc = "Number of bins to use for granular binning", required = false) + int nBins = 499; + /** * Do not tabulate the sample summary statistics (total, mean, median, quartile coverage per sample) */ @@ -174,27 +207,22 @@ public class DepthOfCoverageWalker extends LocusWalker partitionTypes = EnumSet.of(DoCOutputType.Partition.sample); + /** * Consider a spanning deletion as contributing to coverage. Also enables deletion counts in per-base output. */ + @Advanced @Argument(fullName = "includeDeletions", shortName = "dels", doc = "Include information on deletions", required = false) boolean includeDeletions = false; + + @Advanced @Argument(fullName = "ignoreDeletionSites", doc = "Ignore sites consisting only of deletions", required = false) boolean ignoreDeletionSites = false; - /** - * Path to the RefSeq file for use in aggregating coverage statistics over genes - */ - @Argument(fullName = "calculateCoverageOverGenes", shortName = "geneList", doc = "Calculate the coverage statistics over this list of genes. Currently accepts RefSeq.", required = false) - File refSeqGeneList = null; - /** - * The format of the output file - */ - @Argument(fullName = "outputFormat", doc = "the format of the output file (e.g. csv, table, rtable); defaults to r-readable table", required = false) - String outputFormat = "rtable"; /** * A coverage threshold for summarizing (e.g. % bases >= CT for each sample) */ + @Advanced @Argument(fullName = "summaryCoverageThreshold", shortName = "ct", doc = "for summary file outputs, report the % of bases coverd to >= this number. Defaults to 15; can take multiple arguments.", required = false) int[] coverageThresholds = {15}; @@ -334,24 +362,29 @@ public CoveragePartitioner reduceInit() { } public Map> map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) { + if (includeRefNBases || BaseUtils.isRegularBase(ref.getBase())) { + if ( ! omitDepthOutput ) { + getCorrectStream(null, DoCOutputType.Aggregation.locus, DoCOutputType.FileType.summary).printf("%s",ref.getLocus()); // yes: print locus in map, and the rest of the info in reduce (for eventual cumulatives) + //System.out.printf("\t[log]\t%s",ref.getLocus()); + } - if ( ! omitDepthOutput ) { - getCorrectStream(null, DoCOutputType.Aggregation.locus, DoCOutputType.FileType.summary).printf("%s",ref.getLocus()); // yes: print locus in map, and the rest of the info in reduce (for eventual cumulatives) - //System.out.printf("\t[log]\t%s",ref.getLocus()); + return CoverageUtils.getBaseCountsByPartition(context,minMappingQuality,maxMappingQuality,minBaseQuality,maxBaseQuality,partitionTypes); + } else { + return null; } - - return CoverageUtils.getBaseCountsByPartition(context,minMappingQuality,maxMappingQuality,minBaseQuality,maxBaseQuality,partitionTypes); } public CoveragePartitioner reduce(Map> thisMap, CoveragePartitioner prevReduce) { - if ( ! omitDepthOutput ) { - //checkOrder(prevReduce); // tests prevReduce.getIdentifiersByType().get(t) against the initialized header order - printDepths(getCorrectStream(null, DoCOutputType.Aggregation.locus, DoCOutputType.FileType.summary),thisMap,prevReduce.getIdentifiersByType()); - // this is an additional iteration through thisMap, plus dealing with IO, so should be much slower without - // turning on omit - } + if ( thisMap != null ) { // skip sites we didn't want to include in the calculation (ref Ns) + if ( ! omitDepthOutput ) { + //checkOrder(prevReduce); // tests prevReduce.getIdentifiersByType().get(t) against the initialized header order + printDepths(getCorrectStream(null, DoCOutputType.Aggregation.locus, DoCOutputType.FileType.summary),thisMap,prevReduce.getIdentifiersByType()); + // this is an additional iteration through thisMap, plus dealing with IO, so should be much slower without + // turning on omit + } - prevReduce.update(thisMap); // note that in "useBoth" cases, this method alters the thisMap object + prevReduce.update(thisMap); // note that in "useBoth" cases, this method alters the thisMap object + } return prevReduce; } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageIntegrationTest.java index 1c58346b4f..6f13700082 100644 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageIntegrationTest.java @@ -94,4 +94,14 @@ public void testNoCoverageDueToFiltering() { execute("testNoCoverageDueToFiltering",spec); } + + public void testRefNHandling(boolean includeNs, final String md5) { + String command = "-R " + b37KGReference + " -L 20:26,319,565-26,319,575 -I " + validationDataLocation + "NA12878.HiSeq.WGS.bwa.cleaned.recal.hg19.20.bam -T DepthOfCoverage -baseCounts --omitIntervalStatistics --omitLocusTable --omitPerSampleStats -o %s"; + if ( includeNs ) command += " --includeRefNSites"; + WalkerTestSpec spec = new WalkerTestSpec(command, 1, Arrays.asList(md5)); + executeTest("Testing DoC " + (includeNs ? "with" : "without") + " reference Ns", spec); + } + + @Test public void testRefNWithNs() { testRefNHandling(true, "24cd2da2e4323ce6fd76217ba6dc2834"); } + @Test public void testRefNWithoutNs() { testRefNHandling(false, "4fc0f1a2e968f777d693abcefd4fb7af"); } } From dea35943d17830496645dcab9cd59a063b258e34 Mon Sep 17 00:00:00 2001 From: Guillermo del Angel Date: Sat, 25 Feb 2012 13:57:28 -0500 Subject: [PATCH 282/356] a) Bug fix in calling new functions that give indel bases and length from regular pileup in LocusIteratorByState, b) Added unit test to cover these. --- .../gatk/iterators/LocusIteratorByState.java | 8 ++- .../LocusIteratorByStateUnitTest.java | 50 +++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java b/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java index b8dd033177..a47c61d0b8 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java +++ b/public/java/src/org/broadinstitute/sting/gatk/iterators/LocusIteratorByState.java @@ -465,7 +465,7 @@ else if (op != CigarOperator.D || readInfo.includeReadsWithDeletionAtLoci()) { final CigarElement nextElement = state.peekForwardOnGenome(); // next cigar element final CigarOperator nextOp = nextElement.getOperator(); final int readOffset = state.getReadOffset(); // the base offset on this read - byte[] insertedBases = Arrays.copyOfRange(read.getReadBases(), readOffset + 1, readOffset + 1 + nextElement.getLength()); + int nextElementLength = nextElement.getLength(); if (op == CigarOperator.N) // N's are never added to any pileup @@ -483,8 +483,12 @@ else if (op != CigarOperator.D || readInfo.includeReadsWithDeletionAtLoci()) { } else { if (!filterBaseInRead(read, location.getStart())) { + String insertedBaseString = null; + if (nextOp == CigarOperator.I) { + insertedBaseString = new String(Arrays.copyOfRange(read.getReadBases(), readOffset + 1, readOffset + 1 + nextElement.getLength())); + } pile.add(new PileupElement(read, readOffset, false, nextOp == CigarOperator.D, nextOp == CigarOperator.I, nextOp == CigarOperator.S || (state.getGenomeOffset() == 0 && read.getSoftStart() != read.getAlignmentStart()), - new String(insertedBases),nextElementLength)); + insertedBaseString,nextElementLength)); size++; if (read.getMappingQuality() == 0) nMQ0Reads++; diff --git a/public/java/test/org/broadinstitute/sting/gatk/iterators/LocusIteratorByStateUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/iterators/LocusIteratorByStateUnitTest.java index 04e11db541..7282d6c485 100644 --- a/public/java/test/org/broadinstitute/sting/gatk/iterators/LocusIteratorByStateUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/iterators/LocusIteratorByStateUnitTest.java @@ -6,6 +6,7 @@ import net.sf.samtools.util.CloseableIterator; import org.broadinstitute.sting.gatk.filters.ReadFilter; import org.broadinstitute.sting.utils.Utils; +import org.broadinstitute.sting.utils.pileup.PileupElement; import org.broadinstitute.sting.utils.pileup.ReadBackedPileup; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import org.testng.Assert; @@ -85,6 +86,55 @@ public void testIndelBaseQualityFiltering() { Assert.assertTrue(foundExtendedEventPileup,"Extended event pileup not found"); } + @Test + public void testIndelsInRegularPileup() { + final byte[] bases = new byte[] {'A','A','A','A','A','A','A','A','A','A'}; + final byte[] indelBases = new byte[] {'A','A','A','A','C','T','A','A','A','A','A','A'}; + + // create a test version of the Reads object + ReadProperties readAttributes = createTestReadProperties(); + JVMUtils.setFieldValue(JVMUtils.findField(ReadProperties.class,"generateExtendedEvents"),readAttributes,true); + + SAMRecord before = ArtificialSAMUtils.createArtificialRead(header,"before",0,1,10); + before.setReadBases(bases); + before.setBaseQualities(new byte[] {20,20,20,20,20,20,20,20,20,20}); + before.setCigarString("10M"); + + SAMRecord during = ArtificialSAMUtils.createArtificialRead(header,"during",0,2,10); + during.setReadBases(indelBases); + during.setBaseQualities(new byte[] {20,20,20,20,20,20,20,20,20,20,20,20}); + during.setCigarString("4M2I6M"); + + SAMRecord after = ArtificialSAMUtils.createArtificialRead(header,"after",0,3,10); + after.setReadBases(bases); + after.setBaseQualities(new byte[] {20,20,20,20,20,20,20,20,20,20}); + after.setCigarString("10M"); + + List reads = Arrays.asList(before,during,after); + + // create the iterator by state with the fake reads and fake records + li = makeLTBS(reads,readAttributes); + + boolean foundIndel = false; + while (li.hasNext()) { + AlignmentContext context = li.next(); + if(!context.hasBasePileup()) + continue; + + ReadBackedPileup pileup = context.getBasePileup().getBaseFilteredPileup(10); + for (PileupElement p : pileup) { + if (p.isBeforeInsertion()) { + foundIndel = true; + Assert.assertEquals(p.getEventLength(), 2, "Wrong event length"); + Assert.assertEquals(p.getEventBases(), "CT", "Inserted bases are incorrect"); + break; + } + } + + } + + Assert.assertTrue(foundIndel,"Indel in pileup not found"); + } /** * Right now, the GATK's extended event pileup DOES NOT include reads which stop immediately before an insertion From dfdf4f989ba550d14319accca19fcfd5bf5a8de5 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Mon, 27 Feb 2012 09:50:09 -0500 Subject: [PATCH 283/356] Enabling Fisher Strand for multi-allelics: use the alt allele with max AC. Added minor optimization to the method in the VC. --- .../sting/gatk/walkers/annotator/FisherStrand.java | 10 +++++----- .../sting/utils/variantcontext/VariantContext.java | 14 ++++++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java index 987579ab86..6a825cba79 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java @@ -59,10 +59,10 @@ public Map annotate(RefMetaDataTracker tracker, AnnotatorCompati int[][] table; - if (vc.isBiallelic() && vc.isSNP()) - table = getSNPContingencyTable(stratifiedContexts, vc.getReference(), vc.getAlternateAllele(0)); - else if (vc.isIndel() || vc.isMixed()) { - table = getIndelContingencyTable(stratifiedContexts, vc); + if ( vc.isSNP() ) + table = getSNPContingencyTable(stratifiedContexts, vc.getReference(), vc.getAltAlleleWithHighestAlleleCount()); + else if ( vc.isIndel() || vc.isMixed() ) { + table = getIndelContingencyTable(stratifiedContexts); if (table == null) return null; } @@ -234,7 +234,7 @@ private static int[][] getSNPContingencyTable(Map stra * allele2 # # * @return a 2x2 contingency table */ - private static int[][] getIndelContingencyTable(Map stratifiedContexts, VariantContext vc) { + private static int[][] getIndelContingencyTable(Map stratifiedContexts) { final double INDEL_LIKELIHOOD_THRESH = 0.3; final HashMap> indelLikelihoodMap = IndelGenotypeLikelihoodsCalculationModel.getIndelLikelihoodMap(); diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java index 27721be95b..f5c57ca44f 100755 --- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java +++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java @@ -1229,12 +1229,15 @@ public boolean hasSymbolicAlleles() { } public Allele getAltAlleleWithHighestAlleleCount() { - // first idea: get two alleles with highest AC + // optimization: for bi-allelic sites, just return the 1only alt allele + if ( isBiallelic() ) + return getAlternateAllele(0); + Allele best = null; int maxAC1 = 0; - for (Allele a:this.getAlternateAlleles()) { - int ac = this.getCalledChrCount(a); - if (ac >=maxAC1) { + for ( Allele a : getAlternateAlleles() ) { + final int ac = getCalledChrCount(a); + if ( ac >= maxAC1 ) { maxAC1 = ac; best = a; } @@ -1244,6 +1247,9 @@ public Allele getAltAlleleWithHighestAlleleCount() { } public int[] getGLIndecesOfAllele(Allele inputAllele) { + + // TODO -- this information is cached statically by the UnifiedGenotyperEngine; pull it out into a common utils class for all to use + int[] idxVector = new int[3]; int numAlleles = this.getAlleles().size(); From 850c5d0db2908828f08554be7dc557a460a75855 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Mon, 27 Feb 2012 09:59:36 -0500 Subject: [PATCH 284/356] Enabling Rank Sum Tests for multi-allelics: use ref vs any alt allele. --- .../annotator/BaseQualityRankSumTest.java | 4 ++-- .../annotator/MappingQualityRankSumTest.java | 4 ++-- .../gatk/walkers/annotator/RankSumTest.java | 20 +++++++++++-------- .../walkers/annotator/ReadPosRankSumTest.java | 7 +++---- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java index 507a6559c2..97a4ac4680 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java @@ -22,12 +22,12 @@ public class BaseQualityRankSumTest extends RankSumTest { public List getDescriptions() { return Arrays.asList(new VCFInfoHeaderLine("BaseQRankSum", 1, VCFHeaderLineType.Float, "Z-score from Wilcoxon rank sum test of Alt Vs. Ref base qualities")); } - protected void fillQualsFromPileup(byte ref, byte alt, ReadBackedPileup pileup, List refQuals, List altQuals) { + protected void fillQualsFromPileup(byte ref, List alts, ReadBackedPileup pileup, List refQuals, List altQuals) { for ( final PileupElement p : pileup ) { if( isUsableBase(p) ) { if ( p.getBase() == ref ) refQuals.add((double)p.getQual()); - else if ( p.getBase() == alt ) + else if ( alts.contains(p.getBase()) ) altQuals.add((double)p.getQual()); } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityRankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityRankSumTest.java index 9857c339f3..aa4f26ef3d 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityRankSumTest.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityRankSumTest.java @@ -24,12 +24,12 @@ public class MappingQualityRankSumTest extends RankSumTest { public List getDescriptions() { return Arrays.asList(new VCFInfoHeaderLine("MQRankSum", 1, VCFHeaderLineType.Float, "Z-score From Wilcoxon rank sum test of Alt vs. Ref read mapping qualities")); } - protected void fillQualsFromPileup(byte ref, byte alt, ReadBackedPileup pileup, List refQuals, List altQuals) { + protected void fillQualsFromPileup(byte ref, List alts, ReadBackedPileup pileup, List refQuals, List altQuals) { for ( final PileupElement p : pileup ) { if ( isUsableBase(p) ) { if ( p.getBase() == ref ) { refQuals.add((double)p.getMappingQual()); - } else if ( p.getBase() == alt ) { + } else if ( alts.contains(p.getBase()) ) { altQuals.add((double)p.getMappingQual()); } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java index e0e62cdb0f..3f555f780d 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java @@ -12,6 +12,7 @@ import org.broadinstitute.sting.utils.collections.Pair; import org.broadinstitute.sting.utils.pileup.PileupElement; import org.broadinstitute.sting.utils.pileup.ReadBackedPileup; +import org.broadinstitute.sting.utils.variantcontext.Allele; import org.broadinstitute.sting.utils.variantcontext.Genotype; import org.broadinstitute.sting.utils.variantcontext.GenotypesContext; import org.broadinstitute.sting.utils.variantcontext.VariantContext; @@ -41,16 +42,19 @@ public Map annotate(RefMetaDataTracker tracker, AnnotatorCompati final ArrayList refQuals = new ArrayList(); final ArrayList altQuals = new ArrayList(); - if (vc.isSNP() && vc.isBiallelic()) { - // todo - no current support for multiallelic snps - for (final Genotype genotype : genotypes.iterateInSampleNameOrder()) { + if ( vc.isSNP() ) { + for ( final Genotype genotype : genotypes.iterateInSampleNameOrder() ) { final AlignmentContext context = stratifiedContexts.get(genotype.getSampleName()); - if (context == null) { + if ( context == null ) continue; - } - fillQualsFromPileup(ref.getBase(), vc.getAlternateAllele(0).getBases()[0], context.getBasePileup(), refQuals, altQuals); + + final List altAlleles = new ArrayList(); + for ( final Allele a : vc.getAlternateAlleles() ) + altAlleles.add(a.getBases()[0]); + + fillQualsFromPileup(ref.getBase(), altAlleles, context.getBasePileup(), refQuals, altQuals); } - } else if (vc.isIndel() || vc.isMixed()) { + } else if ( vc.isIndel() || vc.isMixed() ) { for (final Genotype genotype : genotypes.iterateInSampleNameOrder()) { final AlignmentContext context = stratifiedContexts.get(genotype.getSampleName()); @@ -105,7 +109,7 @@ else if (context.hasBasePileup()) } - protected abstract void fillQualsFromPileup(byte ref, byte alt, ReadBackedPileup pileup, List refQuals, List altQuals); + protected abstract void fillQualsFromPileup(byte ref, List alts, ReadBackedPileup pileup, List refQuals, List altQuals); protected abstract void fillIndelQualsFromPileup(ReadBackedPileup pileup, List refQuals, List altQuals); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java index b0039d1a00..a998cd08b4 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java @@ -32,7 +32,7 @@ public List getDescriptions() { return Arrays.asList(new VCFInfoHeaderLine("ReadPosRankSum", 1, VCFHeaderLineType.Float, "Z-score from Wilcoxon rank sum test of Alt vs. Ref read position bias")); } - protected void fillQualsFromPileup(byte ref, byte alt, ReadBackedPileup pileup, List refQuals, List altQuals) { + protected void fillQualsFromPileup(byte ref, List alts, ReadBackedPileup pileup, List refQuals, List altQuals) { for (final PileupElement p : pileup) { if (isUsableBase(p)) { int readPos = AlignmentUtils.calcAlignmentByteArrayOffset(p.getRead().getCigar(), p, 0, 0); @@ -41,11 +41,10 @@ protected void fillQualsFromPileup(byte ref, byte alt, ReadBackedPileup pileup, readPos = numAlignedBases - (readPos + 1); - if (p.getBase() == ref) + if ( p.getBase() == ref ) refQuals.add((double) readPos); - else if (p.getBase() == alt) + else if ( alts.contains(p.getBase()) ) altQuals.add((double) readPos); - } } } From 1ea34058c2143b2650a058d213f8698b512f6d7e Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Mon, 27 Feb 2012 11:32:26 -0500 Subject: [PATCH 285/356] Updating integration tests now that standard annotations support multiple alleles --- .../walkers/annotator/VariantAnnotatorIntegrationTest.java | 6 +++--- .../walkers/genotyper/UnifiedGenotyperIntegrationTest.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java index 7984a00c0a..02026b375e 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java @@ -32,7 +32,7 @@ public void testHasAnnotsNotAsking2() { public void testHasAnnotsAsking1() { WalkerTestSpec spec = new WalkerTestSpec( baseTestString() + " -G Standard --variant:VCF3 " + validationDataLocation + "vcfexample2.vcf -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -L 1:10,020,000-10,021,000", 1, - Arrays.asList("604328867fc9aaf3e71fa0f9ca2ba5c9")); + Arrays.asList("3b7796fa7c7dc94878bedadf7938db4c")); executeTest("test file has annotations, asking for annotations, #1", spec); } @@ -66,7 +66,7 @@ public void testNoAnnotsNotAsking2() { public void testNoAnnotsAsking1() { WalkerTestSpec spec = new WalkerTestSpec( baseTestString() + " -G Standard --variant:VCF3 " + validationDataLocation + "vcfexample2empty.vcf -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -L 1:10,020,000-10,021,000", 1, - Arrays.asList("bbde8c92d27ad2a7ec1ff2d095d459eb")); + Arrays.asList("279cace364f747f9bae7fe391b5026f0")); executeTest("test file doesn't have annotations, asking for annotations, #1", spec); } @@ -82,7 +82,7 @@ public void testNoAnnotsAsking2() { public void testExcludeAnnotations() { WalkerTestSpec spec = new WalkerTestSpec( baseTestString() + " -G Standard -XA FisherStrand -XA ReadPosRankSumTest --variant:VCF3 " + validationDataLocation + "vcfexample2empty.vcf -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -L 1:10,020,000-10,021,000", 1, - Arrays.asList("8ec9f79cab84f26d8250f00d99d18aac")); + Arrays.asList("e488abd05d6162758698a3a7579866a6")); executeTest("test exclude annotations", spec); } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java index fc4f0f46bd..823eeeeb97 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java @@ -60,7 +60,7 @@ public void testSingleSamplePilot2() { public void testMultipleSNPAlleles() { WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec( "-T UnifiedGenotyper -R " + b37KGReference + " -nosl -NO_HEADER -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + validationDataLocation + "multiallelic.snps.bam -o %s -L " + validationDataLocation + "multiallelic.snps.intervals", 1, - Arrays.asList("b53cb55a5f868663068812b13578af57")); + Arrays.asList("10027d13befaa07b7900a7af0ae0791c")); executeTest("test Multiple SNP alleles", spec); } From 998ed8fff39cfe499ed08a461b4cd3701f1d9868 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Mon, 27 Feb 2012 14:56:10 -0500 Subject: [PATCH 286/356] Bug fix to deal with VCF records that don't have GTs. While in there, optimized a bunch of related functions (including removing a copy of the method calculateChromosomeCounts(); why did we have 2 copies? very dangerous). --- .../walkers/variantutils/SelectVariants.java | 2 +- .../sting/utils/codecs/vcf/VCFCodec.java | 2 +- .../sting/utils/variantcontext/Genotype.java | 12 ++-- .../variantcontext/VariantContextUtils.java | 58 +++++-------------- 4 files changed, 21 insertions(+), 53 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java index 5eef7fb66c..204851e1fd 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java @@ -682,7 +682,7 @@ private VariantContext subsetRecord(VariantContext vc, Set samples) { for (String sample : sub.getSampleNames()) { Genotype g = sub.getGenotype(sample); - if (g.isNotFiltered() && g.isCalled()) { + if ( g.isNotFiltered() ) { String dp = (String) g.getAttribute("DP"); if (dp != null && ! dp.equals(VCFConstants.MISSING_DEPTH_v3) && ! dp.equals(VCFConstants.MISSING_VALUE_v4) ) { diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFCodec.java b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFCodec.java index 453155be7e..01cc367c44 100755 --- a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFCodec.java +++ b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFCodec.java @@ -203,7 +203,7 @@ public LazyGenotypesContext.LazyData createGenotypeMap(String str, List if ( genotypeAlleleLocation > 0 ) generateException("Saw GT field at position " + genotypeAlleleLocation + ", but it must be at the first position for genotypes when present"); - List GTalleles = (genotypeAlleleLocation == -1 ? null : parseGenotypeAlleles(GTValueArray[genotypeAlleleLocation], alleles, alleleMap)); + List GTalleles = (genotypeAlleleLocation == -1 ? new ArrayList(0) : parseGenotypeAlleles(GTValueArray[genotypeAlleleLocation], alleles, alleleMap)); boolean phased = genotypeAlleleLocation != -1 && GTValueArray[genotypeAlleleLocation].indexOf(VCFConstants.PHASED) != -1; // add it to the list diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java index 1691129c94..13c4ff3d81 100755 --- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java +++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java @@ -89,9 +89,6 @@ public List getAlleles() { } public List getAlleles(Allele allele) { - if ( getType() == Type.UNAVAILABLE ) - throw new ReviewedStingException("Requesting alleles for an UNAVAILABLE genotype"); - List al = new ArrayList(); for ( Allele a : alleles ) if ( a.equals(allele) ) @@ -112,7 +109,7 @@ public Allele getAllele(int i) { * @return the ploidy of this genotype */ public int getPloidy() { - if ( alleles == null ) + if ( alleles.size() == 0 ) throw new ReviewedStingException("Requesting ploidy for an UNAVAILABLE genotype"); return alleles.size(); } @@ -134,7 +131,7 @@ public Type getType() { } protected Type determineType() { - if ( alleles == null ) + if ( alleles.size() == 0 ) return Type.UNAVAILABLE; boolean sawNoCall = false, sawMultipleAlleles = false; @@ -234,8 +231,7 @@ private GenotypeLikelihoods getLikelihoods(String key, boolean asPL) { } public void validate() { - if ( alleles == null ) return; - if ( alleles.size() == 0) throw new IllegalArgumentException("BUG: alleles cannot be of size 0"); + if ( alleles.size() == 0) return; // int nNoCalls = 0; for ( Allele allele : alleles ) { @@ -254,7 +250,7 @@ public String getGenotypeString() { } public String getGenotypeString(boolean ignoreRefState) { - if ( alleles == null ) + if ( alleles.size() == 0 ) return null; // Notes: diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java index c79bbaace7..fc50df3a59 100755 --- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java @@ -65,8 +65,10 @@ public class VariantContextUtils { * @return the attributes map provided as input, returned for programming convenience */ public static Map calculateChromosomeCounts(VariantContext vc, Map attributes, boolean removeStaleValues) { + final int AN = vc.getCalledChrCount(); + // if everyone is a no-call, remove the old attributes if requested - if ( vc.getCalledChrCount() == 0 && removeStaleValues ) { + if ( AN == 0 && removeStaleValues ) { if ( attributes.containsKey(VCFConstants.ALLELE_COUNT_KEY) ) attributes.remove(VCFConstants.ALLELE_COUNT_KEY); if ( attributes.containsKey(VCFConstants.ALLELE_FREQUENCY_KEY) ) @@ -77,19 +79,22 @@ public static Map calculateChromosomeCounts(VariantContext vc, M } if ( vc.hasGenotypes() ) { - attributes.put(VCFConstants.ALLELE_NUMBER_KEY, vc.getCalledChrCount()); + attributes.put(VCFConstants.ALLELE_NUMBER_KEY, AN); // if there are alternate alleles, record the relevant tags if ( vc.getAlternateAlleles().size() > 0 ) { - ArrayList alleleFreqs = new ArrayList(); - ArrayList alleleCounts = new ArrayList(); - double totalChromosomes = (double)vc.getCalledChrCount(); + final ArrayList alleleFreqs = new ArrayList(); + final ArrayList alleleCounts = new ArrayList(); for ( Allele allele : vc.getAlternateAlleles() ) { int altChromosomes = vc.getCalledChrCount(allele); alleleCounts.add(altChromosomes); - // todo -- this is a performance problem - String freq = String.format(makePrecisionFormatStringFromDenominatorValue(totalChromosomes), ((double)altChromosomes / totalChromosomes)); - alleleFreqs.add(freq); + if ( AN == 0 ) { + alleleFreqs.add("0.0"); + } else { + // todo -- this is a performance problem + final String freq = String.format(makePrecisionFormatStringFromDenominatorValue((double)AN), ((double)altChromosomes / (double)AN)); + alleleFreqs.add(freq); + } } attributes.put(VCFConstants.ALLELE_COUNT_KEY, alleleCounts.size() == 1 ? alleleCounts.get(0) : alleleCounts); @@ -113,41 +118,8 @@ public static Map calculateChromosomeCounts(VariantContext vc, M */ public static void calculateChromosomeCounts(VariantContextBuilder builder, boolean removeStaleValues) { final VariantContext vc = builder.make(); - - // if everyone is a no-call, remove the old attributes if requested - if ( vc.getCalledChrCount() == 0 && removeStaleValues ) { - if ( vc.hasAttribute(VCFConstants.ALLELE_COUNT_KEY) ) - builder.rmAttribute(VCFConstants.ALLELE_COUNT_KEY); - if ( vc.hasAttribute(VCFConstants.ALLELE_FREQUENCY_KEY) ) - builder.rmAttribute(VCFConstants.ALLELE_FREQUENCY_KEY); - if ( vc.hasAttribute(VCFConstants.ALLELE_NUMBER_KEY) ) - builder.rmAttribute(VCFConstants.ALLELE_NUMBER_KEY); - return; - } - - if ( vc.hasGenotypes() ) { - builder.attribute(VCFConstants.ALLELE_NUMBER_KEY, vc.getCalledChrCount()); - - // if there are alternate alleles, record the relevant tags - if ( vc.getAlternateAlleles().size() > 0 ) { - ArrayList alleleFreqs = new ArrayList(); - ArrayList alleleCounts = new ArrayList(); - double totalChromosomes = (double)vc.getCalledChrCount(); - for ( Allele allele : vc.getAlternateAlleles() ) { - int altChromosomes = vc.getCalledChrCount(allele); - alleleCounts.add(altChromosomes); - String freq = String.format(makePrecisionFormatStringFromDenominatorValue(totalChromosomes), ((double)altChromosomes / totalChromosomes)); - alleleFreqs.add(freq); - } - - builder.attribute(VCFConstants.ALLELE_COUNT_KEY, alleleCounts.size() == 1 ? alleleCounts.get(0) : alleleCounts); - builder.attribute(VCFConstants.ALLELE_FREQUENCY_KEY, alleleFreqs.size() == 1 ? alleleFreqs.get(0) : alleleFreqs); - } - else { - builder.attribute(VCFConstants.ALLELE_COUNT_KEY, 0); - builder.attribute(VCFConstants.ALLELE_FREQUENCY_KEY, 0.0); - } - } + final Map attrs = calculateChromosomeCounts(vc, new HashMap(vc.getAttributes()), removeStaleValues); + builder.attributes(attrs); } private static String makePrecisionFormatStringFromDenominatorValue(double maxValue) { From 52871187d78eefc313466fd0fd56aa8632c4fe83 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Mon, 27 Feb 2012 15:09:56 -0500 Subject: [PATCH 287/356] Adding integration test for file with no GTs. Also updated md5 for one other test (since we no longer print out 'NaN' for the AF). --- .../SelectVariantsIntegrationTest.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariantsIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariantsIntegrationTest.java index 9577966b72..900e3d489e 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariantsIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariantsIntegrationTest.java @@ -45,7 +45,7 @@ public void testRepeatedLineSelection() { WalkerTestSpec spec = new WalkerTestSpec( baseTestString(" -sn A -sn B -sn C --variant " + testfile), 1, - Arrays.asList("b74038779fe6485dbb8734ae48178356") + Arrays.asList("5085a2f8cddfeae9f6274f905025184f") ); executeTest("testRepeatedLineSelection--" + testfile, spec); @@ -129,6 +129,19 @@ public void testMultipleRecordsAtOnePosition() { executeTest("testMultipleRecordsAtOnePositionFirstIsFiltered--" + testFile, spec); } + @Test + public void testNoGTs() { + String testFile = validationDataLocation + "vcf4.1.example.vcf"; + + WalkerTestSpec spec = new WalkerTestSpec( + "-T SelectVariants -R " + b37KGReference + " --variant " + testFile + " -o %s -NO_HEADER", + 1, + Arrays.asList("f17885e5cbd5387edb99112047ea43c1") + ); + + executeTest("testMultipleRecordsAtOnePositionFirstIsFiltered--" + testFile, spec); + } + @Test public void testParallelization() { String testfile = validationDataLocation + "test.filtered.maf_annotated.vcf"; From bd944ab04fa5aae49b8fb68864e66824bf9002aa Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Mon, 27 Feb 2012 15:19:08 -0500 Subject: [PATCH 288/356] Another test where we no longer print out 'NaN' for the AF. --- .../walkers/variantutils/CombineVariantsIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/CombineVariantsIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/CombineVariantsIntegrationTest.java index 5a4d6e6a1b..d74aac79d4 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/CombineVariantsIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/CombineVariantsIntegrationTest.java @@ -110,7 +110,7 @@ public void combinePLs(String file1, String file2, String md5) { " -priority NA19240_BGI,NA19240_ILLUMINA,NA19240_WUGSC,denovoInfo" + " -genotypeMergeOptions UNIQUIFY -L 1"), 1, - Arrays.asList("b14f8cbb5d03a2e613b12da4da9efd9a")); + Arrays.asList("ab72f4bfb16d3894942149173a087647")); executeTest("threeWayWithRefs", spec); } From 33cf1368ba88f14dfe0b61853a689f1916029ba7 Mon Sep 17 00:00:00 2001 From: Menachem Fromer Date: Mon, 27 Feb 2012 16:03:02 -0500 Subject: [PATCH 289/356] Added options to add XHMM command-line parameters for discovery and genotyping From ca0931c01f3a19d7abdd766520ee5b4b6d3f65b4 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Mon, 27 Feb 2012 17:05:32 -0500 Subject: [PATCH 290/356] Adding test for reading samtools VCF file --- .../sting/utils/codecs/vcf/VCFIntegrationTest.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/VCFIntegrationTest.java b/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/VCFIntegrationTest.java index ca5fcf4195..b7bbae68da 100644 --- a/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/VCFIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/VCFIntegrationTest.java @@ -39,4 +39,14 @@ public void testReadingAndWritingBreakpointAlleles() { executeTest("Test reading and writing breakpoint VCF", spec1); } + @Test + public void testReadingAndWritingSamtools() { + String testVCF = validationDataLocation + "samtools.vcf"; + + String baseCommand = "-R " + b37KGReference + " -NO_HEADER -o %s "; + + String test1 = baseCommand + "-T SelectVariants -V " + testVCF; + WalkerTestSpec spec1 = new WalkerTestSpec(test1, 1, Arrays.asList("87d5b180ef5f9dc5aaee4b02601b43a2")); + executeTest("Test reading and writing samtools vcf", spec1); + } } From 1245a3c86840f50cba2269c999a5f1461866cf8b Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Mon, 27 Feb 2012 19:36:21 -0500 Subject: [PATCH 291/356] Tool for diagnosing new technology error modes From d7928ad669b0a6877923eeb3f4a724fae464eca5 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Mon, 27 Feb 2012 21:31:54 -0500 Subject: [PATCH 292/356] Drat, missed one: handle null alleles being passed in. --- .../broadinstitute/sting/utils/variantcontext/Genotype.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java index 13c4ff3d81..8712788bf2 100755 --- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java +++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java @@ -29,8 +29,9 @@ public Genotype(String sampleName, List alleles, double log10PError, Set } public Genotype(String sampleName, List alleles, double log10PError, Set filters, Map attributes, boolean isPhased, double[] log10Likelihoods) { - if ( alleles != null ) - this.alleles = Collections.unmodifiableList(alleles); + if ( alleles == null ) + alleles = new ArrayList(0); + this.alleles = Collections.unmodifiableList(alleles); commonInfo = new CommonInfo(sampleName, log10PError, filters, attributes); if ( log10Likelihoods != null ) commonInfo.putAttribute(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY, GenotypeLikelihoods.fromLog10Likelihoods(log10Likelihoods)); From 40bdadbda581e8cc91e8ec0919e2133f4164eb73 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Tue, 28 Feb 2012 09:24:07 -0500 Subject: [PATCH 293/356] Minor optimization as per Mark --- .../broadinstitute/sting/utils/variantcontext/Genotype.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java index 8712788bf2..c3cf233039 100755 --- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java +++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java @@ -30,8 +30,9 @@ public Genotype(String sampleName, List alleles, double log10PError, Set public Genotype(String sampleName, List alleles, double log10PError, Set filters, Map attributes, boolean isPhased, double[] log10Likelihoods) { if ( alleles == null ) - alleles = new ArrayList(0); - this.alleles = Collections.unmodifiableList(alleles); + this.alleles = Collections.emptyList(); + else + this.alleles = Collections.unmodifiableList(alleles); commonInfo = new CommonInfo(sampleName, log10PError, filters, attributes); if ( log10Likelihoods != null ) commonInfo.putAttribute(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY, GenotypeLikelihoods.fromLog10Likelihoods(log10Likelihoods)); From bd398e30fd72b9f0b3002d6a909d422f656135a5 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Tue, 28 Feb 2012 09:25:35 -0500 Subject: [PATCH 294/356] Another quick optimization --- .../org/broadinstitute/sting/utils/variantcontext/Genotype.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java index c3cf233039..ea86af442c 100755 --- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java +++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java @@ -29,7 +29,7 @@ public Genotype(String sampleName, List alleles, double log10PError, Set } public Genotype(String sampleName, List alleles, double log10PError, Set filters, Map attributes, boolean isPhased, double[] log10Likelihoods) { - if ( alleles == null ) + if ( alleles == null || alleles.size() == 0 ) this.alleles = Collections.emptyList(); else this.alleles = Collections.unmodifiableList(alleles); From 0681bea5a57a39229ef350c7cec6c8ad06d83062 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Tue, 28 Feb 2012 09:25:38 -0500 Subject: [PATCH 295/356] Changed DoC from PartitionType.INTERVAL to PartitionType.NONE since it doesn't have a way to gather scattered outputs. Added MultiallelicSummary to HSP eval. --- .../sting/gatk/walkers/coverage/DepthOfCoverageWalker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageWalker.java index 7d1858a634..94f9eb6c53 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageWalker.java @@ -114,7 +114,7 @@ // todo -- allow for user to set linear binning (default is logarithmic) // todo -- formatting --> do something special for end bins in getQuantile(int[] foo), this gets mushed into the end+-1 bins for now @By(DataSource.REFERENCE) -@PartitionBy(PartitionType.INTERVAL) +@PartitionBy(PartitionType.NONE) public class DepthOfCoverageWalker extends LocusWalker>, CoveragePartitioner> implements TreeReducible { @Output @Multiplex(value=DoCOutputMultiplexer.class,arguments={"partitionTypes","refSeqGeneList","omitDepthOutput","omitIntervals","omitSampleSummary","omitLocusTable"}) From a4a279ce808d02082f1bcbba8d4df1226177264e Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Tue, 28 Feb 2012 10:09:09 -0500 Subject: [PATCH 296/356] Damn you, Mark --- .../org/broadinstitute/sting/utils/variantcontext/Genotype.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java index ea86af442c..747d83e6f9 100755 --- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java +++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java @@ -29,7 +29,7 @@ public Genotype(String sampleName, List alleles, double log10PError, Set } public Genotype(String sampleName, List alleles, double log10PError, Set filters, Map attributes, boolean isPhased, double[] log10Likelihoods) { - if ( alleles == null || alleles.size() == 0 ) + if ( alleles == null || alleles.isEmpty() ) this.alleles = Collections.emptyList(); else this.alleles = Collections.unmodifiableList(alleles); From 3ddcd6879ff3184a77a6a5c1983c36be5a40d73e Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Tue, 28 Feb 2012 10:36:58 -0500 Subject: [PATCH 297/356] Bugfix for fullRefresh mode for gsafolkLogsForTableau From cf6472eea633f7e01158369c371dd54eb88c19a6 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Tue, 28 Feb 2012 10:52:13 -0500 Subject: [PATCH 298/356] Silly build fix. From 30abe123f118512815dff52054ed6e7d8fcc90d2 Mon Sep 17 00:00:00 2001 From: Menachem Fromer Date: Tue, 28 Feb 2012 14:54:45 -0500 Subject: [PATCH 299/356] Added space before command-line parameters From a6735d1d880c7891ad19744b85d8b7987a1934b8 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Tue, 28 Feb 2012 20:41:52 -0500 Subject: [PATCH 300/356] UnitTest framework for HaplotypeCaller likelihood function -- Currently disabled as the likelihood function doesn't pass basic unit tests -- Also make low-level function in LikelihoodCalculationEngine protected From 18c91e3cb3e1307a53aac717b288ba39f8779c8d Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Wed, 29 Feb 2012 08:49:12 -0500 Subject: [PATCH 301/356] Massively expanded haplotype caller likelihood testing -- Now include combinatorial testing for all input parameters: base quality, indel quality, continuation penalty, base identity, and indel length -- Disabled default the results coming back are not correct From d379c3763a1dcbde79356fa7c4d659554d28c36d Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Wed, 29 Feb 2012 18:04:02 -0500 Subject: [PATCH 302/356] DNA Sequence to BitSet and vice-versa conversion tools * Turns DNA sequences (for context covariates) into bit sets for maximum compression * Allows variable context size representation guaranteeing uniqueness. * Works with long precision, so it is limited to a context size of 31 bases (can be extended with BigNumber precision if necessary). * Unit Tests added --- .../broadinstitute/sting/utils/MathUtils.java | 111 ++++++++++++++++-- .../sting/utils/MathUtilsUnitTest.java | 32 ++++- 2 files changed, 127 insertions(+), 16 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java index c9ab3b58e5..6f2db67ee9 100644 --- a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java @@ -29,6 +29,7 @@ import com.google.java.contract.Requires; import net.sf.samtools.SAMRecord; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; import java.math.BigDecimal; @@ -1619,30 +1620,120 @@ public static Double[] vectorLog10(Double v1[]) { * @param bitSet the bitset * @return an integer with the bitset representation */ - public static int intFrom(final BitSet bitSet) { - int integer = 0; + public static long intFrom(final BitSet bitSet) { + long number = 0; for (int bitIndex = bitSet.nextSetBit(0); bitIndex >= 0; bitIndex = bitSet.nextSetBit(bitIndex+1)) - integer |= 1 << bitIndex; + number |= 1L << bitIndex; - return integer; + return number; } /** * Creates a BitSet representation of a given integer * - * @param integer the number to turn into a bitset + * @param number the number to turn into a bitset * @return a bitset representation of the integer */ - public static BitSet bitSetFrom(int integer) { - BitSet bitSet = new BitSet((int) Math.ceil(Math.sqrt(integer))); + public static BitSet bitSetFrom(long number) { + BitSet bitSet = new BitSet(); int bitIndex = 0; - while (integer > 0) { - if (integer%2 > 0) + while (number > 0) { + if (number%2 > 0) bitSet.set(bitIndex); bitIndex++; - integer /= 2; + number /= 2; } return bitSet; } + /** + * Converts a BitSet into the dna string representation. + * + * Warning: This conversion is limited to long precision, therefore the dna sequence cannot + * be longer than 31 bases. To increase this limit, use BigNumbers instead of long and create + * a bitSetFrom(BigNumber) method. + * + * We calculate the length of the resulting DNA sequence by looking at the sum(4^i) that exceeds the + * base_10 representation of the sequence. This is important for us to know how to bring the number + * to a quasi-canonical base_4 representation, and to fill in leading A's (since A's are represented + * as 0's and leading 0's are omitted). + * + * quasi-canonical because A is represented by a 0, therefore, + * instead of : 0, 1, 2, 3, 10, 11, 12, ... + * we have : 0, 1, 2, 3, 00, 01, 02, ... + * + * but we can correctly decode it because we know the final length. + * + * @param bitSet the bitset representation of the dna sequence + * @return the dna sequence represented by the bitset + */ + public static String dnaFrom(final BitSet bitSet) { + long number = intFrom(bitSet); // the base_10 representation of the bit set + long preContext = 0; // the number of combinations skipped to get to the quasi-canonical representation (we keep it to subtract later) + long nextContext = 4; // the next context (we advance it so we know which one was preceding it). + int i = 1; // the calculated length of the DNA sequence given the base_10 representation of its BitSet. + while (nextContext <= number) { // find the length of the dna string (i) + preContext = nextContext; // keep track of the number of combinations in the preceding context + nextContext += Math.pow(4, ++i);// calculate the next context + } + number -= preContext; // subtract the the number of combinations of the preceding context from the number to get to the quasi-canonical representation + + String dna = ""; + while (number > 0) { // perform a simple base_10 to base_4 conversion (quasi-canonical) + byte base = (byte) (number % 4); + switch (base) { + case 0 : dna = "A" + dna; break; + case 1 : dna = "C" + dna; break; + case 2 : dna = "G" + dna; break; + case 3 : dna = "T" + dna; break; + } + number /= 4; + } + for (int j = dna.length(); j < i; j++) + dna = "A" + dna; // add leading A's as necessary (due to the "quasi" canonical status, see description above) + + return dna; + } + + /** + * Creates a BitSet representation of a given dna string. + * + * Warning: This conversion is limited to long precision, therefore the dna sequence cannot + * be longer than 31 bases. To increase this limit, use BigNumbers instead of long and create + * a bitSetFrom(BigNumber) method. + * + * The bit representation of a dna string is the simple: + * 0 A 4 AA 8 CA + * 1 C 5 AC ... + * 2 G 6 AG 1343 TTGGT + * 3 T 7 AT 1364 TTTTT + * + * To convert from dna to number, we convert the dna string to base10 and add all combinations that + * preceded the string (with smaller lengths). + * + * @param dna the dna sequence + * @return the bitset representing the dna sequence + */ + public static BitSet bitSetFrom(String dna) { + if (dna.length() > 31) + throw new ReviewedStingException(String.format("DNA Length cannot be bigger than 31. dna: %s (%d)", dna, dna.length())); + + long baseTen = 0; // the number in base_10 that we are going to use to generate the bit set + long preContext = 0; // the sum of all combinations that preceded the length of the dna string + for (int i=0; i0) + preContext += Math.pow(4, i); // each length will have 4^i combinations (e.g 1 = 4, 2 = 16, 3 = 64, ...) + } + + return bitSetFrom(baseTen+preContext); // the number representing this DNA string is the base_10 representation plus all combinations that preceded this string length. + } + + } diff --git a/public/java/test/org/broadinstitute/sting/utils/MathUtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/MathUtilsUnitTest.java index 5b50c91a60..75fc44873d 100755 --- a/public/java/test/org/broadinstitute/sting/utils/MathUtilsUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/MathUtilsUnitTest.java @@ -207,14 +207,34 @@ public void testArrayShuffle() { @Test(enabled = true) public void testIntAndBitSetConversion() { - Assert.assertEquals(428, MathUtils.intFrom(MathUtils.bitSetFrom(428))); - Assert.assertEquals(239847, MathUtils.intFrom(MathUtils.bitSetFrom(239847))); - Assert.assertEquals(12726, MathUtils.intFrom(MathUtils.bitSetFrom(12726))); - Assert.assertEquals(0, MathUtils.intFrom(MathUtils.bitSetFrom(0))); - Assert.assertEquals(1, MathUtils.intFrom(MathUtils.bitSetFrom(1))); - Assert.assertEquals(65536, MathUtils.intFrom(MathUtils.bitSetFrom(65536))); + Assert.assertEquals(MathUtils.intFrom(MathUtils.bitSetFrom(428)), 428); + Assert.assertEquals(MathUtils.intFrom(MathUtils.bitSetFrom(239847)), 239847); + Assert.assertEquals(MathUtils.intFrom(MathUtils.bitSetFrom(12726)), 12726); + Assert.assertEquals(MathUtils.intFrom(MathUtils.bitSetFrom(0)), 0); + Assert.assertEquals(MathUtils.intFrom(MathUtils.bitSetFrom(1)), 1); + Assert.assertEquals(MathUtils.intFrom(MathUtils.bitSetFrom(65536)), 65536); + Assert.assertEquals(MathUtils.intFrom(MathUtils.bitSetFrom(Long.MAX_VALUE)), Long.MAX_VALUE); } + @Test(enabled = true) + public void testDNAAndBitSetConversion() { + Assert.assertEquals(MathUtils.dnaFrom(MathUtils.bitSetFrom("ACGT")), "ACGT"); + Assert.assertEquals(MathUtils.dnaFrom(MathUtils.bitSetFrom("AGGTGTTGT")), "AGGTGTTGT"); + Assert.assertEquals(MathUtils.dnaFrom(MathUtils.bitSetFrom("A")), "A"); + Assert.assertEquals(MathUtils.dnaFrom(MathUtils.bitSetFrom("C")), "C"); + Assert.assertEquals(MathUtils.dnaFrom(MathUtils.bitSetFrom("G")), "G"); + Assert.assertEquals(MathUtils.dnaFrom(MathUtils.bitSetFrom("T")), "T"); + Assert.assertEquals(MathUtils.dnaFrom(MathUtils.bitSetFrom("CC")), "CC"); + Assert.assertEquals(MathUtils.dnaFrom(MathUtils.bitSetFrom("AA")), "AA"); + Assert.assertEquals(MathUtils.dnaFrom(MathUtils.bitSetFrom("AAAA")), "AAAA"); + Assert.assertEquals(MathUtils.dnaFrom(MathUtils.bitSetFrom("CCCCCCCCCCCCCC")), "CCCCCCCCCCCCCC"); + Assert.assertEquals(MathUtils.dnaFrom(MathUtils.bitSetFrom("GGGGGGGGGGGGGG")), "GGGGGGGGGGGGGG"); + Assert.assertEquals(MathUtils.dnaFrom(MathUtils.bitSetFrom("TTTTTTTTTTTTTT")), "TTTTTTTTTTTTTT"); + Assert.assertEquals(MathUtils.dnaFrom(MathUtils.bitSetFrom("GTAGACCGATCTCAGCTAGT")), "GTAGACCGATCTCAGCTAGT"); + Assert.assertEquals(MathUtils.dnaFrom(MathUtils.bitSetFrom("AACGTCAATGCAGTCAAGTCAGACGTGGGTT")), "AACGTCAATGCAGTCAAGTCAGACGTGGGTT"); // testing max precision (length == 31) + } + + private boolean hasUniqueElements(Object[] x) { for (int i = 0; i < x.length; i++) for (int j = i + 1; j < x.length; j++) From 9e95b10789ae09600675c2da11a3ef6e8b76fac0 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Wed, 29 Feb 2012 18:56:11 -0500 Subject: [PATCH 303/356] Context covariate now operates as a highly compressed bitset * All contexts with 'N' bases are now collapsed as uninformative * Context size is now represented internally as a BitSet but output as a dna string * Temporarily disabled sorted outputs because of null objects --- .../gatk/walkers/bqsr/ContextCovariate.java | 54 +++++++++---------- .../sting/gatk/walkers/bqsr/Covariate.java | 2 +- .../gatk/walkers/bqsr/CovariateValues.java | 14 ++--- .../gatk/walkers/bqsr/CycleCovariate.java | 2 +- .../walkers/bqsr/QualityScoreCovariate.java | 4 +- .../gatk/walkers/bqsr/ReadGroupCovariate.java | 2 +- 6 files changed, 35 insertions(+), 43 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java index 89a30e4f5f..a1ab733418 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariate.java @@ -26,10 +26,12 @@ package org.broadinstitute.sting.gatk.walkers.bqsr; import org.broadinstitute.sting.utils.BaseUtils; +import org.broadinstitute.sting.utils.MathUtils; import org.broadinstitute.sting.utils.exceptions.UserException; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import java.util.Arrays; +import java.util.BitSet; /** * Created by IntelliJ IDEA. @@ -43,10 +45,6 @@ public class ContextCovariate implements StandardCovariate { private int insertionsContextSize; private int deletionsContextSize; - private String mismatchesNoContext = ""; - private String insertionsNoContext = ""; - private String deletionsNoContext = ""; - // Initialize any member variables using the command-line arguments passed to the walkers @Override public void initialize(final RecalibrationArgumentCollection RAC) { @@ -57,29 +55,26 @@ public void initialize(final RecalibrationArgumentCollection RAC) { if (mismatchesContextSize <= 0 || insertionsContextSize <= 0 || deletionsContextSize <= 0) throw new UserException(String.format("Context Size must be positive, if you don't want to use the context covariate, just turn it off instead. Mismatches: %d Insertions: %d Deletions:%d", mismatchesContextSize, insertionsContextSize, deletionsContextSize)); - // initialize no context strings given the size of the context for each covariate type - mismatchesNoContext = makeAllNStringWithLength(mismatchesContextSize); - insertionsNoContext = makeAllNStringWithLength(insertionsContextSize); - deletionsNoContext = makeAllNStringWithLength( deletionsContextSize); } @Override public CovariateValues getValues(final GATKSAMRecord read) { int l = read.getReadLength(); - String[] mismatches = new String [l]; - String[] insertions = new String [l]; - String[] deletions = new String [l]; + BitSet[] mismatches = new BitSet[l]; + BitSet[] insertions = new BitSet[l]; + BitSet[] deletions = new BitSet[l]; final boolean negativeStrand = read.getReadNegativeStrandFlag(); byte[] bases = read.getReadBases(); - if (negativeStrand) { - bases = BaseUtils.simpleReverseComplement(bases); //this is NOT in-place - } + if (negativeStrand) + bases = BaseUtils.simpleReverseComplement(bases); + for (int i = 0; i < read.getReadLength(); i++) { - mismatches[i] = contextWith(bases, i, mismatchesContextSize, mismatchesNoContext); - insertions[i] = contextWith(bases, i, insertionsContextSize, insertionsNoContext); - deletions[i] = contextWith(bases, i, deletionsContextSize, deletionsNoContext); + mismatches[i] = contextWith(bases, i, mismatchesContextSize); + insertions[i] = contextWith(bases, i, insertionsContextSize); + deletions[i] = contextWith(bases, i, deletionsContextSize); } + if (negativeStrand) { reverse(mismatches); reverse(insertions); @@ -90,7 +85,7 @@ public CovariateValues getValues(final GATKSAMRecord read) { // Used to get the covariate's value from input csv file during on-the-fly recalibration @Override - public final Comparable getValue(final String str) { + public final Object getValue(final String str) { return str; } @@ -100,29 +95,28 @@ public final Comparable getValue(final String str) { * @param bases the bases in the read to build the context from * @param offset the position in the read to calculate the context for * @param contextSize context size to use building the context - * @param noContextString string to return if the position is not far enough in the read to have a full context before. * @return */ - private String contextWith(byte [] bases, int offset, int contextSize, String noContextString) { - return (offset < contextSize) ? noContextString : new String(Arrays.copyOfRange(bases, offset - contextSize, offset)); + private BitSet contextWith(byte [] bases, int offset, int contextSize) { + if (offset < contextSize) + return null; + + String context = new String(Arrays.copyOfRange(bases, offset - contextSize, offset)); + if (context.contains("N")) + return null; + + return MathUtils.bitSetFrom(context); } - - private String makeAllNStringWithLength(int length) { - String s = ""; - for (int i=0; i Date: Thu, 1 Mar 2012 09:05:55 -0500 Subject: [PATCH 304/356] HaplotypeCaller integration test goes back to using flat base insertion and base deletion quality scores while the BQSR is in flux. From ed70d9b380c86f74fa18535596c0949829ba9e3f Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Thu, 1 Mar 2012 10:55:08 -0500 Subject: [PATCH 305/356] The reduced reads + calling script I am using; adding now for Khalid's benefit From aff508e09137318579887ca757c638dc78211b17 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Thu, 1 Mar 2012 15:01:11 -0500 Subject: [PATCH 306/356] ReadGroupProperties walker and associated infrastructure -- ReadGroupProperties: Emits a GATKReport containing read group, sample, library, platform, center, median insert size and median read length for each read group in every BAM file. -- Median tool that collects up to a given maximum number of elements and returns the median of the elements. -- Unit and integration tests for everything. -- Making name of TestProvider protected so subclasses and override name more easily --- .../diagnostics/ReadGroupProperties.java | 192 ++++++++++++++++++ .../broadinstitute/sting/utils/Median.java | 93 +++++++++ .../org/broadinstitute/sting/BaseTest.java | 2 +- .../broadinstitute/sting/MedianUnitTest.java | 123 +++++++++++ .../ReadGroupPropertiesIntegrationTest.java | 44 ++++ 5 files changed, 453 insertions(+), 1 deletion(-) create mode 100644 public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java create mode 100644 public/java/src/org/broadinstitute/sting/utils/Median.java create mode 100644 public/java/test/org/broadinstitute/sting/MedianUnitTest.java create mode 100644 public/java/test/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupPropertiesIntegrationTest.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java new file mode 100644 index 0000000000..85f587aaf2 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.walkers.diagnostics; + +import net.sf.samtools.SAMReadGroupRecord; +import org.broadinstitute.sting.commandline.Argument; +import org.broadinstitute.sting.commandline.Output; +import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.gatk.refdata.ReadMetaDataTracker; +import org.broadinstitute.sting.gatk.report.GATKReport; +import org.broadinstitute.sting.gatk.report.GATKReportTable; +import org.broadinstitute.sting.gatk.walkers.ReadWalker; +import org.broadinstitute.sting.utils.Median; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; + +import java.io.PrintStream; +import java.util.HashMap; +import java.util.Map; + +/** + * Emits a GATKReport containing read group, sample, library, platform, center, median insert size and + * median read length for each read group in every BAM file. + * + * Note that this walker stops when all read groups have been observed at least a few thousand times so that + * the median statistics are well determined. It is safe to run it WG and it'll finish in an appropriate + * timeframe. + * + *

Input

+ *

+ * Any number of BAM files + *

+ * + *

Output

+ *

+ * GATKReport containing read group, sample, library, platform, center, median insert size and median read length. + * + * For example, running this tool on the NA12878 data sets: + * + * 

+ *      ##:GATKReport.v0.2 ReadGroupProperties : Table of read group properties
+ *      readgroup  sample   library       platform  center  median.read.length  median.insert.size
+ *      20FUK.1    NA12878  Solexa-18483  illumina  BI                     101                 387
+ *      20FUK.2    NA12878  Solexa-18484  illumina  BI                     101                 415
+ *      20FUK.3    NA12878  Solexa-18483  illumina  BI                     101                 388
+ *      20FUK.4    NA12878  Solexa-18484  illumina  BI                     101                 415
+ *      20FUK.5    NA12878  Solexa-18483  illumina  BI                     101                 387
+ *      20FUK.6    NA12878  Solexa-18484  illumina  BI                     101                 415
+ *      20FUK.7    NA12878  Solexa-18483  illumina  BI                     101                 388
+ *      20FUK.8    NA12878  Solexa-18484  illumina  BI                     101                 415
+ *      20GAV.1    NA12878  Solexa-18483  illumina  BI                     101                 388
+ *      20GAV.2    NA12878  Solexa-18484  illumina  BI                     101                 415
+ *      20GAV.3    NA12878  Solexa-18483  illumina  BI                     101                 388
+ *      20GAV.4    NA12878  Solexa-18484  illumina  BI                     101                 416
+ *      20GAV.5    NA12878  Solexa-18483  illumina  BI                     101                 388
+ *      20GAV.6    NA12878  Solexa-18484  illumina  BI                     101                 415
+ *      20GAV.7    NA12878  Solexa-18483  illumina  BI                     101                 387
+ *      20GAV.8    NA12878  Solexa-18484  illumina  BI                     101                 414
+ *
+ *

+ * + *

Examples

+ * 
+ *    java
+ *      -jar GenomeAnalysisTK.jar
+ *      -T ReadGroupProperties
+ *      -I example1.bam -I example2.bam etc
+ *      -R reference.fasta
+ *      -o example.gatkreport.txt
+ *
+ * + * @author Mark DePristo + */ + + + +public class ReadGroupProperties extends ReadWalker { + @Output + public PrintStream out; + + @Argument(shortName="maxElementsForMedian", doc="Calculate median from the first maxElementsForMedian values observed", required=false) + public int MAX_VALUES_FOR_MEDIAN = 10000; + + private final static String TABLE_NAME = "ReadGroupProperties"; + private final Map> readLengths = new HashMap>(); + private final Map> insertSizes = new HashMap>(); + + @Override + public void initialize() { + for ( final SAMReadGroupRecord rg : getToolkit().getSAMFileHeader().getReadGroups() ) { + readLengths.put(rg.getId(), new Median(MAX_VALUES_FOR_MEDIAN)); + insertSizes.put(rg.getId(), new Median(MAX_VALUES_FOR_MEDIAN)); + } + } + + @Override + public boolean filter(ReferenceContext ref, GATKSAMRecord read) { + return ! (read.getReadFailsVendorQualityCheckFlag() || read.getReadUnmappedFlag()); + } + + @Override + public boolean isDone() { + // TODO -- this is far too slow! + return ! (anyMedianNeedsData(readLengths) || anyMedianNeedsData(insertSizes)); + } + + private final boolean anyMedianNeedsData(Map> medianMap) { + for ( Median median : medianMap.values() ) { + if ( ! median.isFull() ) + return true; + } + + return false; + } + + private final void updateMedian(final Median median, final int value) { + median.add(value); + } + + @Override + public Integer map(ReferenceContext referenceContext, GATKSAMRecord read, ReadMetaDataTracker readMetaDataTracker) { + final String rg = read.getReadGroup().getId(); + + updateMedian(readLengths.get(rg), read.getReadLength()); + if ( read.getReadPairedFlag() && read.getInferredInsertSize() != 0) { + //logger.info(rg + " => " + Math.abs(read.getInferredInsertSize())); + updateMedian(insertSizes.get(rg), Math.abs(read.getInferredInsertSize())); + } + + return null; + } + + @Override + public Integer reduceInit() { + return null; + } + + @Override + public Integer reduce(Integer integer, Integer integer1) { + return null; + } + + @Override + public void onTraversalDone(Integer sum) { + final GATKReport report = new GATKReport(); + report.addTable(TABLE_NAME, "Table of read group properties"); + GATKReportTable table = report.getTable(TABLE_NAME); + + table.addPrimaryKey("readgroup"); + //* Emits a GATKReport containing read group, sample, library, platform, center, median insert size and + //* median read length for each read group in every BAM file. + table.addColumn("sample", "NA"); + table.addColumn("library", "NA"); + table.addColumn("platform", "NA"); + table.addColumn("center", "NA"); + table.addColumn("median.read.length", Integer.valueOf(0)); + table.addColumn("median.insert.size", Integer.valueOf(0)); + + for ( final SAMReadGroupRecord rg : getToolkit().getSAMFileHeader().getReadGroups() ) { + final String rgID = rg.getId(); + table.set(rgID, "sample", rg.getSample()); + table.set(rgID, "library", rg.getLibrary()); + table.set(rgID, "platform", rg.getPlatform()); + table.set(rgID, "center", rg.getSequencingCenter()); + table.set(rgID, "median.read.length", readLengths.get(rgID).getMedian(0)); + table.set(rgID, "median.insert.size", insertSizes.get(rgID).getMedian(0)); + } + + report.print(out); + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/Median.java b/public/java/src/org/broadinstitute/sting/utils/Median.java new file mode 100644 index 0000000000..7ebe8d2d72 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/Median.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils; + +import java.util.*; + +/** + * Utility class for calculating median from a data set, potentially limiting the size of data to a + * fixed amount + * + * @author Your Name + * @since Date created + */ +public class Median { + final List values; + final int maxValuesToKeep; + boolean sorted = false; + + public Median() { + this(Integer.MAX_VALUE); + } + + public Median(final int maxValuesToKeep) { + this.maxValuesToKeep = maxValuesToKeep; + this.values = new ArrayList(); + } + + public boolean isFull() { + return values.size() >= maxValuesToKeep; + } + + public int size() { + return values.size(); + } + + public boolean isEmpty() { + return values.isEmpty(); + } + + public T getMedian() { + if ( isEmpty() ) + throw new IllegalStateException("Cannot get median value from empty array"); + return getMedian(null); // note that value null will never be used + } + + /** + * Returns the floor(n + 1 / 2) item from the list of values if the list + * has values, or defaultValue is the list is empty. + */ + public T getMedian(final T defaultValue) { + if ( isEmpty() ) + return defaultValue; + + if ( ! sorted ) { + sorted = true; + Collections.sort(values); + } + + final int offset = (int)Math.floor((values.size() + 1) * 0.5) - 1; + return values.get(offset); + } + + public boolean add(final T value) { + if ( ! isFull() ) { + sorted = false; + return values.add(value); + } + else + return false; + } +} diff --git a/public/java/test/org/broadinstitute/sting/BaseTest.java b/public/java/test/org/broadinstitute/sting/BaseTest.java index 626b91cbfe..ac3a970f97 100755 --- a/public/java/test/org/broadinstitute/sting/BaseTest.java +++ b/public/java/test/org/broadinstitute/sting/BaseTest.java @@ -141,7 +141,7 @@ public abstract class BaseTest { */ public static class TestDataProvider { private static final Map> tests = new HashMap>(); - private String name; + protected String name; /** * Create a new TestDataProvider instance bound to the class variable C diff --git a/public/java/test/org/broadinstitute/sting/MedianUnitTest.java b/public/java/test/org/broadinstitute/sting/MedianUnitTest.java new file mode 100644 index 0000000000..c12db9b77a --- /dev/null +++ b/public/java/test/org/broadinstitute/sting/MedianUnitTest.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +// our package +package org.broadinstitute.sting; + + +// the imports for unit testing. + + +import org.broadinstitute.sting.gatk.walkers.haplotypecaller.LikelihoodCalculationEngine; +import org.broadinstitute.sting.utils.Median; +import org.testng.Assert; +import org.testng.annotations.BeforeSuite; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + + +public class MedianUnitTest extends BaseTest { + LikelihoodCalculationEngine engine; + + @BeforeSuite + public void before() { + engine = new LikelihoodCalculationEngine(0, 0, false); + } + + // -------------------------------------------------------------------------------- + // + // Provider + // + // -------------------------------------------------------------------------------- + + private class MedianTestProvider extends TestDataProvider { + final List values = new ArrayList(); + final int cap; + final Integer expected; + + public MedianTestProvider(int expected, int cap, Integer ... values) { + super(MedianTestProvider.class); + this.expected = expected; + this.cap = cap; + this.values.addAll(Arrays.asList(values)); + this.name = String.format("values=%s expected=%d cap=%d", this.values, expected, cap); + } + } + + @DataProvider(name = "MedianTestProvider") + public Object[][] makeMedianTestProvider() { + new MedianTestProvider(1, 1000, 0, 1, 2); + new MedianTestProvider(1, 1000, 1, 0, 1, 2); + new MedianTestProvider(1, 1000, 0, 1, 2, 3); + new MedianTestProvider(2, 1000, 0, 1, 2, 3, 4); + new MedianTestProvider(2, 1000, 4, 1, 2, 3, 0); + new MedianTestProvider(1, 1000, 1); + new MedianTestProvider(2, 1000, 2); + new MedianTestProvider(1, 1000, 1, 2); + + new MedianTestProvider(1, 3, 1); + new MedianTestProvider(1, 3, 1, 2); + new MedianTestProvider(2, 3, 1, 2, 3); + new MedianTestProvider(2, 3, 1, 2, 3, 4); + new MedianTestProvider(2, 3, 1, 2, 3, 4, 5); + + new MedianTestProvider(1, 3, 1); + new MedianTestProvider(1, 3, 1, 2); + new MedianTestProvider(2, 3, 3, 2, 1); + new MedianTestProvider(3, 3, 4, 3, 2, 1); + new MedianTestProvider(4, 3, 5, 4, 3, 2, 1); + + return MedianTestProvider.getTests(MedianTestProvider.class); + } + + @Test(dataProvider = "MedianTestProvider") + public void testBasicLikelihoods(MedianTestProvider cfg) { + final Median median = new Median(cfg.cap); + + int nAdded = 0; + for ( final int value : cfg.values ) + if ( median.add(value) ) + nAdded++; + + Assert.assertEquals(nAdded, median.size()); + + Assert.assertEquals(cfg.values.isEmpty(), median.isEmpty()); + Assert.assertEquals(cfg.values.size() >= cfg.cap, median.isFull()); + Assert.assertEquals(median.getMedian(), cfg.expected, cfg.toString()); + } + + @Test(expectedExceptions = IllegalStateException.class) + public void testEmptyMedian() { + final Median median = new Median(); + Assert.assertTrue(median.isEmpty()); + final Integer d = 100; + Assert.assertEquals(median.getMedian(d), d); + median.getMedian(); + } + +} \ No newline at end of file diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupPropertiesIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupPropertiesIntegrationTest.java new file mode 100644 index 0000000000..84f7fa3639 --- /dev/null +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupPropertiesIntegrationTest.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.walkers.diagnostics; + +import org.broadinstitute.sting.WalkerTest; +import org.testng.annotations.Test; + +import java.util.Arrays; + +/** + * Tests ReadGroupProperties + */ +public class ReadGroupPropertiesIntegrationTest extends WalkerTest { + @Test + public void basicTest() { + WalkerTestSpec spec = new WalkerTestSpec( + "-T ReadGroupProperties -R " + b37KGReference + " -I " + b37GoodBAM + " -L 20:10,000,000-11,000,000 -o %s", + 1, + Arrays.asList("1795e3157ab23e7e597acec334e29525")); + executeTest("ReadGroupProperties:", spec); + } +} \ No newline at end of file From 29f74b658bde8012abfbc69699693053b5fd0ebd Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Thu, 1 Mar 2012 17:11:05 -0500 Subject: [PATCH 307/356] Unit tests for the context covariate this is simple, but it's the infra-structure to start messing around with the context. --- .../bqsr/ContextCovariateUnitTest.java | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 public/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariateUnitTest.java diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariateUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariateUnitTest.java new file mode 100644 index 0000000000..aa6a72ef97 --- /dev/null +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/bqsr/ContextCovariateUnitTest.java @@ -0,0 +1,103 @@ +package org.broadinstitute.sting.gatk.walkers.bqsr; + +import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; +import org.broadinstitute.sting.utils.MathUtils; +import org.broadinstitute.sting.utils.sam.ArtificialSAMUtils; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.BitSet; +import java.util.Random; + +/** + * Short one line description of the walker. + * + *

+ * [Long description of the walker] + *

+ * + * + *

Input

+ *

+ * [Description of the Input] + *

+ * + *

Output

+ *

+ * [Description of the Output] + *

+ * + *

Examples

+ * 
+ *    java
+ *      -jar GenomeAnalysisTK.jar
+ *      -T [walker name]
+ *
+ * + * @author Mauricio Carneiro + * @since 3/1/12 + */ +public class ContextCovariateUnitTest { + ContextCovariate covariate; + RecalibrationArgumentCollection RAC; + Random random; + + @BeforeClass + public void init() { + RAC = new RecalibrationArgumentCollection(); + covariate = new ContextCovariate(); + random = GenomeAnalysisEngine.getRandomGenerator(); + covariate.initialize(RAC); + + } + + @Test(enabled = true) + public void testSimpleContexts() { + byte [] quals = createRandomReadQuals(101); + byte [] bbases = createRandomReadBases(101); + String bases = stringFrom(bbases); + GATKSAMRecord read = ArtificialSAMUtils.createArtificialRead(bbases, quals, bbases.length + "M"); + CovariateValues values = covariate.getValues(read); + verifyCovariateArray((BitSet []) values.getMismatches(), RAC.MISMATCHES_CONTEXT_SIZE, bases); + verifyCovariateArray((BitSet []) values.getInsertions(), RAC.INSERTIONS_CONTEXT_SIZE, bases); + verifyCovariateArray((BitSet []) values.getDeletions(), RAC.DELETIONS_CONTEXT_SIZE, bases); + } + + private void verifyCovariateArray(BitSet[] values, int contextSize, String bases) { + for (int i=0; i= contextSize) + Assert.assertEquals(MathUtils.dnaFrom(values[i]), bases.substring(i-contextSize, i)); + else + Assert.assertNull(values[i]); + } + } + + private String stringFrom(byte [] array) { + String s = ""; + for (byte value : array) + s += (char) value; + return s; + } + + private byte [] createRandomReadQuals(int length) { + byte [] quals = new byte[length]; + for (int i=0; i Date: Thu, 1 Mar 2012 17:31:36 -0500 Subject: [PATCH 308/356] get rid of the sorting parameter From 486712bfc2f69d356da739098b8710142fe1bf47 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Thu, 1 Mar 2012 17:55:28 -0500 Subject: [PATCH 309/356] ugly RG encoding --- .../gatk/walkers/bqsr/ReadGroupCovariate.java | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ReadGroupCovariate.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ReadGroupCovariate.java index 6076a7a20b..aecdd3d4b8 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ReadGroupCovariate.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/ReadGroupCovariate.java @@ -3,6 +3,7 @@ import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import java.util.Arrays; +import java.util.HashMap; /* * Copyright (c) 2009 The Broad Institute @@ -38,6 +39,10 @@ */ public class ReadGroupCovariate implements RequiredCovariate { + + private final HashMap readGroupLookupTable = new HashMap(); + private final HashMap readGroupReverseLookupTable = new HashMap(); + private short nextId = 0; // Initialize any member variables using the command-line arguments passed to the walkers @Override @@ -48,8 +53,17 @@ public void initialize(final RecalibrationArgumentCollection RAC) { public CovariateValues getValues(final GATKSAMRecord read) { final int l = read.getReadLength(); final String readGroupId = read.getReadGroup().getReadGroupId(); - String [] readGroups = new String[l]; - Arrays.fill(readGroups, readGroupId); + short shortId; + if (readGroupLookupTable.containsKey(readGroupId)) + shortId = readGroupLookupTable.get(readGroupId); + else { + shortId = nextId; + readGroupLookupTable.put(readGroupId, nextId); + readGroupReverseLookupTable.put(nextId, readGroupId); + nextId++; + } + Short [] readGroups = new Short[l]; + Arrays.fill(readGroups, shortId); return new CovariateValues(readGroups, readGroups, readGroups); } @@ -58,6 +72,10 @@ public CovariateValues getValues(final GATKSAMRecord read) { public final Object getValue(final String str) { return str; } + + public final String decodeReadGroup(final short id) { + return readGroupReverseLookupTable.get(id); + } } From 2f334a57c2433aade864d1c28fa08b6fc5f28dec Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Thu, 1 Mar 2012 18:43:23 -0500 Subject: [PATCH 310/356] ReadGroupProperties mk2 -- Includes paired end status (T/F) -- Includes count of reads used in calculation -- Includes simple read type (2x76 for example) -- Better handling of insert size, read length when there's no data, or the data isn't paired end by emitting NA not 0 --- .../diagnostics/ReadGroupProperties.java | 109 +++++++++++------- .../ReadGroupPropertiesIntegrationTest.java | 2 +- 2 files changed, 66 insertions(+), 45 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java index 85f587aaf2..c192d04e7b 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java @@ -40,8 +40,9 @@ import java.util.Map; /** - * Emits a GATKReport containing read group, sample, library, platform, center, median insert size and - * median read length for each read group in every BAM file. + * Emits a GATKReport containing read group, sample, library, platform, center, paired end status, + * simple read type name (e.g. 2x76) median insert size and median read length for each read group + * in every provided BAM file * * Note that this walker stops when all read groups have been observed at least a few thousand times so that * the median statistics are well determined. It is safe to run it WG and it'll finish in an appropriate @@ -60,23 +61,23 @@ * * 
  *      ##:GATKReport.v0.2 ReadGroupProperties : Table of read group properties
- *      readgroup  sample   library       platform  center  median.read.length  median.insert.size
- *      20FUK.1    NA12878  Solexa-18483  illumina  BI                     101                 387
- *      20FUK.2    NA12878  Solexa-18484  illumina  BI                     101                 415
- *      20FUK.3    NA12878  Solexa-18483  illumina  BI                     101                 388
- *      20FUK.4    NA12878  Solexa-18484  illumina  BI                     101                 415
- *      20FUK.5    NA12878  Solexa-18483  illumina  BI                     101                 387
- *      20FUK.6    NA12878  Solexa-18484  illumina  BI                     101                 415
- *      20FUK.7    NA12878  Solexa-18483  illumina  BI                     101                 388
- *      20FUK.8    NA12878  Solexa-18484  illumina  BI                     101                 415
- *      20GAV.1    NA12878  Solexa-18483  illumina  BI                     101                 388
- *      20GAV.2    NA12878  Solexa-18484  illumina  BI                     101                 415
- *      20GAV.3    NA12878  Solexa-18483  illumina  BI                     101                 388
- *      20GAV.4    NA12878  Solexa-18484  illumina  BI                     101                 416
- *      20GAV.5    NA12878  Solexa-18483  illumina  BI                     101                 388
- *      20GAV.6    NA12878  Solexa-18484  illumina  BI                     101                 415
- *      20GAV.7    NA12878  Solexa-18483  illumina  BI                     101                 387
- *      20GAV.8    NA12878  Solexa-18484  illumina  BI                     101                 414
+ *      readgroup  sample   library       platform  center  has.any.reads  is.paired.end  n.reads.analyzed  simple.read.type  median.read.length  median.insert.size
+ *      20FUK.1    NA12878  Solexa-18483  illumina  BI      true           true                      10100  2x101                            101                 387
+ *      20FUK.2    NA12878  Solexa-18484  illumina  BI      true           true                      10115  2x101                            101                 415
+ *      20FUK.3    NA12878  Solexa-18483  illumina  BI      true           true                      10090  2x101                            101                 388
+ *      20FUK.4    NA12878  Solexa-18484  illumina  BI      true           true                      10081  2x101                            101                 415
+ *      20FUK.5    NA12878  Solexa-18483  illumina  BI      true           true                      10078  2x101                            101                 387
+ *      20FUK.6    NA12878  Solexa-18484  illumina  BI      true           true                      10072  2x101                            101                 415
+ *      20FUK.7    NA12878  Solexa-18483  illumina  BI      true           true                      10086  2x101                            101                 388
+ *      20FUK.8    NA12878  Solexa-18484  illumina  BI      true           true                      10097  2x101                            101                 415
+ *      20GAV.1    NA12878  Solexa-18483  illumina  BI      true           true                      10135  2x101                            101                 388
+ *      20GAV.2    NA12878  Solexa-18484  illumina  BI      true           true                      10172  2x101                            101                 415
+ *      20GAV.3    NA12878  Solexa-18483  illumina  BI      true           true                      10141  2x101                            101                 388
+ *      20GAV.4    NA12878  Solexa-18484  illumina  BI      true           true                      10251  2x101                            101                 416
+ *      20GAV.5    NA12878  Solexa-18483  illumina  BI      true           true                      10145  2x101                            101                 388
+ *      20GAV.6    NA12878  Solexa-18484  illumina  BI      true           true                      10184  2x101                            101                 415
+ *      20GAV.7    NA12878  Solexa-18483  illumina  BI      true           true                      10174  2x101                            101                 387
+ *      20GAV.8    NA12878  Solexa-18484  illumina  BI      true           true                      10141  2x101                            101                 414
  *
*

* @@ -103,14 +104,22 @@ public class ReadGroupProperties extends ReadWalker { public int MAX_VALUES_FOR_MEDIAN = 10000; private final static String TABLE_NAME = "ReadGroupProperties"; - private final Map> readLengths = new HashMap>(); - private final Map> insertSizes = new HashMap>(); + private final Map readGroupInfo = new HashMap(); + + private class PerReadGroupInfo { + public final Median readLength = new Median(MAX_VALUES_FOR_MEDIAN); + public final Median insertSize = new Median(MAX_VALUES_FOR_MEDIAN); + public int nReadsSeen = 0, nReadsPaired = 0; + + public boolean needsMoreData() { + return ! readLength.isFull() || ! insertSize.isFull(); + } + } @Override public void initialize() { for ( final SAMReadGroupRecord rg : getToolkit().getSAMFileHeader().getReadGroups() ) { - readLengths.put(rg.getId(), new Median(MAX_VALUES_FOR_MEDIAN)); - insertSizes.put(rg.getId(), new Median(MAX_VALUES_FOR_MEDIAN)); + readGroupInfo.put(rg.getId(), new PerReadGroupInfo()); } } @@ -121,31 +130,28 @@ public boolean filter(ReferenceContext ref, GATKSAMRecord read) { @Override public boolean isDone() { - // TODO -- this is far too slow! - return ! (anyMedianNeedsData(readLengths) || anyMedianNeedsData(insertSizes)); - } - - private final boolean anyMedianNeedsData(Map> medianMap) { - for ( Median median : medianMap.values() ) { - if ( ! median.isFull() ) - return true; + for ( PerReadGroupInfo info : readGroupInfo.values() ) { + if ( info.needsMoreData() ) + return false; } - return false; - } - - private final void updateMedian(final Median median, final int value) { - median.add(value); + return true; } @Override public Integer map(ReferenceContext referenceContext, GATKSAMRecord read, ReadMetaDataTracker readMetaDataTracker) { - final String rg = read.getReadGroup().getId(); - - updateMedian(readLengths.get(rg), read.getReadLength()); - if ( read.getReadPairedFlag() && read.getInferredInsertSize() != 0) { - //logger.info(rg + " => " + Math.abs(read.getInferredInsertSize())); - updateMedian(insertSizes.get(rg), Math.abs(read.getInferredInsertSize())); + final String rgID = read.getReadGroup().getId(); + final PerReadGroupInfo info = readGroupInfo.get(rgID); + + if ( info.needsMoreData() ) { + info.readLength.add(read.getReadLength()); + info.nReadsSeen++; + if ( read.getReadPairedFlag() ) { + info.nReadsPaired++; + if ( read.getInferredInsertSize() != 0) { + info.insertSize.add(Math.abs(read.getInferredInsertSize())); + } + } } return null; @@ -174,17 +180,32 @@ public void onTraversalDone(Integer sum) { table.addColumn("library", "NA"); table.addColumn("platform", "NA"); table.addColumn("center", "NA"); + table.addColumn("has.any.reads", "false"); + table.addColumn("is.paired.end", "false"); + table.addColumn("n.reads.analyzed", "NA"); + table.addColumn("simple.read.type", "NA"); table.addColumn("median.read.length", Integer.valueOf(0)); table.addColumn("median.insert.size", Integer.valueOf(0)); for ( final SAMReadGroupRecord rg : getToolkit().getSAMFileHeader().getReadGroups() ) { final String rgID = rg.getId(); + PerReadGroupInfo info = readGroupInfo.get(rgID); + + // we are paired if > 25% of reads are paired + final boolean isPaired = info.nReadsPaired / (1.0 * (info.nReadsSeen+1)) > 0.25; + final boolean hasAnyReads = info.nReadsSeen > 0; + final int readLength = info.readLength.getMedian(0); + table.set(rgID, "sample", rg.getSample()); table.set(rgID, "library", rg.getLibrary()); table.set(rgID, "platform", rg.getPlatform()); table.set(rgID, "center", rg.getSequencingCenter()); - table.set(rgID, "median.read.length", readLengths.get(rgID).getMedian(0)); - table.set(rgID, "median.insert.size", insertSizes.get(rgID).getMedian(0)); + table.set(rgID, "has.any.reads", hasAnyReads); + table.set(rgID, "is.paired.end", isPaired); + table.set(rgID, "n.reads.analyzed", info.nReadsSeen); + table.set(rgID, "simple.read.type", hasAnyReads ? String.format("%dx%d", isPaired ? 2 : 1, readLength) : "NA"); + table.set(rgID, "median.read.length", hasAnyReads ? readLength : "NA" ); + table.set(rgID, "median.insert.size", hasAnyReads && isPaired ? info.insertSize.getMedian(0) : "NA" ); } report.print(out); diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupPropertiesIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupPropertiesIntegrationTest.java index 84f7fa3639..04c30a8fe4 100644 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupPropertiesIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupPropertiesIntegrationTest.java @@ -38,7 +38,7 @@ public void basicTest() { WalkerTestSpec spec = new WalkerTestSpec( "-T ReadGroupProperties -R " + b37KGReference + " -I " + b37GoodBAM + " -L 20:10,000,000-11,000,000 -o %s", 1, - Arrays.asList("1795e3157ab23e7e597acec334e29525")); + Arrays.asList("8e4d09665c0b65c971bd5dead24f95fe")); executeTest("ReadGroupProperties:", spec); } } \ No newline at end of file From 0ad7d5fbc1129569c7a9ce633e0bd8eb9fb5c072 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Thu, 1 Mar 2012 22:41:13 -0500 Subject: [PATCH 311/356] Standalone common Pair HMM utility class with associated unit tests. --- .../broadinstitute/sting/utils/collections/NestedHashMap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/collections/NestedHashMap.java b/public/java/src/org/broadinstitute/sting/utils/collections/NestedHashMap.java index d280ac8049..8652d3c282 100755 --- a/public/java/src/org/broadinstitute/sting/utils/collections/NestedHashMap.java +++ b/public/java/src/org/broadinstitute/sting/utils/collections/NestedHashMap.java @@ -34,7 +34,7 @@ * Date: Dec 29, 2009 */ -public class NestedHashMap{ +public class NestedHashMap { public final Map data = new HashMap(); From 0a7137616c6cddaa39cac62f71fc276dcdef36f7 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 2 Mar 2012 09:11:59 -0500 Subject: [PATCH 312/356] Now converts gatkreports to properly typed R data types in gsa.read.gatkreport -- use the general function type.convert from read.table to automagically convert the string data to booleans, factors, and numeric types as appropriate. Vastly better than the previous behavior which only worked for numerics, in some cases. --- .../sting/utils/R/gsalib/R/gsa.read.gatkreport.R | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.read.gatkreport.R b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.read.gatkreport.R index 46bbf7eda5..876cf5cbc9 100644 --- a/public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.read.gatkreport.R +++ b/public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.read.gatkreport.R @@ -4,11 +4,9 @@ colnames(d) = tableHeader; for (i in 1:ncol(d)) { - v = suppressWarnings(as.numeric(d[,i])); - - if (length(na.omit(as.numeric(v))) == length(d[,i])) { - d[,i] = v; - } + # use the general type.convert infrastructure of read.table to convert column data to R types + v = type.convert(d[,i]) + d[,i] = v; } usedNames = ls(envir=tableEnv, pattern=tableName); From 1e07e97b589eb73cebe5052fb656b3c54295b618 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Fri, 2 Mar 2012 13:30:17 -0500 Subject: [PATCH 313/356] Optimization: create allele list just once, not for each genotype --- .../sting/gatk/walkers/annotator/RankSumTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java index 3f555f780d..00968943d8 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java @@ -43,15 +43,15 @@ public Map annotate(RefMetaDataTracker tracker, AnnotatorCompati final ArrayList altQuals = new ArrayList(); if ( vc.isSNP() ) { + final List altAlleles = new ArrayList(); + for ( final Allele a : vc.getAlternateAlleles() ) + altAlleles.add(a.getBases()[0]); + for ( final Genotype genotype : genotypes.iterateInSampleNameOrder() ) { final AlignmentContext context = stratifiedContexts.get(genotype.getSampleName()); if ( context == null ) continue; - final List altAlleles = new ArrayList(); - for ( final Allele a : vc.getAlternateAlleles() ) - altAlleles.add(a.getBases()[0]); - fillQualsFromPileup(ref.getBase(), altAlleles, context.getBasePileup(), refQuals, altQuals); } } else if ( vc.isIndel() || vc.isMixed() ) { From fc1c0a9d8f1467d78a7929d119c52d91b5435bcd Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Fri, 2 Mar 2012 14:15:56 -0500 Subject: [PATCH 314/356] Minor change: switched HSP default fasta from bundle/g1k to Picard since in all oneoff runs of the HSP the BAMs were aligned by Picard to Picard's reference. From ba71b0aee4267be72ed32eaaa0b89cc6434f1a1a Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 2 Mar 2012 14:51:23 -0500 Subject: [PATCH 315/356] ReadGroupProperties mk3 -- Includes sequencing date --- .../diagnostics/ReadGroupProperties.java | 44 ++++++++++--------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java index c192d04e7b..2eb3b5e85f 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java @@ -36,13 +36,14 @@ import org.broadinstitute.sting.utils.sam.GATKSAMRecord; import java.io.PrintStream; +import java.text.DateFormat; import java.util.HashMap; import java.util.Map; /** - * Emits a GATKReport containing read group, sample, library, platform, center, paired end status, - * simple read type name (e.g. 2x76) median insert size and median read length for each read group - * in every provided BAM file + * Emits a GATKReport containing read group, sample, library, platform, center, sequencing data, + * paired end status, simple read type name (e.g. 2x76) median insert size and median read length + * for each read group in every provided BAM file * * Note that this walker stops when all read groups have been observed at least a few thousand times so that * the median statistics are well determined. It is safe to run it WG and it'll finish in an appropriate @@ -61,23 +62,23 @@ * * 
  *      ##:GATKReport.v0.2 ReadGroupProperties : Table of read group properties
- *      readgroup  sample   library       platform  center  has.any.reads  is.paired.end  n.reads.analyzed  simple.read.type  median.read.length  median.insert.size
- *      20FUK.1    NA12878  Solexa-18483  illumina  BI      true           true                      10100  2x101                            101                 387
- *      20FUK.2    NA12878  Solexa-18484  illumina  BI      true           true                      10115  2x101                            101                 415
- *      20FUK.3    NA12878  Solexa-18483  illumina  BI      true           true                      10090  2x101                            101                 388
- *      20FUK.4    NA12878  Solexa-18484  illumina  BI      true           true                      10081  2x101                            101                 415
- *      20FUK.5    NA12878  Solexa-18483  illumina  BI      true           true                      10078  2x101                            101                 387
- *      20FUK.6    NA12878  Solexa-18484  illumina  BI      true           true                      10072  2x101                            101                 415
- *      20FUK.7    NA12878  Solexa-18483  illumina  BI      true           true                      10086  2x101                            101                 388
- *      20FUK.8    NA12878  Solexa-18484  illumina  BI      true           true                      10097  2x101                            101                 415
- *      20GAV.1    NA12878  Solexa-18483  illumina  BI      true           true                      10135  2x101                            101                 388
- *      20GAV.2    NA12878  Solexa-18484  illumina  BI      true           true                      10172  2x101                            101                 415
- *      20GAV.3    NA12878  Solexa-18483  illumina  BI      true           true                      10141  2x101                            101                 388
- *      20GAV.4    NA12878  Solexa-18484  illumina  BI      true           true                      10251  2x101                            101                 416
- *      20GAV.5    NA12878  Solexa-18483  illumina  BI      true           true                      10145  2x101                            101                 388
- *      20GAV.6    NA12878  Solexa-18484  illumina  BI      true           true                      10184  2x101                            101                 415
- *      20GAV.7    NA12878  Solexa-18483  illumina  BI      true           true                      10174  2x101                            101                 387
- *      20GAV.8    NA12878  Solexa-18484  illumina  BI      true           true                      10141  2x101                            101                 414
+ *      readgroup  sample   library       platform  center  date     has.any.reads  is.paired.end  n.reads.analyzed  simple.read.type  median.read.length  median.insert.size
+ *      20FUK.1    NA12878  Solexa-18483  illumina  BI      2/2/10   true           true                        498  2x101                            101                 386
+ *      20FUK.2    NA12878  Solexa-18484  illumina  BI      2/2/10   true           true                        476  2x101                            101                 417
+ *      20FUK.3    NA12878  Solexa-18483  illumina  BI      2/2/10   true           true                        407  2x101                            101                 387
+ *      20FUK.4    NA12878  Solexa-18484  illumina  BI      2/2/10   true           true                        389  2x101                            101                 415
+ *      20FUK.5    NA12878  Solexa-18483  illumina  BI      2/2/10   true           true                        433  2x101                            101                 386
+ *      20FUK.6    NA12878  Solexa-18484  illumina  BI      2/2/10   true           true                        480  2x101                            101                 418
+ *      20FUK.7    NA12878  Solexa-18483  illumina  BI      2/2/10   true           true                        450  2x101                            101                 386
+ *      20FUK.8    NA12878  Solexa-18484  illumina  BI      2/2/10   true           true                        438  2x101                            101                 418
+ *      20GAV.1    NA12878  Solexa-18483  illumina  BI      1/26/10  true           true                        490  2x101                            101                 391
+ *      20GAV.2    NA12878  Solexa-18484  illumina  BI      1/26/10  true           true                        485  2x101                            101                 417
+ *      20GAV.3    NA12878  Solexa-18483  illumina  BI      1/26/10  true           true                        460  2x101                            101                 392
+ *      20GAV.4    NA12878  Solexa-18484  illumina  BI      1/26/10  true           true                        434  2x101                            101                 415
+ *      20GAV.5    NA12878  Solexa-18483  illumina  BI      1/26/10  true           true                        479  2x101                            101                 389
+ *      20GAV.6    NA12878  Solexa-18484  illumina  BI      1/26/10  true           true                        461  2x101                            101                 416
+ *      20GAV.7    NA12878  Solexa-18483  illumina  BI      1/26/10  true           true                        509  2x101                            101                 386
+ *      20GAV.8    NA12878  Solexa-18484  illumina  BI      1/26/10  true           true                        476  2x101                            101                 410                           101                 414
  *
*

* @@ -172,6 +173,7 @@ public void onTraversalDone(Integer sum) { final GATKReport report = new GATKReport(); report.addTable(TABLE_NAME, "Table of read group properties"); GATKReportTable table = report.getTable(TABLE_NAME); + DateFormat dateFormatter = DateFormat.getDateInstance(DateFormat.SHORT); table.addPrimaryKey("readgroup"); //* Emits a GATKReport containing read group, sample, library, platform, center, median insert size and @@ -180,6 +182,7 @@ public void onTraversalDone(Integer sum) { table.addColumn("library", "NA"); table.addColumn("platform", "NA"); table.addColumn("center", "NA"); + table.addColumn("date", "NA"); table.addColumn("has.any.reads", "false"); table.addColumn("is.paired.end", "false"); table.addColumn("n.reads.analyzed", "NA"); @@ -200,6 +203,7 @@ public void onTraversalDone(Integer sum) { table.set(rgID, "library", rg.getLibrary()); table.set(rgID, "platform", rg.getPlatform()); table.set(rgID, "center", rg.getSequencingCenter()); + table.set(rgID, "date", dateFormatter.format(rg.getRunDate())); table.set(rgID, "has.any.reads", hasAnyReads); table.set(rgID, "is.paired.end", isPaired); table.set(rgID, "n.reads.analyzed", info.nReadsSeen); From 914c23da51fe56a962a42386dbc2ea566dab268b Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 2 Mar 2012 16:09:38 -0500 Subject: [PATCH 316/356] Generic infrastructure for quantizing quality scores -- Just infrastructure at this point (but with UnitTests!). -- Capable of taking a histogram of quality scores and a target number of levels (8 for example), and mapping the full range of input quality scores down to only 8. -- The selected quality scores are chosen to minimize the miscalibration rate of the resulting bins. I believe this adaptive approach is vastly better than the current systems being developed by EBI and NCBI -- This infrastructure is designed to work with BQSRv2. I envision a system where we feed in the projected empirical quality score distribution from the BQSRv2 table, compute the required deleveling for each of the B, I, and D qualities, and on the fly emit calibrated, compressed quality scores. -- Note the algorithm right now for determining the best intervals is both greedy (i.e., will miss the best overall choice) and potentially extremely slow. But it is enough for me to play with. From 69611af7d303246b1087c5081aaeada66fbaded5 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 2 Mar 2012 18:53:45 -0500 Subject: [PATCH 317/356] Workaround for bug in Picard in ReadGroupProperties -- NPE caused when you call getRunDate on a read group without a date. --- .../diagnostics/ReadGroupProperties.java | 31 ++++++++++++------- .../ReadGroupPropertiesIntegrationTest.java | 2 +- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java index 2eb3b5e85f..d7a48d321e 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java @@ -199,19 +199,28 @@ public void onTraversalDone(Integer sum) { final boolean hasAnyReads = info.nReadsSeen > 0; final int readLength = info.readLength.getMedian(0); - table.set(rgID, "sample", rg.getSample()); - table.set(rgID, "library", rg.getLibrary()); - table.set(rgID, "platform", rg.getPlatform()); - table.set(rgID, "center", rg.getSequencingCenter()); - table.set(rgID, "date", dateFormatter.format(rg.getRunDate())); - table.set(rgID, "has.any.reads", hasAnyReads); - table.set(rgID, "is.paired.end", isPaired); - table.set(rgID, "n.reads.analyzed", info.nReadsSeen); - table.set(rgID, "simple.read.type", hasAnyReads ? String.format("%dx%d", isPaired ? 2 : 1, readLength) : "NA"); - table.set(rgID, "median.read.length", hasAnyReads ? readLength : "NA" ); - table.set(rgID, "median.insert.size", hasAnyReads && isPaired ? info.insertSize.getMedian(0) : "NA" ); + setTableValue(table, rgID, "sample", rg.getSample()); + setTableValue(table, rgID, "library", rg.getLibrary()); + setTableValue(table, rgID, "platform", rg.getPlatform()); + setTableValue(table, rgID, "center", rg.getSequencingCenter()); + try { + setTableValue(table, rgID, "date", rg.getRunDate() != null ? dateFormatter.format(rg.getRunDate()) : "NA"); + } catch ( NullPointerException e ) { + // TODO: remove me when bug in Picard is fixed that causes NPE when date isn't present + setTableValue(table, rgID, "date", "NA"); + } + setTableValue(table, rgID, "has.any.reads", hasAnyReads); + setTableValue(table, rgID, "is.paired.end", isPaired); + setTableValue(table, rgID, "n.reads.analyzed", info.nReadsSeen); + setTableValue(table, rgID, "simple.read.type", hasAnyReads ? String.format("%dx%d", isPaired ? 2 : 1, readLength) : "NA"); + setTableValue(table, rgID, "median.read.length", hasAnyReads ? readLength : "NA" ); + setTableValue(table, rgID, "median.insert.size", hasAnyReads && isPaired ? info.insertSize.getMedian(0) : "NA" ); } report.print(out); } + + private final void setTableValue(GATKReportTable table, final String rgID, final String key, final Object value) { + table.set(rgID, key, value == null ? "NA" : value); + } } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupPropertiesIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupPropertiesIntegrationTest.java index 04c30a8fe4..1a4c8db305 100644 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupPropertiesIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupPropertiesIntegrationTest.java @@ -38,7 +38,7 @@ public void basicTest() { WalkerTestSpec spec = new WalkerTestSpec( "-T ReadGroupProperties -R " + b37KGReference + " -I " + b37GoodBAM + " -L 20:10,000,000-11,000,000 -o %s", 1, - Arrays.asList("8e4d09665c0b65c971bd5dead24f95fe")); + Arrays.asList("6b8cce223af28cbadcfe87a3b841fc56")); executeTest("ReadGroupProperties:", spec); } } \ No newline at end of file From 3b5a7c34d74518739ca588be62e21f591d053aa1 Mon Sep 17 00:00:00 2001 From: Guillermo del Angel Date: Sun, 4 Mar 2012 10:24:29 -0500 Subject: [PATCH 318/356] Added argument to ValidationAmplicons to only output valid sequences - useful for not having to post-filter or grep resulting files before delivering downstream --- .../validation/ValidationAmplicons.java | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/ValidationAmplicons.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/ValidationAmplicons.java index b27bef2650..e812fb53ae 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/ValidationAmplicons.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/ValidationAmplicons.java @@ -110,6 +110,13 @@ public class ValidationAmplicons extends RodWalker { @Argument(doc="Lower case SNPs rather than replacing with 'N'",fullName="lowerCaseSNPs",required=false) boolean lowerCaseSNPs = false; + /** + * If onlyOutputValidAmplicons is true, the output fasta file will contain only valid sequences. + * Useful for producing delivery-ready files. + */ + @Argument(doc="Only output valid sequences.",fullName="onlyOutputValidAmplicons",required=false) + boolean onlyOutputValidAmplicons = false; + /** * BWA single-end alignment is used as a primer specificity proxy. Low-complexity regions (that don't align back to themselves as a best hit) are lowercased. * This changes the size of the k-mer used for alignment. @@ -486,14 +493,16 @@ public void print() { valid = "Valid"; } - String seqIdentity = sequence.toString().replace('n', 'N').replace('i', 'I').replace('d', 'D'); - if (!sequenomOutput) - out.printf(">%s %s %s%n%s%n", allelePos != null ? allelePos.toString() : "multiple", valid, probeName, seqIdentity); - else { - seqIdentity = seqIdentity.replace("*",""); // identifier < 20 letters long, no * in ref allele, one line per record - probeName = probeName.replace("amplicon_","a"); - out.printf("%s_%s %s%n", allelePos != null ? allelePos.toString() : "multiple", probeName, seqIdentity); + if (!onlyOutputValidAmplicons || !sequenceInvalid) { + String seqIdentity = sequence.toString().replace('n', 'N').replace('i', 'I').replace('d', 'D'); + if (!sequenomOutput) + out.printf(">%s %s %s%n%s%n", allelePos != null ? allelePos.toString() : "multiple", valid, probeName, seqIdentity); + else { + seqIdentity = seqIdentity.replace("*",""); // identifier < 20 letters long, no * in ref allele, one line per record + probeName = probeName.replace("amplicon_","a"); + out.printf("%s_%s %s%n", allelePos != null ? allelePos.toString() : "multiple", probeName, seqIdentity); + } } } } From d6871967aea416f87b26dee1ce173155214e029d Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Mon, 5 Mar 2012 08:28:42 -0500 Subject: [PATCH 319/356] Adding more unit tests and contracts to PairHMM util class. Updating HaplotypeCaller to use the new PairHMM util class. Now that the HMM result isn't dependent on the length of the haplotype there is no reason to ensure all haplotypes have the save length which simplifies the code considerably. --- .../broadinstitute/sting/utils/Haplotype.java | 36 +++++++++---------- .../broadinstitute/sting/MedianUnitTest.java | 7 ---- 2 files changed, 16 insertions(+), 27 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/Haplotype.java b/public/java/src/org/broadinstitute/sting/utils/Haplotype.java index def2fc349d..41b73d1576 100755 --- a/public/java/src/org/broadinstitute/sting/utils/Haplotype.java +++ b/public/java/src/org/broadinstitute/sting/utils/Haplotype.java @@ -113,24 +113,27 @@ public boolean isReference() { return isReference; } - public byte[] insertAllele( final Allele refAllele, final Allele altAllele, int refInsertLocation, final byte[] paddedRef, final int refStart, - final Cigar haplotypeCigar, final int numBasesAddedToStartOfHaplotype, final int refHaplotypeLength ) { + public byte[] insertAllele( final Allele refAllele, final Allele altAllele, int refInsertLocation, final int hapStart, final Cigar haplotypeCigar ) { if( refAllele.length() != altAllele.length() ) { refInsertLocation++; } - int haplotypeInsertLocation = getHaplotypeCoordinateForReferenceCoordinate(refStart + numBasesAddedToStartOfHaplotype, haplotypeCigar, refInsertLocation); + int haplotypeInsertLocation = getHaplotypeCoordinateForReferenceCoordinate(hapStart, haplotypeCigar, refInsertLocation); if( haplotypeInsertLocation == -1 ) { // desired change falls inside deletion so don't bother creating a new haplotype - return getBases().clone(); + return bases.clone(); } - haplotypeInsertLocation += numBasesAddedToStartOfHaplotype; - final byte[] newHaplotype = getBases().clone(); + byte[] newHaplotype; try { if( refAllele.length() == altAllele.length() ) { // SNP or MNP + newHaplotype = bases.clone(); for( int iii = 0; iii < altAllele.length(); iii++ ) { newHaplotype[haplotypeInsertLocation+iii] = altAllele.getBases()[iii]; } - } else if( refAllele.length() < altAllele.length() ) { // insertion + } else if( refAllele.length() < altAllele.length() ) { // insertion final int altAlleleLength = altAllele.length(); + newHaplotype = new byte[bases.length + altAlleleLength]; + for( int iii = 0; iii < bases.length; iii++ ) { + newHaplotype[iii] = bases[iii]; + } for( int iii = newHaplotype.length - 1; iii > haplotypeInsertLocation + altAlleleLength - 1; iii-- ) { newHaplotype[iii] = newHaplotype[iii-altAlleleLength]; } @@ -138,24 +141,17 @@ public byte[] insertAllele( final Allele refAllele, final Allele altAllele, int newHaplotype[haplotypeInsertLocation+iii] = altAllele.getBases()[iii]; } } else { // deletion - int refHaplotypeOffset = 0; - for( final CigarElement ce : haplotypeCigar.getCigarElements()) { - if(ce.getOperator() == CigarOperator.D) { refHaplotypeOffset += ce.getLength(); } - else if(ce.getOperator() == CigarOperator.I) { refHaplotypeOffset -= ce.getLength(); } - } - for( int iii = 0; iii < altAllele.length(); iii++ ) { - newHaplotype[haplotypeInsertLocation+iii] = altAllele.getBases()[iii]; - } final int shift = refAllele.length() - altAllele.length(); - for( int iii = haplotypeInsertLocation + altAllele.length(); iii < newHaplotype.length - shift; iii++ ) { - newHaplotype[iii] = newHaplotype[iii+shift]; + newHaplotype = new byte[bases.length - shift]; + for( int iii = 0; iii < haplotypeInsertLocation + altAllele.length(); iii++ ) { + newHaplotype[iii] = bases[iii]; } - for( int iii = 0; iii < shift; iii++ ) { - newHaplotype[iii+newHaplotype.length-shift] = paddedRef[refStart+refHaplotypeLength+refHaplotypeOffset+iii]; + for( int iii = haplotypeInsertLocation + altAllele.length(); iii < newHaplotype.length; iii++ ) { + newHaplotype[iii] = bases[iii+shift]; } } } catch (Exception e) { // event already on haplotype is too large/complex to insert another allele, most likely because of not enough reference padding - return getBases().clone(); + return bases.clone(); } return newHaplotype; diff --git a/public/java/test/org/broadinstitute/sting/MedianUnitTest.java b/public/java/test/org/broadinstitute/sting/MedianUnitTest.java index c12db9b77a..db89aee78b 100644 --- a/public/java/test/org/broadinstitute/sting/MedianUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/MedianUnitTest.java @@ -29,7 +29,6 @@ // the imports for unit testing. -import org.broadinstitute.sting.gatk.walkers.haplotypecaller.LikelihoodCalculationEngine; import org.broadinstitute.sting.utils.Median; import org.testng.Assert; import org.testng.annotations.BeforeSuite; @@ -42,12 +41,6 @@ public class MedianUnitTest extends BaseTest { - LikelihoodCalculationEngine engine; - - @BeforeSuite - public void before() { - engine = new LikelihoodCalculationEngine(0, 0, false); - } // -------------------------------------------------------------------------------- // From dfbffc95a3412d68f0f8da3c24289d6fb7b78e69 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Fri, 2 Mar 2012 18:54:35 -0500 Subject: [PATCH 320/356] getting rid of the old Indel BQSR From a1d6b3818c3737aa42f42b6c87bd3d074c44aea5 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Fri, 2 Mar 2012 19:10:50 -0500 Subject: [PATCH 321/356] dont include deletions in the pileup From e9ad382e749773827d78f90a9c5d6595c77aca72 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Sat, 3 Mar 2012 18:01:55 -0500 Subject: [PATCH 322/356] unifying the BQSR argument collection --- .../bqsr/RecalibrationArgumentCollection.java | 67 ++++++++++++++++++- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalibrationArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalibrationArgumentCollection.java index 38e7051e48..cc6f67cc9a 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalibrationArgumentCollection.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/bqsr/RecalibrationArgumentCollection.java @@ -25,8 +25,14 @@ package org.broadinstitute.sting.gatk.walkers.bqsr; -import org.broadinstitute.sting.commandline.Argument; -import org.broadinstitute.sting.commandline.Hidden; +import org.broad.tribble.Feature; +import org.broadinstitute.sting.commandline.*; +import org.broadinstitute.sting.gatk.walkers.recalibration.CountCovariatesGatherer; + +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; /** * Created by IntelliJ IDEA. @@ -39,6 +45,63 @@ public class RecalibrationArgumentCollection { + /** + * This algorithm treats every reference mismatch as an indication of error. However, real genetic variation is expected to mismatch the reference, + * so it is critical that a database of known polymorphic sites is given to the tool in order to skip over those sites. This tool accepts any number of RodBindings (VCF, Bed, etc.) + * for use as this database. For users wishing to exclude an interval list of known variation simply use -XL my.interval.list to skip over processing those sites. + * Please note however that the statistics reported by the tool will not accurately reflected those sites skipped by the -XL argument. + */ + @Input(fullName = "knownSites", shortName = "knownSites", doc = "A database of known polymorphic sites to skip over in the recalibration algorithm", required = false) + protected List> knownSites = Collections.emptyList(); + + /** + * After the header, data records occur one per line until the end of the file. The first several items on a line are the + * values of the individual covariates and will change depending on which covariates were specified at runtime. The last + * three items are the data- that is, number of observations for this combination of covariates, number of reference mismatches, + * and the raw empirical quality score calculated by phred-scaling the mismatch rate. + */ + @Gather(CountCovariatesGatherer.class) + @Output + protected PrintStream RECAL_FILE; + + /** + * List all implemented covariates. + */ + @Argument(fullName = "list", shortName = "ls", doc = "List the available covariates and exit", required = false) + protected boolean LIST_ONLY = false; + + /** + * Covariates to be used in the recalibration. Each covariate is given as a separate cov parameter. ReadGroup and ReportedQuality are required covariates and are already added for you. See the list of covariates with -list. + */ + @Argument(fullName = "covariate", shortName = "cov", doc = "Covariates to be used in the recalibration. Each covariate is given as a separate cov parameter. ReadGroup and ReportedQuality are required covariates and are already added for you.", required = false) + protected String[] COVARIATES = null; + + /* + * Use the standard set of covariates in addition to the ones listed using the -cov argument + */ + @Argument(fullName = "standard_covs", shortName = "standard", doc = "Use the standard set of covariates in addition to the ones listed using the -cov argument", required = false) + protected boolean USE_STANDARD_COVARIATES = true; + + ///////////////////////////// + // Debugging-only Arguments + ///////////////////////////// + /** + * This calculation is critically dependent on being able to skip over known polymorphic sites. Please be sure that you know what you are doing if you use this option. + */ + @Hidden + @Argument(fullName = "run_without_dbsnp_potentially_ruining_quality", shortName = "run_without_dbsnp_potentially_ruining_quality", required = false, doc = "If specified, allows the recalibrator to be used without a dbsnp rod. Very unsafe and for expert users only.") + protected boolean RUN_WITHOUT_DBSNP = false; + + ///////////////////////////// + // protected Member Variables + ///////////////////////////// + protected final RecalDataManager dataManager = new RecalDataManager(); // Holds the data HashMap used to create collapsed data hashmaps (delta delta tables) + protected final ArrayList requestedCovariates = new ArrayList();// A list to hold the covariate objects that were requested + + protected final String SKIP_RECORD_ATTRIBUTE = "SKIP"; // used to label reads that should be skipped. + protected final String SEEN_ATTRIBUTE = "SEEN"; // used to label reads as processed. + + /** * CountCovariates and TableRecalibration accept a --solid_recal_mode flag which governs how the recalibrator handles the * reads which have had the reference inserted because of color space inconsistencies. From 14a77b1e717d699efbc63796d699b5317fd6a2ae Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Mon, 5 Mar 2012 12:28:32 -0500 Subject: [PATCH 323/356] Getting rid of redundant methods in MathUtils. Adding unit tests for approximateLog10SumLog10 and normalizeFromLog10. Increasing the precision of the Jacobian approximation used by approximateLog10SumLog which changes the UG+HC integration tests ever so slightly. --- .../indels/HaplotypeIndelErrorModel.java | 3 +- .../indels/PairHMMIndelErrorModel.java | 17 +-- .../GaussianMixtureModel.java | 11 +- .../broadinstitute/sting/utils/MathUtils.java | 128 ++++-------------- .../UnifiedGenotyperIntegrationTest.java | 32 ++--- .../sting/utils/MathUtilsUnitTest.java | 97 ++++++++++--- 6 files changed, 133 insertions(+), 155 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/HaplotypeIndelErrorModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/HaplotypeIndelErrorModel.java index 200a250f24..26023bd2ff 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/HaplotypeIndelErrorModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/HaplotypeIndelErrorModel.java @@ -454,8 +454,7 @@ public double[] computeReadHaplotypeLikelihoods(ReadBackedPileup pileup, HashMap // Compute log10(10^x1/2 + 10^x2/2) = log10(10^x1+x0^x2)-log10(2) // First term is approximated by Jacobian log with table lookup. // Second term is a constant added to both likelihoods so will be ignored - haplotypeLikehoodMatrix[i][j] += MathUtils.softMax(readLikelihood[0], - readLikelihood[1]); + haplotypeLikehoodMatrix[i][j] += MathUtils.approximateLog10SumLog10(readLikelihood[0], readLikelihood[1]); } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java index 6410d619d4..64993b43ae 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java @@ -166,18 +166,17 @@ private void updateCell(final int indI, final int indJ, final int X_METRIC_LENGT final double pBaseRead = (x == y)? baseMatchArray[(int)qual]:baseMismatchArray[(int)qual]; - matchMetricArray[indI][indJ] = MathUtils.softMax(matchMetricArray[im1][jm1] + pBaseRead, XMetricArray[im1][jm1] + pBaseRead, - YMetricArray[im1][jm1] + pBaseRead); + matchMetricArray[indI][indJ] = pBaseRead + MathUtils.approximateLog10SumLog10(new double[]{matchMetricArray[im1][jm1], XMetricArray[im1][jm1], YMetricArray[im1][jm1]}); final double c1 = indJ == Y_METRIC_LENGTH-1 ? END_GAP_COST : currentGOP[jm1]; final double d1 = indJ == Y_METRIC_LENGTH-1 ? END_GAP_COST : currentGCP[jm1]; - XMetricArray[indI][indJ] = MathUtils.softMax(matchMetricArray[im1][indJ] + c1, XMetricArray[im1][indJ] + d1); + XMetricArray[indI][indJ] = MathUtils.approximateLog10SumLog10(matchMetricArray[im1][indJ] + c1, XMetricArray[im1][indJ] + d1); // update Y array final double c2 = indI == X_METRIC_LENGTH-1 ? END_GAP_COST : currentGOP[jm1]; final double d2 = indI == X_METRIC_LENGTH-1 ? END_GAP_COST : currentGCP[jm1]; - YMetricArray[indI][indJ] = MathUtils.softMax(matchMetricArray[indI][jm1] + c2, YMetricArray[indI][jm1] + d2); + YMetricArray[indI][indJ] = MathUtils.approximateLog10SumLog10(matchMetricArray[indI][jm1] + c2, YMetricArray[indI][jm1] + d2); } } @@ -316,9 +315,7 @@ else if (bestMetric < maxElementInDiag - DIAG_TOL) final int bestI = X_METRIC_LENGTH - 1, bestJ = Y_METRIC_LENGTH - 1; - final double bestMetric = MathUtils.softMax(matchMetricArray[bestI][bestJ], - XMetricArray[bestI][bestJ], - YMetricArray[bestI][bestJ]); + final double bestMetric = MathUtils.approximateLog10SumLog10(new double[]{ matchMetricArray[bestI][bestJ], XMetricArray[bestI][bestJ], YMetricArray[bestI][bestJ] }); /* if (DEBUG) { @@ -651,7 +648,7 @@ private int computeFirstDifferingPosition(double[] b1, double[] b2) { private final static double[] getHaplotypeLikelihoods(final int numHaplotypes, final int readCounts[], final double readLikelihoods[][]) { final double[][] haplotypeLikehoodMatrix = new double[numHaplotypes][numHaplotypes]; - // todo: MAD 09/26/11 -- I'm almost certain this calculation can be simplied to just a single loop without the intermediate NxN matrix + // todo: MAD 09/26/11 -- I'm almost certain this calculation can be simplified to just a single loop without the intermediate NxN matrix for (int i=0; i < numHaplotypes; i++) { for (int j=i; j < numHaplotypes; j++){ // combine likelihoods of haplotypeLikelihoods[i], haplotypeLikelihoods[j] @@ -665,7 +662,7 @@ private final static double[] getHaplotypeLikelihoods(final int numHaplotypes, f final double li = readLikelihoods[readIdx][i]; final double lj = readLikelihoods[readIdx][j]; final int readCount = readCounts[readIdx]; - haplotypeLikehoodMatrix[i][j] += readCount * (MathUtils.softMax(li, lj) + LOG_ONE_HALF); + haplotypeLikehoodMatrix[i][j] += readCount * (MathUtils.approximateLog10SumLog10(li, lj) + LOG_ONE_HALF); } } } @@ -678,7 +675,7 @@ private final static double[] getHaplotypeLikelihoods(final int numHaplotypes, f } } - // renormalize so that max element is zero. + // renormalize so that max element is zero. return MathUtils.normalizeFromLog10(genotypeLikelihoods, false, true); } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/GaussianMixtureModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/GaussianMixtureModel.java index 82776ca2e4..6f0ae7c259 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/GaussianMixtureModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/GaussianMixtureModel.java @@ -140,15 +140,16 @@ public void expectationStep( final List data ) { } for( final VariantDatum datum : data ) { - final ArrayList pVarInGaussianLog10 = new ArrayList( gaussians.size() ); + final double[] pVarInGaussianLog10 = new double[gaussians.size()]; + int gaussianIndex = 0; for( final MultivariateGaussian gaussian : gaussians ) { final double pVarLog10 = gaussian.evaluateDatumLog10( datum ); - pVarInGaussianLog10.add( pVarLog10 ); + pVarInGaussianLog10[gaussianIndex++] = pVarLog10; } - final double[] pVarInGaussianNormalized = MathUtils.normalizeFromLog10( pVarInGaussianLog10 ); - int iii = 0; + final double[] pVarInGaussianNormalized = MathUtils.normalizeFromLog10( pVarInGaussianLog10, false ); + gaussianIndex = 0; for( final MultivariateGaussian gaussian : gaussians ) { - gaussian.assignPVarInGaussian( pVarInGaussianNormalized[iii++] ); //BUGBUG: to clean up + gaussian.assignPVarInGaussian( pVarInGaussianNormalized[gaussianIndex++] ); } } } diff --git a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java index 6f2db67ee9..a96cbffc5f 100644 --- a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java @@ -41,14 +41,6 @@ * @author Kiran Garimella */ public class MathUtils { - /** Public constants - used for the Lanczos approximation to the factorial function - * (for the calculation of the binomial/multinomial probability in logspace) - * @param LANC_SEQ[] - an array holding the constants which correspond to the product - * of Chebyshev Polynomial coefficients, and points on the Gamma function (for interpolation) - * [see A Precision Approximation of the Gamma Function J. SIAM Numer. Anal. Ser. B, Vol. 1 1964. pp. 86-96] - * @param LANC_G - a value for the Lanczos approximation to the gamma function that works to - * high precision - */ /** * Private constructor. No instantiating this class! @@ -56,6 +48,28 @@ public class MathUtils { private MathUtils() { } + public static final double[] log10Cache; + private static final double[] jacobianLogTable; + private static final double JACOBIAN_LOG_TABLE_STEP = 0.001; + private static final double MAX_JACOBIAN_TOLERANCE = 10.0; + private static final int JACOBIAN_LOG_TABLE_SIZE = (int) (MAX_JACOBIAN_TOLERANCE / JACOBIAN_LOG_TABLE_STEP) + 1; + private static final int MAXN = 11000; + private static final int LOG10_CACHE_SIZE = 4 * MAXN; // we need to be able to go up to 2*(2N) when calculating some of the coefficients + + static { + log10Cache = new double[LOG10_CACHE_SIZE]; + jacobianLogTable = new double[JACOBIAN_LOG_TABLE_SIZE]; + + log10Cache[0] = Double.NEGATIVE_INFINITY; + for (int k = 1; k < LOG10_CACHE_SIZE; k++) + log10Cache[k] = Math.log10(k); + + for (int k = 0; k < JACOBIAN_LOG_TABLE_SIZE; k++) { + jacobianLogTable[k] = Math.log10(1.0 + Math.pow(10.0, -((double) k) * JACOBIAN_LOG_TABLE_STEP)); + + } + } + // A fast implementation of the Math.round() method. This method does not perform // under/overflow checking, so this shouldn't be used in the general case (but is fine // if one is already make those checks before calling in to the rounding). @@ -536,7 +550,7 @@ public static double[] normalizeFromLog10(double[] array, boolean takeLog10OfOut // all negative) the largest value; also, we need to convert to normal-space. double maxValue = Utils.findMaxEntry(array); - // we may decide to just normalize in log space with converting to linear space + // we may decide to just normalize in log space without converting to linear space if (keepInLogSpace) { for (int i = 0; i < array.length; i++) array[i] -= maxValue; @@ -563,29 +577,6 @@ public static double[] normalizeFromLog10(double[] array, boolean takeLog10OfOut return normalized; } - public static double[] normalizeFromLog10(List array, boolean takeLog10OfOutput) { - double[] normalized = new double[array.size()]; - - // for precision purposes, we need to add (or really subtract, since they're - // all negative) the largest value; also, we need to convert to normal-space. - double maxValue = MathUtils.arrayMaxDouble(array); - for (int i = 0; i < array.size(); i++) - normalized[i] = Math.pow(10, array.get(i) - maxValue); - - // normalize - double sum = 0.0; - for (int i = 0; i < array.size(); i++) - sum += normalized[i]; - for (int i = 0; i < array.size(); i++) { - double x = normalized[i] / sum; - if (takeLog10OfOutput) - x = Math.log10(x); - normalized[i] = x; - } - - return normalized; - } - /** * normalizes the log10-based array. ASSUMES THAT ALL ARRAY ENTRIES ARE <= 0 (<= 1 IN REAL-SPACE). * @@ -596,10 +587,6 @@ public static double[] normalizeFromLog10(double[] array) { return normalizeFromLog10(array, false); } - public static double[] normalizeFromLog10(List array) { - return normalizeFromLog10(array, false); - } - public static int maxElementIndex(final double[] array) { return maxElementIndex(array, array.length); } @@ -1207,78 +1194,11 @@ public static double ratio(long num, long denom) { return ((double) num) / (Math.max(denom, 1)); } - public static final double[] log10Cache; - public static final double[] jacobianLogTable; - public static final int JACOBIAN_LOG_TABLE_SIZE = 101; - public static final double JACOBIAN_LOG_TABLE_STEP = 0.1; - public static final double INV_JACOBIAN_LOG_TABLE_STEP = 1.0 / JACOBIAN_LOG_TABLE_STEP; - public static final double MAX_JACOBIAN_TOLERANCE = 10.0; - private static final int MAXN = 11000; - private static final int LOG10_CACHE_SIZE = 4 * MAXN; // we need to be able to go up to 2*(2N) when calculating some of the coefficients - - static { - log10Cache = new double[LOG10_CACHE_SIZE]; - jacobianLogTable = new double[JACOBIAN_LOG_TABLE_SIZE]; - - log10Cache[0] = Double.NEGATIVE_INFINITY; - for (int k = 1; k < LOG10_CACHE_SIZE; k++) - log10Cache[k] = Math.log10(k); - - for (int k = 0; k < JACOBIAN_LOG_TABLE_SIZE; k++) { - jacobianLogTable[k] = Math.log10(1.0 + Math.pow(10.0, -((double) k) * JACOBIAN_LOG_TABLE_STEP)); - - } - } - - static public double softMax(final double[] vec) { - double acc = vec[0]; - for (int k = 1; k < vec.length; k++) - acc = softMax(acc, vec[k]); - - return acc; - - } - static public double max(double x0, double x1, double x2) { double a = Math.max(x0, x1); return Math.max(a, x2); } - static public double softMax(final double x0, final double x1, final double x2) { - // compute naively log10(10^x[0] + 10^x[1]+...) - // return Math.log10(MathUtils.sumLog10(vec)); - - // better approximation: do Jacobian logarithm function on data pairs - double a = softMax(x0, x1); - return softMax(a, x2); - } - - static public double softMax(final double x, final double y) { - // we need to compute log10(10^x + 10^y) - // By Jacobian logarithm identity, this is equal to - // max(x,y) + log10(1+10^-abs(x-y)) - // we compute the second term as a table lookup - // with integer quantization - - // slow exact version: - // return Math.log10(Math.pow(10.0,x) + Math.pow(10.0,y)); - - double diff = x - y; - - if (diff > MAX_JACOBIAN_TOLERANCE) - return x; - else if (diff < -MAX_JACOBIAN_TOLERANCE) - return y; - else if (diff >= 0) { - int ind = (int) (diff * INV_JACOBIAN_LOG_TABLE_STEP + 0.5); - return x + jacobianLogTable[ind]; - } - else { - int ind = (int) (-diff * INV_JACOBIAN_LOG_TABLE_STEP + 0.5); - return y + jacobianLogTable[ind]; - } - } - public static double phredScaleToProbability(byte q) { return Math.pow(10, (-q) / 10.0); } @@ -1734,6 +1654,4 @@ public static BitSet bitSetFrom(String dna) { return bitSetFrom(baseTen+preContext); // the number representing this DNA string is the base_10 representation plus all combinations that preceded this string length. } - - } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java index 823eeeeb97..cfb0d11a1f 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java @@ -28,7 +28,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest { public void testMultiSamplePilot1() { WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec( baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000", 1, - Arrays.asList("202b337ebbea3def1be8495eb363dfa8")); + Arrays.asList("8f81a14fffc1a59b4b066f8595dc1232")); executeTest("test MultiSample Pilot1", spec); } @@ -52,7 +52,7 @@ public void testWithAllelesPassedIn2() { public void testSingleSamplePilot2() { WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec( baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1, - Arrays.asList("ae29b9c9aacce8046dc780430540cd62")); + Arrays.asList("c5b53231f4f6d9524bc4ec8115f44f5c")); executeTest("test SingleSample Pilot2", spec); } @@ -60,7 +60,7 @@ public void testSingleSamplePilot2() { public void testMultipleSNPAlleles() { WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec( "-T UnifiedGenotyper -R " + b37KGReference + " -nosl -NO_HEADER -glm BOTH --dbsnp " + b37dbSNP129 + " -I " + validationDataLocation + "multiallelic.snps.bam -o %s -L " + validationDataLocation + "multiallelic.snps.intervals", 1, - Arrays.asList("10027d13befaa07b7900a7af0ae0791c")); + Arrays.asList("5af005255240a2186f04cb50851b8b6f")); executeTest("test Multiple SNP alleles", spec); } @@ -70,7 +70,7 @@ public void testMultipleSNPAlleles() { // // -------------------------------------------------------------------------------------------------------------- - private final static String COMPRESSED_OUTPUT_MD5 = "fda341de80b3f6fd42a83352b18b1d65"; + private final static String COMPRESSED_OUTPUT_MD5 = "a08df9aea2b3df09cf90ff8e6e3be3ea"; @Test public void testCompressedOutput() { @@ -91,7 +91,7 @@ public void testParallelization() { // Note that we need to turn off any randomization for this to work, so no downsampling and no annotations - String md5 = "32a34362dff51d8b73a3335048516d82"; + String md5 = "6358934c1c26345013a38261b8c45aa4"; WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec( baseCommand + " -dt NONE -G none -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,075,000", 1, @@ -179,8 +179,8 @@ public void testConfidence2() { @Test public void testHeterozyosity() { HashMap e = new HashMap(); - e.put( 0.01, "2cb2544739e01f6c08fd820112914317" ); - e.put( 1.0 / 1850, "730b2b83a4b1f6d46fc3b5cd7d90756c" ); + e.put( 0.01, "926b58038dd4989bf7eda697a847eea9" ); + e.put( 1.0 / 1850, "93f44105b43b65730a3b821e27b0fa16" ); for ( Map.Entry entry : e.entrySet() ) { WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec( @@ -204,7 +204,7 @@ public void testMultiTechnologies() { " -o %s" + " -L 1:10,000,000-10,100,000", 1, - Arrays.asList("f0fbe472f155baf594b1eeb58166edef")); + Arrays.asList("a1b75a7e12b160b0be823228c958573f")); executeTest(String.format("test multiple technologies"), spec); } @@ -223,7 +223,7 @@ public void testCallingWithBAQ() { " -L 1:10,000,000-10,100,000" + " -baq CALCULATE_AS_NECESSARY", 1, - Arrays.asList("8c87c749a7bb5a76ed8504d4ec254272")); + Arrays.asList("3bda1279cd6dcb47885f3e19466f11b9")); executeTest(String.format("test calling with BAQ"), spec); } @@ -242,7 +242,7 @@ public void testSimpleIndels() { " -o %s" + " -L 1:10,000,000-10,500,000", 1, - Arrays.asList("bd9d3d50a1f49605d7cd592a0f446899")); + Arrays.asList("d9fc3ba94a0d46029778c7b457e7292a")); executeTest(String.format("test indel caller in SLX"), spec); } @@ -257,7 +257,7 @@ public void testIndelsWithLowMinAlleleCnt() { " -minIndelCnt 1" + " -L 1:10,000,000-10,100,000", 1, - Arrays.asList("2ad52c2e75b3ffbfd8f03237c444e8e6")); + Arrays.asList("b2e30ae3e5ffa6108f9f6178b1d2e679")); executeTest(String.format("test indel caller in SLX with low min allele count"), spec); } @@ -270,7 +270,7 @@ public void testMultiTechnologyIndels() { " -o %s" + " -L 1:10,000,000-10,500,000", 1, - Arrays.asList("91cd6d2e3972b0b8e4064bb35a33241f")); + Arrays.asList("2cd182a84613fa91a6020466d2d327e2")); executeTest(String.format("test indel calling, multiple technologies"), spec); } @@ -280,7 +280,7 @@ public void testWithIndelAllelesPassedIn1() { WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec( baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "indelAllelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,100,000", 1, - Arrays.asList("c60a44ba94a80a0cb1fba8b6f90a13cd")); + Arrays.asList("9cd08dc412a007933381e9c76c073899")); executeTest("test MultiSample Pilot2 indels with alleles passed in", spec1); } @@ -290,7 +290,7 @@ public void testWithIndelAllelesPassedIn2() { baseCommandIndels + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "indelAllelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,100,000", 1, - Arrays.asList("320f61c87253aba77d6dc782242cba8b")); + Arrays.asList("5ef1f007d3ef77c1b8f31e5e036eff53")); executeTest("test MultiSample Pilot2 indels with alleles passed in and emitting all sites", spec2); } @@ -300,7 +300,7 @@ public void testWithIndelAllelesPassedIn3() { WalkerTest.WalkerTestSpec spec3 = new WalkerTest.WalkerTestSpec( baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "ALL.wgs.union_v2.20101123.indels.sites.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,080,000", 1, - Arrays.asList("c9897b80615c53a4ea10a4b193d56d9c")); + Arrays.asList("2609675a356f2dfc86f8a1d911210978")); executeTest("test MultiSample Pilot2 indels with complicated records", spec3); } @@ -309,7 +309,7 @@ public void testWithIndelAllelesPassedIn4() { WalkerTest.WalkerTestSpec spec4 = new WalkerTest.WalkerTestSpec( baseCommandIndelsb37 + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "ALL.wgs.union_v2_chr20_100_110K.20101123.indels.sites.vcf -I " + validationDataLocation + "phase1_GBR_realigned.chr20.100K-110K.bam -o %s -L 20:100,000-110,000", 1, - Arrays.asList("5282fdb1711a532d726c13507bf80a21")); + Arrays.asList("4fdd8da77167881b71b3547da5c13f94")); executeTest("test MultiSample Phase1 indels with complicated records", spec4); } diff --git a/public/java/test/org/broadinstitute/sting/utils/MathUtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/MathUtilsUnitTest.java index 75fc44873d..1ba6c74d46 100755 --- a/public/java/test/org/broadinstitute/sting/utils/MathUtilsUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/MathUtilsUnitTest.java @@ -205,6 +205,24 @@ public void testArrayShuffle() { } } + /** + * Private functions used by testArrayShuffle() + */ + private boolean hasUniqueElements(Object[] x) { + for (int i = 0; i < x.length; i++) + for (int j = i + 1; j < x.length; j++) + if (x[i].equals(x[j]) || x[i] == x[j]) + return false; + return true; + } + + private boolean hasAllElements(final Object[] expected, final Object[] actual) { + HashSet set = new HashSet(); + set.addAll(Arrays.asList(expected)); + set.removeAll(Arrays.asList(actual)); + return set.isEmpty(); + } + @Test(enabled = true) public void testIntAndBitSetConversion() { Assert.assertEquals(MathUtils.intFrom(MathUtils.bitSetFrom(428)), 428); @@ -234,26 +252,71 @@ public void testDNAAndBitSetConversion() { Assert.assertEquals(MathUtils.dnaFrom(MathUtils.bitSetFrom("AACGTCAATGCAGTCAAGTCAGACGTGGGTT")), "AACGTCAATGCAGTCAAGTCAGACGTGGGTT"); // testing max precision (length == 31) } - - private boolean hasUniqueElements(Object[] x) { - for (int i = 0; i < x.length; i++) - for (int j = i + 1; j < x.length; j++) - if (x[i].equals(x[j]) || x[i] == x[j]) - return false; - return true; + @Test + public void testApproximateLog10SumLog10() { + Assert.assertEquals(MathUtils.approximateLog10SumLog10(0.0, 0.0), Math.log10(Math.pow(10.0, 0.0) + Math.pow(10.0, 0.0)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(-1.0, 0.0), Math.log10(Math.pow(10.0, -1.0) + Math.pow(10.0, 0.0)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(0.0, -1.0), Math.log10(Math.pow(10.0, 0.0) + Math.pow(10.0, -1.0)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(-2.2, -3.5), Math.log10(Math.pow(10.0, -2.2) + Math.pow(10.0, -3.5)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(-1.0, -7.1), Math.log10(Math.pow(10.0, -1.0) + Math.pow(10.0, -7.1)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(5.0, 6.2), Math.log10(Math.pow(10.0, 5.0) + Math.pow(10.0, 6.2)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(38.1, 16.2), Math.log10(Math.pow(10.0, 38.1) + Math.pow(10.0, 16.2)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(-38.1, 6.2), Math.log10(Math.pow(10.0, -38.1) + Math.pow(10.0, 6.2)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(-19.1, -37.1), Math.log10(Math.pow(10.0, -19.1) + Math.pow(10.0, -37.1)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(-29.1, -27.6), Math.log10(Math.pow(10.0, -29.1) + Math.pow(10.0, -27.6)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(-0.12345, -0.23456), Math.log10(Math.pow(10.0, -0.12345) + Math.pow(10.0, -0.23456)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(-15.7654, -17.0101), Math.log10(Math.pow(10.0, -15.7654) + Math.pow(10.0, -17.0101)), 1e-3); + + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{0.0, 0.0}), Math.log10(Math.pow(10.0, 0.0) + Math.pow(10.0, 0.0)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{-1.0, 0.0}), Math.log10(Math.pow(10.0, -1.0) + Math.pow(10.0, 0.0)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{0.0, -1.0}), Math.log10(Math.pow(10.0, 0.0) + Math.pow(10.0, -1.0)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{-2.2, -3.5}), Math.log10(Math.pow(10.0, -2.2) + Math.pow(10.0, -3.5)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{-1.0, -7.1}), Math.log10(Math.pow(10.0, -1.0) + Math.pow(10.0, -7.1)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{5.0, 6.2}), Math.log10(Math.pow(10.0, 5.0) + Math.pow(10.0, 6.2)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{38.1, 16.2}), Math.log10(Math.pow(10.0, 38.1) + Math.pow(10.0, 16.2)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{-38.1, 6.2}), Math.log10(Math.pow(10.0, -38.1) + Math.pow(10.0, 6.2)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{-19.1, -37.1}), Math.log10(Math.pow(10.0, -19.1) + Math.pow(10.0, -37.1)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{-29.1, -27.6}), Math.log10(Math.pow(10.0, -29.1) + Math.pow(10.0, -27.6)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{-0.12345, -0.23456}), Math.log10(Math.pow(10.0, -0.12345) + Math.pow(10.0, -0.23456)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{-15.7654, -17.0101}), Math.log10(Math.pow(10.0, -15.7654) + Math.pow(10.0, -17.0101)), 1e-3); + + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{0.0, 0.0, 0.0}), Math.log10(Math.pow(10.0, 0.0) + Math.pow(10.0, 0.0) + Math.pow(10.0, 0.0)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{-1.0, 0.0, 0.0}), Math.log10(Math.pow(10.0, -1.0) + Math.pow(10.0, 0.0) + Math.pow(10.0, 0.0)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{0.0, -1.0, -2.5}), Math.log10(Math.pow(10.0, 0.0) + Math.pow(10.0, -1.0) + Math.pow(10.0, -2.5)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{-2.2, -3.5, -1.1}), Math.log10(Math.pow(10.0, -2.2) + Math.pow(10.0, -3.5) + Math.pow(10.0, -1.1)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{-1.0, -7.1, 0.5}), Math.log10(Math.pow(10.0, -1.0) + Math.pow(10.0, -7.1) + Math.pow(10.0, 0.5)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{5.0, 6.2, 1.3}), Math.log10(Math.pow(10.0, 5.0) + Math.pow(10.0, 6.2) + Math.pow(10.0, 1.3)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{38.1, 16.2, 18.1}), Math.log10(Math.pow(10.0, 38.1) + Math.pow(10.0, 16.2) + Math.pow(10.0, 18.1)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{-38.1, 6.2, 26.6}), Math.log10(Math.pow(10.0, -38.1) + Math.pow(10.0, 6.2) + Math.pow(10.0, 26.6)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{-19.1, -37.1, -45.1}), Math.log10(Math.pow(10.0, -19.1) + Math.pow(10.0, -37.1) + Math.pow(10.0, -45.1)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{-29.1, -27.6, -26.2}), Math.log10(Math.pow(10.0, -29.1) + Math.pow(10.0, -27.6) + Math.pow(10.0, -26.2)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{-0.12345, -0.23456, -0.34567}), Math.log10(Math.pow(10.0, -0.12345) + Math.pow(10.0, -0.23456) + Math.pow(10.0, -0.34567)), 1e-3); + Assert.assertEquals(MathUtils.approximateLog10SumLog10(new double[]{-15.7654, -17.0101, -17.9341}), Math.log10(Math.pow(10.0, -15.7654) + Math.pow(10.0, -17.0101) + Math.pow(10.0, -17.9341)), 1e-3); } - private boolean hasAllElements(final Object[] expected, final Object[] actual) { - HashSet set = new HashSet(); - set.addAll(Arrays.asList(expected)); - set.removeAll(Arrays.asList(actual)); - return set.isEmpty(); + @Test + public void testNormalizeFromLog10() { + Assert.assertTrue(compareDoubleArrays(MathUtils.normalizeFromLog10(new double[]{0.0, 0.0, -1.0, -1.1, -7.8}, false, true), new double[]{0.0, 0.0, -1.0, -1.1, -7.8})); + Assert.assertTrue(compareDoubleArrays(MathUtils.normalizeFromLog10(new double[]{-1.0, -1.0, -1.0, -1.1, -7.8}, false, true), new double[]{0.0, 0.0, 0.0, -0.1, -6.8})); + Assert.assertTrue(compareDoubleArrays(MathUtils.normalizeFromLog10(new double[]{-10.0, -7.8, -10.5, -1.1, -10.0}, false, true), new double[]{-8.9, -6.7, -9.4, 0.0, -8.9})); + + Assert.assertTrue(compareDoubleArrays(MathUtils.normalizeFromLog10(new double[]{-1.0, -1.0, -1.0, -1.0}), new double[]{0.25, 0.25, 0.25, 0.25})); + Assert.assertTrue(compareDoubleArrays(MathUtils.normalizeFromLog10(new double[]{-1.0, -3.0, -1.0, -1.0}), new double[]{0.1 * 1.0 / 0.301, 0.001 * 1.0 / 0.301, 0.1 * 1.0 / 0.301, 0.1 * 1.0 / 0.301})); + Assert.assertTrue(compareDoubleArrays(MathUtils.normalizeFromLog10(new double[]{-1.0, -3.0, -1.0, -2.0}), new double[]{0.1 * 1.0 / 0.211, 0.001 * 1.0 / 0.211, 0.1 * 1.0 / 0.211, 0.01 * 1.0 / 0.211})); } - private void p (Object []x) { - for (Object v: x) - System.out.print((Integer) v + " "); - System.out.println(); - } + /** + * Private function used by testNormalizeFromLog10() + */ + private boolean compareDoubleArrays(double[] b1, double[] b2) { + if( b1.length != b2.length ) { + return false; // sanity check + } + for( int i=0; i < b1.length; i++ ){ + if ( MathUtils.compareDoubles(b1[i], b2[i]) != 0 ) + return false; + } + return true; + } } From c6ded4d23c69247700504ac5314fb56e27dfb2e1 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Mon, 5 Mar 2012 17:54:42 -0500 Subject: [PATCH 324/356] Bug fix for hard clipping reads when base insertion and base deletion qualities are present in the read. Updating HaplotypeCaller integration tests to reflect all the recent changes. --- .../sting/utils/clipping/ClippingOp.java | 10 ++++++ .../sting/utils/fragments/FragmentUtils.java | 36 +++++++++++-------- .../sting/utils/sam/GATKSAMRecord.java | 4 +++ 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/clipping/ClippingOp.java b/public/java/src/org/broadinstitute/sting/utils/clipping/ClippingOp.java index fb133d902f..62a67a1f2e 100644 --- a/public/java/src/org/broadinstitute/sting/utils/clipping/ClippingOp.java +++ b/public/java/src/org/broadinstitute/sting/utils/clipping/ClippingOp.java @@ -4,6 +4,7 @@ import net.sf.samtools.Cigar; import net.sf.samtools.CigarElement; import net.sf.samtools.CigarOperator; +import org.broadinstitute.sting.gatk.walkers.bqsr.RecalDataManager; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; import org.broadinstitute.sting.utils.sam.GATKSAMRecord; @@ -314,6 +315,15 @@ private GATKSAMRecord hardClip(GATKSAMRecord read, int start, int stop) { if (start == 0) hardClippedRead.setAlignmentStart(read.getAlignmentStart() + calculateAlignmentStartShift(read.getCigar(), cigarShift.cigar)); + if (read.hasBaseIndelQualities()) { + byte[] newBaseInsertionQuals = new byte[newLength]; + byte[] newBaseDeletionQuals = new byte[newLength]; + System.arraycopy(read.getBaseInsertionQualities(), copyStart, newBaseInsertionQuals, 0, newLength); + System.arraycopy(read.getBaseDeletionQualities(), copyStart, newBaseDeletionQuals, 0, newLength); + hardClippedRead.setBaseQualities(newBaseInsertionQuals, RecalDataManager.BaseRecalibrationType.BASE_INSERTION); + hardClippedRead.setBaseQualities(newBaseDeletionQuals, RecalDataManager.BaseRecalibrationType.BASE_DELETION); + } + return hardClippedRead; } diff --git a/public/java/src/org/broadinstitute/sting/utils/fragments/FragmentUtils.java b/public/java/src/org/broadinstitute/sting/utils/fragments/FragmentUtils.java index 7104b1eddb..eea45567f9 100644 --- a/public/java/src/org/broadinstitute/sting/utils/fragments/FragmentUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/fragments/FragmentUtils.java @@ -151,23 +151,16 @@ public final static List mergeOverlappingPairedFragments( List MIN_QUAL_BAD_OVERLAP && secondReadQuals[iii-firstReadStop] > MIN_QUAL_BAD_OVERLAP && firstReadBases[iii] != secondReadBases[iii-firstReadStop] ) { @@ -175,22 +168,16 @@ public final static List mergeOverlappingPairedFragments( List secondReadQuals[iii-firstReadStop] ? firstReadBases[iii] : secondReadBases[iii-firstReadStop] ); quals[iii] = ( firstReadQuals[iii] > secondReadQuals[iii-firstReadStop] ? firstReadQuals[iii] : secondReadQuals[iii-firstReadStop] ); - insertionQuals[iii] = ( firstReadQuals[iii] > secondReadQuals[iii-firstReadStop] ? firstReadInsertionQuals[iii] : secondReadInsertionQuals[iii-firstReadStop] ); // Purposefully checking the highest base quality score - deletionQuals[iii] = ( firstReadQuals[iii] > secondReadQuals[iii-firstReadStop] ? firstReadDeletionQuals[iii] : secondReadDeletionQuals[iii-firstReadStop] ); // Purposefully checking the highest base quality score } for(int iii = firstRead.getReadLength(); iii < numBases; iii++) { bases[iii] = secondReadBases[iii-firstReadStop]; quals[iii] = secondReadQuals[iii-firstReadStop]; - insertionQuals[iii] = secondReadInsertionQuals[iii-firstReadStop]; - deletionQuals[iii] = secondReadDeletionQuals[iii-firstReadStop]; } final GATKSAMRecord returnRead = new GATKSAMRecord(firstRead.getHeader()); returnRead.setAlignmentStart(firstRead.getUnclippedStart()); returnRead.setReadBases( bases ); - returnRead.setBaseQualities( quals, RecalDataManager.BaseRecalibrationType.BASE_SUBSTITUTION ); - returnRead.setBaseQualities( insertionQuals, RecalDataManager.BaseRecalibrationType.BASE_INSERTION ); - returnRead.setBaseQualities( deletionQuals, RecalDataManager.BaseRecalibrationType.BASE_DELETION ); + returnRead.setBaseQualities( quals ); returnRead.setReadGroup( firstRead.getReadGroup() ); returnRead.setReferenceName( firstRead.getReferenceName() ); final CigarElement c = new CigarElement(bases.length, CigarOperator.M); @@ -199,6 +186,27 @@ public final static List mergeOverlappingPairedFragments( List secondReadQuals[iii-firstReadStop] ? firstReadInsertionQuals[iii] : secondReadInsertionQuals[iii-firstReadStop] ); // Purposefully checking the highest *base* quality score + deletionQuals[iii] = ( firstReadQuals[iii] > secondReadQuals[iii-firstReadStop] ? firstReadDeletionQuals[iii] : secondReadDeletionQuals[iii-firstReadStop] ); // Purposefully checking the highest *base* quality score + } + for(int iii = firstRead.getReadLength(); iii < numBases; iii++) { + insertionQuals[iii] = secondReadInsertionQuals[iii-firstReadStop]; + deletionQuals[iii] = secondReadDeletionQuals[iii-firstReadStop]; + } + returnRead.setBaseQualities( insertionQuals, RecalDataManager.BaseRecalibrationType.BASE_INSERTION ); + returnRead.setBaseQualities( deletionQuals, RecalDataManager.BaseRecalibrationType.BASE_DELETION ); + } + final ArrayList returnList = new ArrayList(); returnList.add(returnRead); return returnList; diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java index f5a9b2f456..648dafb816 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/GATKSAMRecord.java @@ -194,6 +194,10 @@ public byte[] getBaseQualities( final RecalDataManager.BaseRecalibrationType err } } + public boolean hasBaseIndelQualities() { + return getAttribute( BQSR_BASE_INSERTION_QUALITIES ) != null || getAttribute( BQSR_BASE_DELETION_QUALITIES ) != null; + } + public byte[] getBaseInsertionQualities() { byte[] quals = SAMUtils.fastqToPhred( getStringAttribute( BQSR_BASE_INSERTION_QUALITIES ) ); if( quals == null ) { From 9b53250bef91b2f9f3d18752d210e1d4f1bd85a2 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Mon, 5 Mar 2012 21:07:36 -0500 Subject: [PATCH 325/356] Adding Unit test for Haplotype class. Used in HC's genotype given alleles mode. --- .../java/src/org/broadinstitute/sting/utils/Haplotype.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/Haplotype.java b/public/java/src/org/broadinstitute/sting/utils/Haplotype.java index 41b73d1576..085794babc 100755 --- a/public/java/src/org/broadinstitute/sting/utils/Haplotype.java +++ b/public/java/src/org/broadinstitute/sting/utils/Haplotype.java @@ -24,6 +24,7 @@ package org.broadinstitute.sting.utils; +import com.google.java.contract.Requires; import net.sf.samtools.Cigar; import net.sf.samtools.CigarElement; import net.sf.samtools.CigarOperator; @@ -113,10 +114,11 @@ public boolean isReference() { return isReference; } - public byte[] insertAllele( final Allele refAllele, final Allele altAllele, int refInsertLocation, final int hapStart, final Cigar haplotypeCigar ) { + @Requires({"refInsertLocation >= 0", "hapStartInRefCoords >= 0"}) + public byte[] insertAllele( final Allele refAllele, final Allele altAllele, int refInsertLocation, final int hapStartInRefCoords, final Cigar haplotypeCigar ) { if( refAllele.length() != altAllele.length() ) { refInsertLocation++; } - int haplotypeInsertLocation = getHaplotypeCoordinateForReferenceCoordinate(hapStart, haplotypeCigar, refInsertLocation); + int haplotypeInsertLocation = getHaplotypeCoordinateForReferenceCoordinate(hapStartInRefCoords, haplotypeCigar, refInsertLocation); if( haplotypeInsertLocation == -1 ) { // desired change falls inside deletion so don't bother creating a new haplotype return bases.clone(); } From f6905630bb2a860364ed8684513a70290272d179 Mon Sep 17 00:00:00 2001 From: Ryan Poplin Date: Mon, 5 Mar 2012 21:08:07 -0500 Subject: [PATCH 326/356] Adding Unit test for Haplotype class. Used in HC's genotype given alleles mode. --- .../sting/utils/HaplotypeUnitTest.java | 163 ++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 public/java/test/org/broadinstitute/sting/utils/HaplotypeUnitTest.java diff --git a/public/java/test/org/broadinstitute/sting/utils/HaplotypeUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/HaplotypeUnitTest.java new file mode 100644 index 0000000000..25bd7a2ebd --- /dev/null +++ b/public/java/test/org/broadinstitute/sting/utils/HaplotypeUnitTest.java @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2010 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR + * THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils; + + +import net.sf.samtools.Cigar; +import net.sf.samtools.CigarElement; +import net.sf.samtools.CigarOperator; +import org.broadinstitute.sting.BaseTest; +import org.broadinstitute.sting.utils.variantcontext.Allele; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.*; + +/** + * Basic unit test for Haplotype Class + */ +public class HaplotypeUnitTest extends BaseTest { + @BeforeClass + public void init() { + } + + @Test + public void testSimpleInsertionAllele() { + final String bases = "ACTGGTCAACTGGTCAACTGGTCAACTGGTCA"; + + final ArrayList h1CigarList = new ArrayList(); + h1CigarList.add(new CigarElement(bases.length(), CigarOperator.M)); + final Cigar h1Cigar = new Cigar(h1CigarList); + String h1bases = "AACTTCTGGTCAACTGGTCAACTGGTCAACTGGTCA"; + basicInsertTest("-", "ACTT", 1, h1Cigar, bases, h1bases); + h1bases = "ACTGGTCACTTAACTGGTCAACTGGTCAACTGGTCA"; + basicInsertTest("-", "ACTT", 7, h1Cigar, bases, h1bases); + h1bases = "ACTGGTCAACTGGTCAAACTTCTGGTCAACTGGTCA"; + basicInsertTest("-", "ACTT", 17, h1Cigar, bases, h1bases); + } + + @Test + public void testSimpleDeletionAllele() { + final String bases = "ACTGGTCAACTGGTCAACTGGTCAACTGGTCA"; + + final ArrayList h1CigarList = new ArrayList(); + h1CigarList.add(new CigarElement(bases.length(), CigarOperator.M)); + final Cigar h1Cigar = new Cigar(h1CigarList); + String h1bases = "ATCAACTGGTCAACTGGTCAACTGGTCA"; + basicInsertTest("ACTT", "-", 1, h1Cigar, bases, h1bases); + h1bases = "ACTGGTCGGTCAACTGGTCAACTGGTCA"; + basicInsertTest("ACTT", "-", 7, h1Cigar, bases, h1bases); + h1bases = "ACTGGTCAACTGGTCAATCAACTGGTCA"; + basicInsertTest("ACTT", "-", 17, h1Cigar, bases, h1bases); + } + + @Test + public void testSimpleSNPAllele() { + final String bases = "ACTGGTCAACTGGTCAACTGGTCAACTGGTCA"; + + final ArrayList h1CigarList = new ArrayList(); + h1CigarList.add(new CigarElement(bases.length(), CigarOperator.M)); + final Cigar h1Cigar = new Cigar(h1CigarList); + String h1bases = "AGTGGTCAACTGGTCAACTGGTCAACTGGTCA"; + basicInsertTest("C", "G", 1, h1Cigar, bases, h1bases); + h1bases = "ACTGGTCTACTGGTCAACTGGTCAACTGGTCA"; + basicInsertTest("A", "T", 7, h1Cigar, bases, h1bases); + h1bases = "ACTGGTCAACTGGTCAAATGGTCAACTGGTCA"; + basicInsertTest("C", "A", 17, h1Cigar, bases, h1bases); + } + + @Test + public void testComplexInsertionAllele() { + final String bases = "ATCG" + "CCGGCCGGCC" + "ATCGATCG" + "AGGGGGA" + "AGGC"; + + final ArrayList h1CigarList = new ArrayList(); + h1CigarList.add(new CigarElement(4, CigarOperator.M)); + h1CigarList.add(new CigarElement(10, CigarOperator.I)); + h1CigarList.add(new CigarElement(8, CigarOperator.M)); + h1CigarList.add(new CigarElement(3, CigarOperator.D)); + h1CigarList.add(new CigarElement(7, CigarOperator.M)); + h1CigarList.add(new CigarElement(4, CigarOperator.M)); + final Cigar h1Cigar = new Cigar(h1CigarList); + String h1bases = "AACTTTCG" + "CCGGCCGGCC" + "ATCGATCG" + "AGGGGGA" + "AGGC"; + basicInsertTest("-", "ACTT", 1, h1Cigar, bases, h1bases); + h1bases = "ATCG" + "CCGGCCGGCC" + "ATCACTTGATCG" + "AGGGGGA" + "AGGC"; + basicInsertTest("-", "ACTT", 7, h1Cigar, bases, h1bases); + h1bases = "ATCG" + "CCGGCCGGCC" + "ATCGATCG" + "AGACTTGGGGA" + "AGGC"; + basicInsertTest("-", "ACTT", 17, h1Cigar, bases, h1bases); + } + + @Test + public void testComplexDeletionAllele() { + final String bases = "ATCG" + "CCGGCCGGCC" + "ATCGATCG" + "AGGGGGA" + "AGGC"; + + final ArrayList h1CigarList = new ArrayList(); + h1CigarList.add(new CigarElement(4, CigarOperator.M)); + h1CigarList.add(new CigarElement(10, CigarOperator.I)); + h1CigarList.add(new CigarElement(8, CigarOperator.M)); + h1CigarList.add(new CigarElement(3, CigarOperator.D)); + h1CigarList.add(new CigarElement(7, CigarOperator.M)); + h1CigarList.add(new CigarElement(4, CigarOperator.M)); + final Cigar h1Cigar = new Cigar(h1CigarList); + String h1bases = "A" + "CGGCCGGCC" + "ATCGATCG" + "AGGGGGA" + "AGGC"; + basicInsertTest("ACTT", "-", 1, h1Cigar, bases, h1bases); + h1bases = "ATCG" + "CCGGCCGGCC" + "ATCG" + "AGGGGGA" + "AGGC"; + basicInsertTest("ACTT", "-", 7, h1Cigar, bases, h1bases); + h1bases = "ATCG" + "CCGGCCGGCC" + "ATCGATCG" + "AGA" + "AGGC"; + basicInsertTest("ACTT", "-", 17, h1Cigar, bases, h1bases); + } + + @Test + public void testComplexSNPAllele() { + final String bases = "ATCG" + "CCGGCCGGCC" + "ATCGATCG" + "AGGGGGA" + "AGGC"; + + final ArrayList h1CigarList = new ArrayList(); + h1CigarList.add(new CigarElement(4, CigarOperator.M)); + h1CigarList.add(new CigarElement(10, CigarOperator.I)); + h1CigarList.add(new CigarElement(8, CigarOperator.M)); + h1CigarList.add(new CigarElement(3, CigarOperator.D)); + h1CigarList.add(new CigarElement(7, CigarOperator.M)); + h1CigarList.add(new CigarElement(4, CigarOperator.M)); + final Cigar h1Cigar = new Cigar(h1CigarList); + String h1bases = "AGCG" + "CCGGCCGGCC" + "ATCGATCG" + "AGGGGGA" + "AGGC"; + basicInsertTest("T", "G", 1, h1Cigar, bases, h1bases); + h1bases = "ATCG" + "CCGGCCGGCC" + "ATCTATCG" + "AGGGGGA" + "AGGC"; + basicInsertTest("G", "T", 7, h1Cigar, bases, h1bases); + h1bases = "ATCG" + "CCGGCCGGCC" + "ATCGATCG" + "AGCGGGA" + "AGGC"; + basicInsertTest("G", "C", 17, h1Cigar, bases, h1bases); + } + + private void basicInsertTest(String ref, String alt, int loc, Cigar cigar, String hap, String newHap) { + final int INDEL_PADDING_BASE = (ref.length() == alt.length() ? 0 : 1); + final Haplotype h = new Haplotype(hap.getBytes()); + final Allele h1refAllele = Allele.create(ref, true); + final Allele h1altAllele = Allele.create(alt, false); + final Haplotype h1 = new Haplotype( h.insertAllele(h1refAllele, h1altAllele, loc - INDEL_PADDING_BASE, 0, cigar) ); + final Haplotype h1expected = new Haplotype(newHap.getBytes()); + Assert.assertEquals(h1, h1expected); + + } +} From 027843d7913aa2fc58bfcd3f6604d6caf42bfe1c Mon Sep 17 00:00:00 2001 From: Lechu Date: Sun, 4 Mar 2012 03:32:30 +0100 Subject: [PATCH 327/356] I've simply added a "library(grid)" call at the beginning of the R script generation since R 2.14.2 doesn't seem to load the "grid" package as default. I haven't tested it on previous R versions (you may edit the R version comment to be more precise if desired), but I'm almost certain that this library call shouldn't do any harm on them. Signed-off-by: Ryan Poplin --- .../gatk/walkers/variantrecalibration/VariantRecalibrator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrator.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrator.java index 7cc5b16252..3cdcf4982e 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrator.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrator.java @@ -372,6 +372,8 @@ private void createVisualizationScript( final ExpandingArrayList r stream.println("library(ggplot2)"); // For compactPDF in R 2.13+ stream.println("library(tools)"); + // For graphical functions R 2.14.2+ + stream.println("library(grid)"); createArrangeFunction( stream ); From 0702ee15871475b527ad44ab587af18378ba994c Mon Sep 17 00:00:00 2001 From: David Roazen Date: Wed, 22 Feb 2012 16:45:20 -0500 Subject: [PATCH 328/356] Public-key authorization scheme to restrict use of NO_ET -Running the GATK with the -et NO_ET or -et STDOUT options now requires a key issued by us. Our reasons for doing this, and the procedure for our users to request keys, are documented here: http://www.broadinstitute.org/gsa/wiki/index.php/Phone_home -A GATK user key is an email address plus a cryptographic signature signed using our private key, all wrapped in a GZIP container. User keys are validated using the public key we now distribute with the GATK. Our private key is kept in a secure location. -Keys are cryptographically secure in that valid keys definitely came from us and keys cannot be fabricated, however keys are not "copy-protected" in any way. -Includes private, standalone utilities to create a new GATK user key (GenerateGATKUserKey) and to create a new master public/private key pair (GenerateKeyPair). Usage of these tools will be documented on the internal wiki shortly. -Comprehensive unit/integration tests, including tests to ensure the continued integrity of the GATK master public/private key pair. -Generation of new user keys and the new unit/integration tests both require access to the GATK private key, which can only be read by members of the group "gsagit". --- build.xml | 6 + .../sting/gatk/CommandLineExecutable.java | 30 +- .../arguments/GATKArgumentCollection.java | 5 +- .../sting/gatk/phonehome/GATKRunReport.java | 9 +- .../sting/utils/crypt/CryptUtils.java | 390 ++++++++++++++++++ .../sting/utils/crypt/GATKKey.java | 349 ++++++++++++++++ .../sting/utils/exceptions/UserException.java | 36 ++ .../sting/utils/io/IOUtils.java | 172 ++++++++ .../org/broadinstitute/sting/BaseTest.java | 4 + .../org/broadinstitute/sting/WalkerTest.java | 25 +- .../sting/utils/crypt/CryptUtilsUnitTest.java | 177 ++++++++ .../utils/crypt/GATKKeyIntegrationTest.java | 156 +++++++ .../sting/utils/crypt/GATKKeyUnitTest.java | 113 +++++ .../sting/utils/io/IOUtilsUnitTest.java | 92 +++++ public/keys/GATK_public.key | Bin 0 -> 294 bytes public/packages/GATKEngine.xml | 2 + 16 files changed, 1551 insertions(+), 15 deletions(-) create mode 100644 public/java/src/org/broadinstitute/sting/utils/crypt/CryptUtils.java create mode 100644 public/java/src/org/broadinstitute/sting/utils/crypt/GATKKey.java create mode 100644 public/java/test/org/broadinstitute/sting/utils/crypt/CryptUtilsUnitTest.java create mode 100644 public/java/test/org/broadinstitute/sting/utils/crypt/GATKKeyIntegrationTest.java create mode 100644 public/java/test/org/broadinstitute/sting/utils/crypt/GATKKeyUnitTest.java create mode 100644 public/keys/GATK_public.key diff --git a/build.xml b/build.xml index 1df75cd1db..d3e25d4244 100644 --- a/build.xml +++ b/build.xml @@ -47,6 +47,7 @@ + @@ -567,6 +568,7 @@ + @@ -615,6 +617,9 @@ + + + @@ -879,6 +884,7 @@ + diff --git a/public/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java b/public/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java index 32002e0936..e5aaf23385 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java +++ b/public/java/src/org/broadinstitute/sting/gatk/CommandLineExecutable.java @@ -35,9 +35,12 @@ import org.broadinstitute.sting.gatk.phonehome.GATKRunReport; import org.broadinstitute.sting.gatk.refdata.utils.RMDTriplet; import org.broadinstitute.sting.gatk.walkers.Walker; -import org.broadinstitute.sting.utils.classloader.JVMUtils; +import org.broadinstitute.sting.utils.crypt.CryptUtils; +import org.broadinstitute.sting.utils.crypt.GATKKey; +import org.broadinstitute.sting.utils.exceptions.UserException; import org.broadinstitute.sting.utils.text.ListFileUtils; +import java.security.PublicKey; import java.util.*; /** @@ -78,6 +81,9 @@ protected int execute() throws Exception { Walker walker = engine.getWalkerByName(getAnalysisName()); try { + // Make sure a valid GATK user key is present, if required. + authorizeGATKRun(); + engine.setArguments(getArgumentCollection()); // File lists can require a bit of additional expansion. Set these explicitly by the engine. @@ -130,6 +136,28 @@ protected int execute() throws Exception { return 0; } + /** + * Authorizes this run of the GATK by checking for a valid GATK user key, if required. + * Currently, a key is required only if running with the -et NO_ET or -et STDOUT options. + */ + private void authorizeGATKRun() { + if ( getArgumentCollection().phoneHomeType == GATKRunReport.PhoneHomeOption.NO_ET || + getArgumentCollection().phoneHomeType == GATKRunReport.PhoneHomeOption.STDOUT ) { + if ( getArgumentCollection().gatkKeyFile == null ) { + throw new UserException("Running with the -et NO_ET or -et STDOUT option requires a GATK Key file. " + + "Please see http://www.broadinstitute.org/gsa/wiki/index.php/Phone_home " + + "for more information and instructions on how to obtain a key."); + } + else { + PublicKey gatkPublicKey = CryptUtils.loadGATKDistributedPublicKey(); + GATKKey gatkUserKey = new GATKKey(gatkPublicKey, getArgumentCollection().gatkKeyFile); + + if ( ! gatkUserKey.isValid() ) { + throw new UserException.KeySignatureVerificationException(getArgumentCollection().gatkKeyFile); + } + } + } + } /** * Generate the GATK run report for this walker using the current GATKEngine, if -et is enabled. diff --git a/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java index 8ec7078010..02d211a0cb 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java +++ b/public/java/src/org/broadinstitute/sting/gatk/arguments/GATKArgumentCollection.java @@ -65,9 +65,12 @@ public GATKArgumentCollection() { @Argument(fullName = "read_buffer_size", shortName = "rbs", doc="Number of reads per SAM file to buffer in memory", required = false) public Integer readBufferSize = null; - @Argument(fullName = "phone_home", shortName = "et", doc="What kind of GATK run report should we generate? Standard is the default, can be verbose or NO_ET so nothing is posted to the run repository", required = false) + @Argument(fullName = "phone_home", shortName = "et", doc="What kind of GATK run report should we generate? STANDARD is the default, can be NO_ET so nothing is posted to the run repository. Please see http://www.broadinstitute.org/gsa/wiki/index.php/Phone_home for details.", required = false) public GATKRunReport.PhoneHomeOption phoneHomeType = GATKRunReport.PhoneHomeOption.STANDARD; + @Argument(fullName = "gatk_key", shortName = "K", doc="GATK Key file. Required if running with -et NO_ET. Please see http://www.broadinstitute.org/gsa/wiki/index.php/Phone_home for details.", required = false) + public File gatkKeyFile = null; + @Argument(fullName = "read_filter", shortName = "rf", doc = "Specify filtration criteria to apply to each read individually", required = false) public List readFilters = new ArrayList(); diff --git a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java index e8627ef4c9..f1f74069fb 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java +++ b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java @@ -154,9 +154,7 @@ public enum PhoneHomeOption { /** Standard option. Writes to local repository if it can be found, or S3 otherwise */ STANDARD, /** Force output to STDOUT. For debugging only */ - STDOUT, - /** Force output to S3. For debugging only */ - AWS_S3 // todo -- remove me -- really just for testing purposes + STDOUT } private static final DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH.mm.ss"); @@ -239,11 +237,8 @@ public void postReport(PhoneHomeOption type) { case STDOUT: postReportToStream(System.out); break; - case AWS_S3: - postReportToAWSS3(); - break; default: - exceptDuringRunReport("BUG: unexcepted PhoneHomeOption "); + exceptDuringRunReport("BUG: unexpected PhoneHomeOption "); break; } } diff --git a/public/java/src/org/broadinstitute/sting/utils/crypt/CryptUtils.java b/public/java/src/org/broadinstitute/sting/utils/crypt/CryptUtils.java new file mode 100644 index 0000000000..e84b1432e0 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/crypt/CryptUtils.java @@ -0,0 +1,390 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.crypt; + +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import org.broadinstitute.sting.utils.io.IOUtils; + +import javax.crypto.Cipher; +import java.io.File; +import java.io.InputStream; +import java.security.*; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.KeySpec; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.X509EncodedKeySpec; +import java.util.Arrays; + +/** + * A set of cryptographic utility methods and constants. + * + * Contains methods to: + * + * -Create a public/private key pair + * -Read and write public/private keys to/from files/streams + * -Load the GATK master private/public keys + * -Encrypt/decrypt data + * + * Also contains constants that control the cryptographic defaults + * throughout the GATK. + * + * @author David Roazen + */ +public class CryptUtils { + + // --------------------------------------------------------------------------------- + // Constants (these control the default cryptographic settings throughout the GATK): + // --------------------------------------------------------------------------------- + + /** + * Default key length in bits of newly-created keys. 2048 bits provides a good balance between + * security and speed. + */ + public static final int DEFAULT_KEY_LENGTH = 2048; + + /** + * Default encryption algorithm to use, when none is specified. + */ + public static final String DEFAULT_ENCRYPTION_ALGORITHM = "RSA"; + + /** + * Default random-number generation algorithm to use, when none is specified. + */ + public static final String DEFAULT_RANDOM_NUMBER_GENERATION_ALGORITHM = "SHA1PRNG"; + + /** + * Name of the public key file distributed with the GATK. This file is packaged + * into the GATK jar, and we use the system ClassLoader to find it. + */ + public static final String GATK_DISTRIBUTED_PUBLIC_KEY_FILE_NAME = "GATK_public.key"; + + /** + * Location of the master copy of the GATK private key. + */ + public static final String GATK_MASTER_PRIVATE_KEY_FILE = "/humgen/gsa-hpprojects/GATK/data/gatk_master_keys/GATK_private.key"; + + /** + * Location of the master copy of the GATK public key. This file should always be the same as + * the public key file distributed with the GATK (and there are automated tests to ensure that it is). + */ + public static final String GATK_MASTER_PUBLIC_KEY_FILE = "/humgen/gsa-hpprojects/GATK/data/gatk_master_keys/GATK_public.key"; + + /** + * Directory where generated GATK user keys are stored. See the GATKKey class for more information. + */ + public static final String GATK_USER_KEY_DIRECTORY = "/humgen/gsa-hpprojects/GATK/data/gatk_user_keys/"; + + + // ----------------------- + // Utility Methods: + // ----------------------- + + /** + * Generate a new public/private key pair using the default encryption settings defined above. + * + * @return A new public/private key pair created using the default settings + */ + public static KeyPair generateKeyPair() { + return generateKeyPair(DEFAULT_KEY_LENGTH, DEFAULT_ENCRYPTION_ALGORITHM, DEFAULT_RANDOM_NUMBER_GENERATION_ALGORITHM); + } + + /** + * Generate a new public/private key pair using custom encryption settings. + * + * @param keyLength Length of the key in bits + * @param encryptionAlgorithm Encryption algorithm to use + * @param randNumberAlgorithm Random-number generation algorithm to use + * @return A new public/private key pair, created according to the specified parameters + */ + public static KeyPair generateKeyPair( int keyLength, String encryptionAlgorithm, String randNumberAlgorithm ) { + try { + KeyPairGenerator keyGen = KeyPairGenerator.getInstance(encryptionAlgorithm); + SecureRandom randomnessSource = createRandomnessSource(randNumberAlgorithm); + + keyGen.initialize(keyLength, randomnessSource); + return keyGen.generateKeyPair(); + } + catch ( NoSuchAlgorithmException e ) { + throw new ReviewedStingException(String.format("Could not find an implementation of the requested encryption algorithm %s", encryptionAlgorithm), e); + } + catch ( Exception e ) { + throw new ReviewedStingException("Error while generating key pair", e); + } + } + + /** + * Create a source of randomness using the default random-number generation algorithm. + * + * @return A randomness source that uses the default algorithm + */ + public static SecureRandom createRandomnessSource() { + return createRandomnessSource(DEFAULT_RANDOM_NUMBER_GENERATION_ALGORITHM); + } + + /** + * Create a source of randomness using a custom random-number generation algorithm. + * + * @param randAlgorithm The random-number generation algorithm to use + * @return A randomness sources that uses the specified algorithm + */ + public static SecureRandom createRandomnessSource ( String randAlgorithm ) { + try { + return SecureRandom.getInstance(randAlgorithm); + } + catch ( NoSuchAlgorithmException e ) { + throw new ReviewedStingException(String.format("Could not find an implementation of the requested random-number generation algorithm %s", randAlgorithm), e); + } + } + + /** + * Writes a public/private key pair to disk + * + * @param keyPair The key pair we're writing to disk + * @param privateKeyFile Location to write the private key + * @param publicKeyFile Location to write the public key + */ + public static void writeKeyPair ( KeyPair keyPair, File privateKeyFile, File publicKeyFile ) { + writeKey(keyPair.getPrivate(), privateKeyFile); + writeKey(keyPair.getPublic(), publicKeyFile); + } + + /** + * Writes an arbitrary key to disk + * + * @param key The key to write + * @param destination Location to write the key to + */ + public static void writeKey ( Key key, File destination ) { + IOUtils.writeByteArrayToFile(key.getEncoded(), destination); + } + + /** + * Reads in a public key created using the default encryption algorithm from a file. + * + * @param source File containing the public key + * @return The public key read + */ + public static PublicKey readPublicKey ( File source ) { + return decodePublicKey(IOUtils.readFileIntoByteArray(source), DEFAULT_ENCRYPTION_ALGORITHM); + } + + /** + * Reads in a public key created using the default encryption algorithm from a stream. + * + * @param source Stream attached to the public key + * @return The public key read + */ + public static PublicKey readPublicKey ( InputStream source ) { + return decodePublicKey(IOUtils.readStreamIntoByteArray(source), DEFAULT_ENCRYPTION_ALGORITHM); + } + + /** + * Decodes the raw bytes of a public key into a usable object. + * + * @param rawKey The encoded bytes of a public key as read from, eg., a file. The + * key must be in the standard X.509 format for a public key. + * @param encryptionAlgorithm The encryption algorithm used to create the public key + * @return The public key as a usable object + */ + public static PublicKey decodePublicKey ( byte[] rawKey, String encryptionAlgorithm ) { + try { + KeySpec keySpec = new X509EncodedKeySpec(rawKey); + KeyFactory keyFactory = KeyFactory.getInstance(encryptionAlgorithm); + return keyFactory.generatePublic(keySpec); + } + catch ( NoSuchAlgorithmException e ) { + throw new ReviewedStingException(String.format("Could not find an implementation of the requested encryption algorithm %s", encryptionAlgorithm), e); + } + catch ( InvalidKeySpecException e ) { + throw new ReviewedStingException("Unable to use X.509 key specification to decode the given key", e); + } + } + + /** + * Reads in a private key created using the default encryption algorithm from a file. + * + * @param source File containing the private key + * @return The private key read + */ + public static PrivateKey readPrivateKey ( File source ) { + return decodePrivateKey(IOUtils.readFileIntoByteArray(source), DEFAULT_ENCRYPTION_ALGORITHM); + } + + /** + * Reads in a private key created using the default encryption algorithm from a stream. + * + * @param source Stream attached to the private key + * @return The private key read + */ + public static PrivateKey readPrivateKey ( InputStream source ) { + return decodePrivateKey(IOUtils.readStreamIntoByteArray(source), DEFAULT_ENCRYPTION_ALGORITHM); + } + + /** + * Decodes the raw bytes of a private key into a usable object. + * + * @param rawKey The encoded bytes of a private key as read from, eg., a file. The + * key must be in the standard PKCS #8 format for a private key. + * @param encryptionAlgorithm The encryption algorithm used to create the private key + * @return The private key as a usable object + */ + public static PrivateKey decodePrivateKey ( byte[] rawKey, String encryptionAlgorithm ) { + try { + KeySpec keySpec = new PKCS8EncodedKeySpec(rawKey); + KeyFactory keyFactory = KeyFactory.getInstance(encryptionAlgorithm); + return keyFactory.generatePrivate(keySpec); + } + catch ( NoSuchAlgorithmException e ) { + throw new ReviewedStingException(String.format("Could not find an implementation of the requested encryption algorithm %s", encryptionAlgorithm), e); + } + catch ( InvalidKeySpecException e ) { + throw new ReviewedStingException("Unable to use the PKCS #8 key specification to decode the given key", e); + } + } + + /** + * Loads the copy of the GATK public key that is distributed with the GATK. Uses the system + * ClassLoader to locate the public key file, which should be stored at the root of the GATK + * jar file. + * + * @return The GATK public key as a usable object + */ + public static PublicKey loadGATKDistributedPublicKey() { + InputStream publicKeyInputStream = ClassLoader.getSystemResourceAsStream(GATK_DISTRIBUTED_PUBLIC_KEY_FILE_NAME); + + if ( publicKeyInputStream == null ) { + throw new ReviewedStingException(String.format("Could not locate the GATK public key %s in the classpath", + GATK_DISTRIBUTED_PUBLIC_KEY_FILE_NAME)); + } + + return readPublicKey(publicKeyInputStream); + } + + /** + * Loads the master copy of the GATK private key. You must have the appropriate UNIX permissions + * to do this! + * + * @return The GATK master private key as a usable object + */ + public static PrivateKey loadGATKMasterPrivateKey() { + return readPrivateKey(new File(GATK_MASTER_PRIVATE_KEY_FILE)); + } + + /** + * Loads the master copy of the GATK public key. This should always be the same as the + * public key distributed with the GATK returned by loadGATKDistributedPublicKey(). + * + * @return The GATK master public key as a usable object + */ + public static PublicKey loadGATKMasterPublicKey() { + return readPublicKey(new File(GATK_MASTER_PUBLIC_KEY_FILE)); + } + + /** + * Encrypts the given data using the key provided. + * + * @param data The data to encrypt, as a byte array + * @param encryptKey The key with which to encrypt the data + * @return The encrypted version of the provided data + */ + public static byte[] encryptData ( byte[] data, Key encryptKey ) { + return transformDataUsingCipher(data, encryptKey, Cipher.ENCRYPT_MODE); + } + + /** + * Decrypts the given data using the key provided. + * + * @param encryptedData Data to decrypt, as a byte array + * @param decryptKey The key with which to decrypt the data + * @return The decrypted version of the provided data + */ + public static byte[] decryptData ( byte[] encryptedData, Key decryptKey ) { + return transformDataUsingCipher(encryptedData, decryptKey, Cipher.DECRYPT_MODE); + } + + /** + * Helper method for encryption/decryption that takes data and processes it using + * the given key + * + * @param data Data to encrypt/decrypt + * @param key Key to use to encrypt/decrypt the data + * @param cipherMode Specifies whether we are encrypting or decrypting + * @return The encrypted/decrypted data + */ + private static byte[] transformDataUsingCipher ( byte[] data, Key key, int cipherMode ) { + try { + Cipher cipher = Cipher.getInstance(key.getAlgorithm()); + cipher.init(cipherMode, key); + return cipher.doFinal(data); + } + catch ( NoSuchAlgorithmException e ) { + throw new ReviewedStingException(String.format("Could not find an implementation of the requested algorithm %s", + key.getAlgorithm()), e); + } + catch ( InvalidKeyException e ) { + throw new ReviewedStingException("Key is invalid", e); + } + catch ( GeneralSecurityException e ) { + throw new ReviewedStingException("Error during encryption", e); + } + } + + /** + * Tests whether the public/private keys provided can each decrypt data encrypted by + * the other key -- ie., tests whether these two keys are part of the same public/private + * key pair. + * + * @param privateKey The private key to test + * @param publicKey The public key to test + * @return True if the keys are part of the same key pair and can decrypt each other's + * encrypted data, otherwise false. + */ + public static boolean keysDecryptEachOther ( PrivateKey privateKey, PublicKey publicKey ) { + byte[] plainText = "Test PlainText".getBytes(); + + byte[] dataEncryptedUsingPrivateKey = CryptUtils.encryptData(plainText, privateKey); + byte[] dataEncryptedUsingPublicKey = CryptUtils.encryptData(plainText, publicKey); + + byte[] privateKeyDataDecryptedWithPublicKey = CryptUtils.decryptData(dataEncryptedUsingPrivateKey, publicKey); + byte[] publicKeyDataDecryptedWithPrivateKey = CryptUtils.decryptData(dataEncryptedUsingPublicKey, privateKey); + + // Make sure we actually transformed the data during encryption: + if ( Arrays.equals(plainText, dataEncryptedUsingPrivateKey) || + Arrays.equals(plainText, dataEncryptedUsingPublicKey) || + Arrays.equals(dataEncryptedUsingPrivateKey, dataEncryptedUsingPublicKey) ) { + return false; + } + + // Make sure that we were able to recreate the original plaintext using + // both the public key on the private-key-encrypted data and the private + // key on the public-key-encrypted data: + if ( ! Arrays.equals(plainText, privateKeyDataDecryptedWithPublicKey) || + ! Arrays.equals(plainText, publicKeyDataDecryptedWithPrivateKey) ) { + return false; + } + + return true; + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/crypt/GATKKey.java b/public/java/src/org/broadinstitute/sting/utils/crypt/GATKKey.java new file mode 100644 index 0000000000..408cb56aab --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/crypt/GATKKey.java @@ -0,0 +1,349 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.crypt; + +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import org.broadinstitute.sting.utils.exceptions.UserException; +import org.broadinstitute.sting.utils.io.IOUtils; + +import java.io.*; +import java.security.*; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +/** + * Class to represent a GATK user key. + * + * A GATK user key contains an email address and a cryptographic signature. + * The signature is the SHA-1 hash of the email address encrypted using + * the GATK master private key. The GATK master public key (distributed + * with the GATK) is used to decrypt the signature and validate the key + * at the start of each GATK run that requires a key. + * + * Keys are cryptographically secure in that valid keys definitely come + * from us and cannot be fabricated, however nothing prevents keys from + * being shared between users. + * + * GATK user keys have the following on-disk format: + * + * GZIP Container: + * Email address + * NUL byte (delimiter) + * Cryptographic Signature (encrypted SHA-1 hash of email address) + * + * The key data is wrapped within a GZIP container to placate over-zealous + * email filters (since keys must often be emailed) and also to provide an + * additional integrity check via the built-in GZIP CRC. + * + * @author David Roazen + */ +public class GATKKey { + + /** + * Private key used to sign the GATK key. Required only when creating a new + * key from scratch, not when loading an existing key from disk. + */ + private PrivateKey privateKey; + + /** + * Public key used to validate the GATK key. + */ + private PublicKey publicKey; + + /** + * The user's email address, stored within the key and signed. + */ + private String emailAddress; + + /** + * The cryptographic signature of the email address. By default, this is + * the SHA-1 hash of the email address encrypted using the RSA algorithm. + */ + private byte[] signature; + + /** + * The combination of hash/encryption algorithms to use to generate the signature. + * By default this is "SHA1withRSA" + */ + private String signingAlgorithm; + + /** + * Default hash/encryption algorithms to use to sign the key. + */ + public static final String DEFAULT_SIGNING_ALGORITHM = "SHA1withRSA"; + + /** + * Byte value used to separate the email address from its signature in the key file. + */ + public static final byte GATK_KEY_SECTIONAL_DELIMITER = 0; + + + // ----------------------- + // Constructors: + // ----------------------- + + /** + * Constructor to create a new GATK key from scratch using an email address + * and public/private key pair. The private key is used for signing, and the + * public key is used to validate the newly-created key. + * + * @param privateKey Private key used to sign the new GATK key + * @param publicKey Public key used to validate the new GATK key + * @param emailAddress The user's email address, which we will store in the key and sign + */ + public GATKKey ( PrivateKey privateKey, PublicKey publicKey, String emailAddress ) { + this(privateKey, publicKey, emailAddress, DEFAULT_SIGNING_ALGORITHM); + } + + /** + * Constructor to create a new GATK key from scratch using an email address + * and public/private key pair, and additionally specify the signing algorithm + * to use. The private key is used for signing, and the public key is used to + * validate the newly-created key. + * + * @param privateKey Private key used to sign the new GATK key + * @param publicKey Public key used to validate the new GATK key + * @param emailAddress The user's email address, which we will store in the key and sign + * @param signingAlgorithm The combination of hash and encryption algorithms to use to sign the key + */ + public GATKKey ( PrivateKey privateKey, PublicKey publicKey, String emailAddress, String signingAlgorithm ) { + if ( privateKey == null || publicKey == null || emailAddress == null || emailAddress.length() == 0 || signingAlgorithm == null ) { + throw new ReviewedStingException("Cannot construct GATKKey using null/empty arguments"); + } + + this.privateKey = privateKey; + this.publicKey = publicKey; + this.emailAddress = emailAddress; + this.signingAlgorithm = signingAlgorithm; + + validateEmailAddress(); + generateSignature(); + + if ( ! isValid() ) { + throw new ReviewedStingException("Newly-generated GATK key fails validation -- this should never happen!"); + } + } + + /** + * Constructor to load an existing GATK key from a file. + * + * During loading, the key file is checked for integrity, but not cryptographic + * validity (which must be done through a subsequent call to isValid()). + * + * @param publicKey Public key that will be used to validate the loaded GATK key + * in subsequent calls to isValid() + * @param keyFile File containing the GATK key to load + */ + public GATKKey ( PublicKey publicKey, File keyFile ) { + this(publicKey, keyFile, DEFAULT_SIGNING_ALGORITHM); + } + + /** + * Constructor to load an existing GATK key from a file, and additionally specify + * the signing algorithm used to sign the key being loaded. + * + * During loading, the key file is checked for integrity, but not cryptographic + * validity (which must be done through a subsequent call to isValid()). + * + * @param publicKey Public key that will be used to validate the loaded GATK key + * in subsequent calls to isValid() + * @param keyFile File containing the GATK key to load + * @param signingAlgorithm The combination of hash and encryption algorithms used to sign the key + */ + public GATKKey ( PublicKey publicKey, File keyFile, String signingAlgorithm ) { + if ( publicKey == null || keyFile == null || signingAlgorithm == null ) { + throw new ReviewedStingException("Cannot construct GATKKey using null arguments"); + } + + this.publicKey = publicKey; + this.signingAlgorithm = signingAlgorithm; + + readKey(keyFile); + } + + // ----------------------- + // Public API Methods: + // ----------------------- + + /** + * Writes out this key to a file in the format described at the top of this class, + * encapsulating the key within a GZIP container. + * + * @param destination File to write the key to + */ + public void writeKey ( File destination ) { + try { + byte[] keyBytes = marshalKeyData(); + IOUtils.writeByteArrayToStream(keyBytes, new GZIPOutputStream(new FileOutputStream(destination))); + } + catch ( IOException e ) { + throw new UserException.CouldNotCreateOutputFile(destination, e); + } + } + + /** + * Checks whether the signature of this key is cryptographically valid (ie., can be + * decrypted by the public key to produce a valid SHA-1 hash of the email address + * in the key). + * + * @return True if the key's signature passes validation, otherwise false + */ + public boolean isValid() { + try { + Signature sig = Signature.getInstance(signingAlgorithm); + sig.initVerify(publicKey); + sig.update(emailAddress.getBytes()); + return sig.verify(signature); + } + catch ( NoSuchAlgorithmException e ) { + throw new ReviewedStingException(String.format("Signing algorithm %s not found", signingAlgorithm), e); + } + catch ( InvalidKeyException e ) { + // If the GATK public key is invalid, it's likely our problem, not the user's: + throw new ReviewedStingException(String.format("Public key %s is invalid", publicKey), e); + } + catch ( SignatureException e ) { + throw new UserException.UnreadableKeyException("Signature is invalid or signing algorithm was unable to process the input data", e); + } + } + + // ----------------------- + // Private Helper Methods: + // ----------------------- + + /** + * Helper method that creates a signature for this key using the combination of + * hash/encryption algorithms specified at construction time. + */ + private void generateSignature() { + try { + Signature sig = Signature.getInstance(signingAlgorithm); + sig.initSign(privateKey, CryptUtils.createRandomnessSource()); + sig.update(emailAddress.getBytes()); + signature = sig.sign(); + } + catch ( NoSuchAlgorithmException e ) { + throw new ReviewedStingException(String.format("Signing algorithm %s not found", signingAlgorithm), e); + } + catch ( InvalidKeyException e ) { + throw new ReviewedStingException(String.format("Private key %s is invalid", privateKey), e); + } + catch ( SignatureException e ) { + throw new ReviewedStingException(String.format("Error creating signature for email address %s", emailAddress), e); + } + } + + /** + * Helper method that reads in a GATK key from a file. Should not be called directly -- + * use the appropriate constructor above. + * + * @param source File to read the key from + */ + private void readKey ( File source ) { + try { + byte[] keyBytes = IOUtils.readStreamIntoByteArray(new GZIPInputStream(new FileInputStream(source))); + + // As a sanity check, compare the number of bytes read to the uncompressed file size + // stored in the GZIP ISIZE field. If they don't match, the key must be corrupt: + if ( keyBytes.length != IOUtils.getGZIPFileUncompressedSize(source) ) { + throw new UserException.UnreadableKeyException("Number of bytes read does not match the uncompressed size specified in the GZIP ISIZE field"); + } + + unmarshalKeyData(keyBytes); + } + catch ( FileNotFoundException e ) { + throw new UserException.CouldNotReadInputFile(source, e); + } + catch ( IOException e ) { + throw new UserException.UnreadableKeyException(source, e); + } + catch ( UserException.CouldNotReadInputFile e ) { + throw new UserException.UnreadableKeyException(source, e); + } + } + + /** + * Helper method that assembles the email address and signature into a format + * suitable for writing to disk. + * + * @return The aggregated key data, ready to be written to disk + */ + private byte[] marshalKeyData() { + byte[] emailAddressBytes = emailAddress.getBytes(); + byte[] assembledKey = new byte[emailAddressBytes.length + 1 + signature.length]; + + System.arraycopy(emailAddressBytes, 0, assembledKey, 0, emailAddressBytes.length); + assembledKey[emailAddressBytes.length] = GATK_KEY_SECTIONAL_DELIMITER; + System.arraycopy(signature, 0, assembledKey, emailAddressBytes.length + 1, signature.length); + + return assembledKey; + } + + /** + * Helper method that parses the raw key data from disk into its component + * email address and signature. Performs some basic validation in the process. + * + * @param keyBytes The raw, uncompressed key data read from disk + */ + private void unmarshalKeyData ( byte[] keyBytes ) { + int delimiterPosition = -1; + + for ( int i = 0; i < keyBytes.length; i++ ) { + if ( keyBytes[i] == GATK_KEY_SECTIONAL_DELIMITER ) { + delimiterPosition = i; + break; + } + } + + if ( delimiterPosition == -1 ) { + throw new UserException.UnreadableKeyException("Malformed GATK key contains no sectional delimiter"); + } + else if ( delimiterPosition == 0 ) { + throw new UserException.UnreadableKeyException("Malformed GATK key contains no email address"); + } + else if ( delimiterPosition == keyBytes.length - 1 ) { + throw new UserException.UnreadableKeyException("Malformed GATK key contains no signature"); + } + + byte[] emailAddressBytes = new byte[delimiterPosition]; + System.arraycopy(keyBytes, 0, emailAddressBytes, 0, delimiterPosition); + emailAddress = new String(emailAddressBytes); + + signature = new byte[keyBytes.length - delimiterPosition - 1]; + System.arraycopy(keyBytes, delimiterPosition + 1, signature, 0, keyBytes.length - delimiterPosition - 1); + } + + /** + * Helper method that ensures that the user's email address does not contain the NUL byte, which we + * reserve as a delimiter within each key file. + */ + private void validateEmailAddress() { + for ( byte b : emailAddress.getBytes() ) { + if ( b == GATK_KEY_SECTIONAL_DELIMITER ) { + throw new UserException(String.format("Email address must not contain a byte with value %d", GATK_KEY_SECTIONAL_DELIMITER)); + } + } + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java b/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java index 2ece3b077d..6cc8008d21 100755 --- a/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java +++ b/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java @@ -132,6 +132,10 @@ public CouldNotReadInputFile(File file, String message, Exception e) { public CouldNotReadInputFile(File file, Exception e) { this(file, e.getMessage()); } + + public CouldNotReadInputFile(String message) { + super(message); + } } @@ -151,6 +155,10 @@ public CouldNotCreateOutputFile(String filename, String message, Exception e) { public CouldNotCreateOutputFile(File file, Exception e) { super(String.format("Couldn't write file %s because exception %s", file.getAbsolutePath(), e.getMessage())); } + + public CouldNotCreateOutputFile(String message, Exception e) { + super(message, e); + } } public static class MissortedBAM extends UserException { @@ -319,4 +327,32 @@ public CouldNotCreateReferenceIndexFileBecauseOfLock(File f) { "and try again.", null); } } + + public static class UnreadableKeyException extends UserException { + public UnreadableKeyException ( File f, Exception e ) { + super(String.format("Key file %s cannot be read (possibly the key file is corrupt?). Error was: %s. " + + "Please see http://www.broadinstitute.org/gsa/wiki/index.php/Phone_home for help.", + f.getAbsolutePath(), e.getMessage())); + } + + public UnreadableKeyException ( String message, Exception e ) { + this(String.format("%s. Error was: %s", message, e.getMessage())); + } + + public UnreadableKeyException ( String message ) { + super(String.format("Key file cannot be read (possibly the key file is corrupt?): %s. " + + "Please see http://www.broadinstitute.org/gsa/wiki/index.php/Phone_home for help.", + message)); + } + } + + public static class KeySignatureVerificationException extends UserException { + public KeySignatureVerificationException ( File f ) { + super(String.format("The signature in key file %s failed cryptographic verification. " + + "If this key was valid in the past, it's likely been revoked. " + + "Please see http://www.broadinstitute.org/gsa/wiki/index.php/Phone_home " + + "for help.", + f.getAbsolutePath())); + } + } } diff --git a/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java b/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java index a5ba857efd..160df0e510 100644 --- a/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java @@ -29,10 +29,13 @@ import org.apache.commons.io.LineIterator; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.StingException; import org.broadinstitute.sting.utils.exceptions.UserException; import java.io.*; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.*; public class IOUtils { @@ -400,4 +403,173 @@ public static LineIterator lineIterator(File file) { public static boolean isSpecialFile(File file) { return file != null && (file.getAbsolutePath().startsWith("/dev/") || file.equals(DEV_DIR)); } + + /** + * Reads the entirety of the given file into a byte array. Uses a read buffer size of 4096 bytes. + * + * @param source File to read + * @return The contents of the file as a byte array + */ + public static byte[] readFileIntoByteArray ( File source ) { + return readFileIntoByteArray(source, 4096); + } + + /** + * Reads the entirety of the given file into a byte array using the requested read buffer size. + * + * @param source File to read + * @param readBufferSize Number of bytes to read in at one time + * @return The contents of the file as a byte array + */ + public static byte[] readFileIntoByteArray ( File source, int readBufferSize ) { + if ( source == null ) { + throw new ReviewedStingException("Source file was null"); + } + + byte[] fileContents; + + try { + fileContents = readStreamIntoByteArray(new FileInputStream(source), readBufferSize); + } + catch ( FileNotFoundException e ) { + throw new UserException.CouldNotReadInputFile(source, e); + } + + if ( fileContents.length != source.length() ) { + throw new UserException.CouldNotReadInputFile(String.format("Unable to completely read file %s: read only %d/%d bytes", + source.getAbsolutePath(), fileContents.length, source.length())); + } + + return fileContents; + } + + /** + * Reads all data from the given stream into a byte array. Uses a read buffer size of 4096 bytes. + * + * @param in Stream to read data from + * @return The contents of the stream as a byte array + */ + public static byte[] readStreamIntoByteArray ( InputStream in ) { + return readStreamIntoByteArray(in, 4096); + } + + /** + * Reads all data from the given stream into a byte array using the requested read buffer size. + * + * @param in Stream to read data from + * @param readBufferSize Number of bytes to read in at one time + * @return The contents of the stream as a byte array + */ + public static byte[] readStreamIntoByteArray ( InputStream in, int readBufferSize ) { + if ( in == null ) { + throw new ReviewedStingException("Input stream was null"); + } + else if ( readBufferSize <= 0 ) { + throw new ReviewedStingException("Read buffer size must be > 0"); + } + + // Use a fixed-size buffer for each read, but a dynamically-growing buffer + // to hold the accumulated contents of the file/stream: + byte[] readBuffer = new byte[readBufferSize]; + ByteArrayOutputStream fileBuffer = new ByteArrayOutputStream(readBufferSize * 4); + + try { + try { + int currentBytesRead; + + while ( (currentBytesRead = in.read(readBuffer, 0, readBuffer.length)) >= 0 ) { + fileBuffer.write(readBuffer, 0, currentBytesRead); + } + } + finally { + in.close(); + } + } + catch ( IOException e ) { + throw new UserException.CouldNotReadInputFile("I/O error reading from input stream", e); + } + + return fileBuffer.toByteArray(); + } + + /** + * Writes the given array of bytes to a file + * + * @param bytes Data to write + * @param destination File to write the data to + */ + public static void writeByteArrayToFile ( byte[] bytes, File destination ) { + if ( destination == null ) { + throw new ReviewedStingException("Destination file was null"); + } + + try { + writeByteArrayToStream(bytes, new FileOutputStream(destination)); + } + catch ( FileNotFoundException e ) { + throw new UserException.CouldNotCreateOutputFile(destination, e); + } + } + + /** + * Writes the given array of bytes to a stream + * + * @param bytes Data to write + * @param out Stream to write the data to + */ + public static void writeByteArrayToStream ( byte[] bytes, OutputStream out ) { + if ( bytes == null || out == null ) { + throw new ReviewedStingException("Data to write or output stream was null"); + } + + try { + try { + out.write(bytes); + } + finally { + out.close(); + } + } + catch ( IOException e ) { + throw new UserException.CouldNotCreateOutputFile("I/O error writing to output stream", e); + } + } + + /** + * Determines the uncompressed size of a GZIP file. Uses the GZIP ISIZE field in the last + * 4 bytes of the file to get this information. + * + * @param gzipFile GZIP-format file whose uncompressed size to determine + * @return The uncompressed size (in bytes) of the GZIP file + */ + public static int getGZIPFileUncompressedSize ( File gzipFile ) { + if ( gzipFile == null ) { + throw new ReviewedStingException("GZIP file to examine was null"); + } + + try { + // The GZIP ISIZE field holds the uncompressed size of the compressed data. + // It occupies the last 4 bytes of any GZIP file: + RandomAccessFile in = new RandomAccessFile(gzipFile, "r"); + in.seek(gzipFile.length() - 4); + byte[] sizeBytes = new byte[4]; + in.read(sizeBytes, 0, 4); + + ByteBuffer byteBuf = ByteBuffer.wrap(sizeBytes); + byteBuf.order(ByteOrder.LITTLE_ENDIAN); // The GZIP spec mandates little-endian byte order + int uncompressedSize = byteBuf.getInt(); + + // If the size read in is negative, we've overflowed our signed integer: + if ( uncompressedSize < 0 ) { + throw new UserException.CouldNotReadInputFile(String.format("Cannot accurately determine the uncompressed size of file %s " + + "because it's either larger than %d bytes or the GZIP ISIZE field is corrupt", + gzipFile.getAbsolutePath(), Integer.MAX_VALUE)); + } + + return uncompressedSize; + } + catch ( IOException e ) { + throw new UserException.CouldNotReadInputFile(gzipFile, e); + } + } } diff --git a/public/java/test/org/broadinstitute/sting/BaseTest.java b/public/java/test/org/broadinstitute/sting/BaseTest.java index ac3a970f97..bc4ce098be 100755 --- a/public/java/test/org/broadinstitute/sting/BaseTest.java +++ b/public/java/test/org/broadinstitute/sting/BaseTest.java @@ -6,6 +6,7 @@ import org.apache.log4j.PatternLayout; import org.apache.log4j.spi.LoggingEvent; import org.broadinstitute.sting.commandline.CommandLineUtils; +import org.broadinstitute.sting.utils.crypt.CryptUtils; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.io.IOUtils; @@ -87,6 +88,9 @@ public abstract class BaseTest { public static final File testDirFile = new File("public/testdata/"); public static final String testDir = testDirFile.getAbsolutePath() + "/"; + public static final String keysDataLocation = validationDataLocation + "keys/"; + public static final String gatkKeyFile = CryptUtils.GATK_USER_KEY_DIRECTORY + "gsamembers_broadinstitute.org.key"; + /** before the class starts up */ static { // setup a basic log configuration diff --git a/public/java/test/org/broadinstitute/sting/WalkerTest.java b/public/java/test/org/broadinstitute/sting/WalkerTest.java index ca7653b580..c9e3b6b1b7 100755 --- a/public/java/test/org/broadinstitute/sting/WalkerTest.java +++ b/public/java/test/org/broadinstitute/sting/WalkerTest.java @@ -30,6 +30,7 @@ import org.broad.tribble.Tribble; import org.broad.tribble.index.Index; import org.broad.tribble.index.IndexFactory; +import org.broadinstitute.sting.gatk.phonehome.GATKRunReport; import org.broadinstitute.sting.utils.codecs.vcf.VCFCodec; import org.broadinstitute.sting.gatk.CommandLineExecutable; import org.broadinstitute.sting.gatk.CommandLineGATK; @@ -45,7 +46,7 @@ import java.util.*; public class WalkerTest extends BaseTest { - private static final boolean ENABLE_REPORTING = false; + private static final boolean ENABLE_PHONE_HOME_FOR_TESTS = false; @BeforeMethod public void initializeRandomGenerator() { @@ -121,11 +122,19 @@ public String buildCommandLine(String... arguments) { } public class WalkerTestSpec { + + // Arguments implicitly included in all Walker command lines, unless explicitly + // disabled using the disableImplicitArgs() method below. + final String IMPLICIT_ARGS = ENABLE_PHONE_HOME_FOR_TESTS ? + String.format("-et %s", GATKRunReport.PhoneHomeOption.STANDARD) : + String.format("-et %s -K %s", GATKRunReport.PhoneHomeOption.NO_ET, gatkKeyFile); + String args = ""; int nOutputFiles = -1; List md5s = null; List exts = null; Class expectedException = null; + boolean includeImplicitArgs = true; // the default output path for the integration test private File outputFileLocation = null; @@ -159,6 +168,10 @@ public WalkerTestSpec(String args, int nOutputFiles, Class expectedException) { this.expectedException = expectedException; } + public String getArgsWithImplicitArgs() { + return args + (includeImplicitArgs ? " " + IMPLICIT_ARGS : ""); + } + public void setOutputFileLocation(File outputFileLocation) { this.outputFileLocation = outputFileLocation; } @@ -180,6 +193,9 @@ public void addAuxFile(String expectededMD5sum, File outputfile) { auxillaryFiles.put(expectededMD5sum, outputfile); } + public void disableImplicitArgs() { + includeImplicitArgs = false; + } } protected boolean parameterize() { @@ -213,7 +229,7 @@ protected Pair, List> executeTest(final String name, WalkerTe tmpFiles.add(fl); } - final String args = String.format(spec.args, tmpFiles.toArray()); + final String args = String.format(spec.getArgsWithImplicitArgs(), tmpFiles.toArray()); System.out.println(Utils.dupString('-', 80)); if ( spec.expectsException() ) { @@ -277,13 +293,10 @@ private Pair, List> executeTest(String name, File outputFileL * @param args the argument list * @param expectedException the expected exception or null */ - public static void executeTest(String name, String args, Class expectedException) { + private void executeTest(String name, String args, Class expectedException) { CommandLineGATK instance = new CommandLineGATK(); String[] command = Utils.escapeExpressions(args); - // add the logging level to each of the integration test commands - command = Utils.appendArray(command, "-et", ENABLE_REPORTING ? "STANDARD" : "NO_ET"); - // run the executable boolean gotAnException = false; try { diff --git a/public/java/test/org/broadinstitute/sting/utils/crypt/CryptUtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/crypt/CryptUtilsUnitTest.java new file mode 100644 index 0000000000..eae4486c64 --- /dev/null +++ b/public/java/test/org/broadinstitute/sting/utils/crypt/CryptUtilsUnitTest.java @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.crypt; + +import org.broadinstitute.sting.BaseTest; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import org.broadinstitute.sting.utils.exceptions.UserException; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; +import org.testng.Assert; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.security.Key; +import java.security.KeyPair; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.util.Arrays; + +public class CryptUtilsUnitTest extends BaseTest { + + @Test + public void testGenerateValidKeyPairWithDefaultSettings() { + KeyPair keyPair = CryptUtils.generateKeyPair(); + Assert.assertTrue(CryptUtils.keysDecryptEachOther(keyPair.getPrivate(), keyPair.getPublic())); + } + + @DataProvider( name = "InvalidKeyPairSettings" ) + public Object[][] invalidKeyPairSettingsDataProvider() { + return new Object[][] { + { -1, CryptUtils.DEFAULT_ENCRYPTION_ALGORITHM, CryptUtils.DEFAULT_RANDOM_NUMBER_GENERATION_ALGORITHM}, + { CryptUtils.DEFAULT_KEY_LENGTH, "Made-up algorithm", CryptUtils.DEFAULT_RANDOM_NUMBER_GENERATION_ALGORITHM}, + { CryptUtils.DEFAULT_KEY_LENGTH, CryptUtils.DEFAULT_ENCRYPTION_ALGORITHM, "Made-up algorithm"} + }; + } + + @Test( dataProvider = "InvalidKeyPairSettings", expectedExceptions = ReviewedStingException.class ) + public void testGenerateKeyPairWithInvalidSettings( int keyLength, String encryptionAlgorithm, String randomNumberGenerationAlgorithm ) { + KeyPair keyPair = CryptUtils.generateKeyPair(keyLength, encryptionAlgorithm, randomNumberGenerationAlgorithm); + } + + @Test + public void testGATKMasterKeyPairMutualDecryption() { + Assert.assertTrue(CryptUtils.keysDecryptEachOther(CryptUtils.loadGATKMasterPrivateKey(), CryptUtils.loadGATKMasterPublicKey())); + } + + @Test + public void testGATKMasterPrivateKeyWithDistributedPublicKeyMutualDecryption() { + Assert.assertTrue(CryptUtils.keysDecryptEachOther(CryptUtils.loadGATKMasterPrivateKey(), CryptUtils.loadGATKDistributedPublicKey())); + } + + @Test + public void testKeyPairWriteThenRead() { + KeyPair keyPair = CryptUtils.generateKeyPair(); + File privateKeyFile = createTempFile("testKeyPairWriteThenRead_private", "key"); + File publicKeyFile = createTempFile("testKeyPairWriteThenRead_public", "key"); + + CryptUtils.writeKeyPair(keyPair, privateKeyFile, publicKeyFile); + + assertKeysAreEqual(keyPair.getPrivate(), CryptUtils.readPrivateKey(privateKeyFile)); + assertKeysAreEqual(keyPair.getPublic(), CryptUtils.readPublicKey(publicKeyFile)); + } + + @Test + public void testPublicKeyWriteThenReadFromFile() { + File keyFile = createTempFile("testPublicKeyWriteThenReadFromFile", "key"); + PublicKey publicKey = CryptUtils.generateKeyPair().getPublic(); + + CryptUtils.writeKey(publicKey, keyFile); + + assertKeysAreEqual(publicKey, CryptUtils.readPublicKey(keyFile)); + } + + @Test + public void testPublicKeyWriteThenReadFromStream() throws IOException { + File keyFile = createTempFile("testPublicKeyWriteThenReadFromStream", "key"); + PublicKey publicKey = CryptUtils.generateKeyPair().getPublic(); + + CryptUtils.writeKey(publicKey, keyFile); + + assertKeysAreEqual(publicKey, CryptUtils.readPublicKey(new FileInputStream(keyFile))); + } + + @Test + public void testPrivateKeyWriteThenReadFromFile() { + File keyFile = createTempFile("testPrivateKeyWriteThenReadFromFile", "key"); + PrivateKey privateKey = CryptUtils.generateKeyPair().getPrivate(); + + CryptUtils.writeKey(privateKey, keyFile); + + assertKeysAreEqual(privateKey, CryptUtils.readPrivateKey(keyFile)); + } + + @Test + public void testPrivateKeyWriteThenReadFromStream() throws IOException { + File keyFile = createTempFile("testPrivateKeyWriteThenReadFromStream", "key"); + PrivateKey privateKey = CryptUtils.generateKeyPair().getPrivate(); + + CryptUtils.writeKey(privateKey, keyFile); + + assertKeysAreEqual(privateKey, CryptUtils.readPrivateKey(new FileInputStream(keyFile))); + } + + @Test( expectedExceptions = UserException.CouldNotReadInputFile.class ) + public void testReadNonExistentPublicKey() { + File nonExistentFile = new File("jdshgkdfhg.key"); + Assert.assertFalse(nonExistentFile.exists()); + + CryptUtils.readPublicKey(nonExistentFile); + } + + @Test( expectedExceptions = UserException.CouldNotReadInputFile.class ) + public void testReadNonExistentPrivateKey() { + File nonExistentFile = new File("jdshgkdfhg.key"); + Assert.assertFalse(nonExistentFile.exists()); + + CryptUtils.readPrivateKey(nonExistentFile); + } + + @Test + public void testDecodePublicKey() { + PublicKey originalKey = CryptUtils.generateKeyPair().getPublic(); + PublicKey decodedKey = CryptUtils.decodePublicKey(originalKey.getEncoded(), CryptUtils.DEFAULT_ENCRYPTION_ALGORITHM); + assertKeysAreEqual(originalKey, decodedKey); + } + + @Test + public void testDecodePrivateKey() { + PrivateKey originalKey = CryptUtils.generateKeyPair().getPrivate(); + PrivateKey decodedKey = CryptUtils.decodePrivateKey(originalKey.getEncoded(), CryptUtils.DEFAULT_ENCRYPTION_ALGORITHM); + assertKeysAreEqual(originalKey, decodedKey); + } + + @Test + public void testLoadGATKMasterPrivateKey() { + PrivateKey gatkMasterPrivateKey = CryptUtils.loadGATKMasterPrivateKey(); + } + + @Test + public void testLoadGATKMasterPublicKey() { + PublicKey gatkMasterPublicKey = CryptUtils.loadGATKMasterPublicKey(); + } + + @Test + public void testLoadGATKDistributedPublicKey() { + PublicKey gatkDistributedPublicKey = CryptUtils.loadGATKDistributedPublicKey(); + } + + private void assertKeysAreEqual( Key originalKey, Key keyFromDisk ) { + Assert.assertTrue(Arrays.equals(originalKey.getEncoded(), keyFromDisk.getEncoded())); + Assert.assertEquals(originalKey.getAlgorithm(), keyFromDisk.getAlgorithm()); + Assert.assertEquals(originalKey.getFormat(), keyFromDisk.getFormat()); + } +} diff --git a/public/java/test/org/broadinstitute/sting/utils/crypt/GATKKeyIntegrationTest.java b/public/java/test/org/broadinstitute/sting/utils/crypt/GATKKeyIntegrationTest.java new file mode 100644 index 0000000000..8fb75ef381 --- /dev/null +++ b/public/java/test/org/broadinstitute/sting/utils/crypt/GATKKeyIntegrationTest.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.crypt; + +import org.broadinstitute.sting.WalkerTest; +import org.broadinstitute.sting.gatk.phonehome.GATKRunReport; +import org.broadinstitute.sting.utils.exceptions.UserException; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.util.Arrays; + +public class GATKKeyIntegrationTest extends WalkerTest { + + public static final String BASE_COMMAND = String.format("-T PrintReads -R %s -I %s -o %%s", + testDir + "exampleFASTA.fasta", + testDir + "exampleBAM.bam"); + public static final String MD5_UPON_SUCCESSFUL_RUN = "b9dc5bf6753ca2819e70b056eaf61258"; + + + private void runGATKKeyTest ( String testName, String etArg, String keyArg, Class expectedException, String md5 ) { + String command = BASE_COMMAND + String.format(" %s %s", etArg, keyArg); + + WalkerTestSpec spec = expectedException != null ? + new WalkerTestSpec(command, 1, expectedException) : + new WalkerTestSpec(command, 1, Arrays.asList(md5)); + + spec.disableImplicitArgs(); // Turn off automatic inclusion of -et/-K args by WalkerTest + executeTest(testName, spec); + } + + @Test + public void testValidKeyNoET() { + runGATKKeyTest("testValidKeyNoET", + "-et " + GATKRunReport.PhoneHomeOption.NO_ET, + "-K " + keysDataLocation + "valid.key", + null, + MD5_UPON_SUCCESSFUL_RUN); + } + + @Test + public void testValidKeyETStdout() { + runGATKKeyTest("testValidKeyETStdout", + "-et " + GATKRunReport.PhoneHomeOption.STDOUT, + "-K " + keysDataLocation + "valid.key", + null, + MD5_UPON_SUCCESSFUL_RUN); + } + + @Test + public void testValidKeyETStandard() { + runGATKKeyTest("testValidKeyETStandard", + "", + "-K " + keysDataLocation + "valid.key", + null, + MD5_UPON_SUCCESSFUL_RUN); + } + + @Test + public void testNoKeyNoET() { + runGATKKeyTest("testNoKeyNoET", + "-et " + GATKRunReport.PhoneHomeOption.NO_ET, + "", + UserException.class, + null); + } + + @Test + public void testNoKeyETStdout() { + runGATKKeyTest("testNoKeyETStdout", + "-et " + GATKRunReport.PhoneHomeOption.STDOUT, + "", + UserException.class, + null); + } + + @Test + public void testNoKeyETStandard() { + runGATKKeyTest("testNoKeyETStandard", + "", + "", + null, + MD5_UPON_SUCCESSFUL_RUN); + } + + @Test + public void testRevokedKey() { + runGATKKeyTest("testRevokedKey", + "-et " + GATKRunReport.PhoneHomeOption.NO_ET, + "-K " + keysDataLocation + "revoked.key", + UserException.KeySignatureVerificationException.class, + null); + } + + @DataProvider(name = "CorruptKeyTestData") + public Object[][] corruptKeyDataProvider() { + return new Object[][] { + { "corrupt_empty.key", UserException.UnreadableKeyException.class }, + { "corrupt_single_byte_file.key", UserException.UnreadableKeyException.class }, + { "corrupt_random_contents.key", UserException.UnreadableKeyException.class }, + { "corrupt_single_byte_deletion.key", UserException.UnreadableKeyException.class }, + { "corrupt_single_byte_insertion.key", UserException.UnreadableKeyException.class }, + { "corrupt_single_byte_change.key", UserException.UnreadableKeyException.class }, + { "corrupt_multi_byte_deletion.key", UserException.UnreadableKeyException.class }, + { "corrupt_multi_byte_insertion.key", UserException.UnreadableKeyException.class }, + { "corrupt_multi_byte_change.key", UserException.UnreadableKeyException.class }, + { "corrupt_bad_isize_field.key", UserException.UnreadableKeyException.class }, + { "corrupt_bad_crc.key", UserException.UnreadableKeyException.class }, + { "corrupt_no_email_address.key", UserException.UnreadableKeyException.class }, + { "corrupt_no_sectional_delimiter.key", UserException.KeySignatureVerificationException.class }, + { "corrupt_no_signature.key", UserException.UnreadableKeyException.class }, + { "corrupt_bad_signature.key", UserException.KeySignatureVerificationException.class }, + { "corrupt_non_gzipped_valid_key.key", UserException.UnreadableKeyException.class } + }; + } + + @Test(dataProvider = "CorruptKeyTestData") + public void testCorruptKey ( String corruptKeyName, Class expectedException ) { + runGATKKeyTest(String.format("testCorruptKey (%s)", corruptKeyName), + "-et " + GATKRunReport.PhoneHomeOption.NO_ET, + "-K " + keysDataLocation + corruptKeyName, + expectedException, + null); + } + + @Test + public void testCorruptButNonRequiredKey() { + runGATKKeyTest("testCorruptButNonRequiredKey", + "", + "-K " + keysDataLocation + "corrupt_random_contents.key", + null, + MD5_UPON_SUCCESSFUL_RUN); + } +} diff --git a/public/java/test/org/broadinstitute/sting/utils/crypt/GATKKeyUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/crypt/GATKKeyUnitTest.java new file mode 100644 index 0000000000..5e7b07a1ea --- /dev/null +++ b/public/java/test/org/broadinstitute/sting/utils/crypt/GATKKeyUnitTest.java @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.crypt; + +import org.broadinstitute.sting.BaseTest; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import org.broadinstitute.sting.utils.exceptions.UserException; +import org.testng.annotations.Test; +import org.testng.Assert; + +import java.io.File; +import java.security.KeyPair; +import java.security.PrivateKey; +import java.security.PublicKey; + +public class GATKKeyUnitTest extends BaseTest { + + @Test + public void testCreateGATKKeyUsingMasterKeyPair() { + PrivateKey masterPrivateKey = CryptUtils.loadGATKMasterPrivateKey(); + PublicKey masterPublicKey = CryptUtils.loadGATKMasterPublicKey(); + + // We should be able to create a valid GATKKey using our master key pair: + GATKKey key = new GATKKey(masterPrivateKey, masterPublicKey, "foo@bar.com"); + Assert.assertTrue(key.isValid()); + } + + @Test + public void testCreateGATKKeyUsingMasterPrivateKeyAndDistributedPublicKey() { + PrivateKey masterPrivateKey = CryptUtils.loadGATKMasterPrivateKey(); + PublicKey distributedPublicKey = CryptUtils.loadGATKDistributedPublicKey(); + + // We should also be able to create a valid GATKKey using our master private + // key and the public key we distribute with the GATK: + GATKKey key = new GATKKey(masterPrivateKey, distributedPublicKey, "foo@bar.com"); + Assert.assertTrue(key.isValid()); + } + + @Test( expectedExceptions = ReviewedStingException.class ) + public void testKeyPairMismatch() { + KeyPair firstKeyPair = CryptUtils.generateKeyPair(); + KeyPair secondKeyPair = CryptUtils.generateKeyPair(); + + // Attempting to create a GATK Key with private and public keys that aren't part of the + // same key pair should immediately trigger a validation failure: + GATKKey key = new GATKKey(firstKeyPair.getPrivate(), secondKeyPair.getPublic(), "foo@bar.com"); + } + + @Test( expectedExceptions = ReviewedStingException.class ) + public void testEncryptionAlgorithmMismatch() { + KeyPair keyPair = CryptUtils.generateKeyPair(CryptUtils.DEFAULT_KEY_LENGTH, "DSA", CryptUtils.DEFAULT_RANDOM_NUMBER_GENERATION_ALGORITHM); + + // Attempting to use a DSA private key to create an RSA signature should throw an error: + GATKKey key = new GATKKey(keyPair.getPrivate(), keyPair.getPublic(), "foo@bar.com", "SHA1withRSA"); + } + + @Test( expectedExceptions = UserException.class ) + public void testInvalidEmailAddress() { + String emailAddressWithNulByte = new String(new byte[] { 0 }); + KeyPair keyPair = CryptUtils.generateKeyPair(); + + // Email addresses cannot contain the NUL byte, since it's used as a sectional delimiter in the key file: + GATKKey key = new GATKKey(CryptUtils.loadGATKMasterPrivateKey(), CryptUtils.loadGATKDistributedPublicKey(), + emailAddressWithNulByte); + } + + @Test + public void testCreateGATKKeyFromValidKeyFile() { + GATKKey key = new GATKKey(CryptUtils.loadGATKDistributedPublicKey(), new File(keysDataLocation + "valid.key")); + Assert.assertTrue(key.isValid()); + } + + @Test( expectedExceptions = UserException.UnreadableKeyException.class ) + public void testCreateGATKKeyFromCorruptKeyFile() { + GATKKey key = new GATKKey(CryptUtils.loadGATKDistributedPublicKey(), new File(keysDataLocation + "corrupt_random_contents.key")); + } + + @Test + public void testCreateGATKKeyFromRevokedKeyFile() { + GATKKey key = new GATKKey(CryptUtils.loadGATKDistributedPublicKey(), new File(keysDataLocation + "revoked.key")); + Assert.assertFalse(key.isValid()); + } + + @Test( expectedExceptions = UserException.CouldNotReadInputFile.class ) + public void testCreateGATKKeyFromNonExistentFile() { + File nonExistentFile = new File("ghfdkgsdhg.key"); + Assert.assertFalse(nonExistentFile.exists()); + + GATKKey key = new GATKKey(CryptUtils.loadGATKDistributedPublicKey(), nonExistentFile); + } +} diff --git a/public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java index 757e6efdf0..941d2b14c5 100644 --- a/public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java @@ -27,12 +27,18 @@ import org.apache.commons.io.FileUtils; import org.broadinstitute.sting.BaseTest; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Random; +import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; import org.testng.Assert; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class IOUtilsUnitTest extends BaseTest { @@ -230,4 +236,90 @@ public void testIsSpecialFile() { Assert.assertFalse(IOUtils.isSpecialFile(new File("/home/user/my.file"))); Assert.assertFalse(IOUtils.isSpecialFile(new File("/devfake/null"))); } + + @DataProvider( name = "ByteArrayIOTestData") + public Object[][] byteArrayIOTestDataProvider() { + return new Object[][] { + // file size, read buffer size + { 0, 4096 }, + { 1, 4096 }, + { 2000, 4096 }, + { 4095, 4096 }, + { 4096, 4096 }, + { 4097, 4096 }, + { 6000, 4096 }, + { 8191, 4096 }, + { 8192, 4096 }, + { 8193, 4096 }, + { 10000, 4096 } + }; + } + + @Test( dataProvider = "ByteArrayIOTestData" ) + public void testWriteThenReadFileIntoByteArray ( int fileSize, int readBufferSize ) throws Exception { + File tempFile = createTempFile(String.format("testWriteThenReadFileIntoByteArray_%d_%d", fileSize, readBufferSize), "tmp"); + + byte[] dataWritten = getDeterministicRandomData(fileSize); + IOUtils.writeByteArrayToFile(dataWritten, tempFile); + byte[] dataRead = IOUtils.readFileIntoByteArray(tempFile, readBufferSize); + + Assert.assertEquals(dataRead.length, dataWritten.length); + Assert.assertTrue(Arrays.equals(dataRead, dataWritten)); + } + + @Test( dataProvider = "ByteArrayIOTestData" ) + public void testWriteThenReadStreamIntoByteArray ( int fileSize, int readBufferSize ) throws Exception { + File tempFile = createTempFile(String.format("testWriteThenReadStreamIntoByteArray_%d_%d", fileSize, readBufferSize), "tmp"); + + byte[] dataWritten = getDeterministicRandomData(fileSize); + IOUtils.writeByteArrayToStream(dataWritten, new FileOutputStream(tempFile)); + byte[] dataRead = IOUtils.readStreamIntoByteArray(new FileInputStream(tempFile), readBufferSize); + + Assert.assertEquals(dataRead.length, dataWritten.length); + Assert.assertTrue(Arrays.equals(dataRead, dataWritten)); + } + + @Test( expectedExceptions = UserException.CouldNotReadInputFile.class ) + public void testReadNonExistentFileIntoByteArray() { + File nonExistentFile = new File("djfhsdkjghdfk"); + Assert.assertFalse(nonExistentFile.exists()); + + IOUtils.readFileIntoByteArray(nonExistentFile); + } + + @Test( expectedExceptions = ReviewedStingException.class ) + public void testReadNullStreamIntoByteArray() { + IOUtils.readStreamIntoByteArray(null); + } + + @Test( expectedExceptions = ReviewedStingException.class ) + public void testReadStreamIntoByteArrayInvalidBufferSize() throws Exception { + IOUtils.readStreamIntoByteArray(new FileInputStream(createTempFile("testReadStreamIntoByteArrayInvalidBufferSize", "tmp")), + -1); + } + + @Test( expectedExceptions = UserException.CouldNotCreateOutputFile.class ) + public void testWriteByteArrayToUncreatableFile() { + IOUtils.writeByteArrayToFile(new byte[]{0}, new File("/dev/foo/bar")); + } + + @Test( expectedExceptions = ReviewedStingException.class ) + public void testWriteNullByteArrayToFile() { + IOUtils.writeByteArrayToFile(null, createTempFile("testWriteNullByteArrayToFile", "tmp")); + } + + @Test( expectedExceptions = ReviewedStingException.class ) + public void testWriteByteArrayToNullStream() { + IOUtils.writeByteArrayToStream(new byte[]{0}, null); + } + + private byte[] getDeterministicRandomData ( int size ) { + GenomeAnalysisEngine.resetRandomGenerator(); + Random rand = GenomeAnalysisEngine.getRandomGenerator(); + + byte[] randomData = new byte[size]; + rand.nextBytes(randomData); + + return randomData; + } } diff --git a/public/keys/GATK_public.key b/public/keys/GATK_public.key new file mode 100644 index 0000000000000000000000000000000000000000..05cdde1c2710421e9c2524eee66f9ea206825091 GIT binary patch literal 294 zcmV+>0ondAf&n5h4F(A+hDe6@4FLfG1potr0S^E$f&mHwf&l>ly(1=10wm#ywI0GW z7t#>rw@^Wh@%LJdafJ!>EoaqBa(Pg_^`O72XYthVZUN1uv@&=X)+9g)nCpEZaBh4O|8L{(%)!MYikf8$T$)3@RaKt;0@_6(dn8yVGeOo;j0s{d60cT=_8~^|S literal 0 HcmV?d00001 diff --git a/public/packages/GATKEngine.xml b/public/packages/GATKEngine.xml index 283b5eabfa..68459f6d2d 100644 --- a/public/packages/GATKEngine.xml +++ b/public/packages/GATKEngine.xml @@ -36,6 +36,8 @@ + + From 20c1fbaf0f7b71a1ee16e463e0f464bfdd8c0158 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Tue, 6 Mar 2012 14:22:45 -0500 Subject: [PATCH 329/356] Fixing a merge (turning off downsampling on DoC) --- .../sting/gatk/walkers/coverage/DepthOfCoverageWalker.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageWalker.java index 94f9eb6c53..833dce9328 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageWalker.java @@ -29,6 +29,7 @@ import org.broadinstitute.sting.commandline.Advanced; import org.broadinstitute.sting.commandline.Argument; import org.broadinstitute.sting.commandline.Output; +import org.broadinstitute.sting.gatk.DownsampleType; import org.broadinstitute.sting.gatk.contexts.AlignmentContext; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; @@ -115,6 +116,7 @@ // todo -- formatting --> do something special for end bins in getQuantile(int[] foo), this gets mushed into the end+-1 bins for now @By(DataSource.REFERENCE) @PartitionBy(PartitionType.NONE) +@Downsample(by= DownsampleType.NONE, toCoverage=Integer.MAX_VALUE) public class DepthOfCoverageWalker extends LocusWalker>, CoveragePartitioner> implements TreeReducible { @Output @Multiplex(value=DoCOutputMultiplexer.class,arguments={"partitionTypes","refSeqGeneList","omitDepthOutput","omitIntervals","omitSampleSummary","omitLocusTable"}) From 811f871f7875d302f1876d97ed3247974afdd00b Mon Sep 17 00:00:00 2001 From: David Roazen Date: Tue, 6 Mar 2012 15:25:19 -0500 Subject: [PATCH 330/356] Do not fail tests that require the GATK private key if the user does not have permission to read it Several of the unit tests for the new key authorization feature require read access to the GATK master private key file. Since this file is only readable by members of the group gsagit, this makes it hard for people outside the group to run the test suite. Now, we skip tests that require the master private key if the private key exists (since not existing would be a true error) but is not readable by the user running the test suite Bamboo, of course, will always be able to run these tests. --- .../sting/utils/crypt/CryptUtilsUnitTest.java | 21 +++++++++++++++++++ .../sting/utils/crypt/GATKKeyUnitTest.java | 19 +++++++++++++++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/public/java/test/org/broadinstitute/sting/utils/crypt/CryptUtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/crypt/CryptUtilsUnitTest.java index eae4486c64..f5cfea148e 100644 --- a/public/java/test/org/broadinstitute/sting/utils/crypt/CryptUtilsUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/crypt/CryptUtilsUnitTest.java @@ -27,6 +27,7 @@ import org.broadinstitute.sting.BaseTest; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; +import org.testng.SkipException; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import org.testng.Assert; @@ -64,11 +65,21 @@ public void testGenerateKeyPairWithInvalidSettings( int keyLength, String encryp @Test public void testGATKMasterKeyPairMutualDecryption() { + if ( gatkPrivateKeyExistsButReadPermissionDenied() ) { + throw new SkipException(String.format("Skipping test %s because we do not have permission to read the GATK private key", + "testGATKMasterKeyPairMutualDecryption")); + } + Assert.assertTrue(CryptUtils.keysDecryptEachOther(CryptUtils.loadGATKMasterPrivateKey(), CryptUtils.loadGATKMasterPublicKey())); } @Test public void testGATKMasterPrivateKeyWithDistributedPublicKeyMutualDecryption() { + if ( gatkPrivateKeyExistsButReadPermissionDenied() ) { + throw new SkipException(String.format("Skipping test %s because we do not have permission to read the GATK private key", + "testGATKMasterPrivateKeyWithDistributedPublicKeyMutualDecryption")); + } + Assert.assertTrue(CryptUtils.keysDecryptEachOther(CryptUtils.loadGATKMasterPrivateKey(), CryptUtils.loadGATKDistributedPublicKey())); } @@ -156,6 +167,11 @@ public void testDecodePrivateKey() { @Test public void testLoadGATKMasterPrivateKey() { + if ( gatkPrivateKeyExistsButReadPermissionDenied() ) { + throw new SkipException(String.format("Skipping test %s because we do not have permission to read the GATK private key", + "testLoadGATKMasterPrivateKey")); + } + PrivateKey gatkMasterPrivateKey = CryptUtils.loadGATKMasterPrivateKey(); } @@ -174,4 +190,9 @@ private void assertKeysAreEqual( Key originalKey, Key keyFromDisk ) { Assert.assertEquals(originalKey.getAlgorithm(), keyFromDisk.getAlgorithm()); Assert.assertEquals(originalKey.getFormat(), keyFromDisk.getFormat()); } + + private boolean gatkPrivateKeyExistsButReadPermissionDenied() { + File gatkPrivateKey = new File(CryptUtils.GATK_MASTER_PRIVATE_KEY_FILE); + return gatkPrivateKey.exists() && ! gatkPrivateKey.canRead(); + } } diff --git a/public/java/test/org/broadinstitute/sting/utils/crypt/GATKKeyUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/crypt/GATKKeyUnitTest.java index 5e7b07a1ea..660f957969 100644 --- a/public/java/test/org/broadinstitute/sting/utils/crypt/GATKKeyUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/crypt/GATKKeyUnitTest.java @@ -27,6 +27,7 @@ import org.broadinstitute.sting.BaseTest; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; +import org.testng.SkipException; import org.testng.annotations.Test; import org.testng.Assert; @@ -39,6 +40,11 @@ public class GATKKeyUnitTest extends BaseTest { @Test public void testCreateGATKKeyUsingMasterKeyPair() { + if ( gatkPrivateKeyExistsButReadPermissionDenied() ) { + throw new SkipException(String.format("Skipping test %s because we do not have permission to read the GATK private key", + "testCreateGATKKeyUsingMasterKeyPair")); + } + PrivateKey masterPrivateKey = CryptUtils.loadGATKMasterPrivateKey(); PublicKey masterPublicKey = CryptUtils.loadGATKMasterPublicKey(); @@ -49,6 +55,11 @@ public void testCreateGATKKeyUsingMasterKeyPair() { @Test public void testCreateGATKKeyUsingMasterPrivateKeyAndDistributedPublicKey() { + if ( gatkPrivateKeyExistsButReadPermissionDenied() ) { + throw new SkipException(String.format("Skipping test %s because we do not have permission to read the GATK private key", + "testCreateGATKKeyUsingMasterPrivateKeyAndDistributedPublicKey")); + } + PrivateKey masterPrivateKey = CryptUtils.loadGATKMasterPrivateKey(); PublicKey distributedPublicKey = CryptUtils.loadGATKDistributedPublicKey(); @@ -82,8 +93,7 @@ public void testInvalidEmailAddress() { KeyPair keyPair = CryptUtils.generateKeyPair(); // Email addresses cannot contain the NUL byte, since it's used as a sectional delimiter in the key file: - GATKKey key = new GATKKey(CryptUtils.loadGATKMasterPrivateKey(), CryptUtils.loadGATKDistributedPublicKey(), - emailAddressWithNulByte); + GATKKey key = new GATKKey(keyPair.getPrivate(), keyPair.getPublic(), emailAddressWithNulByte); } @Test @@ -110,4 +120,9 @@ public void testCreateGATKKeyFromNonExistentFile() { GATKKey key = new GATKKey(CryptUtils.loadGATKDistributedPublicKey(), nonExistentFile); } + + private boolean gatkPrivateKeyExistsButReadPermissionDenied() { + File gatkPrivateKey = new File(CryptUtils.GATK_MASTER_PRIVATE_KEY_FILE); + return gatkPrivateKey.exists() && ! gatkPrivateKey.canRead(); + } } From b7089a3b05455aa43e68a789f69ff978988ef02d Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sun, 4 Mar 2012 18:44:11 -0500 Subject: [PATCH 331/356] Improvements to QualQuantizer; Walker to quantize quals in BAM file -- QualQuantizer now tracks merge order and level in the QualInterval for debugging / visualization -- Write out QualIntervals tree for visualization -- visualizeQuantizedQuals.R r script for basic visualization of the quality score quantization From 8d2db3f24914236dc7d1eb85a82fd7f835805a7c Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sun, 4 Mar 2012 20:07:50 -0500 Subject: [PATCH 332/356] Emit and visualize quality histogram in QualQuantizer From 5f35f5d3380d8a74086d6439c73d0c45cf5271a6 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Mon, 5 Mar 2012 15:49:56 -0500 Subject: [PATCH 333/356] QualQuantizer scales the penalty by the log of the two error rates -- Old equation was |E1 - E*| * N1. New equation is |log10(E1) - log10(E2)| * N1 which is equivalent to E1 * N1/E2 From 569be953b905405dea6faaf5b0f44b9bce60e76a Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Tue, 6 Mar 2012 16:54:59 -0500 Subject: [PATCH 334/356] Bugfix for VariantEval -- We weren't properly handling the case where a site had both a SNP and indel in both eval and comp. These would naturally pair off as SNP x SNP and INDEL x INDEL in eval, but we'd still invoke update2 with (null, SNP) and (null, INDEL) resulting most conspicously as incorrect false negatives in the validation report. -- Updating misc. integrationtests, as the counting of comps (in particular for dbSNP) was inflated because of this effect. --- .../varianteval/VariantEvalWalker.java | 69 ++++++++++++------- .../varianteval/util/VariantEvalUtils.java | 2 - .../VariantEvalIntegrationTest.java | 10 +-- 3 files changed, 50 insertions(+), 31 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java index 74291e025b..d18c7e10a3 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java @@ -1,5 +1,6 @@ package org.broadinstitute.sting.gatk.walkers.varianteval; +import com.google.java.contract.Requires; import net.sf.picard.reference.IndexedFastaSequenceFile; import net.sf.picard.util.IntervalTree; import net.sf.samtools.SAMSequenceRecord; @@ -19,11 +20,8 @@ import org.broadinstitute.sting.gatk.walkers.Window; import org.broadinstitute.sting.gatk.walkers.varianteval.evaluators.VariantEvaluator; import org.broadinstitute.sting.gatk.walkers.varianteval.stratifications.IntervalStratification; -import org.broadinstitute.sting.gatk.walkers.varianteval.stratifications.JexlExpression; import org.broadinstitute.sting.gatk.walkers.varianteval.stratifications.VariantStratifier; import org.broadinstitute.sting.gatk.walkers.varianteval.util.*; -import org.broadinstitute.sting.gatk.walkers.variantrecalibration.Tranche; -import org.broadinstitute.sting.gatk.walkers.variantrecalibration.VariantRecalibrator; import org.broadinstitute.sting.utils.GenomeLoc; import org.broadinstitute.sting.utils.GenomeLocParser; import org.broadinstitute.sting.utils.SampleUtils; @@ -32,7 +30,6 @@ import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.StingException; import org.broadinstitute.sting.utils.exceptions.UserException; -import org.broadinstitute.sting.utils.interval.IntervalUtils; import org.broadinstitute.sting.utils.variantcontext.Allele; import org.broadinstitute.sting.utils.variantcontext.VariantContext; import org.broadinstitute.sting.utils.variantcontext.VariantContextBuilder; @@ -389,9 +386,9 @@ public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentCo nec.apply(tracker, ref, context, comp, eval); } - // eval=null against all comps of different type + // eval=null against all comps of different type that aren't bound to another eval for ( VariantContext otherComp : compSet ) { - if ( otherComp != comp ) { + if ( otherComp != comp && ! compHasMatchingEval(otherComp, evalSetBySample) ) { synchronized (nec) { nec.apply(tracker, ref, context, otherComp, null); } @@ -409,6 +406,35 @@ public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentCo return null; } + @Requires({"comp != null", "evals != null"}) + private boolean compHasMatchingEval(final VariantContext comp, final Collection evals) { + // find all of the matching comps + for ( final VariantContext eval : evals ) { + if ( eval != null && doEvalAndCompMatch(comp, eval, requireStrictAlleleMatch) != EvalCompMatchType.NO_MATCH ) + return true; + } + + // nothing matched + return false; + } + + private enum EvalCompMatchType { NO_MATCH, STRICT, LENIENT } + + @Requires({"eval != null", "comp != null"}) + private EvalCompMatchType doEvalAndCompMatch(final VariantContext eval, final VariantContext comp, boolean requireStrictAlleleMatch) { + // find all of the matching comps + if ( comp.getType() != eval.getType() ) + return EvalCompMatchType.NO_MATCH; + + // find the comp which matches both the reference allele and alternate allele from eval + final Allele altEval = eval.getAlternateAlleles().size() == 0 ? null : eval.getAlternateAllele(0); + final Allele altComp = comp.getAlternateAlleles().size() == 0 ? null : comp.getAlternateAllele(0); + if ((altEval == null && altComp == null) || (altEval != null && altEval.equals(altComp) && eval.getReference().equals(comp.getReference()))) + return EvalCompMatchType.STRICT; + else + return requireStrictAlleleMatch ? EvalCompMatchType.NO_MATCH : EvalCompMatchType.LENIENT; + } + private VariantContext findMatchingComp(final VariantContext eval, final Collection comps) { // if no comps, return null if ( comps == null || comps.isEmpty() ) @@ -419,26 +445,21 @@ private VariantContext findMatchingComp(final VariantContext eval, final Collect return comps.iterator().next(); // find all of the matching comps - List matchingComps = new ArrayList(comps.size()); - for ( VariantContext comp : comps ) { - if ( comp.getType() == eval.getType() ) - matchingComps.add(comp); - } - - // if no matching comp, return null - if ( matchingComps.size() == 0 ) - return null; - - // find the comp which matches both the reference allele and alternate allele from eval - Allele altEval = eval.getAlternateAlleles().size() == 0 ? null : eval.getAlternateAllele(0); - for ( VariantContext comp : matchingComps ) { - Allele altComp = comp.getAlternateAlleles().size() == 0 ? null : comp.getAlternateAllele(0); - if ( (altEval == null && altComp == null) || (altEval != null && altEval.equals(altComp) && eval.getReference().equals(comp.getReference())) ) - return comp; + VariantContext lenientMatch = null; + for ( final VariantContext comp : comps ) { + switch ( doEvalAndCompMatch(comp, eval, requireStrictAlleleMatch) ) { + case STRICT: + return comp; + case LENIENT: + if ( lenientMatch == null ) lenientMatch = comp; + break; + case NO_MATCH: + ; + } } - // if none match, just return the first one unless we require a strict match - return (requireStrictAlleleMatch ? null : matchingComps.get(0)); + // nothing matched, just return lenientMatch, which might be null + return lenientMatch; } public Integer treeReduce(Integer lhs, Integer rhs) { return null; } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/util/VariantEvalUtils.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/util/VariantEvalUtils.java index cb44ca5222..fdeb6919dd 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/util/VariantEvalUtils.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/util/VariantEvalUtils.java @@ -417,8 +417,6 @@ public ArrayList initializeStateKeys(HashMap Date: Tue, 6 Mar 2012 16:56:17 -0500 Subject: [PATCH 335/356] Minor improvements to QuantizeQuals -- Commenting out excessive debugging in the walker -- Scala script to quantize BAM, run calibrate genotype likelihoods, call snps, and compare them to the full bam call set for 1, 2, 4, 8, 16, 32, and 64 quantization levels From 26dcec08d5d47a9c3ea2b2dd866e7593905ccbf7 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Wed, 7 Mar 2012 09:30:03 -0500 Subject: [PATCH 336/356] Bugfix for QualQuantizerUnitTest -- Enabled failing provider -- Fixed incorrect expectation in unit test From 155839e901b86ba3b8bdc56eccffcf0b4b16aa43 Mon Sep 17 00:00:00 2001 From: Christopher Hartl Date: Wed, 7 Mar 2012 09:46:43 -0500 Subject: [PATCH 337/356] Commit of VQSRV3 with Random Forest Bridge and Decision Tree engines. Lots of code duplication with the variant recalibrator in public, but also some subtle changes (i.e. to the engines and data manager). Code worked when it overwrote the stuff in public, but couldn't commit that. Will push if it works for private as well. From c4824a77f50010a0c5e5a020245a52b7a52d7ec0 Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Wed, 7 Mar 2012 10:03:10 -0500 Subject: [PATCH 338/356] Some to-do items for the reduced reads calling script From 0376d73ece4e337c1b29935f200fe9e478945d2a Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Wed, 7 Mar 2012 13:08:52 -0500 Subject: [PATCH 339/356] Improved, public version of ErrorRateByCycle -- A cleaner table output (molten). For those interested in seeing how this can be done with GATKReports look here for a nice clean example -- Integration tests -- Minor improvements to GATKReportTable with methods to getPrimaryKeys --- .../sting/gatk/report/GATKReportTable.java | 4 + .../diagnostics/ErrorRatePerCycle.java | 162 ++++++++++++++++++ .../diagnostics/ReadGroupProperties.java | 3 - .../ErrorRatePerCycleIntegrationTest.java | 41 +++++ 4 files changed, 207 insertions(+), 3 deletions(-) create mode 100755 public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ErrorRatePerCycle.java create mode 100644 public/java/test/org/broadinstitute/sting/gatk/walkers/diagnostics/ErrorRatePerCycleIntegrationTest.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java index b72b20e0b7..b59b550e1c 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java +++ b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java @@ -296,6 +296,10 @@ public boolean containsKey(Object primaryKey) { return primaryKeyColumn.contains(primaryKey); } + public Collection getPrimaryKeys() { + return Collections.unmodifiableCollection(primaryKeyColumn); + } + /** * Set the value for a given position in the table * diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ErrorRatePerCycle.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ErrorRatePerCycle.java new file mode 100755 index 0000000000..e7a2f74e23 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ErrorRatePerCycle.java @@ -0,0 +1,162 @@ +package org.broadinstitute.sting.gatk.walkers.diagnostics; + +import net.sf.samtools.SAMReadGroupRecord; +import org.broadinstitute.sting.commandline.Argument; +import org.broadinstitute.sting.commandline.Output; +import org.broadinstitute.sting.gatk.contexts.AlignmentContext; +import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; +import org.broadinstitute.sting.gatk.report.GATKReport; +import org.broadinstitute.sting.gatk.report.GATKReportTable; +import org.broadinstitute.sting.gatk.walkers.LocusWalker; +import org.broadinstitute.sting.utils.BaseUtils; +import org.broadinstitute.sting.utils.QualityUtils; +import org.broadinstitute.sting.utils.pileup.PileupElement; +import org.broadinstitute.sting.utils.sam.GATKSAMRecord; + +import java.io.PrintStream; + +/** + * Computes the read error rate per position in read (in the original 5'->3' orientation that the read had coming off the machine) + * + * Emits a GATKReport containing readgroup, cycle, mismatches, counts, qual, and error rate for each read + * group in the input BAMs FOR ONLY THE FIRST OF PAIR READS. + * + *

Input

+ *

+ * Any number of BAM files + *

+ * + *

Output

+ *

+ * GATKReport containing readgroup, cycle, mismatches, counts, qual, and error rate. + * + * For example, running this tool on the NA12878 data sets: + * + * 

+ *      ##:GATKReport.v0.2 ErrorRatePerCycle : The error rate per sequenced position in the reads
+ *      readgroup  cycle  mismatches  counts  qual  errorrate
+ *      20FUK.1        0          80   23368    25   3.47e-03
+ *      20FUK.1        1          40   23433    28   1.75e-03
+ *      20FUK.1        2          36   23453    28   1.58e-03
+ *      20FUK.1        3          26   23476    29   1.15e-03
+ *      20FUK.1        4          32   23495    29   1.40e-03
+ *      up to 101 cycles
+ *      20FUK.2        0          77   20886    24   3.73e-03
+ *      20FUK.2        1          28   20920    29   1.39e-03
+ *      20FUK.2        2          24   20931    29   1.19e-03
+ *      20FUK.2        3          30   20940    28   1.48e-03
+ *      20FUK.2        4          25   20948    29   1.24e-03
+ *      up to 101 cycles
+ *      20FUK.3        0          78   22038    24   3.58e-03
+ *      20FUK.3        1          40   22091    27   1.86e-03
+ *      20FUK.3        2          23   22108    30   1.09e-03
+ *      20FUK.3        3          36   22126    28   1.67e-03
+ *
+ *

+ * + *

Examples

+ * 
+ *    java
+ *      -jar GenomeAnalysisTK.jar
+ *      -T ErrorRatePerCycle
+ *      -I bundle/current/b37/NA12878.HiSeq.WGS.bwa.cleaned.recal.hg19.20.bam
+ *      -R bundle/current/b37/human_g1k_v37.fasta
+ *      -o example.gatkreport.txt
+ *
+ * + * @author Kiran Garimella, Mark DePristo + */ +public class ErrorRatePerCycle extends LocusWalker { + @Output PrintStream out; + @Argument(fullName="min_base_quality_score", shortName="mbq", doc="Minimum base quality required to consider a base for calling", required=false) + public Integer MIN_BASE_QUAL = 0; + @Argument(fullName="min_mapping_quality_score", shortName="mmq", doc="Minimum read mapping quality required to consider a read for calling", required=false) + public Integer MIN_MAPPING_QUAL = 20; + + private GATKReport report; + private GATKReportTable table; + private final static String reportName = "ErrorRatePerCycle"; + private final static String reportDescription = "The error rate per sequenced position in the reads"; + + /** + * Allows us to use multiple records for the key (read group x cycle) + */ + private static class TableKey implements Comparable { + final String readGroup; + final int cycle; + + private TableKey(final String readGroup, final int cycle) { + this.readGroup = readGroup; + this.cycle = cycle; + } + + @Override + public int compareTo(final TableKey tableKey) { + final int scmp = readGroup.compareTo(tableKey.readGroup); + if ( scmp == 0 ) + return Integer.valueOf(cycle).compareTo(tableKey.cycle); + else + return scmp; + } + } + + public void initialize() { + report = new GATKReport(); + report.addTable(reportName, reportDescription); + table = report.getTable(reportName); + table.addPrimaryKey("key", false); + table.addColumn("readgroup", 0); + table.addColumn("cycle", 0); + table.addColumn("mismatches", 0); + table.addColumn("counts", 0); + table.addColumn("qual", 0); + table.addColumn("errorrate", 0.0f, "%.2e"); + } + + public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) { + for ( final PileupElement p : context.getBasePileup() ) { + final GATKSAMRecord read = p.getRead(); + final int offset = p.getOffset(); + final boolean firstOfPair = ! read.getReadPairedFlag() || read.getFirstOfPairFlag(); + + if ( firstOfPair && read.getMappingQuality() >= MIN_MAPPING_QUAL && p.getQual() >= MIN_BASE_QUAL ) { + final byte readBase = p.getBase(); + final byte refBase = ref.getBase(); + final int cycle = offset; + + if ( BaseUtils.isRegularBase(readBase) && BaseUtils.isRegularBase(refBase) ) { + final TableKey key = new TableKey(read.getReadGroup().getReadGroupId(), cycle); + + if ( ! table.containsKey(key) ) { + table.set(key, "cycle", cycle); + table.set(key, "readgroup", read.getReadGroup().getReadGroupId()); + } + + table.increment(key, "counts"); + if (readBase != refBase) + table.increment(key, "mismatches"); + } + } + } + + return null; + } + + public Integer reduceInit() { return null; } + + public Integer reduce(Integer value, Integer sum) { return null; } + + public void onTraversalDone(Integer sum) { + for ( final Object key : table.getPrimaryKeys() ) { + final int mismatches = (Integer)table.get(key, "mismatches"); + final int count = (Integer)table.get(key, "counts"); + final double errorRate = (mismatches + 1) / (1.0*(count + 1)); + final int qual = QualityUtils.probToQual(1-errorRate, 0.0); + table.set(key, "qual", qual); + table.set(key, "errorrate", errorRate); + } + + report.print(out); + } +} diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java index d7a48d321e..14985907d4 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diagnostics/ReadGroupProperties.java @@ -94,9 +94,6 @@ * * @author Mark DePristo */ - - - public class ReadGroupProperties extends ReadWalker { @Output public PrintStream out; diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/diagnostics/ErrorRatePerCycleIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/diagnostics/ErrorRatePerCycleIntegrationTest.java new file mode 100644 index 0000000000..accb9c0cf0 --- /dev/null +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/diagnostics/ErrorRatePerCycleIntegrationTest.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.walkers.diagnostics; + +import org.broadinstitute.sting.WalkerTest; +import org.testng.annotations.Test; + +import java.util.Arrays; + +public class ErrorRatePerCycleIntegrationTest extends WalkerTest { + @Test + public void basicTest() { + WalkerTestSpec spec = new WalkerTestSpec( + "-T ErrorRatePerCycle -R " + b37KGReference + " -I " + b37GoodBAM + " -L 20:10,000,000-10,100,000 -o %s", + 1, + Arrays.asList("0cc212ecb6df300e321784039ff29f13")); + executeTest("ErrorRatePerCycle:", spec); + } +} \ No newline at end of file From 20d10dfa35ac39814abfc66756ac33737b32d9ab Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Wed, 7 Mar 2012 13:09:48 -0500 Subject: [PATCH 340/356] EvalQuantizedQuals now tests the impact on reduced reads as well From fbd2f04a04c63726024516ddeadf711afea7e8f5 Mon Sep 17 00:00:00 2001 From: Andrey Sivachenko Date: Wed, 7 Mar 2012 17:29:42 -0500 Subject: [PATCH 341/356] JEXL support added; intermediate commit, not yet functional --- .../indels/SomaticIndelDetectorWalker.java | 66 ++++++++++++++++++- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetectorWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetectorWalker.java index aa9ae1517e..4247ab7a2b 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetectorWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetectorWalker.java @@ -26,6 +26,10 @@ package org.broadinstitute.sting.gatk.walkers.indels; import net.sf.samtools.*; +import org.apache.commons.jexl2.Expression; +import org.apache.commons.jexl2.JexlContext; +import org.apache.commons.jexl2.JexlEngine; +import org.apache.commons.jexl2.MapContext; import org.broad.tribble.Feature; import org.broadinstitute.sting.commandline.*; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; @@ -178,6 +182,10 @@ public class SomaticIndelDetectorWalker extends ReadWalker { "GENOMIC/UTR/INTRON/CODING and with the gene name", required=false) String RefseqFileName = null; + + @Argument(shortName="filter", doc="One or more criteria to use when selecting the data", required=false) + public ArrayList FILTER_EXPRESSIONS = new ArrayList(); + //@Argument(fullName="blacklistedLanes", shortName="BL", // doc="Name of lanes (platform units) that should be ignored. Reads coming from these lanes will never be seen "+ // "by this application, so they will not contribute indels to consider and will not be counted.", required=false) @@ -221,7 +229,7 @@ public class SomaticIndelDetectorWalker extends ReadWalker { private Writer verboseWriter = null; - private static String annGenomic = "GENOMIC"; + private static String annGenomic = "GENOMIC\t"; private static String annIntron = "INTRON"; private static String annUTR = "UTR"; private static String annCoding = "CODING"; @@ -245,6 +253,32 @@ enum CallType { private long lastGenotypedPosition = -1; // last position on the currentGenotypeInterval, for which a call was already printed; // can be 1 base before lastGenotyped start + private JexlEngine jexlEngine = new JexlEngine(); + private ArrayList jexlExpressions = new ArrayList(); + + // the following arrays store indel source-specific (normal/tumor) metric names + // for fast access when populating JEXL expression contexts (see IndelPrecall.fillContext()) + private final static String[] normalMetricsCassette = new String[4]; + private final static String[] tumorMetricsCassette = new String[4]; + private final static String[] singleMetricsCassette = new String[4]; + private final static int C_COV=0; + private final static int C_CONS_CNT=1; + private final static int C_INDEL_F=2; + private final static int C_INDEL_CF=3; + static { + normalMetricsCassette[C_COV] = "N_COV"; + tumorMetricsCassette[C_COV] = "T_COV"; + singleMetricsCassette[C_COV] = "COV"; + normalMetricsCassette[C_CONS_CNT] = "N_CONS_CNT"; + tumorMetricsCassette[C_CONS_CNT] = "T_CONS_CNT"; + singleMetricsCassette[C_CONS_CNT] = "CONS_CNT"; + normalMetricsCassette[C_INDEL_F] = "N_INDEL_F"; + tumorMetricsCassette[C_INDEL_F] = "T_INDEL_F"; + singleMetricsCassette[C_INDEL_F] = "INDEL_F"; + normalMetricsCassette[C_INDEL_CF] = "N_INDEL_CF"; + tumorMetricsCassette[C_INDEL_CF] = "T_INDEL_CF"; + singleMetricsCassette[C_INDEL_CF] = "INDEL_CF"; + } // "/humgen/gsa-scr1/GATK_Data/refGene.sorted.txt" @@ -389,6 +423,17 @@ public void initialize() { vcf_writer.writeHeader(new VCFHeader(getVCFHeaderInfo(), SampleUtils.getSAMFileSamples(getToolkit().getSAMFileHeader()))) ; refData = new ReferenceDataSource(getToolkit().getArguments().referenceFile); + + // Now initialize JEXL expressions: + for ( String s : FILTER_EXPRESSIONS ) { + try { + Expression e = jexlEngine.createExpression(s); + jexlExpressions.add(e); + } catch (Exception e) { + throw new UserException.BadArgumentValue("Filter expression", "Invalid expression used (" + s + "). Please see the JEXL docs for correct syntax.") ; + } + + } } @@ -829,6 +874,15 @@ private void emit_somatic(long position, boolean force) { IndelPrecall tumorCall = new IndelPrecall(tumor_context,pos,NQS_WIDTH); IndelPrecall normalCall = new IndelPrecall(normal_context,pos,NQS_WIDTH); + JexlContext jc = new MapContext(); + tumorCall.fillContext(jc,tumorMetricsCassette); + normalCall.fillContext(jc,normalMetricsCassette); + boolean result = false; + + for ( Expression e : jexlExpressions ) { + if ( ((Boolean)e.evaluate(jc)).booleanValue() ) { result=true; break; } + } + if ( tumorCall.getCoverage() < minCoverage && ! genotype ) { if ( DEBUG ) { System.out.println("DEBUG>> Indel in tumor at "+pos+"; coverare in tumor="+tumorCall.getCoverage()+" (SKIPPED)"); @@ -1602,6 +1656,13 @@ public double getNQSRefAvQual() { public IndelVariant getVariant() { return consensus_indel; } + public void fillContext(JexlContext context,String[] cassette) { + context.set(cassette[C_INDEL_F],((double)consensus_indel_count)/total_coverage); + context.set(cassette[C_INDEL_CF],((double)consensus_indel_count/all_indel_count)); + context.set(cassette[C_COV],total_coverage); + context.set(cassette[C_CONS_CNT],consensus_indel_count); + } + public boolean isCall() { boolean ret = ( consensus_indel_count >= minIndelCount && (double)consensus_indel_count > minFraction * total_coverage && @@ -1610,8 +1671,9 @@ public boolean isCall() { " total_count="+all_indel_count+" cov="+total_coverage+ " minConsensusF="+((double)consensus_indel_count)/all_indel_count+ " minF="+((double)consensus_indel_count)/total_coverage); - return ret; + return ret; +// return true; } /** Utility method: finds the indel variant with the largest count (ie consensus) among all the observed From 497a1b059ef906d2f085b77a88732ff9eeca03d3 Mon Sep 17 00:00:00 2001 From: Andrey Sivachenko Date: Wed, 7 Mar 2012 18:34:11 -0500 Subject: [PATCH 342/356] transition to JEXL completed, old parameters setting individual cutoffs now deprecated --- .../indels/SomaticIndelDetectorWalker.java | 155 ++++++++++-------- 1 file changed, 90 insertions(+), 65 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetectorWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetectorWalker.java index 4247ab7a2b..733d32e3d0 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetectorWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetectorWalker.java @@ -151,30 +151,44 @@ public class SomaticIndelDetectorWalker extends ReadWalker { doc="Lightweight bed output file (only positions and events, no stats/annotations)", required=false) java.io.File bedOutput = null; + @Deprecated @Argument(fullName="minCoverage", shortName="minCoverage", doc="indel calls will be made only at sites with tumor coverage of minCoverage or more reads; "+ - "with --unpaired (single sample) option, this value is used for minimum sample coverage", required=false) + "with --unpaired (single sample) option, this value is used for minimum sample coverage. "+ + "INSTEAD USE: T_COV { String RefseqFileName = null; - @Argument(shortName="filter", doc="One or more criteria to use when selecting the data", required=false) + @Argument(shortName="filter", doc="One or more logical expressions. If any of the expressions is TRUE, " + + "putative indel will be discarded and nothing will be printed into the output (unless genotyping "+ + "at the specific position is explicitly requested, see -genotype). "+ + "Default: T_COV<6||N_COV<4||T_INDEL_F<0.3||T_INDEL_CF<0.7", required=false) public ArrayList FILTER_EXPRESSIONS = new ArrayList(); //@Argument(fullName="blacklistedLanes", shortName="BL", @@ -425,6 +442,13 @@ public void initialize() { refData = new ReferenceDataSource(getToolkit().getArguments().referenceFile); // Now initialize JEXL expressions: + if ( FILTER_EXPRESSIONS.size() == 0 ) { + if ( call_unpaired ) { + FILTER_EXPRESSIONS.add("COV<6||INDEL_F<0.3||INDEL_CF<0.7"); + } else { + FILTER_EXPRESSIONS.add("T_COV<6||N_COV<4||T_INDEL_F<0.3||T_INDEL_CF<0.7"); + } + } for ( String s : FILTER_EXPRESSIONS ) { try { Expression e = jexlEngine.createExpression(s); @@ -706,14 +730,26 @@ private void emit(long position, boolean force) { if ( normal_context.indelsAt(pos).size() == 0 && ! genotype ) continue; IndelPrecall normalCall = new IndelPrecall(normal_context,pos,NQS_WIDTH); + JexlContext jc = new MapContext(); + normalCall.fillContext(jc,singleMetricsCassette); + boolean discard_event = false; - if ( normalCall.getCoverage() < minCoverage && ! genotype ) { - if ( DEBUG ) { - System.out.println("DEBUG>> Indel at "+pos+"; coverare in normal="+normalCall.getCoverage()+" (SKIPPED)"); - } - continue; // low coverage + for ( Expression e : jexlExpressions ) { + if ( ((Boolean)e.evaluate(jc)).booleanValue() ) { discard_event=true; break; } } + if ( discard_event && ! genotype ) { + normal_context.indelsAt(pos).clear(); + continue; //* + } + +// if ( normalCall.getCoverage() < minCoverage && ! genotype ) { +// if ( DEBUG ) { +// System.out.println("DEBUG>> Indel at "+pos+"; coverare in normal="+normalCall.getCoverage()+" (SKIPPED)"); +// } +// continue; // low coverage +// } + if ( DEBUG ) System.out.println("DEBUG>> "+(normalCall.getAllVariantCount() == 0?"No Indel":"Indel")+" at "+pos); long left = Math.max( pos-NQS_WIDTH, normal_context.getStart() ); @@ -742,24 +778,16 @@ private void emit(long position, boolean force) { location = getToolkit().getGenomeLocParser().createGenomeLoc(location.getContig(), pos); - boolean haveCall = normalCall.isCall(); // cache the value - - if ( haveCall || genotype) { - if ( haveCall ) normalCallsMade++; - printVCFLine(vcf_writer,normalCall); - if ( bedWriter != null ) normalCall.printBedLine(bedWriter); - if ( verboseWriter != null ) printVerboseLine(verboseWriter, normalCall); - lastGenotypedPosition = pos; - } + if ( ! discard_event ) normalCallsMade++; + printVCFLine(vcf_writer,normalCall, discard_event); + if ( bedWriter != null ) normalCall.printBedLine(bedWriter); + if ( verboseWriter != null ) printVerboseLine(verboseWriter, normalCall, discard_event); + lastGenotypedPosition = pos; normal_context.indelsAt(pos).clear(); // we dealt with this indel; don't want to see it again // (we might otherwise in the case when 1) there is another indel that follows // within MISMATCH_WIDTH bases and 2) we'd need to wait for more coverage for that next indel) - -// for ( IndelVariant var : variants ) { -// System.out.print("\t"+var.getType()+"\t"+var.getBases()+"\t"+var.getCount()); -// } } if ( DEBUG ) System.out.println("DEBUG>> Actual shift to " + move_to + " ("+adjustedPosition+")"); @@ -877,24 +905,29 @@ private void emit_somatic(long position, boolean force) { JexlContext jc = new MapContext(); tumorCall.fillContext(jc,tumorMetricsCassette); normalCall.fillContext(jc,normalMetricsCassette); - boolean result = false; + boolean discard_event = false; for ( Expression e : jexlExpressions ) { - if ( ((Boolean)e.evaluate(jc)).booleanValue() ) { result=true; break; } + if ( ((Boolean)e.evaluate(jc)).booleanValue() ) { discard_event=true; break; } } - if ( tumorCall.getCoverage() < minCoverage && ! genotype ) { - if ( DEBUG ) { - System.out.println("DEBUG>> Indel in tumor at "+pos+"; coverare in tumor="+tumorCall.getCoverage()+" (SKIPPED)"); - } - continue; // low coverage - } - if ( normalCall.getCoverage() < minNormalCoverage && ! genotype ) { - if ( DEBUG ) { - System.out.println("DEBUG>> Indel in tumor at "+pos+"; coverare in normal="+normalCall.getCoverage()+" (SKIPPED)"); - } - continue; // low coverage + if ( discard_event && ! genotype ) { + tumor_context.indelsAt(pos).clear(); + normal_context.indelsAt(pos).clear(); + continue; //* } +// if ( tumorCall.getCoverage() < minCoverage && ! genotype ) { +// if ( DEBUG ) { +// System.out.println("DEBUG>> Indel in tumor at "+pos+"; coverare in tumor="+tumorCall.getCoverage()+" (SKIPPED)"); +// } +// continue; // low coverage +// } +// if ( normalCall.getCoverage() < minNormalCoverage && ! genotype ) { +// if ( DEBUG ) { +// System.out.println("DEBUG>> Indel in tumor at "+pos+"; coverare in normal="+normalCall.getCoverage()+" (SKIPPED)"); +// } +// continue; // low coverage +// } if ( DEBUG ) { System.out.print("DEBUG>> "+(tumorCall.getAllVariantCount() == 0?"No Indel":"Indel")+" in tumor, "); @@ -922,32 +955,24 @@ private void emit_somatic(long position, boolean force) { if ( right > tumor_context.getStop() ) right = tumor_context.getStop(); // if indel is too close to the end of the window but we need to emit anyway (force-shift), adjust right -// location = getToolkit().getGenomeLocParser().setStart(location,pos); -// location = getToolkit().getGenomeLocParser().setStop(location,pos); // retrieve annotation data - location = getToolkit().getGenomeLocParser().createGenomeLoc(location.getContig(),pos); // retrieve annotation data - boolean haveCall = tumorCall.isCall(); // cache the value +// boolean haveCall = tumorCall.isCall(); // cache the value - if ( haveCall || genotype ) { - if ( haveCall ) tumorCallsMade++; + if ( ! discard_event ) tumorCallsMade++; - printVCFLine(vcf_writer,normalCall,tumorCall); + printVCFLine(vcf_writer,normalCall,tumorCall,discard_event); - if ( bedWriter != null ) tumorCall.printBedLine(bedWriter); + if ( bedWriter != null ) tumorCall.printBedLine(bedWriter); + + if ( verboseWriter != null ) printVerboseLine(verboseWriter, normalCall, tumorCall, discard_event ); + lastGenotypedPosition = pos; - if ( verboseWriter != null ) printVerboseLine(verboseWriter, normalCall, tumorCall ); - lastGenotypedPosition = pos; - } tumor_context.indelsAt(pos).clear(); normal_context.indelsAt(pos).clear(); // we dealt with this indel; don't want to see it again // (we might otherwise in the case when 1) there is another indel that follows // within MISMATCH_WIDTH bases and 2) we'd need to wait for more coverage for that next indel) - -// for ( IndelVariant var : variants ) { -// System.out.print("\t"+var.getType()+"\t"+var.getBases()+"\t"+var.getCount()); -// } } if ( DEBUG ) System.out.println("DEBUG>> Actual shift to " + move_to + " ("+adjustedPosition+")"); @@ -1001,14 +1026,14 @@ private String getAnnotationString(RODRecordList ann) { } - public void printVerboseLine(Writer verboseWriter, IndelPrecall normalCall) { + public void printVerboseLine(Writer verboseWriter, IndelPrecall normalCall, boolean discard_event) { RODRecordList annotationList = (refseqIterator == null ? null : refseqIterator.seekForward(location)); String annotationString = (refseqIterator == null ? "" : getAnnotationString(annotationList)); StringBuilder fullRecord = new StringBuilder(); fullRecord.append(makeFullRecord(normalCall)); fullRecord.append(annotationString); - if ( ! normalCall.isCall() && normalCall.getVariant() != null ) fullRecord.append("\tFILTERED_NOCALL"); + if ( discard_event && normalCall.getVariant() != null ) fullRecord.append("\tFILTERED_NOCALL"); try { verboseWriter.write(fullRecord.toString()); verboseWriter.write('\n'); @@ -1019,7 +1044,7 @@ public void printVerboseLine(Writer verboseWriter, IndelPrecall normalCall) { } - public void printVerboseLine(Writer verboseWriter, IndelPrecall normalCall, IndelPrecall tumorCall) { + public void printVerboseLine(Writer verboseWriter, IndelPrecall normalCall, IndelPrecall tumorCall, boolean discard_event) { RODRecordList annotationList = (refseqIterator == null ? null : refseqIterator.seekForward(location)); String annotationString = (refseqIterator == null ? "" : getAnnotationString(annotationList)); @@ -1067,7 +1092,7 @@ public void printVerboseLine(Writer verboseWriter, IndelPrecall normalCall, Inde fullRecord.append('\t'); fullRecord.append(annotationString); - if ( ! tumorCall.isCall() && tumorCall.getVariant() != null ) fullRecord.append("\tFILTERED_NOCALL"); + if ( discard_event && tumorCall.getVariant() != null ) fullRecord.append("\tFILTERED_NOCALL"); try { verboseWriter.write(fullRecord.toString()); @@ -1077,7 +1102,7 @@ public void printVerboseLine(Writer verboseWriter, IndelPrecall normalCall, Inde } } - public void printVCFLine(VCFWriter vcf, IndelPrecall call) { + public void printVCFLine(VCFWriter vcf, IndelPrecall call, boolean discard_event) { long start = call.getPosition()-1; // If the beginning of the chromosome is deleted (possible, however unlikely), it's unclear how to proceed. @@ -1114,14 +1139,14 @@ public void printVCFLine(VCFWriter vcf, IndelPrecall call) { Map attrs = call.makeStatsAttributes(null); - if ( call.isCall() ) // we made a call - put actual het genotype here: + if ( ! discard_event ) // we made a call - put actual het genotype here: genotypes.add(new Genotype(sample,alleles,Genotype.NO_LOG10_PERROR,null,attrs,false)); else // no call: genotype is ref/ref (but alleles still contain the alt if we observed anything at all) genotypes.add(new Genotype(sample, homref_alleles,Genotype.NO_LOG10_PERROR,null,attrs,false)); } Set filters = null; - if ( call.getVariant() != null && ! call.isCall() ) { + if ( call.getVariant() != null && discard_event ) { filters = new HashSet(); filters.add("NoCall"); } @@ -1149,7 +1174,7 @@ private void fillAlleleList(List l, IndelPrecall call) { } } - public void printVCFLine(VCFWriter vcf, IndelPrecall nCall, IndelPrecall tCall) { + public void printVCFLine(VCFWriter vcf, IndelPrecall nCall, IndelPrecall tCall, boolean discard_event) { long start = tCall.getPosition()-1; long stop = start; @@ -1166,7 +1191,7 @@ public void printVCFLine(VCFWriter vcf, IndelPrecall nCall, IndelPrecall tCall) Map attrs = new HashMap(); boolean isSomatic = false; - if ( nCall.getCoverage() >= minNormalCoverage && nCall.getVariant() == null && tCall.getVariant() != null ) { + if ( nCall.getVariant() == null && tCall.getVariant() != null ) { isSomatic = true; attrs.put(VCFConstants.SOMATIC_KEY,true); } @@ -1209,7 +1234,7 @@ public void printVCFLine(VCFWriter vcf, IndelPrecall nCall, IndelPrecall tCall) } Set filters = null; - if ( tCall.getVariant() != null && ! tCall.isCall() ) { + if ( tCall.getVariant() != null && discard_event ) { filters = new HashSet(); filters.add("NoCall"); } @@ -1662,7 +1687,7 @@ public void fillContext(JexlContext context,String[] cassette) { context.set(cassette[C_COV],total_coverage); context.set(cassette[C_CONS_CNT],consensus_indel_count); } - +/* public boolean isCall() { boolean ret = ( consensus_indel_count >= minIndelCount && (double)consensus_indel_count > minFraction * total_coverage && @@ -1675,7 +1700,7 @@ public boolean isCall() { return ret; // return true; } - +*/ /** Utility method: finds the indel variant with the largest count (ie consensus) among all the observed * variants, and sets the counts of consensus observations and all observations of any indels (including non-consensus) * @param variants From 56f074b520e83e13ead8bb12d57541f12491498a Mon Sep 17 00:00:00 2001 From: Andrey Sivachenko Date: Wed, 7 Mar 2012 18:47:15 -0500 Subject: [PATCH 343/356] docs updated --- .../walkers/indels/SomaticIndelDetectorWalker.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetectorWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetectorWalker.java index 733d32e3d0..59a7bd01a2 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetectorWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetectorWalker.java @@ -75,7 +75,7 @@ *

* This is a simple, counts-and-cutoffs based tool for calling indels from aligned (preferrably MSA cleaned) sequencing * data. Supported output formats are: BED format, extended verbose output (tab separated), and VCF. The latter two outputs - * include additional statistics such as mismtaches and base qualitites around the calls, read strandness (how many + * include additional statistics such as mismatches and base qualitites around the calls, read strandness (how many * forward/reverse reads support ref and indel alleles) etc. It is highly recommended to use these additional * statistics to perform post-filtering of the calls as the tool is tuned for sensitivity (in other words it will * attempt to "call" anything remotely reasonable based only on read counts and will generate all the additional @@ -92,6 +92,16 @@ * bam tagging is not required in this case, and tags are completely ignored if still used: all input bams will be merged * on the fly and assumed to represent a single sample - this tool does not check for sample id in the read groups). * + * Which (putative) calls will make it into the output file(s) is controlled by an expression/list of expressions passed with -filter + * flag: if any of the expressions evaluate to TRUE, the site will be discarded. Otherwise the putative call and all the + * associated statistics will be printed into the output. Expressions recognize the following variables(in paired-sample + * somatic mode variables are prefixed with T_ and N_ for Tumor and Normal, e.g. N_COV and T_COV are defined instead of COV): + * COV for coverage at the site, INDEL_F for fraction of reads supporting consensus indel at the site (wrt total coverage), + * INDEL_CF for fraction of reads with consensus indel wrt all reads with an indel at the site, CONS_CNT for the count of + * reads supporting the consensus indel at the site. Conventional arithmetic and logical operations are supported. For instance, + * N_COV<4||T_COV<6||T_INDEL_F<0.3||T_INDEL_CF<0.7 instructs the tool to only output indel calls with at least 30% observed + * allelic fraction and with consensus indel making at least 70% of all indel observations at the site, and only at the sites + * where tumor coverage and normal coverage are at least 6 and 4, respectively. *

Input

*

* Tumor and normal bam files (or single sample bam file(s) in --unpaired mode). From 858acf86165fe0466529a16ddb4d0a25647a69e8 Mon Sep 17 00:00:00 2001 From: Guillermo del Angel Date: Thu, 8 Mar 2012 12:29:44 -0500 Subject: [PATCH 344/356] Hidden mode in ValidationAmplicons to support ILMN output format (same as Sequenom, with just shuffled columns) --- .../validation/ValidationAmplicons.java | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/ValidationAmplicons.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/ValidationAmplicons.java index e812fb53ae..1d7f92242a 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/ValidationAmplicons.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/ValidationAmplicons.java @@ -134,6 +134,10 @@ public class ValidationAmplicons extends RodWalker { @Argument(doc="Use Sequenom output format instead of regular FASTA",fullName="sqnm",required=false) boolean sequenomOutput = false; + @Hidden + @Argument(doc="Use ILMN output format instead of regular FASTA",fullName="ilmn",required=false) + boolean ilmnOutput = false; + GenomeLoc prevInterval; GenomeLoc allelePos; @@ -141,6 +145,7 @@ public class ValidationAmplicons extends RodWalker { StringBuilder sequence; StringBuilder rawSequence; boolean sequenceInvalid; + boolean isSiteSNP; List invReason; int indelCounter; @@ -169,6 +174,9 @@ public void initialize() { header.setSequenceDictionary(referenceDictionary); header.setSortOrder(SAMFileHeader.SortOrder.unsorted); } + + if (ilmnOutput) + out.println("Locus_Name,Target_Type,Sequence,Chromosome,Coordinate,Genome_Build_Version,Source,Source_Version,Sequence_Orientation,Plus_Minus,Force_Infinium_I"); } public Integer reduceInit() { @@ -234,6 +242,8 @@ public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentCo } rawSequence.append(Character.toUpperCase((char) ref.getBase())); } else if ( validate != null ) { + // record variant type in case it's needed in output format + isSiteSNP = (validate.isSNP()); // doesn't matter if there's a mask here too -- this is what we want to validate if ( validate.isFiltered() ) { logger.warn("You are attempting to validate a filtered site. Why are you attempting to validate a filtered site? You should not be attempting to validate a filtered site."); @@ -496,13 +506,19 @@ public void print() { if (!onlyOutputValidAmplicons || !sequenceInvalid) { String seqIdentity = sequence.toString().replace('n', 'N').replace('i', 'I').replace('d', 'D'); - if (!sequenomOutput) - out.printf(">%s %s %s%n%s%n", allelePos != null ? allelePos.toString() : "multiple", valid, probeName, seqIdentity); - else { + if (sequenomOutput) { seqIdentity = seqIdentity.replace("*",""); // identifier < 20 letters long, no * in ref allele, one line per record probeName = probeName.replace("amplicon_","a"); out.printf("%s_%s %s%n", allelePos != null ? allelePos.toString() : "multiple", probeName, seqIdentity); } + else if (ilmnOutput) { + String type = isSiteSNP?"SNP":"INDEL"; + seqIdentity = seqIdentity.replace("*",""); // no * in ref allele + out.printf("%s,%s,%s,%s,%d,37,1000G,ExomePhase1,Forward,Plus,FALSE%n",probeName,type,seqIdentity,allelePos.getContig(),allelePos.getStart()); + } + else{ + out.printf(">%s %s %s%n%s%n", allelePos != null ? allelePos.toString() : "multiple", valid, probeName, seqIdentity); + } } } } From 32dee7ed9bd0c64bdd6dae5de26b751dea05750c Mon Sep 17 00:00:00 2001 From: David Roazen Date: Thu, 8 Mar 2012 12:52:11 -0500 Subject: [PATCH 345/356] Avoid buffer underflow in GATKBAMIndex by detecting premature EOF in BAM indices GATKBAMIndex would allow an extremely confusing BufferUnderflowException to be thrown when a BAM index file was truncated or corrupt. Now, a UserException is thrown in this situation instructing the user to re-index the BAM. Added a unit test for this case as well. --- .../sting/gatk/datasources/reads/GATKBAMIndex.java | 14 +++++++++++++- .../datasources/reads/GATKBAMIndexUnitTest.java | 13 +++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/GATKBAMIndex.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/GATKBAMIndex.java index 244438a593..2bf75b0357 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/GATKBAMIndex.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/GATKBAMIndex.java @@ -25,6 +25,7 @@ import net.sf.samtools.*; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import org.broadinstitute.sting.utils.exceptions.UserException; import java.io.File; import java.io.FileInputStream; @@ -349,7 +350,18 @@ private long[] readLongs(final int count) { private void read(final ByteBuffer buffer) { try { - fileChannel.read(buffer); + int bytesExpected = buffer.limit(); + int bytesRead = fileChannel.read(buffer); + + // We have a rigid expectation here to read in exactly the number of bytes we've limited + // our buffer to -- if we read in fewer bytes than this, or encounter EOF (-1), the index + // must be truncated or otherwise corrupt: + if ( bytesRead < bytesExpected ) { + throw new UserException.MalformedFile(mFile, String.format("Premature end-of-file while reading BAM index file %s. " + + "It's likely that this file is truncated or corrupt -- " + + "Please try re-indexing the corresponding BAM file.", + mFile)); + } } catch(IOException ex) { throw new ReviewedStingException("Index: unable to read bytes from index file " + mFile); diff --git a/public/java/test/org/broadinstitute/sting/gatk/datasources/reads/GATKBAMIndexUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/datasources/reads/GATKBAMIndexUnitTest.java index fde0ce62f5..8cf9f7ce08 100644 --- a/public/java/test/org/broadinstitute/sting/gatk/datasources/reads/GATKBAMIndexUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/datasources/reads/GATKBAMIndexUnitTest.java @@ -27,6 +27,7 @@ import net.sf.samtools.SAMFileReader; import net.sf.samtools.SAMSequenceDictionary; import org.broadinstitute.sting.BaseTest; +import org.broadinstitute.sting.utils.exceptions.UserException; import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -91,4 +92,16 @@ public void testNumberAndSizeOfIndexLevels() { Assert.assertEquals(bamIndex.getLevelSize(5),37448-4681+1); } + @Test( expectedExceptions = UserException.MalformedFile.class ) + public void testDetectTruncatedBamIndexWordBoundary() { + GATKBAMIndex index = new GATKBAMIndex(new File(validationDataLocation + "truncated_at_word_boundary.bai")); + index.readReferenceSequence(0); + } + + @Test( expectedExceptions = UserException.MalformedFile.class ) + public void testDetectTruncatedBamIndexNonWordBoundary() { + GATKBAMIndex index = new GATKBAMIndex(new File(validationDataLocation + "truncated_at_non_word_boundary.bai")); + index.readReferenceSequence(0); + } + } From bc65f6326f79d09757461291c017378e6eaa6ffd Mon Sep 17 00:00:00 2001 From: David Roazen Date: Fri, 9 Mar 2012 12:13:53 -0500 Subject: [PATCH 346/356] Detect incomplete reads from BAM schedule file in BAMSchedule before they become buffer underflows This fix is similar, but distinct from the earlier fix to GATKBAMIndex. If we fail to read in a complete 3-integer bin header from the BAM schedule file that the engine has written, throw a ReviewedStingException (since this is our problem, not the user's) rather than allowing a cryptic buffer underflow error to occur. Note that this change does not fix the underlying problem in the engine, if there is one (there may be an as-yet-undetected bug in the code that writes the bam schedule). It will just make it easier for us to identify what's going wrong in the future. --- .../sting/gatk/datasources/reads/BAMSchedule.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BAMSchedule.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BAMSchedule.java index 657c70aaa3..1d8879d512 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BAMSchedule.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BAMSchedule.java @@ -407,7 +407,14 @@ public BAMScheduleEntry next() { position(currentPosition); // Read data. - read(binHeader); + int binHeaderBytesRead = read(binHeader); + + // Make sure we read in a complete bin header: + if ( binHeaderBytesRead < INT_SIZE_IN_BYTES * 3 ) { + throw new ReviewedStingException(String.format("Unable to read a complete bin header from BAM schedule file %s for BAM file %s. " + + "The BAM schedule file is likely incomplete/corrupt.", + scheduleFile.getAbsolutePath(), reader.getSamFilePath())); + } // Decode contents. binHeader.flip(); From 91d10431d395389d137a9571b3fd75d579f241a3 Mon Sep 17 00:00:00 2001 From: David Roazen Date: Fri, 9 Mar 2012 15:11:59 -0500 Subject: [PATCH 347/356] BAMScheduler: detect contigs from the interval list that are not in the merged BAM header's sequence dictionary This is a quick-and-dirty patch for the null pointer error Mauricio reported earlier. Later on we might want to address in a more general way the fact that we validate user intervals against the reference but not against the merged BAM header produced by the engine at runtime. --- .../sting/gatk/datasources/reads/BAMScheduler.java | 11 ++++++++++- .../sting/utils/exceptions/UserException.java | 13 ++++--------- .../broadinstitute/sting/utils/sam/ReadUtils.java | 8 ++++++++ 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BAMScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BAMScheduler.java index bcb726607f..fdc3d2aa73 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BAMScheduler.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BAMScheduler.java @@ -34,6 +34,8 @@ import org.broadinstitute.sting.utils.GenomeLoc; import org.broadinstitute.sting.utils.GenomeLocParser; import org.broadinstitute.sting.utils.GenomeLocSortedSet; +import org.broadinstitute.sting.utils.exceptions.UserException; +import org.broadinstitute.sting.utils.sam.ReadUtils; import java.util.*; @@ -245,7 +247,14 @@ private BAMScheduleEntry getNextOverlappingBAMScheduleEntry(final GenomeLoc curr // This will ensure that if the two sets of contigs don't quite match (b36 male vs female ref, hg19 Epstein-Barr), then // we'll be using the correct contig index for the BAMs. // TODO: Warning: assumes all BAMs use the same sequence dictionary! Get around this with contig aliasing. - final int currentContigIndex = dataSource.getHeader().getSequence(currentLocus.getContig()).getSequenceIndex(); + SAMSequenceRecord currentContigSequenceRecord = dataSource.getHeader().getSequence(currentLocus.getContig()); + if ( currentContigSequenceRecord == null ) { + throw new UserException(String.format("Contig %s not present in sequence dictionary for merged BAM header: %s", + currentLocus.getContig(), + ReadUtils.prettyPrintSequenceRecords(dataSource.getHeader().getSequenceDictionary()))); + } + + final int currentContigIndex = currentContigSequenceRecord.getSequenceIndex(); // Stale reference sequence or first invocation. (Re)create the binTreeIterator. if(lastReferenceSequenceLoaded == null || lastReferenceSequenceLoaded != currentContigIndex) { diff --git a/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java b/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java index 6cc8008d21..d625cec202 100755 --- a/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java +++ b/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java @@ -30,6 +30,7 @@ import net.sf.samtools.SAMSequenceRecord; import org.broadinstitute.sting.utils.GenomeLoc; import org.broadinstitute.sting.utils.help.DocumentedGATKFeature; +import org.broadinstitute.sting.utils.sam.ReadUtils; import org.broadinstitute.sting.utils.variantcontext.VariantContext; import java.io.File; @@ -273,7 +274,7 @@ public DeprecatedArgument(String param, String doc) { public static class IncompatibleSequenceDictionaries extends UserException { public IncompatibleSequenceDictionaries(String message, String name1, SAMSequenceDictionary dict1, String name2, SAMSequenceDictionary dict2) { super(String.format("Input files %s and %s have incompatible contigs: %s.\n %s contigs = %s\n %s contigs = %s", - name1, name2, message, name1, prettyPrintSequenceRecords(dict1), name2, prettyPrintSequenceRecords(dict2))); + name1, name2, message, name1, ReadUtils.prettyPrintSequenceRecords(dict1), name2, ReadUtils.prettyPrintSequenceRecords(dict2))); } } @@ -284,17 +285,11 @@ public LexicographicallySortedSequenceDictionary(String name, SAMSequenceDiction + "\nThis is because all distributed GATK resources are sorted in karyotypic order, and your processing will fail when you need to use these files." + "\nYou can use the ReorderSam utility to fix this problem: http://www.broadinstitute.org/gsa/wiki/index.php/ReorderSam" + "\n %s contigs = %s", - name, name, prettyPrintSequenceRecords(dict))); + name, name, ReadUtils.prettyPrintSequenceRecords(dict))); } } - private static String prettyPrintSequenceRecords(SAMSequenceDictionary sequenceDictionary) { - String[] sequenceRecordNames = new String[sequenceDictionary.size()]; - int sequenceRecordIndex = 0; - for (SAMSequenceRecord sequenceRecord : sequenceDictionary.getSequences()) - sequenceRecordNames[sequenceRecordIndex++] = sequenceRecord.getSequenceName(); - return Arrays.deepToString(sequenceRecordNames); - } + public static class MissingWalker extends UserException { public MissingWalker(String walkerName, String message) { diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java b/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java index d1e3ce26b4..91389f0bff 100755 --- a/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java @@ -648,4 +648,12 @@ else if (cigarElement.getOperator() != CigarOperator.HARD_CLIP && cigarElement.g } return new Pair>, HashMap>(locusToReadMap, readToLocusMap); } + + public static String prettyPrintSequenceRecords ( SAMSequenceDictionary sequenceDictionary ) { + String[] sequenceRecordNames = new String[sequenceDictionary.size()]; + int sequenceRecordIndex = 0; + for (SAMSequenceRecord sequenceRecord : sequenceDictionary.getSequences()) + sequenceRecordNames[sequenceRecordIndex++] = sequenceRecord.getSequenceName(); + return Arrays.deepToString(sequenceRecordNames); + } } From 8158348e0188f9581a171918003e165c085180f0 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Wed, 7 Mar 2012 17:15:36 -0500 Subject: [PATCH 348/356] Prints xlim = 30 and xlim = 99 in CalibrateGenotypeLikelihoods.R From 1011f3862ba7ac3d7b4cebb4f28e1603c84be6f1 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Thu, 8 Mar 2012 08:57:29 -0500 Subject: [PATCH 349/356] CalibrateGenotypeLikelihoods now emits the position of the variant for debugging -- Refactored some duplicated code (FYI, code duplication = root of all evil) into shared functions -- Added long-missing integrationtests -- CHRIS/RYAN -- it would be very good to add an integration test covering external VCF files as I believe we rely on this functionality and it's not tested at all --- public/java/test/org/broadinstitute/sting/BaseTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/java/test/org/broadinstitute/sting/BaseTest.java b/public/java/test/org/broadinstitute/sting/BaseTest.java index bc4ce098be..e33f6717a0 100755 --- a/public/java/test/org/broadinstitute/sting/BaseTest.java +++ b/public/java/test/org/broadinstitute/sting/BaseTest.java @@ -61,6 +61,8 @@ public abstract class BaseTest { public static final String annotationDataLocation = GATKDataLocation + "Annotations/"; public static final String b37GoodBAM = validationDataLocation + "/CEUTrio.HiSeq.b37.chr20.10_11mb.bam"; + public static final String b37GoodNA12878BAM = validationDataLocation + "/NA12878.HiSeq.WGS.bwa.cleaned.recal.hg19.20.bam"; + public static final String b37_NA12878_OMNI = validationDataLocation + "/NA12878.omni.vcf"; public static final String refseqAnnotationLocation = annotationDataLocation + "refseq/"; public static final String hg18Refseq = refseqAnnotationLocation + "refGene-big-table-hg18.txt"; From 3ba2e5667c30190096504ebe6d505c78d9eee3d9 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Thu, 8 Mar 2012 09:43:24 -0500 Subject: [PATCH 350/356] CalibrateGenotypesLikelihoods include pOfDGivenD now --- .../java/src/org/broadinstitute/sting/utils/QualityUtils.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/public/java/src/org/broadinstitute/sting/utils/QualityUtils.java b/public/java/src/org/broadinstitute/sting/utils/QualityUtils.java index 9722f901b4..7756ac71b3 100755 --- a/public/java/src/org/broadinstitute/sting/utils/QualityUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/QualityUtils.java @@ -10,6 +10,8 @@ */ public class QualityUtils { public final static byte MAX_QUAL_SCORE = SAMUtils.MAX_PHRED_SCORE; + public final static double ERROR_RATE_OF_MAX_QUAL_SCORE = qualToErrorProbRaw(MAX_QUAL_SCORE); + public final static double MIN_REASONABLE_ERROR = 0.0001; public final static byte MAX_REASONABLE_Q_SCORE = 40; public final static byte MIN_USABLE_Q_SCORE = 6; From fceb2bf25bf00f7a6acbd00ad38bdf93e6c3dd7c Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Thu, 8 Mar 2012 10:11:16 -0500 Subject: [PATCH 351/356] Updating CalibrateGenotypeLikelihoods.R to display Q93 not filter them out From e2c62572f9cb5809a24b5ec4e3025cf821fa435b Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Thu, 8 Mar 2012 11:40:23 -0500 Subject: [PATCH 352/356] Further upgrades to CalibrateGenotypeLikelihoods.R - Uses modified yates correction of e + 1 / n + 2 to estimate error rates - Now shows ALL and per read group information - Better limits on diff plots so we can see more information From 4b404cae4843bd3e9e5114bfbf22356412128773 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 9 Mar 2012 12:09:20 -0500 Subject: [PATCH 353/356] Final evaluation script for quantizing quality scores From bd883031a44203bce40422a3cb1f1d80430ce9fd Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 9 Mar 2012 15:59:27 -0500 Subject: [PATCH 354/356] Final version of QualQuantizer -- Docs everywhere -- Contracts everywhere -- More unit tests -- Better error checking -- Marginally nicer interface to QuantizeQualsWalker From 1ee46e5c06a9e364e8d3cae77e4d945657ff6d5b Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sat, 10 Mar 2012 20:27:14 -0500 Subject: [PATCH 355/356] Collect only the bare essentials in the GATKRunReport Now looks like: D7D31ULwTSxlAwnEOSmW6Z4PawXwMxEz 2012/03/10 20.21.19 2012/03/10 20.21.19 0 CountReads 1.4-483-g63ecdb2 85000192 129957888 depristo 10.0.1.10 Apple Inc.-1.6.0_26 Mac OS X-x86_64 105 No longer capturing command line or directory information, to minimize people's concerns with phone home and privacy --- .../sting/gatk/phonehome/GATKRunReport.java | 27 +------------------ 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java index f1f74069fb..bc7d5c6ca8 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java +++ b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java @@ -90,19 +90,12 @@ public class GATKRunReport { protected static Logger logger = Logger.getLogger(GATKRunReport.class); - // the listing of the fields is somewhat important; this is the order that the simple XML will output them - @ElementList(required = true, name = "gatk_header_Information") - private List mGATKHeader; - @Element(required = false, name = "id") private final String id; @Element(required = false, name = "exception") private final ExceptionToXML mException; - @Element(required = true, name = "working_directory") - private String currentPath; - @Element(required = true, name = "start_time") private String startTime = "ND"; @@ -112,9 +105,6 @@ public class GATKRunReport { @Element(required = true, name = "run_time") private long runTime = 0; - @Element(required = true, name = "command_line") - private String cmdLine = "COULD NOT BE DETERMINED"; - @Element(required = true, name = "walker_name") private String walkerName; @@ -127,9 +117,6 @@ public class GATKRunReport { @Element(required = true, name = "max_memory") private long maxMemory; - @Element(required = true, name = "java_tmp_directory") - private String tmpDir; - @Element(required = true, name = "user_name") private String userName; @@ -145,9 +132,6 @@ public class GATKRunReport { @Element(required = true, name = "iterations") private long nIterations; - @Element(required = true, name = "reads") - private long nReads; - public enum PhoneHomeOption { /** Disable phone home */ NO_ET, @@ -172,15 +156,8 @@ public GATKRunReport(Walker walker, Exception e, GenomeAnalysisEngine engin logger.debug("Aggregating data for run report"); - mGATKHeader = CommandLineGATK.createApplicationHeader(); - currentPath = System.getProperty("user.dir"); - // what did we run? id = org.apache.commons.lang.RandomStringUtils.randomAlphanumeric(32); - try { - cmdLine = engine.createApproximateCommandLineArgumentString(engine, walker); - } catch (Exception ignore) { } - walkerName = engine.getWalkerName(walker.getClass()); svnVersion = CommandLineGATK.getVersionNumber(); @@ -191,7 +168,6 @@ public GATKRunReport(Walker walker, Exception e, GenomeAnalysisEngine engin startTime = dateFormat.format(engine.getStartTime()); runTime = (end.getTime() - engine.getStartTime().getTime()) / 1000L; // difference in seconds } - tmpDir = System.getProperty("java.io.tmpdir"); // deal with memory usage Runtime.getRuntime().gc(); // call GC so totalMemory is ~ used memory @@ -202,12 +178,11 @@ public GATKRunReport(Walker walker, Exception e, GenomeAnalysisEngine engin if ( engine.getCumulativeMetrics() != null ) { // it's possible we aborted so early that these data structures arent initialized nIterations = engine.getCumulativeMetrics().getNumIterations(); - nReads = engine.getCumulativeMetrics().getNumReadsSeen(); } // user and hostname -- information about the runner of the GATK userName = System.getProperty("user.name"); - hostName = "unknown"; // resolveHostname(); + hostName = Utils.resolveHostname(); // basic java information java = Utils.join("-", Arrays.asList(System.getProperty("java.vendor"), System.getProperty("java.version"))); From b4749757f81f099dd1086f867de02fb1ebd50f5c Mon Sep 17 00:00:00 2001 From: Eric Banks Date: Mon, 12 Mar 2012 01:07:07 -0400 Subject: [PATCH 356/356] Fixes for SLOD: 1) didn't work properly for multi-allelics (randomly chose an allele, possibly one that wasn't genotyped in the full context); 2) in cases when there were more alt alleles than the max allowed and the user is calculating SB, we would recompute the best alt alleles(s); 3) for some reason, we were recomputing the LOD for the full context when we'd already done that. Given that this passes integration tests on my end, this should be the last commit before the release. --- .../GenotypeLikelihoodsCalculationModel.java | 33 ++++++++++--------- ...elGenotypeLikelihoodsCalculationModel.java | 15 +++++---- ...NPGenotypeLikelihoodsCalculationModel.java | 6 ++-- .../genotyper/UnifiedGenotyperEngine.java | 14 ++++---- 4 files changed, 34 insertions(+), 34 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java index ace780dd0f..fb2428258a 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsCalculationModel.java @@ -38,6 +38,7 @@ import org.broadinstitute.sting.utils.variantcontext.Allele; import org.broadinstitute.sting.utils.variantcontext.VariantContext; +import java.util.List; import java.util.Map; @@ -76,24 +77,24 @@ protected GenotypeLikelihoodsCalculationModel(UnifiedArgumentCollection UAC, Log /** * Can be overridden by concrete subclasses * - * @param tracker rod data - * @param ref reference context - * @param contexts stratified alignment contexts - * @param contextType stratified context type - * @param priors priors to use for GLs - * @param alternateAlleleToUse the alternate allele to use, null if not set - * @param useBAQedPileup should we use the BAQed pileup or the raw one? - * @param locParser Genome Loc Parser + * @param tracker rod data + * @param ref reference context + * @param contexts stratified alignment contexts + * @param contextType stratified context type + * @param priors priors to use for GLs + * @param alternateAllelesToUse the alternate allele to use, null if not set + * @param useBAQedPileup should we use the BAQed pileup or the raw one? + * @param locParser Genome Loc Parser * @return variant context where genotypes are no-called but with GLs */ - public abstract VariantContext getLikelihoods(RefMetaDataTracker tracker, - ReferenceContext ref, - Map contexts, - AlignmentContextUtils.ReadOrientation contextType, - GenotypePriors priors, - Allele alternateAlleleToUse, - boolean useBAQedPileup, - GenomeLocParser locParser); + public abstract VariantContext getLikelihoods(final RefMetaDataTracker tracker, + final ReferenceContext ref, + final Map contexts, + final AlignmentContextUtils.ReadOrientation contextType, + final GenotypePriors priors, + final List alternateAllelesToUse, + final boolean useBAQedPileup, + final GenomeLocParser locParser); protected int getFilteredDepth(ReadBackedPileup pileup) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java index 7ee7b0752c..1b73ef1d70 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java @@ -284,13 +284,14 @@ else if (p.isDeletion()) { private final static EnumSet allowableTypes = EnumSet.of(VariantContext.Type.INDEL, VariantContext.Type.MIXED); - public VariantContext getLikelihoods(RefMetaDataTracker tracker, - ReferenceContext ref, - Map contexts, - AlignmentContextUtils.ReadOrientation contextType, - GenotypePriors priors, - Allele alternateAlleleToUse, - boolean useBAQedPileup, GenomeLocParser locParser) { + public VariantContext getLikelihoods(final RefMetaDataTracker tracker, + final ReferenceContext ref, + final Map contexts, + final AlignmentContextUtils.ReadOrientation contextType, + final GenotypePriors priors, + final List alternateAllelesToUse, + final boolean useBAQedPileup, + final GenomeLocParser locParser) { if (tracker == null) return null; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java index 154612d255..dd21681f04 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java @@ -64,7 +64,7 @@ public VariantContext getLikelihoods(final RefMetaDataTracker tracker, final Map contexts, final AlignmentContextUtils.ReadOrientation contextType, final GenotypePriors priors, - final Allele alternateAlleleToUse, + final List alternateAllelesToUse, final boolean useBAQedPileup, final GenomeLocParser locParser) { @@ -95,8 +95,8 @@ public VariantContext getLikelihoods(final RefMetaDataTracker tracker, } // find the alternate allele(s) that we should be using - if ( alternateAlleleToUse != null ) { - alleles.add(alternateAlleleToUse); + if ( alternateAllelesToUse != null ) { + alleles.addAll(alternateAllelesToUse); } else if ( useAlleleFromVCF ) { final VariantContext vc = UnifiedGenotyperEngine.getVCFromAllelesRod(tracker, ref, ref.getLocus(), true, logger, UAC.alleles); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java index a60cc64f7b..05a977add7 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperEngine.java @@ -237,14 +237,14 @@ public VariantCallContext calculateGenotypes(RefMetaDataTracker tracker, Referen // --------------------------------------------------------------------------------------------------------- // private method called by both UnifiedGenotyper and UGCalcLikelihoods entry points into the engine - private VariantContext calculateLikelihoods(RefMetaDataTracker tracker, ReferenceContext refContext, Map stratifiedContexts, AlignmentContextUtils.ReadOrientation type, Allele alternateAlleleToUse, boolean useBAQedPileup, final GenotypeLikelihoodsCalculationModel.Model model) { + private VariantContext calculateLikelihoods(RefMetaDataTracker tracker, ReferenceContext refContext, Map stratifiedContexts, AlignmentContextUtils.ReadOrientation type, List alternateAllelesToUse, boolean useBAQedPileup, final GenotypeLikelihoodsCalculationModel.Model model) { // initialize the data for this thread if that hasn't been done yet if ( glcm.get() == null ) { glcm.set(getGenotypeLikelihoodsCalculationObject(logger, UAC)); } - return glcm.get().get(model).getLikelihoods(tracker, refContext, stratifiedContexts, type, getGenotypePriors(model), alternateAlleleToUse, useBAQedPileup && BAQEnabledOnCMDLine, genomeLocParser); + return glcm.get().get(model).getLikelihoods(tracker, refContext, stratifiedContexts, type, getGenotypePriors(model), alternateAllelesToUse, useBAQedPileup && BAQEnabledOnCMDLine, genomeLocParser); } private VariantCallContext generateEmptyContext(RefMetaDataTracker tracker, ReferenceContext ref, Map stratifiedContexts, AlignmentContext rawContext) { @@ -398,16 +398,14 @@ else if ( UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_M //final boolean DEBUG_SLOD = false; // the overall lod - VariantContext vcOverall = calculateLikelihoods(tracker, refContext, stratifiedContexts, AlignmentContextUtils.ReadOrientation.COMPLETE, vc.getAlternateAllele(0), false, model); - clearAFarray(AFresult.log10AlleleFrequencyLikelihoods); - clearAFarray(AFresult.log10AlleleFrequencyPosteriors); - afcm.get().getLog10PNonRef(vcOverall, getAlleleFrequencyPriors(model), AFresult); //double overallLog10PofNull = AFresult.log10AlleleFrequencyPosteriors[0]; double overallLog10PofF = MathUtils.log10sumLog10(AFresult.log10AlleleFrequencyPosteriors[0], 0); //if ( DEBUG_SLOD ) System.out.println("overallLog10PofF=" + overallLog10PofF); + List alternateAllelesToUse = builder.make().getAlternateAlleles(); + // the forward lod - VariantContext vcForward = calculateLikelihoods(tracker, refContext, stratifiedContexts, AlignmentContextUtils.ReadOrientation.FORWARD, vc.getAlternateAllele(0), false, model); + VariantContext vcForward = calculateLikelihoods(tracker, refContext, stratifiedContexts, AlignmentContextUtils.ReadOrientation.FORWARD, alternateAllelesToUse, false, model); clearAFarray(AFresult.log10AlleleFrequencyLikelihoods); clearAFarray(AFresult.log10AlleleFrequencyPosteriors); afcm.get().getLog10PNonRef(vcForward, getAlleleFrequencyPriors(model), AFresult); @@ -417,7 +415,7 @@ else if ( UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_M //if ( DEBUG_SLOD ) System.out.println("forwardLog10PofNull=" + forwardLog10PofNull + ", forwardLog10PofF=" + forwardLog10PofF); // the reverse lod - VariantContext vcReverse = calculateLikelihoods(tracker, refContext, stratifiedContexts, AlignmentContextUtils.ReadOrientation.REVERSE, vc.getAlternateAllele(0), false, model); + VariantContext vcReverse = calculateLikelihoods(tracker, refContext, stratifiedContexts, AlignmentContextUtils.ReadOrientation.REVERSE, alternateAllelesToUse, false, model); clearAFarray(AFresult.log10AlleleFrequencyLikelihoods); clearAFarray(AFresult.log10AlleleFrequencyPosteriors); afcm.get().getLog10PNonRef(vcReverse, getAlleleFrequencyPriors(model), AFresult);