Add warning to CheckIlluminaDir if we detect cycles without data. (broadinstitute#874)

* Add warning to CheckIlluminaDir if we detect cycles without data.
CBCLReader will ignore and log a warning for tiles without data.

* Review changes.

Author: jacarey

src/main/java/picard/illumina/CheckIlluminaDirectory.java

@@ -31,6 +31,7 @@
 import java.util.TreeSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import static picard.illumina.BasecallsConverter.TILE_NUMBER_COMPARATOR;
 import static picard.illumina.NewIlluminaBasecallsConverter.getTiledFiles;
@@ -152,7 +153,7 @@ protected int doWork() {
                 tiles.sort(TILE_NUMBER_COMPARATOR);
 
                 //check s.locs
-                final File locsFile = new File(BASECALLS_DIR.getParentFile(), "s.locs");
+                final File locsFile = new File(BASECALLS_DIR.getParentFile(), AbstractIlluminaPositionFileReader.S_LOCS_FILE);
                 final LocsFileReader locsFileReader = new LocsFileReader(locsFile);
                 final List<AbstractIlluminaPositionFileReader.PositionInfo> locs = new ArrayList<>();
                 while (locsFileReader.hasNext()) {
@@ -167,8 +168,19 @@ protected int doWork() {
                 final OutputMapping outputMapping = new OutputMapping(readStructure);
 
                 final CbclReader reader = new CbclReader(cbcls, filterFileMap, readStructure.readLengths, tiles.get(0), locs, outputMapping.getOutputCycles(), true);
-
                 reader.getAllTiles().forEach((key, value) -> {
+                    //we are looking for cycles with compressed data count of 2 bytes (standard gzip header size)
+                    String emptyCycleString = value.stream()
+                            .filter(cycle -> cycle.getCompressedBlockSize() <= 2)
+                            .map(BaseBclReader.TileData::getTileNum)
+                            .map(Object::toString)
+                            .collect(Collectors.joining(", "));
+
+                    if (emptyCycleString.length() > 0) {
+                        log.warn("The following tiles have no data for cycle " + key);
+                        log.warn(emptyCycleString);
+                    }
+
                     final List<File> fileForCycle = reader.getFilesForCycle(key);
                     final long totalFilesSize = fileForCycle.stream().mapToLong(file -> file.length() - reader.getHeaderSize()).sum();
                     final long expectedFileSize = value.stream().mapToLong(BaseBclReader.TileData::getCompressedBlockSize).sum();
@@ -221,7 +233,7 @@ protected int doWork() {
     }
 
     private void createLocFileSymlinks(final IlluminaFileUtil fileUtil, final int lane) {
-        final File baseFile = new File(BASECALLS_DIR.getParentFile().getAbsolutePath() + File.separator + "s.locs");
+        final File baseFile = new File(BASECALLS_DIR.getParentFile().getAbsolutePath() + File.separator + AbstractIlluminaPositionFileReader.S_LOCS_FILE);
         final File newFileBase = new File(baseFile.getParent() + File.separator + IlluminaFileUtil
                 .longLaneStr(lane) + File.separator);
         if (baseFile.exists()) {

src/main/java/picard/illumina/NewIlluminaBasecallsConverter.java

@@ -97,7 +97,7 @@ outputRecordClass, numProcessors, new IlluminaDataProviderFactory(basecallsDir,
         IOUtil.assertFilesAreReadable(cbcls);
 
         //locs
-        final File locsFile = new File(basecallsDir.getParentFile(), "s.locs");
+        final File locsFile = new File(basecallsDir.getParentFile(), AbstractIlluminaPositionFileReader.S_LOCS_FILE);
         try (LocsFileReader locsFileReader = new LocsFileReader(locsFile)) {
             while (locsFileReader.hasNext()) {
                 locs.add(locsFileReader.next());

src/main/java/picard/illumina/parser/readers/BaseBclReader.java

@@ -156,12 +156,17 @@ public class TileData {
             this.filePosition = filePosition;
         }
 
+        public int getTileNum() {
+            return tileNum;
+        }
+
         public int getCompressedBlockSize() {
             return compressedBlockSize;
         }
 
-        public int getNumClustersInTile() {
+        int getNumClustersInTile() {
             return numClustersInTile;
         }
+
     }
 }

src/main/java/picard/illumina/parser/readers/CbclReader.java

@@ -314,10 +314,6 @@ private void cacheFilterAndLocs(final TileData currentTileData, final List<Abstr
 
     private void cacheTile(final int totalCycleCount, final TileData tileData, final CycleData currentCycleData) throws IOException {
         final byte[] tileByteArray = new byte[tileData.compressedBlockSize];
-        //we are going to explode the nibbles in to bytes to make PF filtering easier
-        final byte[] uncompressedByteArray = new byte[tileData.uncompressedBlockSize];
-        // ByteBuffer uncompressedByteArray = ByteBuffer.allocate(tileData.uncompressedBlockSize);
-        final byte[] unNibbledByteArray = new byte[tileData.uncompressedBlockSize * 2];
 
         // Read the whole compressed block into a buffer, then sanity check the length
         final InputStream stream = this.streams[totalCycleCount];
@@ -332,38 +328,18 @@ private void cacheTile(final int totalCycleCount, final TileData tileData, final
                     (totalCycleCount + 1), this.streamFiles[totalCycleCount].getAbsolutePath()));
         }
 
-        // Uncompress the data from the buffer we just wrote - use gzip input stream to write to uncompressed buffer
+        // Decompress the data from the buffer we just wrote - use gzip input stream to write to uncompressed buffer
         final ByteArrayInputStream byteInputStream = new ByteArrayInputStream(Arrays.copyOfRange(tileByteArray, 0, readBytes));
-
-        final GZIPInputStream gzipInputStream = new GZIPInputStream(byteInputStream, uncompressedByteArray.length);
-        int read;
-        int totalRead = 0;
-        try {
-            while ((read = gzipInputStream.read(uncompressedByteArray, totalRead, uncompressedByteArray.length - totalRead)) != -1) {
-                if (read == 0) break;
-                totalRead += read;
-            }
-        } catch (final EOFException eofException) {
-            throw new PicardException("Unexpected end of file " + this.streamFiles[totalCycleCount].getAbsolutePath()
-                    + " this file is likely corrupt or truncated. We have read "
-                    + totalRead + " and were expecting to read "
-                    + uncompressedByteArray.length);
-        }
-        if (totalRead != tileData.uncompressedBlockSize) {
-            throw new PicardException(String.format("Error while decompressing from BCL file for cycle %d. Offending file on disk is %s",
-                    (totalCycleCount + 1), this.streamFiles[totalCycleCount].getAbsolutePath()));
-        }
+        byte[] decompressedByteArray = decompressTile(totalCycleCount, tileData, byteInputStream);
 
         // Read uncompressed data from the buffer and expand each nibble into a full byte for ease of use
-        int index = 0;
-        for (final byte singleByte : uncompressedByteArray) {
-            unNibbledByteArray[index] = (byte) (singleByte & 0x0f);
-            index++;
-            unNibbledByteArray[index] = (byte) ((singleByte >> 4) & 0x0f);
-            index++;
-        }
-        gzipInputStream.close();
+        byte[] unNibbledByteArray = promoteNibblesToBytes(decompressedByteArray);
+        cachedTile[totalCycleCount] = filterNonPfReads(tileData, currentCycleData, unNibbledByteArray);
 
+        cachedTilePosition[totalCycleCount] = 0;
+    }
+
+    private byte[] filterNonPfReads(TileData tileData, CycleData currentCycleData, byte[] unNibbledByteArray) {
         // Write buffer contents to cached tile array
         // if nonPF reads are included we need to strip them out
         if (!currentCycleData.pfExcluded) {
@@ -383,11 +359,52 @@ private void cacheTile(final int totalCycleCount, final TileData tileData, final
                 }
                 filterIndex++;
             }
-            cachedTile[totalCycleCount] = filteredByteArray;
+            return filteredByteArray;
         } else {
-            cachedTile[totalCycleCount] = unNibbledByteArray;
+            return unNibbledByteArray;
         }
-        cachedTilePosition[totalCycleCount] = 0;
+    }
+
+    private byte[] promoteNibblesToBytes(byte[] decompressedByteArray) {
+        //we are going to explode the nibbles in to bytes to make PF filtering easier
+        final byte[] unNibbledByteArray = new byte[decompressedByteArray.length * 2];
+        int index = 0;
+        for (final byte singleByte : decompressedByteArray) {
+            unNibbledByteArray[index] = (byte) (singleByte & 0x0f);
+            index++;
+            unNibbledByteArray[index] = (byte) ((singleByte >> 4) & 0x0f);
+            index++;
+        }
+        return unNibbledByteArray;
+    }
+
+    private byte[] decompressTile(int totalCycleCount, TileData tileData, ByteArrayInputStream byteInputStream) throws IOException {
+        final byte[] decompressedByteArray = new byte[tileData.uncompressedBlockSize];
+        //only decompress the data if we are expecting data.
+        if (decompressedByteArray.length == 0) {
+            log.warn("Ignoring tile " + tileData.tileNum + " there are no PF reads.");
+        } else {
+            int read;
+            int totalRead = 0;
+            try (GZIPInputStream gzipInputStream = new GZIPInputStream(byteInputStream, decompressedByteArray.length)) {
+                while ((read = gzipInputStream.read(decompressedByteArray, totalRead, decompressedByteArray.length - totalRead)) != -1) {
+                    if (read == 0) {
+                        break;
+                    }
+                    totalRead += read;
+                }
+            } catch (final EOFException eofException) {
+                throw new PicardException("Unexpected end of file " + this.streamFiles[totalCycleCount].getAbsolutePath()
+                        + " this file is likely corrupt or truncated. We have read "
+                        + totalRead + " and were expecting to read "
+                        + decompressedByteArray.length);
+            }
+            if (totalRead != tileData.uncompressedBlockSize) {
+                throw new PicardException(String.format("Error while decompressing from BCL file for cycle %d. Offending file on disk is %s",
+                        (totalCycleCount + 1), this.streamFiles[totalCycleCount].getAbsolutePath()));
+            }
+        }
+        return decompressedByteArray;
     }
 
     public CycleData[] getCycleData() {

src/test/java/picard/illumina/CheckIlluminaDirectoryTest.java

@@ -8,12 +8,13 @@
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
-import picard.cmdline.CommandLineProgramTest;
 import picard.PicardException;
+import picard.cmdline.CommandLineProgramTest;
 import picard.cmdline.StandardOptionDefinitions;
 import picard.illumina.parser.IlluminaDataType;
 import picard.illumina.parser.IlluminaFileUtil;
 import picard.illumina.parser.IlluminaFileUtilTest;
+import picard.illumina.parser.readers.AbstractIlluminaPositionFileReader;
 
 import java.io.BufferedWriter;
 import java.io.File;
@@ -373,13 +374,12 @@ public void symlinkLocsTest() {
 
     private void createSingleLocsFile() {
         try {
-            final File singleLocsFile = new File(intensityDir, "s.locs");
+            final File singleLocsFile = new File(intensityDir, AbstractIlluminaPositionFileReader.S_LOCS_FILE);
             final FileWriter writer = new FileWriter(singleLocsFile);
             writer.write("This is a test string.");
             writer.close();
         } catch (final IOException e) {
             e.printStackTrace();
         }
-
     }
 }

src/test/java/picard/illumina/parser/readers/CbclReaderTest.java

@@ -7,6 +7,7 @@
 
 import java.io.File;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -16,8 +17,8 @@ public class CbclReaderTest {
     private static final File TestDataDir = new File("testdata/picard/illumina/readerTests/cbcls");
     private static final File PASSING_CBCL_C1_1 = new File(TestDataDir + "/C1.1", "L001_1.cbcl");
     private static final File PASSING_CBCL_C2_1 = new File(TestDataDir + "/C2.1", "L001_1.cbcl");
+    private static final File CBCL_WITH_EMPTY_TILE = new File(TestDataDir + "/C3.1", "L001_1.cbcl");
     private static final File TILE_1101_FILTER = new File(TestDataDir, "tile_1101.filter");
-    private static final File TILE_1102_FILTER = new File(TestDataDir, "tile_1102.filter");
 
     private static final char[] expectedBases = new char[]{
             'G', 'G', 'C', 'C', 'G', 'A', 'A', 'G'
@@ -63,4 +64,15 @@ public void testMissingTile() {
                 filters, new int[]{2}, 1102, locs, new int[]{1, 2}, false);
 
     }
+
+    @Test
+    public void testEmptyTile() {
+        final Map<Integer, File> filters = new HashMap<>();
+        filters.put(1101, TILE_1101_FILTER);
+        final LocsFileReader locsFileReader = new LocsFileReader(new File("testdata/picard/illumina/readerTests/s_1_6.locs"));
+        List<AbstractIlluminaPositionFileReader.PositionInfo> locs = locsFileReader.toList();
+        CbclReader reader = new CbclReader(Collections.singletonList(CBCL_WITH_EMPTY_TILE),
+                filters, new int[]{1}, 1101, locs, new int[]{3}, false);
+        Assert.assertFalse(reader.hasNext());
+    }
 }