Merge remote-tracking branch 'unstable/master'

build.xml

@@ -1,5 +1,5 @@
 <!--
-  ~ Copyright (c) 2011, The Broad Institute
+  ~ Copyright (c) 2012, The Broad Institute
   ~
   ~ Permission is hereby granted, free of charge, to any person
   ~ obtaining a copy of this software and associated documentation
@@ -47,6 +47,7 @@
     <property name="R.package.path" value="org/broadinstitute/sting/utils/R" />
     <property name="resource.file" value="StingText.properties" />
     <property name="resource.path" value="${java.classes}/StingText.properties" />
+    <property name="key.dir" value="${public.dir}/keys" />
 
     <property name="scala.public.source.dir" value="${public.dir}/scala/src" />
     <property name="scala.private.source.dir" value="${private.dir}/scala/src" />
@@ -215,7 +216,7 @@
 
     <target name="git.describe">
       <exec executable="git" outputproperty="git.describe.output" resultproperty="git.describe.exit.value" failonerror="false">
-        <arg line="describe" />
+        <arg line="describe --long" />
       </exec>
       <condition property="git.describe.succeeded">
         <equals arg1="${git.describe.exit.value}" arg2="0" />
@@ -281,6 +282,10 @@
         <equals arg1="${gatk.target}" arg2="private" casesensitive="false" />
       </condition>
 
+      <condition property="include.external">
+          <available file="${external.dir}"/>
+      </condition>
+
       <condition property="include.contracts">
         <equals arg1="${use.contracts}" arg2="true" />
       </condition>
@@ -331,7 +336,7 @@
         </javac>
     </target>
 
-    <target name="gatk.compile.external.source" depends="gatk.compile.public.source,gatk.compile.private.source">
+    <target name="gatk.compile.external.source" depends="gatk.compile.public.source,gatk.compile.private.source" if="include.external">
         <subant target="compile" genericantfile="build.xml">
             <property name="build.dir" value="${external.build.dir}" />
             <property name="dist.dir" value="${external.dist.dir}" />
@@ -563,6 +568,7 @@
             </fileset>
             <fileset dir="${java.classes}" includes="**/commandline/**/*.class"/>
             <fileset dir="${java.classes}" includes="**/sting/pipeline/**/*.class"/>
+            <fileset dir="${java.classes}" includes="**/sting/tools/**/*.class"/>
             <fileset dir="${java.classes}" includes="**/sting/jna/**/*.class"/>
             <fileset dir="${java.classes}" includes="net/sf/picard/**/*.class"/>
             <fileset dir="${java.classes}" includes="net/sf/samtools/**/*.class"/>
@@ -611,6 +617,9 @@
                 <include name="**/gatk/**/*.R"/>
                 <include name="**/alignment/**/*.R"/>
             </fileset>
+            <fileset dir="${key.dir}">
+                <include name="**/*.key"/>
+            </fileset>
             <manifest>
                 <attribute name="Main-Class" value="org.broadinstitute.sting.gatk.CommandLineGATK" />
             </manifest>
@@ -761,6 +770,7 @@
     <property name="java.test.classes" value="${build.dir}/java/testclasses"/>
     <property name="java.public.test.classes" value="${java.test.classes}/public"/>
     <property name="java.private.test.classes" value="${java.test.classes}/private"/>
+    <property name="java.external.test.classes" value="${java.test.classes}/external"/>
     <property name="java.public.test.sources" value="${public.dir}/java/test"/>
     <property name="java.private.test.sources" value="${private.dir}/java/test"/>
     <property name="scala.test.classes" value="${build.dir}/scala/testclasses"/>
@@ -811,7 +821,23 @@
         </javac>
     </target>
 
-    <target name="test.java.compile" depends="test.java.public.compile, test.java.private.compile"/>
+    <target name="test.java.external.compile" depends="dist,test.init.compile,test.java.public.compile" if="include.external">
+        <mkdir dir="${java.external.test.classes}"/>
+        <echo message="Sting: Compiling external test cases!"/>
+        <javac fork="true" memoryMaximumSize="512m" destdir="${java.external.test.classes}" debug="true" optimize="on" tempdir="${java.io.tmpdir}" srcdir="${external.dir}">
+            <include name="*/test/**/*.java"/>
+            <classpath>
+                <path refid="external.dependencies" />
+                <pathelement location="${java.public.test.classes}"/>
+                <pathelement location="${java.classes}"/>
+                <pathelement location="${java.contracts}"/>
+                <pathelement location="${testng.jar}"/>
+            </classpath>
+            <compilerarg value="-proc:none"/>
+        </javac>
+    </target>
+
+    <target name="test.java.compile" depends="test.java.public.compile, test.java.private.compile, test.java.external.compile"/>
 
     <target name="test.scala.public.compile" depends="test.java.compile,scala.compile" if="scala.include">
         <mkdir dir="${scala.public.test.classes}"/>
@@ -852,11 +878,13 @@
         <pathelement location="${java.contracts}" />
         <pathelement location="${java.public.test.classes}" />
         <pathelement location="${java.private.test.classes}" />
+        <pathelement location="${java.external.test.classes}" />
         <pathelement location="${scala.public.test.classes}" />
         <pathelement location="${scala.private.test.classes}" />
         <pathelement location="${R.tar.dir}" />
         <pathelement location="${R.public.scripts.dir}" />
         <pathelement location="${R.private.scripts.dir}" />
+        <pathelement location="${key.dir}" />
         <path refid="external.dependencies" />
     </path>
 
@@ -934,6 +962,9 @@
                 <classfileset dir="${java.private.test.classes}" erroronmissingdir="false">
                     <include name="**/@{testtype}.class" if="include.private"/>
                 </classfileset>
+                <classfileset dir="${java.external.test.classes}" erroronmissingdir="false">
+                    <include name="**/@{testtype}.class" if="include.external"/>
+                </classfileset>
                 <classfileset dir="${scala.public.test.classes}" erroronmissingdir="false">
                     <include name="**/@{testtype}*.class" if="scala.include"/>
                 </classfileset>
@@ -1210,7 +1241,14 @@
     <!-- Build gsalib R module -->
     <target name="gsalib">
       <exec executable="R" failonerror="true">
-        <arg line="R CMD INSTALL -l ${R.library.dir} ${R.public.src.dir}/${R.package.path}/gsalib" />
+        <arg line="R CMD INSTALL --preclean ${R.public.src.dir}/${R.package.path}/gsalib" />
+      </exec>
+    </target>
+
+    <target name="clean.gsalib">
+        <!-- Currently not cleaning out the lib during 'ant clean' -->
+      <exec executable="R" failonerror="false">
+        <arg line="R CMD REMOVE gsalib" />
       </exec>
     </target>
 </project>

ivy.xml

@@ -1,3 +1,26 @@
+<!--
+  ~ Copyright (c) 2012, The Broad Institute
+  ~
+  ~ Permission is hereby granted, free of charge, to any person
+  ~ obtaining a copy of this software and associated documentation
+  ~ files (the "Software"), to deal in the Software without
+  ~ restriction, including without limitation the rights to use,
+  ~ copy, modify, merge, publish, distribute, sublicense, and/or sell
+  ~ copies of the Software, and to permit persons to whom the
+  ~ Software is furnished to do so, subject to the following
+  ~ conditions:
+  ~
+  ~ The above copyright notice and this permission notice shall be
+  ~ included in all copies or substantial portions of the Software.
+  ~ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+  ~ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+  ~ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+  ~ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+  ~ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+  ~ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+  ~ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+  ~ OTHER DEALINGS IN THE SOFTWARE.
+  -->
 
 <ivy-module version="1.0">
   <info organisation="org.broadinstitute" module="Sting"/>
@@ -18,10 +41,9 @@
     <dependency org="log4j" name="log4j" rev="1.2.15"/>
     <dependency org="javax.mail" name="mail" rev="1.4.4"/>
     <dependency org="colt" name="colt" rev="1.2.0"/>
-    <dependency org="jboss" name="javassist" rev="3.7.ga"/>
+    <!-- <dependency org="jboss" name="javassist" rev="3.7.ga"/> -->
     <dependency org="org.simpleframework" name="simple-xml" rev="2.0.4"/>
     <dependency org="org.apache.bcel" name="bcel" rev="5.2"/>
-    <dependency org="org.yaml" name="snakeyaml" rev="1.7"/>
 
     <!-- Dependencies for reflections mvn repository -->
     <dependency org="org.reflections" name="reflections" rev="0.9.5-RC2"/>
@@ -40,17 +62,17 @@
     <dependency org="org.apache.commons" name="commons-jexl" rev="2.0"/>
     <dependency org="commons-lang" name="commons-lang" rev="2.5"/>
     <dependency org="commons-logging" name="commons-logging" rev="1.1.1"/>
-    <dependency org="commons-io" name="commons-io" rev="2.0"/>
+    <dependency org="commons-io" name="commons-io" rev="2.1"/>
     <dependency org="org.apache.commons" name="commons-math" rev="2.2" />
 
     <!-- Lucene core utilities -->
-    <dependency org="org.apache.lucene" name="lucene-core" rev="3.0.3"/>
+    <!-- <dependency org="org.apache.lucene" name="lucene-core" rev="3.0.3"/> -->
 
     <!-- Dependencies for LSF, DRMAA, and other C libraries -->
     <dependency org="net.java.dev.jna" name="jna" rev="3.2.7"/>
 
     <!-- Dependencies for amazon.com S3 support -->
-    <dependency org="net.java.dev.jets3t" name="jets3t" rev="0.8.0"/>
+    <dependency org="net.java.dev.jets3t" name="jets3t" rev="0.8.1"/>
 
     <!-- Dependencies for GridEngine -->
     <dependency org="net.sf.gridscheduler" name="drmaa" rev="latest.integration"/>

public/R/scripts/org/broadinstitute/sting/queue/util/queueJobReport.R

@@ -1,6 +1,7 @@
 library(gsalib)
-require("ggplot2")
-require("gplots")
+library(ggplot2)
+library(gplots)
+library(tools)
 
 #
 # Standard command line switch.  Can we loaded interactively for development
@@ -201,4 +202,7 @@ for ( group in gatkReportData ) {
 
 if ( ! is.na(outputPDF) ) {
   dev.off()
-} 
+  if (exists("compactPDF")) {
+    compactPDF(outputPDF)
+  }
+}

public/R/src/org/broadinstitute/sting/utils/R/gsalib/R/gsa.read.gatkreport.R

@@ -4,11 +4,9 @@
     colnames(d) = tableHeader;
 
     for (i in 1:ncol(d)) {
-        v = suppressWarnings(as.numeric(d[,i]));
-
-        if (length(na.omit(as.numeric(v))) == length(d[,i])) {
-            d[,i] = v;
-        }
+        # use the general type.convert infrastructure of read.table to convert column data to R types 
+        v = type.convert(d[,i])
+        d[,i] = v;
     }
 
     usedNames = ls(envir=tableEnv, pattern=tableName);

public/c/bwa/build_linux.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-export BWA_HOME="/humgen/gsa-scr1/hanna/src/bwa-trunk/bwa"
+export BWA_HOME="/humgen/gsa-scr1/hanna/src/bio-bwa/bwa"
 export JAVA_INCLUDE="/broad/tools/Linux/x86_64/pkgs/jdk_1.6.0_12/include -I/broad/tools/Linux/x86_64/pkgs/jdk_1.6.0_12/include/linux"
 export TARGET_LIB="libbwa.so"
 export EXTRA_LIBS="-lc -lz -lstdc++ -lpthread"

public/c/bwa/bwa_gateway.cpp

@@ -233,6 +233,8 @@ void BWA::set_disallow_indel_within_range(int indel_range) { options.indel_end_s
 void BWA::set_mismatch_penalty(int penalty) { options.s_mm = penalty; }
 void BWA::set_gap_open_penalty(int penalty) { options.s_gapo = penalty; }
 void BWA::set_gap_extension_penalty(int penalty) { options.s_gape = penalty; }
+void BWA::set_mode_nonstop() { options.mode |= BWA_MODE_NONSTOP; options.max_top2 = 0x7fffffff; }
+void BWA::set_max_entries_in_queue(int max_entries) { options.max_entries = max_entries; }
 
 /**
  * Create a sequence with a set of reasonable initial defaults.  

public/c/bwa/bwa_gateway.h

@@ -60,6 +60,8 @@ class BWA {
   void set_mismatch_penalty(int penalty);
   void set_gap_open_penalty(int penalty);
   void set_gap_extension_penalty(int penalty);
+  void set_mode_nonstop();
+  void set_max_entries_in_queue(int max_entries);
 
   // Perform the alignment
   Alignment* generate_single_alignment(const char* bases, 

public/c/bwa/org_broadinstitute_sting_alignment_bwa_c_BWACAligner.cpp

@@ -8,11 +8,13 @@
 #include "bwa_gateway.h"
 #include "org_broadinstitute_sting_alignment_bwa_c_BWACAligner.h"
 
+typedef void (BWA::*boolean_setter)();
 typedef void (BWA::*int_setter)(int value);
 typedef void (BWA::*float_setter)(float value);
 
 static jobject convert_to_java_alignment(JNIEnv* env, const jbyte* read_bases, const jsize read_length, const Alignment& alignment);
 static jstring get_configuration_file(JNIEnv* env, jobject configuration, const char* field_name);
+static void set_boolean_configuration_param(JNIEnv* env, jobject configuration, const char* field_name, BWA* bwa, boolean_setter setter);
 static void set_int_configuration_param(JNIEnv* env, jobject configuration, const char* field_name, BWA* bwa, int_setter setter);
 static void set_float_configuration_param(JNIEnv* env, jobject configuration, const char* field_name, BWA* bwa, float_setter setter);
 static void throw_config_value_exception(JNIEnv* env, const char* field_name, const char* message);
@@ -100,6 +102,10 @@ JNIEXPORT void JNICALL Java_org_broadinstitute_sting_alignment_bwa_c_BWACAligner
   if(env->ExceptionCheck()) return; 
   set_int_configuration_param(env, configuration, "gapExtensionPenalty", bwa, &BWA::set_gap_extension_penalty);
   if(env->ExceptionCheck()) return;
+  set_boolean_configuration_param(env, configuration, "nonStopMode", bwa, &BWA::set_mode_nonstop);
+  if(env->ExceptionCheck()) return;
+  set_int_configuration_param(env, configuration, "maxEntriesInQueue", bwa, &BWA::set_max_entries_in_queue);
+  if(env->ExceptionCheck()) return;
 }
 
 JNIEXPORT jobjectArray JNICALL Java_org_broadinstitute_sting_alignment_bwa_c_BWACAligner_getPaths(JNIEnv *env, jobject instance, jlong java_bwa, jbyteArray java_bases) 
@@ -357,6 +363,36 @@ static jstring get_configuration_file(JNIEnv* env, jobject configuration, const
   return path;
 }
 
+static void set_boolean_configuration_param(JNIEnv* env, jobject configuration, const char* field_name, BWA* bwa, boolean_setter setter) {
+  jclass configuration_class = env->GetObjectClass(configuration);
+  if(configuration_class == NULL) return;
+
+  jfieldID configuration_field = env->GetFieldID(configuration_class, field_name, "Ljava/lang/Boolean;");
+  if(configuration_field == NULL) return;
+
+  jobject boxed_value = env->GetObjectField(configuration,configuration_field);
+  if(env->ExceptionCheck()) return;
+
+  if(boxed_value != NULL) {
+    jclass boolean_box_class = env->FindClass("java/lang/Boolean");
+    if(boolean_box_class == NULL) return;
+
+    jmethodID boolean_extractor = env->GetMethodID(boolean_box_class,"booleanValue", "()Z");
+    if(boolean_extractor == NULL) return;
+
+    jboolean value = env->CallBooleanMethod(boxed_value,boolean_extractor);
+    if(env->ExceptionCheck()) return;
+
+    if(value)
+      (bwa->*setter)();
+
+    env->DeleteLocalRef(boolean_box_class);
+  }
+
+  env->DeleteLocalRef(boxed_value);
+  env->DeleteLocalRef(configuration_class);
+}
+
 static void set_int_configuration_param(JNIEnv* env, jobject configuration, const char* field_name, BWA* bwa, int_setter setter) {
   jclass configuration_class = env->GetObjectClass(configuration);
   if(configuration_class == NULL) return;