Adapted all experiment scripts to also take the year as a parameter.

hpi-dhc · Jan 28, 2019 · 9565c25 · 9565c25
1 parent b8469f0
commit 9565c25
Show file tree

Hide file tree

Showing 32 changed files with 127 additions and 125 deletions.
diff --git a/scripts/runAllBoostsOptimization.sh b/scripts/runAllBoostsOptimization.sh
@@ -1,12 +1,11 @@
 #!/bin/bash
 
-whats="disease gene fields posneg additional extra"
+whats="genedis fields posneg additional extra pmclass mutation drug"
 
-mvn clean compile
-
-#srun scripts/runBoostOptimizer.sh extra
+mvn compile
 
 for i in $whats; do
     echo "Starting boost optimization of $i"
-    sbatch scripts/runBoostOptimizer.sh $i
+    sbatch scripts/runBoostOptimizer.sh $i 2017
+    sbatch scripts/runBoostOptimizer.sh $i 2018
 done
diff --git a/scripts/runAllPmClassExperimentsLiterature.sh b/scripts/runAllPmClassExperimentsLiterature.sh
@@ -2,8 +2,16 @@
 
 mvn compile
 # For the best_fields, the slop does not do anything, but a argument value is expected
-sbatch --exclude=h5,h6 scripts/runPmClassExperimentsLiterature.sh best_fields OR
-sbatch --exclude=h5,h6  scripts/runPmClassExperimentsLiterature.sh best_fields AND
+sbatch --exclude=h5,h6 scripts/runPmClassExperimentsLiterature.sh best_fields OR 2017
+sbatch --exclude=h5,h6  scripts/runPmClassExperimentsLiterature.sh best_fields AND 2017
 
 # Here, the boolean operator has no effect
-sbatch --exclude=h5,h6  scripts/runPmClassExperimentsLiterature.sh phrase OR
+sbatch --exclude=h5,h6  scripts/runPmClassExperimentsLiterature.sh phrase OR 2017
+
+
+# For the best_fields, the slop does not do anything, but a argument value is expected
+sbatch --exclude=h5,h6 scripts/runPmClassExperimentsLiterature.sh best_fields OR 2018
+sbatch --exclude=h5,h6  scripts/runPmClassExperimentsLiterature.sh best_fields AND 2018
+
+# Here, the boolean operator has no effect
+sbatch --exclude=h5,h6  scripts/runPmClassExperimentsLiterature.sh phrase OR 2018
diff --git a/scripts/runAllTermBoostExperimentsLiterature.sh b/scripts/runAllTermBoostExperimentsLiterature.sh
@@ -2,8 +2,16 @@
 
 mvn compile
 # For the best_fields, the slop does not do anything, but a argument value is expected
-sbatch --exclude=h5,h6 scripts/runTermBoostExperimentsLiterature.sh best_fields OR
-sbatch --exclude=h5,h6  scripts/runTermBoostExperimentsLiterature.sh best_fields AND
+sbatch --exclude=h5,h6 scripts/runTermBoostExperimentsLiterature.sh best_fields OR 2017
+sbatch --exclude=h5,h6  scripts/runTermBoostExperimentsLiterature.sh best_fields AND 2017
 
 # Here, the boolean operator has no effect
-sbatch --exclude=h5,h6  scripts/runTermBoostExperimentsLiterature.sh phrase OR
+sbatch --exclude=h5,h6  scripts/runTermBoostExperimentsLiterature.sh phrase OR 2017
+
+
+# For the best_fields, the slop does not do anything, but a argument value is expected
+sbatch --exclude=h5,h6 scripts/runTermBoostExperimentsLiterature.sh best_fields OR 2018
+sbatch --exclude=h5,h6  scripts/runTermBoostExperimentsLiterature.sh best_fields AND 2018
+
+# Here, the boolean operator has no effect
+sbatch --exclude=h5,h6  scripts/runTermBoostExperimentsLiterature.sh phrase OR 2018
diff --git a/scripts/runBoostOptimizer.sh b/scripts/runBoostOptimizer.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 #SBATCH --mem 10G
 #SBATCH --cpus-per-task 4
-WHAT=$1
+#SBATCH -J boostopt
 
-mvn exec:java -Dexec.mainClass=at.medunigraz.imi.bst.trec.SigirPubmedExperimenterBoostOptimizer -Dexec.args="$1"
+
+mvn exec:java -Dexec.mainClass=at.medunigraz.imi.bst.trec.SigirPubmedExperimenterBoostOptimizer -Dexec.args="$1 $2"
diff --git a/scripts/runPmClassExperimentsLiterature.sh b/scripts/runPmClassExperimentsLiterature.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 #SBATCH --cpus-per-task 2
 #SBATCH --mem 10G
-#SBATCH -J termboostexp
+#SBATCH -J pmclassexp
 
-mvn exec:java -Dexec.mainClass=at.medunigraz.imi.bst.trec.SigirPubmedExperimenterPmClass  -Dexec.args="$1 $2"
+mvn exec:java -Dexec.mainClass=at.medunigraz.imi.bst.trec.SigirPubmedExperimenterPmClass  -Dexec.args="$1 $2 $3"
diff --git a/scripts/runTermBoostExperimentsLiterature.sh b/scripts/runTermBoostExperimentsLiterature.sh
@@ -3,4 +3,4 @@
 #SBATCH --mem 10G
 #SBATCH -J termboostexp
 
-mvn exec:java -Dexec.mainClass=at.medunigraz.imi.bst.trec.SigirPubmedTermBoostExperimenterDefaultBoosting -Dexec.args="$1 $2"
+mvn exec:java -Dexec.mainClass=at.medunigraz.imi.bst.trec.SigirPubmedTermBoostExperimenterDefaultBoosting -Dexec.args="$1 $2 $3"
diff --git a/src/main/java/at/medunigraz/imi/bst/trec/SigirPubmedExperimenterBoostOptimizer.java b/src/main/java/at/medunigraz/imi/bst/trec/SigirPubmedExperimenterBoostOptimizer.java
@@ -2,6 +2,7 @@
 
 import at.medunigraz.imi.bst.trec.experiment.Experiment;
 
+import java.io.File;
 import java.text.DecimalFormat;
 import java.util.*;
 import java.util.stream.IntStream;
@@ -19,8 +20,8 @@ public static void main(String[] args) {
         validParams.add("mutation");
         validParams.add("drug");
 
-        if (args.length != 1 || !validParams.contains(args[0])) {
-            System.err.println("Usage: " + SigirPubmedExperimenterBoostOptimizer.class.getSimpleName() + " <what>");
+        if (args.length != 2 || !validParams.contains(args[0])) {
+            System.err.println("Usage: " + SigirPubmedExperimenterBoostOptimizer.class.getSimpleName() + " <what> <year>");
             System.err.println("Where <what> is one of " + validParams);
             System.exit(1);
         }
@@ -29,7 +30,7 @@ public static void main(String[] args) {
 
         final Experiment.GoldStandard goldStandard = Experiment.GoldStandard.OFFICIAL;
         final Experiment.Task target = Experiment.Task.PUBMED;
-        final int year = 2017;
+        final int year = Integer.parseInt(args[1]);
 
 
         Map<String, String> templateProperties = Collections.unmodifiableMap(SigirParameters.LITERATURE_ES_DEFAULTS);
@@ -58,7 +59,7 @@ public static void main(String[] args) {
                     }
                 }
             }
-            runExperimentsWithParameters(parameters, suffixes, year, what, goldStandard, target);
+            runExperimentsWithParameters("/templates/sigir19_experiments_biomed/baseline.json", parameters, suffixes, year, what, goldStandard, target);
         } else if (what.equals("fields")) {
             List<Map<String, String>> parameters = new ArrayList<>();
             List<String> suffixes = new ArrayList<>();
@@ -81,7 +82,7 @@ public static void main(String[] args) {
                     }
                 }
             }
-            runExperimentsWithParameters(parameters, suffixes, year, what, goldStandard, target);
+            runExperimentsWithParameters("/templates/sigir19_field_boost_optimization", parameters, suffixes, year, what, goldStandard, target);
         } else if (what.equals("posneg")) {
             List<Map<String, String>> parameters = new ArrayList<>();
             List<String> suffixes = new ArrayList<>();
@@ -95,7 +96,7 @@ public static void main(String[] args) {
                     suffixes.add(suffix);
                 }
             }
-            runExperimentsWithParameters(parameters, suffixes, year, what, goldStandard, target);
+            runExperimentsWithParameters("/templates/sigir19_boost_optimization/posneg.json",parameters, suffixes, year, what, goldStandard, target);
         } else if (what.equals("additional")) {
             List<Map<String, String>> parameters = new ArrayList<>();
             List<String> suffixes = new ArrayList<>();
@@ -112,7 +113,7 @@ public static void main(String[] args) {
                     }
                 }
             }
-            runExperimentsWithParameters(parameters, suffixes, year, what, goldStandard, target);
+            runExperimentsWithParameters("/templates/sigir19_boost_optimization/additional.json", parameters, suffixes, year, what, goldStandard, target);
         } else if (what.equals("extra")) {
             List<Map<String, String>> parameters = new ArrayList<>();
             List<String> suffixes = new ArrayList<>();
@@ -123,7 +124,7 @@ public static void main(String[] args) {
                 parameters.add(paramcombination);
                 suffixes.add(suffix);
             }
-            runExperimentsWithParameters(parameters, suffixes, year, what, goldStandard, target);
+            runExperimentsWithParameters("/templates/sigir19_experiments_biomed/extra.json",parameters, suffixes, year, what, goldStandard, target);
         } else if (what.equals("mutation")) {
             List<Map<String, String>> parameters = new ArrayList<>();
             List<String> suffixes = new ArrayList<>();
@@ -134,8 +135,10 @@ public static void main(String[] args) {
                 parameters.add(paramcombination);
                 suffixes.add(suffix);
             }
-            runExperimentsWithParameters(parameters, suffixes, year, what, goldStandard, target);
+            runExperimentsWithParameters("/templates/sigir19_experiments_biomed/mutations.json", parameters, suffixes, year, what, goldStandard, target);
         } else if (what.equals("pmclass")) {
+            List<Map<String, String>> parameters = new ArrayList<>();
+            List<String> suffixes = new ArrayList<>();
             final List<String> pmfields = Arrays.asList("pmclass2017lstm.keyword",
                     "pmclass2017lstmatt.keyword",
                     "pmclass2017lstmgru.keyword",
@@ -144,22 +147,28 @@ public static void main(String[] args) {
                     "pmclass2018lstmgru.keyword",
                     "pmclass2017.keyword",
                     "pmclass2018.keyword");
-            //final List<String> pmfields = Arrays.asList( "pmclass2017.keyword");
-            pmfields.parallelStream().forEach(pmfield -> {
-                Map<String, String> parameters = new HashMap<>(templateProperties);
-                parameters.put("pm_class_field", pmfield);
-                //  runExperiments(parameters, false, goldStandard, target, year, what, "-" + pmfield);
-            });
+            for (String pmfield : pmfields) {
+                for (double extrab = .5; extrab <= 3; extrab += .5) {
+                    Map<String, String> paramcombination = new HashMap<>(templateProperties);
+                    paramcombination.put("pm_boost", String.valueOf(extrab));
+                    paramcombination.put("pm_class_field", pmfield);
+                    String suffix = "--pm" + df.format(extrab) + "-pmf:";
+                    parameters.add(paramcombination);
+                    suffixes.add(suffix);
+                }
+            }
+            runExperimentsWithParameters("/templates/sigir19_pmclass_biomed", parameters, suffixes, year, what, goldStandard, target);
+
         } else throw new IllegalStateException("Unknown mode " + what);
 
 
     }
 
-    private static void runExperimentsWithParameters(List<Map<String, String>> parameters, List<String> suffixes, int year, String what, Experiment.GoldStandard goldStandard, Experiment.Task target) {
+    private static void runExperimentsWithParameters(String template, List<Map<String, String>> parameters, List<String> suffixes, int year, String what, Experiment.GoldStandard goldStandard, Experiment.Task target) {
         IntStream.range(0, parameters.size()).parallel().forEach(i -> {
             Map<String, String> parameterset = parameters.get(i);
             String suffix = suffixes.get(i);
-            // runExperiments(parameterset, false, goldStandard, target, year, what, suffix);
+             runBoostExperiments(template, parameterset, goldStandard, target, year, what, suffix);
         });
     }
 }
diff --git a/src/main/java/at/medunigraz/imi/bst/trec/SigirPubmedExperimenterPmClass.java b/src/main/java/at/medunigraz/imi/bst/trec/SigirPubmedExperimenterPmClass.java
@@ -11,15 +11,15 @@
 public class SigirPubmedExperimenterPmClass extends SuperSigirPubmedRecallExperimenter {
     public static void main(String[] args) {
 
-        if (args.length != 2) {
-            System.err.println("Parameters: <multi match mode> <match default operator>");
+        if (args.length != 3) {
+            System.err.println("Parameters: <multi match mode> <match default operator> <year>");
         }
 
         String what = "pmclass";
 
         final Experiment.GoldStandard goldStandard = Experiment.GoldStandard.OFFICIAL;
         final Experiment.Task target = Experiment.Task.PUBMED;
-        final int year = 2017;
+        final int year = Integer.parseInt(args[2]);
 
 
 

diff --git a/...main/java/at/medunigraz/imi/bst/trec/SigirPubmedTermBoostExperimenterDefaultBoosting.java b/...main/java/at/medunigraz/imi/bst/trec/SigirPubmedTermBoostExperimenterDefaultBoosting.java
@@ -10,13 +10,13 @@
 public class SigirPubmedTermBoostExperimenterDefaultBoosting extends SuperSigirPubmedRecallExperimenter {
     public static void main(String[] args) {
 
-        if (args.length != 2) {
-            System.err.println("Parameters: <multi match mode> <match default operator>");
+        if (args.length != 3) {
+            System.err.println("Parameters: <multi match mode> <match default operator> <year>");
         }
 
         final Experiment.GoldStandard goldStandard = Experiment.GoldStandard.OFFICIAL;
         final Experiment.Task target = Experiment.Task.PUBMED;
-        final int year = 2018;
+        final int year = Integer.parseInt(args[2]);
 
 
         Map<String, String> templateProperties = new HashMap<>(SigirParameters.LITERATURE_ES_DEFAULTS);

diff --git a/src/main/java/at/medunigraz/imi/bst/trec/SuperSigirPubmedRecallExperimenter.java b/src/main/java/at/medunigraz/imi/bst/trec/SuperSigirPubmedRecallExperimenter.java
@@ -2,6 +2,7 @@
 
 import at.medunigraz.imi.bst.trec.experiment.Experiment;
 import at.medunigraz.imi.bst.trec.experiment.ExperimentsBuilder;
+import bsh.commands.dir;
 import com.google.common.collect.Sets;
 import org.apache.commons.lang.StringUtils;
 import org.apache.logging.log4j.LogManager;
@@ -48,6 +49,7 @@ protected static void runPmClassifierExperiments(File template, Map<String, Stri
                 throw new RuntimeException(e);
             }
         }
+        executorService.shutdown();
     }
 
     /**
@@ -61,7 +63,7 @@ protected static void runPmClassifierExperiments(File template, Map<String, Stri
      * @param what
      * @param suffix
      */
-    protected static void runBoostExperiments(File template, Map<String, String> templateProperties, Experiment.GoldStandard goldStandard, Experiment.Task target, int year, String what, String suffix) {
+    protected static void runBoostExperiments(String template, Map<String, String> templateProperties, Experiment.GoldStandard goldStandard, Experiment.Task target, int year, String what, String suffix) {
         if (templateProperties.size() > numProperties)
             throw new IllegalArgumentException("There are more key in the properties map as there are known properties: " + templateProperties.keySet());
 
@@ -70,12 +72,22 @@ protected static void runBoostExperiments(File template, Map<String, String> tem
             builder.setDefaultStatsDir("stats_" + what + "_" + year);
             builder.setDefaultResultsDir("results_" + what + "_" + year);
         }
-        final Map<String, TemplateSet> sigirTemplates = getSigirTemplates("/templates/sigir19_experiments_biomed");
+        final File file = new File(PubmedExperimenter.class.getResource(template).getFile());
+        Map<String, TemplateSet> sigirTemplates;
+        if (file.isDirectory()) {
+            sigirTemplates = getSigirTemplates(template);
+        } else {
+            final TemplateSet templateSet = new TemplateSet();
+            templateSet.setBase(file);
+            sigirTemplates = new HashMap<>();
+            sigirTemplates.put(file.getName(), templateSet);
+        }
+
 
         // Switch everything on - except word removal - so that the boosters actually have a point.
         final Set<Set<Expansion>> expansionSets = new HashSet<>(Arrays.asList(EnumSet.complementOf(EnumSet.of(Expansion.WR))));
 
-        addExperiments(template, sigirTemplates, templateProperties, expansionSets, goldStandard, target, year, builder, suffix);
+        addExperiments(null, sigirTemplates, templateProperties, expansionSets, goldStandard, target, year, builder, suffix);
 
         Set<Experiment> experiments = builder.build();
         final ExecutorService executorService = Executors.newFixedThreadPool(5);
@@ -91,6 +103,7 @@ protected static void runBoostExperiments(File template, Map<String, String> tem
                 throw new RuntimeException(e);
             }
         }
+        executorService.shutdown();
     }
 
     /**
@@ -138,6 +151,7 @@ protected static void runTermBoostExperiments(Map<String, String> templateProper
                 throw new RuntimeException(e);
             }
         }
+        executorService.shutdown();
     }
 
     /**

diff --git a/src/main/resources/templates/sigir19_boost_optimization/additional.json b/src/main/resources/templates/sigir19_boost_optimization/additional.json
@@ -0,0 +1,13 @@
+{
+	"bool": {
+		"must": [
+			{{sigir19_experiments_biomed/disease.json}},
+			{{sigir19_experiments_biomed/gene.json}}
+		],
+		"should": [
+			{{sigir19_experiments_biomed/chemotherapy.json}},
+			{{sigir19_experiments_biomed/cancer.json}},
+			{{sigir19_experiments_biomed/dna.json}}
+		]
+	}
+}
diff --git a/src/main/resources/templates/sigir19_boost_optimization/posneg.json b/src/main/resources/templates/sigir19_boost_optimization/posneg.json
@@ -0,0 +1,12 @@
+{
+  "bool": {
+    "must": [
+      {{sigir19_experiments_biomed/disease.json}},
+      {{sigir19_experiments_biomed/gene.json}}
+    ],
+    "should": [
+      {{sigir19_experiments_biomed/positive_boosters.json}},
+      {{sigir19_experiments_biomed/negative_boosters.json}}
+    ]
+  }
+}
diff --git a/src/main/resources/templates/sigir19_experiments_biomed/cancer.json b/src/main/resources/templates/sigir19_experiments_biomed/cancer.json
@@ -2,7 +2,7 @@
 	"bool": {
 		"must": [
 			{{sigir19_experiments_biomed/disease.json}},
-			{{sigir19_experiments_biomed/gene_plus_genefield.json}}
+			{{sigir19_experiments_biomed/gene.json}}
 		],
 		"should": [
 			{{sigir19_experiments_biomed/cancer.json}},

diff --git a/src/main/resources/templates/sigir19_experiments_biomed/chemo.json b/src/main/resources/templates/sigir19_experiments_biomed/chemo.json
@@ -2,7 +2,7 @@
 	"bool": {
 		"must": [
 			{{sigir19_experiments_biomed/disease.json}},
-			{{sigir19_experiments_biomed/gene_plus_genefield.json}}
+			{{sigir19_experiments_biomed/gene.json}}
 		],
 		"should": [
 			{{sigir19_experiments_biomed/chemotherapy.json}},

diff --git a/src/main/resources/templates/sigir19_experiments_biomed/dna.json b/src/main/resources/templates/sigir19_experiments_biomed/dna.json
@@ -2,7 +2,7 @@
 	"bool": {
 		"must": [
 			{{sigir19_experiments_biomed/disease.json}},
-			{{sigir19_experiments_biomed/gene_plus_genefield.json}}
+			{{sigir19_experiments_biomed/gene.json}}
 		],
 		"should": [
 			{{sigir19_experiments_biomed/dna.json}},

diff --git a/...lass_biomed/baseline_custompm_should.json → ...tes/sigir19_experiments_biomed/extra.json b/...lass_biomed/baseline_custompm_should.json → ...tes/sigir19_experiments_biomed/extra.json
@@ -5,7 +5,7 @@
 			{{sigir19_experiments_biomed/gene.json}}
 		],
 		"should": [
-			{{sigir19_experiments_biomed/pm.json}}
+			{{sigir19_experiments_biomed/extra.json}},
 		]
 	}
 }
diff --git a/src/main/resources/templates/sigir19_experiments_biomed/genefield_custompm_must.json b/src/main/resources/templates/sigir19_experiments_biomed/genefield_custompm_must.json
diff --git a/src/main/resources/templates/sigir19_experiments_biomed/genefield_custompm_should.json b/src/main/resources/templates/sigir19_experiments_biomed/genefield_custompm_should.json