Impulse Response Function addition

.circleci/config.yml

@@ -7,7 +7,7 @@ jobs:
       - checkout
       - run: bash .circleci/setup.sh
       - run: bash .circleci/checkout.sh
-      - run: wget https://dabdceba-6d04-11e5-ba46-22000b92c6ec.e.globus.org/containers/public/e3sm.sif
+      - run: bash .circleci/container.sh
       - run:
           command: singularity exec --hostname singularity e3sm.sif .circleci/run.sh
           no_output_timeout: 60m

.circleci/container.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+wget -t 3  http://portal.nersc.gov/project/e3sm/lukasz/e3sm.sif
+if [ $? -eq 0 ]; then
+    exit 0;
+fi
+
+wget -t 3 https://dabdceba-6d04-11e5-ba46-22000b92c6ec.e.globus.org/containers/public/e3sm.sif
+if [ $? -eq 0 ]; then
+    exit 0;
+fi
+
+exit -1

cime_config/allactive/config_pesall.xml

@@ -8387,12 +8387,12 @@
   <grid name="a%ne30np4.pg.+_oi%EC30to60E2r2">
     <mach name="chrysalis">
       <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="XS">
-        <comment> -compset A_WCYCL* -res ne30pg*EC30to60* on 14 nodes pure-MPI, ~6 sypd </comment>
+        <comment> -compset A_WCYCL* -res ne30pg*EC30to60* on 14 nodes pure-MPI, ~8.1 sypd </comment>
         <ntasks>
           <ntasks_atm>675</ntasks_atm>
-          <ntasks_lnd>128</ntasks_lnd>
-          <ntasks_rof>128</ntasks_rof>
-          <ntasks_ice>576</ntasks_ice>
+          <ntasks_lnd>64</ntasks_lnd>
+          <ntasks_rof>64</ntasks_rof>
+          <ntasks_ice>640</ntasks_ice>
           <ntasks_ocn>192</ntasks_ocn>
           <ntasks_cpl>704</ntasks_cpl>
         </ntasks>
@@ -8406,21 +8406,21 @@
         </nthrds>
         <rootpe>
           <rootpe_atm>0</rootpe_atm>
-          <rootpe_lnd>576</rootpe_lnd>
-          <rootpe_rof>576</rootpe_rof>
+          <rootpe_lnd>640</rootpe_lnd>
+          <rootpe_rof>640</rootpe_rof>
           <rootpe_ice>0</rootpe_ice>
           <rootpe_ocn>704</rootpe_ocn>
           <rootpe_cpl>0</rootpe_cpl>
         </rootpe>
       </pes>
       <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="S">
-        <comment> -compset A_WCYCL* -res ne30pg*EC30to60* on 26 nodes pure-MPI, ~10 sypd </comment>
+        <comment> -compset A_WCYCL* -res ne30pg*EC30to60* on 28 nodes pure-MPI, ~15.3 sypd </comment>
         <ntasks>
           <ntasks_atm>1350</ntasks_atm>
-          <ntasks_lnd>128</ntasks_lnd>
-          <ntasks_rof>128</ntasks_rof>
-          <ntasks_ice>1280</ntasks_ice>
-          <ntasks_ocn>256</ntasks_ocn>
+          <ntasks_lnd>64</ntasks_lnd>
+          <ntasks_rof>64</ntasks_rof>
+          <ntasks_ice>1344</ntasks_ice>
+          <ntasks_ocn>384</ntasks_ocn>
           <ntasks_cpl>1408</ntasks_cpl>
         </ntasks>
         <nthrds>
@@ -8433,21 +8433,21 @@
         </nthrds>
         <rootpe>
           <rootpe_atm>0</rootpe_atm>
-          <rootpe_lnd>1280</rootpe_lnd>
-          <rootpe_rof>1280</rootpe_rof>
+          <rootpe_lnd>1344</rootpe_lnd>
+          <rootpe_rof>1344</rootpe_rof>
           <rootpe_ice>0</rootpe_ice>
           <rootpe_ocn>1408</rootpe_ocn>
           <rootpe_cpl>0</rootpe_cpl>
         </rootpe>
       </pes>
-      <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="M2">
-        <comment> -compset A_WCYCL* -res ne30pg*EC30to60* on 51 nodes pure-MPI, ~14 sypd </comment>
+      <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="M">
+        <comment> -compset A_WCYCL* -res ne30pg*EC30to60* on 53 nodes pure-MPI, ~26.4 sypd </comment>
         <ntasks>
-          <ntasks_atm>2700</ntasks_atm>
+          <ntasks_atm>2752</ntasks_atm>
           <ntasks_lnd>192</ntasks_lnd>
           <ntasks_rof>192</ntasks_rof>
           <ntasks_ice>2560</ntasks_ice>
-          <ntasks_ocn>512</ntasks_ocn>
+          <ntasks_ocn>640</ntasks_ocn>
           <ntasks_cpl>2752</ntasks_cpl>
         </ntasks>
         <nthrds>
@@ -8467,15 +8467,15 @@
           <rootpe_cpl>0</rootpe_cpl>
         </rootpe>
       </pes>
-      <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="M">
-        <comment> -compset A_WCYCL* -res ne30pg*EC30to60* on 60 nodes pure-MPI, ~24 sypd </comment>
+      <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="ML">
+        <comment> -compset A_WCYCL* -res ne30pg*EC30to60* on 71 nodes pure-MPI, ~31.5 sypd </comment>
         <ntasks>
-          <ntasks_atm>3200</ntasks_atm>
-          <ntasks_lnd>640</ntasks_lnd>
-          <ntasks_rof>640</ntasks_rof>
-          <ntasks_ice>2560</ntasks_ice>
-          <ntasks_ocn>640</ntasks_ocn>
-          <ntasks_cpl>3200</ntasks_cpl>
+          <ntasks_atm>3648</ntasks_atm>
+          <ntasks_lnd>192</ntasks_lnd>
+          <ntasks_rof>192</ntasks_rof>
+          <ntasks_ice>3456</ntasks_ice>
+          <ntasks_ocn>896</ntasks_ocn>
+          <ntasks_cpl>3648</ntasks_cpl>
         </ntasks>
         <nthrds>
           <nthrds_atm>1</nthrds_atm>
@@ -8487,22 +8487,22 @@
         </nthrds>
         <rootpe>
           <rootpe_atm>0</rootpe_atm>
-          <rootpe_lnd>2560</rootpe_lnd>
-          <rootpe_rof>2560</rootpe_rof>
+          <rootpe_lnd>3456</rootpe_lnd>
+          <rootpe_rof>3456</rootpe_rof>
           <rootpe_ice>0</rootpe_ice>
-          <rootpe_ocn>3200</rootpe_ocn>
+          <rootpe_ocn>3648</rootpe_ocn>
           <rootpe_cpl>0</rootpe_cpl>
         </rootpe>
       </pes>
-      <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="M80">
-        <comment> -compset A_WCYCL* -res ne30pg*EC30to60* on 80 nodes pure-MPI, ~30 sypd </comment>
+      <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="L">
+        <comment> -compset A_WCYCL* -res ne30pg*EC30to60* on 105 nodes pure-MPI, ~42.3 sypd </comment>
         <ntasks>
-          <ntasks_atm>4160</ntasks_atm>
+          <ntasks_atm>5400</ntasks_atm>
           <ntasks_lnd>320</ntasks_lnd>
           <ntasks_rof>320</ntasks_rof>
-          <ntasks_ice>3840</ntasks_ice>
-          <ntasks_ocn>960</ntasks_ocn>
-          <ntasks_cpl>4160</ntasks_cpl>
+          <ntasks_ice>5120</ntasks_ice>
+          <ntasks_ocn>1280</ntasks_ocn>
+          <ntasks_cpl>5440</ntasks_cpl>
         </ntasks>
         <nthrds>
           <nthrds_atm>1</nthrds_atm>
@@ -8514,22 +8514,53 @@
         </nthrds>
         <rootpe>
           <rootpe_atm>0</rootpe_atm>
-          <rootpe_lnd>3840</rootpe_lnd>
-          <rootpe_rof>3840</rootpe_rof>
+          <rootpe_lnd>5120</rootpe_lnd>
+          <rootpe_rof>5120</rootpe_rof>
+          <rootpe_ice>0</rootpe_ice>
+          <rootpe_ocn>5440</rootpe_ocn>
+          <rootpe_cpl>0</rootpe_cpl>
+        </rootpe>
+      </pes>
+    </mach>
+  </grid>
+  <grid name="a%ne30np4.pg.+_oi%EC30to60E2r2">
+    <mach name="theta">
+      <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="XS">
+        <comment> -compset A_WCYCL* -res ne30pg*EC30to60* on 8 debug Q nodes threaded, 0.8 sypd </comment>
+        <ntasks>
+          <ntasks_atm>384</ntasks_atm>
+          <ntasks_lnd>64</ntasks_lnd>
+          <ntasks_rof>64</ntasks_rof>
+          <ntasks_ice>320</ntasks_ice>
+          <ntasks_ocn>128</ntasks_ocn>
+          <ntasks_cpl>384</ntasks_cpl>
+        </ntasks>
+        <nthrds>
+          <nthrds_atm>2</nthrds_atm>
+          <nthrds_lnd>2</nthrds_lnd>
+          <nthrds_rof>2</nthrds_rof>
+          <nthrds_ice>2</nthrds_ice>
+          <nthrds_ocn>2</nthrds_ocn>
+          <nthrds_cpl>2</nthrds_cpl>
+        </nthrds>
+        <rootpe>
+          <rootpe_atm>0</rootpe_atm>
+          <rootpe_lnd>320</rootpe_lnd>
+          <rootpe_rof>320</rootpe_rof>
           <rootpe_ice>0</rootpe_ice>
-          <rootpe_ocn>4160</rootpe_ocn>
+          <rootpe_ocn>384</rootpe_ocn>
           <rootpe_cpl>0</rootpe_cpl>
         </rootpe>
       </pes>
-      <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="L">
-        <comment> -compset A_WCYCL* -res ne30pg*EC30to60* on 105 nodes pure-MPI, ~38 sypd </comment>
+      <pes compset=".*EAM.+ELM.+MPASSI.+MPASO.+MOSART.+" pesize="any">
+        <comment> -compset A_WCYCL* -res ne30pg*EC30to60* on 128 default Q nodes pure-MPI, 3.8 sypd </comment>
         <ntasks>
           <ntasks_atm>5400</ntasks_atm>
-          <ntasks_lnd>320</ntasks_lnd>
-          <ntasks_rof>320</ntasks_rof>
-          <ntasks_ice>5120</ntasks_ice>
+          <ntasks_lnd>512</ntasks_lnd>
+          <ntasks_rof>512</ntasks_rof>
+          <ntasks_ice>6400</ntasks_ice>
           <ntasks_ocn>1280</ntasks_ocn>
-          <ntasks_cpl>5440</ntasks_cpl>
+          <ntasks_cpl>6912</ntasks_cpl>
         </ntasks>
         <nthrds>
           <nthrds_atm>1</nthrds_atm>
@@ -8541,10 +8572,10 @@
         </nthrds>
         <rootpe>
           <rootpe_atm>0</rootpe_atm>
-          <rootpe_lnd>5120</rootpe_lnd>
-          <rootpe_rof>5120</rootpe_rof>
+          <rootpe_lnd>6400</rootpe_lnd>
+          <rootpe_rof>6400</rootpe_rof>
           <rootpe_ice>0</rootpe_ice>
-          <rootpe_ocn>5440</rootpe_ocn>
+          <rootpe_ocn>6912</rootpe_ocn>
           <rootpe_cpl>0</rootpe_cpl>
         </rootpe>
       </pes>

cime_config/machines/config_batch.xml

@@ -72,7 +72,7 @@
       <arg flag="--cwd" name="CASEROOT"/>
       <arg flag="-A" name="CHARGE_ACCOUNT"/>
       <arg flag="-t" name="JOB_WALLCLOCK_TIME"/>
-      <arg flag="-n" name=" ($TOTALPES + $MAX_MPITASKS_PER_NODE - 1)/$MAX_MPITASKS_PER_NODE"/>
+      <arg flag="-n" name=" $TOTALPES/$MAX_MPITASKS_PER_NODE"/>
       <arg flag="-q" name="JOB_QUEUE"/>
       <arg flag="--mode script"/>
     </submit_args>
@@ -271,6 +271,15 @@
       </queues>
     </batch_system>
 
+    <batch_system MACH="swing" type="slurm">
+      <directives>
+        <directive> --gres=gpu:1</directive>
+      </directives>
+      <queues>
+        <queue walltimemax="01:00:00" nodemax="6" default="true" strict="true">gpu</queue>
+      </queues>
+    </batch_system>
+
     <batch_system MACH="anvil" type="slurm" >
       <queues>
 	<queue walltimemax="48:00:00" nodemax="5" default="true" strict="true">acme-small</queue>