timing

job_template_timing.txt

@@ -0,0 +1,143 @@
+#!/bin/bash
+#SBATCH --mail-user=${job_email}
+#SBATCH --mail-type=${job_email_options}
+#SBATCH --account=${job_rungroup}
+#SBATCH --nodes=1
+#SBATCH --ntasks=${job_ppn}
+#SBATCH --time=${job_walltime}
+#SBATCH --mem=${job_memory}mb
+#SBATCH -o ${job_output_file}
+
+echo "Starting smemwatch"
+smemwatch -k 95 -d 50 $$$$ &
+
+uname -a # outputs node info (name, date&time, type, OS, etc)
+date
+
+#=============================================================================
+#VERSION=
+#JOBDIR=
+#INDIR=
+#OUTDIR=
+#DSTDIR=
+#INLIST=
+#OUTLIST=
+#CONTAINERPATH=
+#MAINCMD=
+#XNATHOST=
+#XNATUSER=
+${job_cmds}
+#=============================================================================
+SESSLIMIT=25
+SLEEPMAX=300
+
+echo $DSTDIR
+echo $INDIR
+echo $OUTDIR
+echo $CONTAINERPATH
+echo $XNATHOST
+echo $XNATUSER
+echo $SESSLIMIT
+echo $SLEEPMAX
+
+mkdir -p $INDIR
+mkdir -p $OUTDIR
+
+# Check number of open sessions on host before we download
+echo "Checking that we can download"
+while true;do
+    SESSID=`curl -s -n "$XNATHOST/data/JSESSION"`
+    #echo "JSESSIONID=$SESSID"
+    SESSCOUNT=`curl -s -b "JSESSIONID=$SESSID" "$XNATHOST/data/user/$XNATUSER/sessions" | cut -s -f2 -d ":" | cut -f1 -d "}"`
+    echo "$SESSCOUNT"
+    if (( "$SESSCOUNT" > "$SESSLIMIT" )); then
+        echo -n "Cannot download yet, too many open sessions, sleeping "
+        curl -s -b "JSESSIONID=$SESSID" -X DELETE "$XNATHOST/data/JSESSION"
+        SLEEPSECS=$[ ( $RANDOM % $SLEEPMAX ) ]s && echo "$SLEEPSECS" && sleep $SLEEPSECS
+    else
+        echo "Looks good, ready to download" 
+        break
+    fi
+done
+#echo "DONE! JSESSIONID=$SESSID"
+echo "DONE!"
+
+# Collect inputs
+date "+%Y%m%d %H%M%S"
+for IN in "${INLIST[@]}"; do
+    IFS=',' read -r col1 col2 col3 <<< "$IN"
+    if [ $col2 == "FILE" ]; then
+        CMD="curl -D - -s -b JSESSIONID=$SESSID \"$col3\" -o $INDIR/$col1"
+        CMD+=" | head -n 1 | awk '{print \$2}'"
+    elif [ $col2 == "DIRJ" ]; then
+        CMD="curl -D - -s -b JSESSIONID=$SESSID \"$col3\"?format=zip -o $INDIR/${col1}.zip"
+        CMD+=" | head -n 1 | awk '{print \$2}'"
+        CMD+=" && unzip -q -j $INDIR/${col1}.zip -d $INDIR/$col1"
+    else
+        CMD="curl -D - -s -b JSESSIONID=$SESSID \"${col3}?format=zip&structure=simplified\" -o $INDIR/${col1}.zip"
+        CMD+=" | head -n 1 | awk '{print \$2}'" 
+        CMD+=" && unzip -q $INDIR/${col1}.zip -d $INDIR/$col1 && mv $INDIR/$col1/*/out/* $INDIR/$col1"
+    fi
+
+    # Show the whole command
+	SAFE_CMD=$(echo ${CMD} | sed "s/$SESSID/SESSID/g")
+    echo $SAFE_CMD
+
+    # Run the full command
+    eval result=\$\($CMD\)
+    echo "result=$result"
+
+    # Check for success
+    if test "$result" != '200' ; then
+        echo "Download failed with code:$result, exiting..."
+        curl -s -b "JSESSIONID=$SESSID" -X DELETE "$XNATHOST/data/JSESSION"
+        exit 1
+    fi
+done
+
+# Disconnect XNAT
+curl -s -b "JSESSIONID=$SESSID" -X DELETE "$XNATHOST/data/JSESSION"
+date "+%Y%m%d %H%M%S"
+
+# Run main command
+echo $_JAVA_OPTIONS
+echo $MAINCMD
+eval $MAINCMD
+
+# Write version files
+mkdir -p $DSTDIR
+echo $VERSION > $DSTDIR/version.txt
+sha256sum $CONTAINERPATH | awk '{print $1}' > $DSTDIR/dax_docker_version.txt
+
+# Handle outputs
+errors=0
+haspdf=0
+for OUT in "${OUTLIST[@]}"; do
+    IFS=',' read -r col1 col2 col3 col4 <<< "$OUT"
+    pathlist=(${OUTDIR}/${col1})
+
+    # Check for missing output
+    if [[ ${#pathlist[@]} == 1 && ! -e $pathlist ]]; then
+        if [[ $col4 != "F" ]]; then errors=1 && echo "ERROR:missing required output:$col1"
+        else echo "WARN:output not required"; fi
+        continue
+    fi
+
+    # Copy output based on type
+    mkdir -p "$DSTDIR/$col3"
+    if [ $col3 == "PDF" ]; then
+        if [ $col2 != "FILE" ]; then errors=1 && echo "ERROR:illegal type for PDF";
+        elif [[ ${#pathlist[@]} != 1 ]]; then errors=1 && echo "ERROR:multiple PDFs";
+        else haspdf=1 && cp $OUTDIR/$col1 $DSTDIR/$col3 || errors=1; fi
+    elif [ $col2 == "FILE" ]; then cp $OUTDIR/$col1 $DSTDIR/$col3 || errors=1;
+    elif [ $col2 == "DIR" ]; then cp -r $OUTDIR/$col1/* $DSTDIR/$col3 || errors=1;
+    else errors=1 && echo "ERROR:invalid type:$col2"; fi
+done
+
+if [ $errors -gt 0 ] || [ $haspdf != 1 ]; then echo "JOB_FAILED" && touch $DSTDIR/JOB_FAILED.txt;
+else echo "COMPLETE" && touch $DSTDIR/READY_TO_UPLOAD.txt; fi
+
+rm -rf $INDIR $OUTDIR
+
+echo "DONE!"
+