From f2b23b3b1640ce176371e3cf29319f0153d5eb18 Mon Sep 17 00:00:00 2001
From: duettwe <61516027+duettwe@users.noreply.github.com>
Date: Thu, 4 Nov 2021 09:47:25 -0500
Subject: [PATCH 1/6] Create job_template_fmriprep-fsdev2.txt

---
 job_template_fmriprep-fsdev2.txt | 192 +++++++++++++++++++++++++++++++
 1 file changed, 192 insertions(+)
 create mode 100644 job_template_fmriprep-fsdev2.txt

diff --git a/job_template_fmriprep-fsdev2.txt b/job_template_fmriprep-fsdev2.txt
new file mode 100644
index 0000000..ae99790
--- /dev/null
+++ b/job_template_fmriprep-fsdev2.txt
@@ -0,0 +1,192 @@
+#!/bin/bash
+#SBATCH --mail-user=${job_email}
+#SBATCH --mail-type=${job_email_options}
+#SBATCH --account=${job_rungroup}
+#SBATCH --nodes=1
+#SBATCH --ntasks=${job_ppn}
+#SBATCH --time=${job_walltime}
+#SBATCH --mem=${job_memory}mb
+#SBATCH -o ${job_output_file}
+#SBATCH --constraint=westmere|sandybridge|haswell|skylake
+
+echo "Starting smemwatch"
+smemwatch -k 95 -d 50 $$$$ &
+
+uname -a # outputs node info (name, date&time, type, OS, etc)
+date
+
+#=============================================================================
+#VERSION=
+#JOBDIR=
+#INDIR=
+#OUTDIR=
+#DSTDIR=
+#INLIST=
+#OUTLIST=
+#CONTAINERPATH=
+#MAINCMD=
+#XNATHOST=
+#XNATUSER=
+${job_cmds}
+#=============================================================================
+SESSLIMIT=25
+SLEEPMAX=300
+
+echo $DSTDIR
+echo $INDIR
+echo $OUTDIR
+echo $CONTAINERPATH
+echo $XNATHOST
+echo $XNATUSER
+echo $SESSLIMIT
+echo $SLEEPMAX
+
+mkdir -p $INDIR
+mkdir -p $OUTDIR
+mkdir -p $JOBDIR/WORK
+# Check number of open sessions on host before we download
+echo "Checking that we can download"
+while true;do
+    SESSID=`curl -s -n "$XNATHOST/data/JSESSION"`
+    #echo "JSESSIONID=$SESSID"
+    SESSCOUNT=`curl -s -b "JSESSIONID=$SESSID" "$XNATHOST/data/user/$XNATUSER/sessions" | cut -s -f2 -d ":" | cut -f1 -d "}"`
+    echo "$SESSCOUNT"
+    if (( "$SESSCOUNT" > "$SESSLIMIT" )); then
+        echo -n "Cannot download yet, too many open sessions, sleeping "
+        curl -s -b "JSESSIONID=$SESSID" -X DELETE "$XNATHOST/data/JSESSION"
+        SLEEPSECS=$[ ( $RANDOM % $SLEEPMAX ) ]s && echo "$SLEEPSECS" && sleep $SLEEPSECS
+    else
+        echo "Looks good, ready to download"
+        break
+    fi
+done
+#echo "DONE! JSESSIONID=$SESSID"
+echo "DONE!"
+
+# Collect inputs
+for IN in "${INLIST[@]}"; do
+    IFS=',' read -r col1 col2 col3 <<< "$IN"
+    if [ $col2 == "FILE" ]; then
+        CMD="curl -D - -s -b "JSESSIONID=$SESSID" $col3 -o $INDIR/$col1"
+        CMD+=" | head -n 1 | awk '{print \$2}'"
+    elif [ $col2 == "DIRJ" ]; then
+        CMD="curl -D - -s -b "JSESSIONID=$SESSID" $col3?format=zip -o $INDIR/${col1}.zip"
+        CMD+=" | head -n 1 | awk '{print \$2}'"
+        CMD+=" && unzip -q -j $INDIR/${col1}.zip -d $INDIR/$col1"
+    else
+        CMD="curl -D - -s -b "JSESSIONID=$SESSID" '$col3?format=zip&structure=simplified' -o $INDIR/${col1}.zip"
+        CMD+=" | head -n 1 | awk '{print \$2}'"
+        CMD+=" && unzip -q $INDIR/${col1}.zip -d $INDIR/$col1 && mv $INDIR/$col1/*/out/* $INDIR/$col1"
+    fi
+
+    # Show the whole command
+	SAFE_CMD=$(echo ${CMD} | sed "s/$SESSID/SESSID/g")
+    echo $SAFE_CMD
+
+    # Run the full command
+    eval result=\$\($CMD\)
+    echo "result=$result"
+
+    # Check for success
+    if test $result != '200' ; then
+        echo "Download failed with code:$result, exiting..."
+        curl -s -b "JSESSIONID=$SESSID" -X DELETE "$XNATHOST/data/JSESSION"
+        exit 1
+    fi
+done
+
+# Disconnect XNAT
+curl -s -b "JSESSIONID=$SESSID" -X DELETE "$XNATHOST/data/JSESSION"
+
+#BIDS Conversion
+for IN in "${INLIST[@]}"; do
+    IFS=',' read -r col1 col2 col3 <<< "$IN"
+    if [[ $col1 == *nii.gz ]]; then
+      echo "Start BIDS conv"
+      PROJECT=$(cut -d'/' -f7 <<<"$col3")
+      echo $PROJECT
+      SCAN_ID=$(cut -d'/' -f13 <<<"$col3")
+      echo $SCAN_ID
+      SUBJ=$(cut -d'/' -f9 <<<"$col3")
+      echo $SUBJ
+      SCAN_FILE=$(cut -d'/' -f17 <<<"$col3")
+      echo $SCAN_FILE
+      SESSION=$(cut -d'/' -f11 <<<"$col3")
+      echo $SESSION
+      BIDS_DIR=$INDIR/BIDS_DATA
+      echo $BIDS_DIR
+      python -c 'import os; from dax import XnatToBids, XnatUtils; XNAT = XnatUtils.get_interface(); XnatToBids.bids_yaml(XNAT, "'$PROJECT'","'$SCAN_ID'","'$SUBJ'", "'$INDIR'","'$SCAN_FILE'", "'$col3'", "'$SESSION'", "'$col1'", "001", "001");'
+    # Use existing FS6_v1 in fmriprep output
+    # col2 will be DIR when it contains FS6_v1 assessor output
+    elif [ $col2 == "DIR" ]; then
+      echo "pre-use freesurfer from XNAT"
+      echo $OUTDIR/freesurfer/sub-01
+      mkdir -p $OUTDIR/freesurfer/sub-01
+      echo $INDIR/$col1
+      echo "movin fs folders"
+      mv $INDIR/$col1/SUBJECT/* $OUTDIR/freesurfer/sub-01/
+      ls $OUTDIR/freesurfer/sub-01/
+    fi
+done
+
+# Run main command
+echo $_JAVA_OPTIONS
+echo $MAINCMD
+eval $MAINCMD
+
+# Write version files
+mkdir -p $DSTDIR
+echo $VERSION > $DSTDIR/version.txt
+sha256sum $CONTAINERPATH | awk '{print $1}' > $DSTDIR/dax_docker_version.txt
+
+# Create the PDF from the OUTLOG
+mkdir -p $DSTDIR/PDF
+RESDIR=${DSTDIR%/*}
+TEXTFILE=$RESDIR/DISKQ/OUTLOG/$(cut -d'/' -f5 <<<"$DSTDIR").txt
+PDFFILE=$DSTDIR/PDF/$(cut -d'/' -f5 <<<"$DSTDIR").pdf
+python -c 'from fpdf import FPDF; pdf = FPDF(); pdf.add_page(); pdf.set_font("Courier", size = 9);
+f = open("'$TEXTFILE'", "r",encoding="latin-1");
+for line in f:
+	x = line.encode("latin1","ignore").decode("latin1")
+	pdf.multi_cell(193, 5, txt = x, ln = 1);
+pdf.output("'$PDFFILE'")'
+
+# Handle outputs
+errors=0
+haspdf=0
+for OUT in "${OUTLIST[@]}"; do
+    IFS=',' read -r col1 col2 col3 col4 <<< "$OUT"
+    pathlist=(${OUTDIR}/${col1})
+
+    # Check for missing output
+    if [[ ${#pathlist[@]} == 1 && ! -e $pathlist ]]; then
+        if [[ $col4 != "F" ]]; then errors=1 && echo "ERROR:missing required output:$col1"
+        else echo "WARN:output not required"; fi
+        continue
+    fi
+
+    # Copy output based on type
+    mkdir -p "$DSTDIR/$col3"
+    if [ $col3 == "PDF" ]; then
+        if [ $col2 != "FILE" ]; then errors=1 && echo "ERROR:illegal type for PDF";
+        elif [[ ${#pathlist[@]} != 1 ]]; then errors=1 && echo "ERROR:multiple PDFs";
+        else cp $OUTDIR/$col1 $DSTDIR/$col3 && haspdf=1 || errors=1; fi
+    elif [ $col2 == "FILE" ]; then cp $OUTDIR/$col1 $DSTDIR/$col3 || errors=1;
+    elif [ $col2 == "DIR" ]; then cp -r $OUTDIR/$col1/* $DSTDIR/$col3 || errors=1;
+    else errors=1 && echo "ERROR:invalid type:$col2"; fi
+done
+
+# Check if the PDF file has been created and is non empty
+if [ -f $PDFFILE ]; then
+    haspdf=1 && echo "PDF exist"
+    else echo "PDF does not exist"; fi
+if [ -s $PDFFILE ]; then
+    echo "PDF not empty"
+    else echo "PDF is empty" && errors=1; fi
+
+if [ $errors -gt 0 ] || [ $haspdf != 1 ]; then echo "JOB_FAILED" && touch $DSTDIR/JOB_FAILED.txt;
+else echo "COMPLETE" && touch $DSTDIR/READY_TO_UPLOAD.txt; fi
+
+rm -rf $INDIR $OUTDIR $JOBDIR/WORK
+
+echo "DONE!"

From 4121cc98f1e6739e636f06752f410e436a55225c Mon Sep 17 00:00:00 2001
From: duettwe <61516027+duettwe@users.noreply.github.com>
Date: Wed, 26 Jan 2022 13:36:45 -0600
Subject: [PATCH 2/6] Create job_template_fmriprep_v3.txt

---
 job_template_fmriprep_v3.txt | 193 +++++++++++++++++++++++++++++++++++
 1 file changed, 193 insertions(+)
 create mode 100644 job_template_fmriprep_v3.txt

diff --git a/job_template_fmriprep_v3.txt b/job_template_fmriprep_v3.txt
new file mode 100644
index 0000000..0fff2e7
--- /dev/null
+++ b/job_template_fmriprep_v3.txt
@@ -0,0 +1,193 @@
+#!/bin/bash
+#SBATCH --mail-user=${job_email}
+#SBATCH --mail-type=${job_email_options}
+#SBATCH --account=${job_rungroup}
+#SBATCH --nodes=1
+#SBATCH --ntasks=${job_ppn}
+#SBATCH --time=${job_walltime}
+#SBATCH --mem=${job_memory}
+#SBATCH -o ${job_output_file}
+#SBATCH --constraint=westmere|sandybridge|haswell|skylake
+
+echo "Starting smemwatch"
+smemwatch -k 95 -d 50 $$$$ &
+
+uname -a # outputs node info (name, date&time, type, OS, etc)
+date
+
+#=============================================================================
+#VERSION=
+#JOBDIR=
+#INDIR=
+#OUTDIR=
+#DSTDIR=
+#INLIST=
+#OUTLIST=
+#CONTAINERPATH=
+#MAINCMD=
+#XNATHOST=
+#XNATUSER=
+${job_cmds}
+#=============================================================================
+SESSLIMIT=25
+SLEEPMAX=300
+
+echo $DSTDIR
+echo $INDIR
+echo $OUTDIR
+echo $CONTAINERPATH
+echo $XNATHOST
+echo $XNATUSER
+echo $SESSLIMIT
+echo $SLEEPMAX
+
+mkdir -p $INDIR
+mkdir -p $OUTDIR
+mkdir -p $JOBDIR/WORK
+
+# Check number of open sessions on host before we download
+echo "Checking that we can download"
+while true;do
+    SESSID=`curl -s -n "$XNATHOST/data/JSESSION"`
+    #echo "JSESSIONID=$SESSID"
+    SESSCOUNT=`curl -s -b "JSESSIONID=$SESSID" "$XNATHOST/data/user/$XNATUSER/sessions" | cut -s -f2 -d ":" | cut -f1 -d "}"`
+    echo "$SESSCOUNT"
+    if (( "$SESSCOUNT" > "$SESSLIMIT" )); then
+        echo -n "Cannot download yet, too many open sessions, sleeping "
+        curl -s -b "JSESSIONID=$SESSID" -X DELETE "$XNATHOST/data/JSESSION"
+        SLEEPSECS=$[ ( $RANDOM % $SLEEPMAX ) ]s && echo "$SLEEPSECS" && sleep $SLEEPSECS
+    else
+        echo "Looks good, ready to download" 
+        break
+    fi
+done
+#echo "DONE! JSESSIONID=$SESSID"
+echo "DONE!"
+
+# Collect inputs
+for IN in "${INLIST[@]}"; do
+    IFS=',' read -r col1 col2 col3 <<< "$IN"
+    if [ $col2 == "FILE" ]; then
+        CMD="curl -D - -s -b JSESSIONID=$SESSID \"$col3\" -o $INDIR/$col1"
+        CMD+=" | head -n 1 | awk '{print \$2}'"
+    elif [ $col2 == "DIRJ" ]; then
+        CMD="curl -D - -s -b JSESSIONID=$SESSID \"$col3\"?format=zip -o $INDIR/${col1}.zip"
+        CMD+=" | head -n 1 | awk '{print \$2}'"
+        CMD+=" && unzip -q -j $INDIR/${col1}.zip -d $INDIR/$col1"
+    else
+        CMD="curl -D - -s -b JSESSIONID=$SESSID \"${col3}?format=zip&structure=simplified\" -o $INDIR/${col1}.zip"
+        CMD+=" | head -n 1 | awk '{print \$2}'" 
+        CMD+=" && unzip -q $INDIR/${col1}.zip -d $INDIR/$col1 && mv $INDIR/$col1/*/out/* $INDIR/$col1"
+    fi
+
+    # Show the whole command
+	SAFE_CMD=$(echo ${CMD} | sed "s/$SESSID/SESSID/g")
+    echo $SAFE_CMD
+
+    # Run the full command
+    eval result=\$\($CMD\)
+    echo "result=$result"
+
+    # Check for success
+    if test "$result" != '200' ; then
+        echo "Download failed with code:$result, exiting..."
+        curl -s -b "JSESSIONID=$SESSID" -X DELETE "$XNATHOST/data/JSESSION"
+        exit 1
+    fi
+done
+
+# Disconnect XNAT
+curl -s -b "JSESSIONID=$SESSID" -X DELETE "$XNATHOST/data/JSESSION"
+
+#BIDS Conversion
+for IN in "${INLIST[@]}"; do
+    IFS=',' read -r col1 col2 col3 <<< "$IN"
+    if [[ $col1 == *nii.gz ]]; then
+      echo "Start BIDS conv"
+      PROJECT=$(cut -d'/' -f7 <<<"$col3")
+      echo $PROJECT
+      SCAN_ID=$(cut -d'/' -f13 <<<"$col3")
+      echo $SCAN_ID
+      SUBJ=$(cut -d'/' -f9 <<<"$col3")
+      echo $SUBJ
+      SCAN_FILE=$(cut -d'/' -f17 <<<"$col3")
+      echo $SCAN_FILE
+      SESSION=$(cut -d'/' -f11 <<<"$col3")
+      echo $SESSION
+      BIDS_DIR=$INDIR/BIDS_DATA
+      echo $BIDS_DIR
+      python -c 'import os; from dax import XnatToBids, XnatUtils; XNAT = XnatUtils.get_interface(); XnatToBids.bids_yaml(XNAT, "'$PROJECT'","'$SCAN_ID'","'$SUBJ'", "'$INDIR'","'$SCAN_FILE'", "'$col3'", "'$SESSION'", "'$col1'", "001", "001");'
+    # Use existing FS6_v1 in fmriprep output
+    # col2 will be DIR when it contains FS6_v1 assessor output
+    elif [ $col2 == "DIR" ]; then
+      echo "pre-use freesurfer from XNAT"
+      echo $OUTDIR/freesurfer/sub-01
+      mkdir -p $OUTDIR/freesurfer/sub-01
+      echo $INDIR/$col1
+      echo "movin fs folders"
+      mv $INDIR/$col1/DATA/* $OUTDIR/freesurfer/sub-01/
+      ls $OUTDIR/freesurfer/sub-01/
+    fi
+done
+
+# Run main command
+echo $_JAVA_OPTIONS
+echo $MAINCMD
+eval $MAINCMD
+
+# Write version files
+mkdir -p $DSTDIR
+echo $VERSION > $DSTDIR/version.txt
+sha256sum $CONTAINERPATH | awk '{print $1}' > $DSTDIR/dax_docker_version.txt
+
+# Create the PDF from the OUTLOG
+mkdir -p $DSTDIR/PDF
+RESDIR=${DSTDIR%/*}
+TEXTFILE=$RESDIR/DISKQ/OUTLOG/$(cut -d'/' -f5 <<<"$DSTDIR").txt
+PDFFILE=$DSTDIR/PDF/$(cut -d'/' -f5 <<<"$DSTDIR").pdf
+python -c 'from fpdf import FPDF; pdf = FPDF(); pdf.add_page(); pdf.set_font("Courier", size = 9);
+f = open("'$TEXTFILE'", "r",encoding="latin-1");
+for line in f:
+	x = line.encode("latin1","ignore").decode("latin1")
+	pdf.multi_cell(193, 5, txt = x, ln = 1);
+pdf.output("'$PDFFILE'")'
+
+# Handle outputs
+errors=0
+haspdf=0
+for OUT in "${OUTLIST[@]}"; do
+    IFS=',' read -r col1 col2 col3 col4 <<< "$OUT"
+    pathlist=(${OUTDIR}/${col1})
+    
+    # Check for missing output
+    if [[ ${#pathlist[@]} == 1 && ! -e $pathlist ]]; then
+        if [[ $col4 != "F" ]]; then errors=1 && echo "ERROR:missing required output:$col1"
+        else echo "WARN:output not required"; fi
+        continue
+    fi
+
+    # Copy output based on type
+    mkdir -p "$DSTDIR/$col3"
+    if [ $col3 == "PDF" ]; then
+        if [ $col2 != "FILE" ]; then errors=1 && echo "ERROR:illegal type for PDF";
+        elif [[ ${#pathlist[@]} != 1 ]]; then errors=1 && echo "ERROR:multiple PDFs";
+        else haspdf=1 && cp $OUTDIR/$col1 $DSTDIR/$col3 || errors=1; fi
+    elif [ $col2 == "FILE" ]; then cp $OUTDIR/$col1 $DSTDIR/$col3 || errors=1;
+    elif [ $col2 == "DIR" ]; then cp -r $OUTDIR/$col1/* $DSTDIR/$col3 || errors=1;
+    else errors=1 && echo "ERROR:invalid type:$col2"; fi
+done
+
+# Check if the PDF file has been created and is non empty
+if [ -f $PDFFILE ]; then
+    haspdf=1 && echo "PDF exist"
+    else echo "PDF does not exist"; fi
+if [ -s $PDFFILE ]; then
+    echo "PDF not empty"
+    else echo "PDF is empty" && errors=1; fi
+
+if [ $errors -gt 0 ]; then echo "JOB_FAILED" && touch $DSTDIR/JOB_FAILED.txt;
+else echo "COMPLETE" && touch $DSTDIR/READY_TO_UPLOAD.txt; fi
+
+rm -rf $INDIR $OUTDIR
+
+echo "DONE!"

From d1ce2b0734b071ceb1b4fc8ba5ad0810da4956bf Mon Sep 17 00:00:00 2001
From: duettwe <61516027+duettwe@users.noreply.github.com>
Date: Wed, 26 Jan 2022 13:39:54 -0600
Subject: [PATCH 3/6] Update job_template_fmriprep_v3.txt

---
 job_template_fmriprep_v3.txt | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/job_template_fmriprep_v3.txt b/job_template_fmriprep_v3.txt
index 0fff2e7..8595f55 100644
--- a/job_template_fmriprep_v3.txt
+++ b/job_template_fmriprep_v3.txt
@@ -140,18 +140,6 @@ mkdir -p $DSTDIR
 echo $VERSION > $DSTDIR/version.txt
 sha256sum $CONTAINERPATH | awk '{print $1}' > $DSTDIR/dax_docker_version.txt
 
-# Create the PDF from the OUTLOG
-mkdir -p $DSTDIR/PDF
-RESDIR=${DSTDIR%/*}
-TEXTFILE=$RESDIR/DISKQ/OUTLOG/$(cut -d'/' -f5 <<<"$DSTDIR").txt
-PDFFILE=$DSTDIR/PDF/$(cut -d'/' -f5 <<<"$DSTDIR").pdf
-python -c 'from fpdf import FPDF; pdf = FPDF(); pdf.add_page(); pdf.set_font("Courier", size = 9);
-f = open("'$TEXTFILE'", "r",encoding="latin-1");
-for line in f:
-	x = line.encode("latin1","ignore").decode("latin1")
-	pdf.multi_cell(193, 5, txt = x, ln = 1);
-pdf.output("'$PDFFILE'")'
-
 # Handle outputs
 errors=0
 haspdf=0

From e13b5d50815ebdc8150c0242ac600b52d093fa12 Mon Sep 17 00:00:00 2001
From: duettwe <61516027+duettwe@users.noreply.github.com>
Date: Wed, 26 Jan 2022 13:40:40 -0600
Subject: [PATCH 4/6] Update job_template_fmriprep_v3.txt

---
 job_template_fmriprep_v3.txt | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/job_template_fmriprep_v3.txt b/job_template_fmriprep_v3.txt
index 8595f55..801fd68 100644
--- a/job_template_fmriprep_v3.txt
+++ b/job_template_fmriprep_v3.txt
@@ -165,14 +165,6 @@ for OUT in "${OUTLIST[@]}"; do
     else errors=1 && echo "ERROR:invalid type:$col2"; fi
 done
 
-# Check if the PDF file has been created and is non empty
-if [ -f $PDFFILE ]; then
-    haspdf=1 && echo "PDF exist"
-    else echo "PDF does not exist"; fi
-if [ -s $PDFFILE ]; then
-    echo "PDF not empty"
-    else echo "PDF is empty" && errors=1; fi
-
 if [ $errors -gt 0 ]; then echo "JOB_FAILED" && touch $DSTDIR/JOB_FAILED.txt;
 else echo "COMPLETE" && touch $DSTDIR/READY_TO_UPLOAD.txt; fi
 

From e8fe3a575662aecb5d737b7c8c9e755898254cb7 Mon Sep 17 00:00:00 2001
From: duettwe <61516027+duettwe@users.noreply.github.com>
Date: Wed, 26 Jan 2022 13:43:21 -0600
Subject: [PATCH 5/6] Delete job_template_fmriprep-fsdev2.txt

---
 job_template_fmriprep-fsdev2.txt | 192 -------------------------------
 1 file changed, 192 deletions(-)
 delete mode 100644 job_template_fmriprep-fsdev2.txt

diff --git a/job_template_fmriprep-fsdev2.txt b/job_template_fmriprep-fsdev2.txt
deleted file mode 100644
index ae99790..0000000
--- a/job_template_fmriprep-fsdev2.txt
+++ /dev/null
@@ -1,192 +0,0 @@
-#!/bin/bash
-#SBATCH --mail-user=${job_email}
-#SBATCH --mail-type=${job_email_options}
-#SBATCH --account=${job_rungroup}
-#SBATCH --nodes=1
-#SBATCH --ntasks=${job_ppn}
-#SBATCH --time=${job_walltime}
-#SBATCH --mem=${job_memory}mb
-#SBATCH -o ${job_output_file}
-#SBATCH --constraint=westmere|sandybridge|haswell|skylake
-
-echo "Starting smemwatch"
-smemwatch -k 95 -d 50 $$$$ &
-
-uname -a # outputs node info (name, date&time, type, OS, etc)
-date
-
-#=============================================================================
-#VERSION=
-#JOBDIR=
-#INDIR=
-#OUTDIR=
-#DSTDIR=
-#INLIST=
-#OUTLIST=
-#CONTAINERPATH=
-#MAINCMD=
-#XNATHOST=
-#XNATUSER=
-${job_cmds}
-#=============================================================================
-SESSLIMIT=25
-SLEEPMAX=300
-
-echo $DSTDIR
-echo $INDIR
-echo $OUTDIR
-echo $CONTAINERPATH
-echo $XNATHOST
-echo $XNATUSER
-echo $SESSLIMIT
-echo $SLEEPMAX
-
-mkdir -p $INDIR
-mkdir -p $OUTDIR
-mkdir -p $JOBDIR/WORK
-# Check number of open sessions on host before we download
-echo "Checking that we can download"
-while true;do
-    SESSID=`curl -s -n "$XNATHOST/data/JSESSION"`
-    #echo "JSESSIONID=$SESSID"
-    SESSCOUNT=`curl -s -b "JSESSIONID=$SESSID" "$XNATHOST/data/user/$XNATUSER/sessions" | cut -s -f2 -d ":" | cut -f1 -d "}"`
-    echo "$SESSCOUNT"
-    if (( "$SESSCOUNT" > "$SESSLIMIT" )); then
-        echo -n "Cannot download yet, too many open sessions, sleeping "
-        curl -s -b "JSESSIONID=$SESSID" -X DELETE "$XNATHOST/data/JSESSION"
-        SLEEPSECS=$[ ( $RANDOM % $SLEEPMAX ) ]s && echo "$SLEEPSECS" && sleep $SLEEPSECS
-    else
-        echo "Looks good, ready to download"
-        break
-    fi
-done
-#echo "DONE! JSESSIONID=$SESSID"
-echo "DONE!"
-
-# Collect inputs
-for IN in "${INLIST[@]}"; do
-    IFS=',' read -r col1 col2 col3 <<< "$IN"
-    if [ $col2 == "FILE" ]; then
-        CMD="curl -D - -s -b "JSESSIONID=$SESSID" $col3 -o $INDIR/$col1"
-        CMD+=" | head -n 1 | awk '{print \$2}'"
-    elif [ $col2 == "DIRJ" ]; then
-        CMD="curl -D - -s -b "JSESSIONID=$SESSID" $col3?format=zip -o $INDIR/${col1}.zip"
-        CMD+=" | head -n 1 | awk '{print \$2}'"
-        CMD+=" && unzip -q -j $INDIR/${col1}.zip -d $INDIR/$col1"
-    else
-        CMD="curl -D - -s -b "JSESSIONID=$SESSID" '$col3?format=zip&structure=simplified' -o $INDIR/${col1}.zip"
-        CMD+=" | head -n 1 | awk '{print \$2}'"
-        CMD+=" && unzip -q $INDIR/${col1}.zip -d $INDIR/$col1 && mv $INDIR/$col1/*/out/* $INDIR/$col1"
-    fi
-
-    # Show the whole command
-	SAFE_CMD=$(echo ${CMD} | sed "s/$SESSID/SESSID/g")
-    echo $SAFE_CMD
-
-    # Run the full command
-    eval result=\$\($CMD\)
-    echo "result=$result"
-
-    # Check for success
-    if test $result != '200' ; then
-        echo "Download failed with code:$result, exiting..."
-        curl -s -b "JSESSIONID=$SESSID" -X DELETE "$XNATHOST/data/JSESSION"
-        exit 1
-    fi
-done
-
-# Disconnect XNAT
-curl -s -b "JSESSIONID=$SESSID" -X DELETE "$XNATHOST/data/JSESSION"
-
-#BIDS Conversion
-for IN in "${INLIST[@]}"; do
-    IFS=',' read -r col1 col2 col3 <<< "$IN"
-    if [[ $col1 == *nii.gz ]]; then
-      echo "Start BIDS conv"
-      PROJECT=$(cut -d'/' -f7 <<<"$col3")
-      echo $PROJECT
-      SCAN_ID=$(cut -d'/' -f13 <<<"$col3")
-      echo $SCAN_ID
-      SUBJ=$(cut -d'/' -f9 <<<"$col3")
-      echo $SUBJ
-      SCAN_FILE=$(cut -d'/' -f17 <<<"$col3")
-      echo $SCAN_FILE
-      SESSION=$(cut -d'/' -f11 <<<"$col3")
-      echo $SESSION
-      BIDS_DIR=$INDIR/BIDS_DATA
-      echo $BIDS_DIR
-      python -c 'import os; from dax import XnatToBids, XnatUtils; XNAT = XnatUtils.get_interface(); XnatToBids.bids_yaml(XNAT, "'$PROJECT'","'$SCAN_ID'","'$SUBJ'", "'$INDIR'","'$SCAN_FILE'", "'$col3'", "'$SESSION'", "'$col1'", "001", "001");'
-    # Use existing FS6_v1 in fmriprep output
-    # col2 will be DIR when it contains FS6_v1 assessor output
-    elif [ $col2 == "DIR" ]; then
-      echo "pre-use freesurfer from XNAT"
-      echo $OUTDIR/freesurfer/sub-01
-      mkdir -p $OUTDIR/freesurfer/sub-01
-      echo $INDIR/$col1
-      echo "movin fs folders"
-      mv $INDIR/$col1/SUBJECT/* $OUTDIR/freesurfer/sub-01/
-      ls $OUTDIR/freesurfer/sub-01/
-    fi
-done
-
-# Run main command
-echo $_JAVA_OPTIONS
-echo $MAINCMD
-eval $MAINCMD
-
-# Write version files
-mkdir -p $DSTDIR
-echo $VERSION > $DSTDIR/version.txt
-sha256sum $CONTAINERPATH | awk '{print $1}' > $DSTDIR/dax_docker_version.txt
-
-# Create the PDF from the OUTLOG
-mkdir -p $DSTDIR/PDF
-RESDIR=${DSTDIR%/*}
-TEXTFILE=$RESDIR/DISKQ/OUTLOG/$(cut -d'/' -f5 <<<"$DSTDIR").txt
-PDFFILE=$DSTDIR/PDF/$(cut -d'/' -f5 <<<"$DSTDIR").pdf
-python -c 'from fpdf import FPDF; pdf = FPDF(); pdf.add_page(); pdf.set_font("Courier", size = 9);
-f = open("'$TEXTFILE'", "r",encoding="latin-1");
-for line in f:
-	x = line.encode("latin1","ignore").decode("latin1")
-	pdf.multi_cell(193, 5, txt = x, ln = 1);
-pdf.output("'$PDFFILE'")'
-
-# Handle outputs
-errors=0
-haspdf=0
-for OUT in "${OUTLIST[@]}"; do
-    IFS=',' read -r col1 col2 col3 col4 <<< "$OUT"
-    pathlist=(${OUTDIR}/${col1})
-
-    # Check for missing output
-    if [[ ${#pathlist[@]} == 1 && ! -e $pathlist ]]; then
-        if [[ $col4 != "F" ]]; then errors=1 && echo "ERROR:missing required output:$col1"
-        else echo "WARN:output not required"; fi
-        continue
-    fi
-
-    # Copy output based on type
-    mkdir -p "$DSTDIR/$col3"
-    if [ $col3 == "PDF" ]; then
-        if [ $col2 != "FILE" ]; then errors=1 && echo "ERROR:illegal type for PDF";
-        elif [[ ${#pathlist[@]} != 1 ]]; then errors=1 && echo "ERROR:multiple PDFs";
-        else cp $OUTDIR/$col1 $DSTDIR/$col3 && haspdf=1 || errors=1; fi
-    elif [ $col2 == "FILE" ]; then cp $OUTDIR/$col1 $DSTDIR/$col3 || errors=1;
-    elif [ $col2 == "DIR" ]; then cp -r $OUTDIR/$col1/* $DSTDIR/$col3 || errors=1;
-    else errors=1 && echo "ERROR:invalid type:$col2"; fi
-done
-
-# Check if the PDF file has been created and is non empty
-if [ -f $PDFFILE ]; then
-    haspdf=1 && echo "PDF exist"
-    else echo "PDF does not exist"; fi
-if [ -s $PDFFILE ]; then
-    echo "PDF not empty"
-    else echo "PDF is empty" && errors=1; fi
-
-if [ $errors -gt 0 ] || [ $haspdf != 1 ]; then echo "JOB_FAILED" && touch $DSTDIR/JOB_FAILED.txt;
-else echo "COMPLETE" && touch $DSTDIR/READY_TO_UPLOAD.txt; fi
-
-rm -rf $INDIR $OUTDIR $JOBDIR/WORK
-
-echo "DONE!"

From 2be58cad4228f632e5ceee785c15ad341dfe9924 Mon Sep 17 00:00:00 2001
From: duettwe <61516027+duettwe@users.noreply.github.com>
Date: Thu, 27 Jan 2022 13:34:58 -0600
Subject: [PATCH 6/6] Update job_template_fmriprep_v3.txt

---
 job_template_fmriprep_v3.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/job_template_fmriprep_v3.txt b/job_template_fmriprep_v3.txt
index 801fd68..9b4876c 100644
--- a/job_template_fmriprep_v3.txt
+++ b/job_template_fmriprep_v3.txt
@@ -76,12 +76,12 @@ for IN in "${INLIST[@]}"; do
         CMD+=" && unzip -q -j $INDIR/${col1}.zip -d $INDIR/$col1"
     else
         CMD="curl -D - -s -b JSESSIONID=$SESSID \"${col3}?format=zip&structure=simplified\" -o $INDIR/${col1}.zip"
-        CMD+=" | head -n 1 | awk '{print \$2}'" 
+        CMD+=" | head -n 1 | awk '{print \$2}'"
         CMD+=" && unzip -q $INDIR/${col1}.zip -d $INDIR/$col1 && mv $INDIR/$col1/*/out/* $INDIR/$col1"
     fi
 
     # Show the whole command
-	SAFE_CMD=$(echo ${CMD} | sed "s/$SESSID/SESSID/g")
+        SAFE_CMD=$(echo ${CMD} | sed "s/$SESSID/SESSID/g")
     echo $SAFE_CMD
 
     # Run the full command
@@ -146,7 +146,7 @@ haspdf=0
 for OUT in "${OUTLIST[@]}"; do
     IFS=',' read -r col1 col2 col3 col4 <<< "$OUT"
     pathlist=(${OUTDIR}/${col1})
-    
+
     # Check for missing output
     if [[ ${#pathlist[@]} == 1 && ! -e $pathlist ]]; then
         if [[ $col4 != "F" ]]; then errors=1 && echo "ERROR:missing required output:$col1"