diff --git a/cld/run_sim.sh b/cld/run_sim.sh index 964af3c..9c8a3e0 100755 --- a/cld/run_sim.sh +++ b/cld/run_sim.sh @@ -9,34 +9,33 @@ set -x env df -h -export NEV=100 -export NUM=$1 #random seed -export SAMPLE=$2 #main card - -#Change these as needed -export OUTDIR=/local/joosep/cld_edm4hep/2024_05_full/ -export SIMDIR=/home/joosep/key4hep-sim/cld/CLDConfig/CLDConfig -export WORKDIR=/scratch/local/$USER/${SAMPLE}_${SLURM_JOB_ID} -export FULLOUTDIR=${OUTDIR}/${SAMPLE} - -mkdir -p $FULLOUTDIR - -mkdir -p $WORKDIR -cd $WORKDIR - -cp $SIMDIR/${SAMPLE}.cmd card.cmd -cp $SIMDIR/pythia.py ./ -cp $SIMDIR/cld_steer.py ./ -cp -R $SIMDIR/PandoraSettingsCLD ./ -cp -R $SIMDIR/CLDReconstruction.py ./ - -echo "Random:seed=${NUM}" >> card.cmd +export NEV=$1 # number of events to generate per rootfile +export SAMPLE=$2 # main card +export JOBID=$3 # random seed +export TAG=$4 # output dir tag on EOS + +# alias for quick access of EOS directory to copy input/output files via xrootd +export EOSDIR=/eos/user/f/fmokhtar/ + +mkdir CLDConfig_tmp +dir_to_bind=$(realpath CLDConfig_tmp) +cd $dir_to_bind + +# copy large input files via xrootd (recommended) +xrdcp root://eosuser.cern.ch/$EOSDIR/key4hep-sim/cld/CLDConfig/CLDConfig/${SAMPLE}.cmd card.cmd +xrdcp root://eosuser.cern.ch/$EOSDIR/key4hep-sim/cld/CLDConfig/CLDConfig/pythia.py pythia.py +xrdcp root://eosuser.cern.ch/$EOSDIR/key4hep-sim/cld/CLDConfig/CLDConfig/cld_steer.py cld_steer.py +xrdcp root://eosuser.cern.ch/$EOSDIR/key4hep-sim/cld/CLDConfig/CLDConfig/CLDReconstruction.py CLDReconstruction.py +xrdcp -r root://eosuser.cern.ch/$EOSDIR/key4hep-sim/cld/CLDConfig/CLDConfig/PandoraSettingsCLD . + +# update the seed in the pythia card +echo "Random:seed=${JOBID}" >> card.cmd cat card.cmd echo " #!/bin/bash set -e -source /cvmfs/sw-nightlies.hsf.org/key4hep/setup.sh +source /cvmfs/sw.hsf.org/key4hep/setup.sh env k4run pythia.py -n $NEV --Dumper.Filename out.hepmc --Pythia8.PythiaInterface.pythiacard card.cmd ddsim -I out.hepmc -N -1 -O out_SIM.root --compactFile \$K4GEO/FCCee/CLD/compact/CLD_o2_v05/CLD_o2_v05.xml --steeringFile cld_steer.py @@ -45,12 +44,9 @@ k4run CLDReconstruction.py --inputFiles out_SIM.root --outputBasename out_RECO - cat sim.sh -# singularity exec -B /cvmfs -B /scratch -B /local docker://ghcr.io/key4hep/key4hep-images/alma9:latest bash sim.sh -singularity exec -B /cvmfs -B /scratch -B /local /home/software/singularity/alma9.simg bash sim.sh - -#Copy the outputs -cp out_RECO_edm4hep.root $FULLOUTDIR/root/reco_${SAMPLE}_${NUM}.root -bzip2 out.hepmc -cp out.hepmc.bz2 $FULLOUTDIR/sim/sim_${SAMPLE}_${NUM}.hepmc.bz2 +# run the event generation and PF reco +singularity exec -B /cvmfs -B $dir_to_bind docker://ghcr.io/key4hep/key4hep-images/alma9:latest bash sim.sh -rm -Rf $WORKDIR +# copy the output files to EOS +# xrdcp out_RECO_edm4hep.root root://eosuser.cern.ch/$EOSDIR/$TAG/reco_${SAMPLE}_${JOBID}.root +xrdcp out_RECO_edm4hep.root root://eosuser.cern.ch//eos/project/c/cern-openlab-coeraise/data/fcc/cld/$TAG/reco_${SAMPLE}_${JOBID}.root \ No newline at end of file diff --git a/cld/run_sim_local.sh b/cld/run_sim_local.sh new file mode 100755 index 0000000..5aa34da --- /dev/null +++ b/cld/run_sim_local.sh @@ -0,0 +1,60 @@ +#!/bin/bash +#SBATCH -p main +#SBATCH --mem-per-cpu=6G +#SBATCH --cpus-per-task=1 +#SBATCH -o logs/slurm-%x-%j-%N.out +set -e +set -x + +env +df -h + +export NEV=$1 # number of events to generate per rootfile +export SAMPLE=$2 # main card +export JOBID=$3 # random seed + +# set the directories +rm -rf CLDConfig_tmp +mkdir CLDConfig_tmp +dir_to_bind=$(realpath CLDConfig_tmp) +cd $dir_to_bind + +# copy inpit files +cp ../CLDConfig/CLDConfig/${SAMPLE}.cmd card.cmd +cp ../CLDConfig/CLDConfig/pythia.py pythia.py +cp ../CLDConfig/CLDConfig/cld_steer.py cld_steer.py +cp ../CLDConfig/CLDConfig/CLDReconstruction.py CLDReconstruction.py +cp -r ../CLDConfig/CLDConfig/PandoraSettingsCLD . + +# update the seed in the pythia card +echo "Random:seed=${JOBID}" >> card.cmd +cat card.cmd + +echo " +#!/bin/bash +set -e +source /cvmfs/sw.hsf.org/key4hep/setup.sh +env +k4run pythia.py -n $NEV --Dumper.Filename out.hepmc --Pythia8.PythiaInterface.pythiacard card.cmd +ddsim -I out.hepmc -N -1 -O out_SIM.root --compactFile \$K4GEO/FCCee/CLD/compact/CLD_o2_v05/CLD_o2_v05.xml --steeringFile cld_steer.py +k4run CLDReconstruction.py --inputFiles out_SIM.root --outputBasename out_RECO --num-events -1 +" > sim.sh + + +# echo " +# #!/bin/bash +# set -e +# source /cvmfs/sw.hsf.org/key4hep/setup.sh +# env +# k4run pythia.py -n $NEV --Dumper.Filename out.hepmc --Pythia8.PythiaInterface.pythiacard card.cmd +# ddsim -I out.hepmc -N -1 -O out_SIM.root --compactFile /afs/cern.ch/user/f/fmokhtar/k4geo/FCCee/ILD_FCCee/compact/ILD_FCCee_v02/ILD_FCCee_v02.xml --steeringFile cld_steer.py +# k4run CLDReconstruction.py --inputFiles out_SIM.root --outputBasename out_RECO --num-events -1 +# " > sim.sh + +# cat sim.sh + +# run the event generation and PF reco +singularity exec -B /cvmfs -B $dir_to_bind docker://ghcr.io/key4hep/key4hep-images/alma9:latest bash sim.sh + +cp out_RECO_edm4hep.root ../reco_${SAMPLE}_${JOBID}.root +cp out.hepmc ../sim_${SAMPLE}_${JOBID}.hepmc \ No newline at end of file diff --git a/cld/submit.py b/cld/submit.py new file mode 100755 index 0000000..3ac0bb6 --- /dev/null +++ b/cld/submit.py @@ -0,0 +1,96 @@ +#!/usr/bin/python + +import argparse +import os + + +def main(args): + + # create logs directory in AFS + if not os.path.exists(f"condor/{args.tag}/logs"): + os.makedirs(f"condor/{args.tag}/logs") + + # create output directory on EOS + # eosdir = "/eos/user/f/fmokhtar/" + eosdir = "/eos/project/c/cern-openlab-coeraise/data/fcc/cld" + + if not os.path.exists(f"{eosdir}/{args.tag}/"): + os.makedirs(f"{eosdir}/{args.tag}/") + + Nevents = args.Nevents + Nevents_per_file = args.Nevents_per_file + + Njobs = Nevents // Nevents_per_file + + for jobid in range(Njobs): # jobid marks the random seed + + jobid += args.seed_begin + + # make condor submit file with the specified jobid and sample + localcondor = f"condor/{args.tag}/{args.sample}_{jobid}.jdl" + + condor_templ_file = open("submit.templ.jdl") + condor_file = open(localcondor, "w") + for line in condor_templ_file: + + line = line.replace("NEV", str(Nevents_per_file)) + line = line.replace("SAMPLE", args.sample) + line = line.replace("JOBID", str(jobid)) + line = line.replace("TAG", args.tag) + + condor_file.write(line) + + condor_file.close() + condor_templ_file.close() + + # submit + if args.submit: + print("Submit ", localcondor) + os.system("condor_submit %s" % localcondor) + + +if __name__ == "__main__": + """ + python3 submit.py --sample p8_ee_tt_ecm365 --Nevents 4 --Nevents-per-file 2 --seed-begin 0 --tag Nov15 --submit # noqa + """ + + parser = argparse.ArgumentParser() + parser.add_argument( + "--sample", + default="", + help="which sample to generate", + type=str, + ) + parser.add_argument( + "--Nevents", + default=4, + help="how many events to generate", + type=int, + ) + parser.add_argument( + "--Nevents-per-file", + default=2, + help="how many events to store per file", + type=int, + ) + parser.add_argument( + "--seed-begin", + default=0, + help="begin count from which random seed", + type=int, + ) + parser.add_argument( + "--submit", + action="store_true", + help="submit jobs when created", + ) + parser.add_argument( + "--tag", + default="", + help="dir tag", + type=str, + ) + + args = parser.parse_args() + + main(args) diff --git a/cld/submit.templ.jdl b/cld/submit.templ.jdl new file mode 100755 index 0000000..2d40015 --- /dev/null +++ b/cld/submit.templ.jdl @@ -0,0 +1,23 @@ +# job.sub + +# Define the executable to run +executable = ./run_sim.sh + +# Arguments for the executable (if any) +arguments = NEV SAMPLE JOBID TAG + +# Specify log, output, and error files for Condor to store job info +log = /afs/cern.ch/user/f/fmokhtar/key4hep-sim/cld/condor/TAG/logs/job_JOBID.log +output = /afs/cern.ch/user/f/fmokhtar/key4hep-sim/cld/condor/TAG/logs/job_JOBID.out +error = /afs/cern.ch/user/f/fmokhtar/key4hep-sim/cld/condor/TAG/logs/job_JOBID.err + +# Request resources (memory, CPUs, etc.) +request_memory = 4 GB +request_cpus = 1 + +WHEN_TO_TRANSFER_OUTPUT = ON_EXIT_OR_EVICT + ++JobFlavour = "workday" + +# If you want to run multiple jobs, set the job count +queue \ No newline at end of file