From e660dba9accf39eb1708d2325bb006691ce5847d Mon Sep 17 00:00:00 2001
From: Anil Panta <47672624+panta-123@users.noreply.github.com>
Date: Mon, 20 May 2024 16:28:42 -0400
Subject: [PATCH] #13 option to use apptainer  (#14)

* add option to use apptainer image

* fix typo

* fix typo

* add option to pass script
---
 hcswif.py           | 12 +++++++++++-
 hcswif_apptainer.sh | 44 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+), 1 deletion(-)
 create mode 100644 hcswif_apptainer.sh

diff --git a/hcswif.py b/hcswif.py
index 06bca61..b0f1bd9 100755
--- a/hcswif.py
+++ b/hcswif.py
@@ -80,6 +80,8 @@ def parseArgs():
             help='max run time per job in seconds allowed before killing jobs')
     parser.add_argument('--shell', nargs=1, dest='shell',
             help='Currently a shell cannot be specified in SWIF2')
+    parser.add_argument('--apptainer', nargs=1, dest='apptainer',
+                    help='Specify path to apptainer image.')
 
     # Check if any args specified
     if len(sys.argv) < 2:
@@ -179,6 +181,11 @@ def getReplayJobs(parsed_args, wf_name):
         batch = os.path.join(hcswif_dir, 'hcswif.sh')
     elif re.search('csh', parsed_args.shell[0]):
         batch = os.path.join(hcswif_dir, 'hcswif.csh')
+    if parsed_args.apptainer:
+        if not os.path.isdir(str(parsed_args.apptainer[0])):
+            warnings.warn("APPTAINER image not found.")
+            sys.exit()
+        batch = os.path.join(hcswif_dir, "hcswif_apptainer.sh")
 
     # Create list of jobs for workflow
     jobs = []
@@ -218,7 +225,10 @@ def getReplayJobs(parsed_args, wf_name):
         job['inputs'][0]['remote'] = coda
 
         # command for job is `/hcswifdir/hcswif.sh REPLAY RUN NUMEVENTS`
-        job['command'] = [" ".join([batch, replay_script, str(run), str(evts)])]
+        if parsed_args.apptainer:
+             job['command'] = [" ".join([batch, replay_script, str(run), str(evts), str(parsed_args.apptainer[0]), str(raw_dir)])]
+        else:
+            job['command'] = [" ".join([batch, replay_script, str(run), str(evts)])]
 
         jobs.append(copy.deepcopy(job))
 
diff --git a/hcswif_apptainer.sh b/hcswif_apptainer.sh
new file mode 100644
index 0000000..c101169
--- /dev/null
+++ b/hcswif_apptainer.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/bash
+ARGC=$#
+if [[ $ARGC -ne 5 ]]; then
+    echo "Usage: hcswif.sh SCRIPT RUN EVENTS APPTAINER RAWDIR"
+    exit 1
+fi;
+
+script=$1 # script to run
+run=$2 # RUN Number
+evt=$3 # Number of events in that run
+apptainer=$4
+rawdir=$5
+
+# Modify as you need
+#--------------------------------------------------
+HALLC_REPLAY_DIR="/home/$USER/hallc_replay"   # my replay directory
+DATA_DIR="${rawdir}"
+ROOT_FILE="/path/to/rootfile/directory"
+REPORT_OUTPUT="/path/to/REPORT_OTUPUT/directory"
+APPTAINER_IMAGE="${apptainer}"
+#--------------------------------------------------
+
+cd $HALLC_REPLAY_DIR
+
+# Check if apptainer is available
+if command -v apptainer > /dev/null 2>&1; then
+    echo "apptainer is already available."
+else
+    # Load apptainer if not available
+    echo "Loading apptainer..."
+   if ! eval module load apptainer; then
+        echo "Failed to load apptainer. Please check if the module is installed and accessible."
+        exit 1  # Exit the script with a non-zero exit code
+    fi
+fi
+
+echo
+echo "---------------------------------------------------------------------------------------------"
+echo "REPLAY for ${runNum}. NEvent=${nEvent} using container=${APPTAINER_IMAGE}"
+echo "----------------------------------------------------------------------------------------------"
+echo
+
+runStr="apptainer exec --bind ${DATA_DIR} --bind ${APPTAINER_IMAGE}--bind ${ROOT_FILE} --bind ${REPORT_OUTPUT} --bind ${HALLC_REPLAY_DIR}  ${APPTAINER_IMAGE} bash -c \"hcana -q ${script}\(${runNum},${nEvent}\)\""
+eval ${runStr}
\ No newline at end of file