Pass a bash script to srun rather than the command directly to circumvent SLURM command-line character limit

Author: dmwnz

snakemake_executor_plugin_slurm_jobstep/__init__.py

@@ -58,6 +58,7 @@ def run_job(self, job: JobExecutorInterface):
         # snakemake_interface_executor_plugins.executors.base.SubmittedJobInfo.
 
         jobsteps = dict()
+        srun_script = None
         # TODO revisit special handling for group job levels via srun at a later stage
         # if job.is_group():
 
@@ -118,14 +119,22 @@ def run_job(self, job: JobExecutorInterface):
 
             call = "srun -n1 --cpu-bind=q "
             call += f" {get_cpu_setting(job, self.gpu_job)} "
-            call += f" {self.format_job_exec(job)}"
+            # format the job to execute with all the snakemake parameters into a bash script
+            srun_script = self.format_job_exec(job)
+            call += " bash -s" # will have the process read the srun bash script from stdin
 
         self.logger.debug(f"This job is a group job: {job.is_group()}")
         self.logger.debug(f"The call for this job is: {call}")
         self.logger.debug(f"Job is running on host: {socket.gethostname()}")
+        srun_script_str = '\n' + srun_script if srun_script else 'N/A'
+        self.logger.debug(f"The script for this job is: {srun_script_str}")
         # this dict is to support the to be implemented feature of oversubscription in
         # "ordinary" group jobs.
-        jobsteps[job] = subprocess.Popen(call, shell=True)
+        jobsteps[job] = subprocess.Popen(call, shell=True, text=True, stdin=subprocess.PIPE)
+        if srun_script is not None:
+            # pass the srun bash script via stdin
+            jobsteps[job].stdin.write(srun_script)
+            jobsteps[job].stdin.close()
 
         job_info = SubmittedJobInfo(job)
         self.report_job_submission(job_info)