Update condor.jl

Author: DilumAluthge

src/condor.jl

@@ -51,6 +51,19 @@ function condor_script(portnum::Integer, np::Integer, params::Dict)
     "$tdir/$jobname.sub"
 end
 
+function _my_wait_without_timeout(f::Function; timeout_seconds)
+    each_sleep_duration = 5
+    for i = 1:each_sleep_duration:timeout_seconds
+        sleep(each_sleep_duration)
+        result = f()
+        if result
+            return nothing
+        end
+    end
+    msg = "Timeout ($(timeout_seconds) seconds) exceeded"
+    error(msg)
+end
+
 function launch(manager::HTCManager, params::Dict, instances_arr::Array, c::Condition)
     let
         mgr_desc = "HTCondor"
@@ -68,13 +81,19 @@ function launch(manager::HTCManager, params::Dict, instances_arr::Array, c::Cond
 
         script = condor_script(portnum, np, params)
         cmd = `condor_submit $script`
-        proc = run(ignorestatus(cmd); wait=false) # run and wait (blocks)
-        while !Base.process_exited(proc)
+        pipeline = Base.pipeline(ignorestatus(cmd); stdout=Base.stdout, stderr=Base.stderr)
+        proc = run(pipeline; wait = false)
+        _my_wait_without_timeout(; timeout_seconds = 5 * 60) do
             run(`condor_q`)
+            return 
+        end
+        if !Base.process_exited(proc)
+            @error "batch queue not available (could not run condor_submit)" Base.process_exited(proc)
+            return nothing
         end
         if !success(proc)
-            println("batch queue not available (could not run condor_submit)")
-            return
+            @error "batch queue not available (could not run condor_submit)" Base.process_exited(proc) success(proc)
+            return nothing
         end
         print("Waiting for $np workers: ")