merge from upstream

Author: Yan Li

examples/disp_host.py

@@ -0,0 +1,16 @@
+from pythonlsf import lsf
+import inspect
+
+if __name__ == '__main__':
+    if lsf.lsb_init("test") > 0:
+        exit - 1;
+
+    for hostInfoEnt in lsf.get_host_info_all():
+        attributes = [d for d in dir(hostInfoEnt)
+                      if not d.startswith('__')]
+        item = {}
+        for a in attributes:
+            v = getattr(hostInfoEnt, a)
+            if not inspect.ismethod(v) and isinstance(v, (int, str, float)):
+                item[a] = v
+        print(item)

examples/disp_rsvInfo.py

@@ -0,0 +1,36 @@
+from pythonlsf import lsf
+
+def printRsvInfo():
+    if lsf.lsb_init("test") > 0:
+        return -1;
+
+    intp_nent = lsf.new_intp()
+    ents = lsf.lsb_reservationinfo(None,intp_nent,0)
+    nent = lsf.intp_value(intp_nent)
+
+    print("{} rsvInfo in the cluster.".format(nent))
+
+    for i in range(nent) :
+        ent = lsf.rsvInfoEntArray_getitem(ents, i)
+        print('No.{} rsvId : {}'.format(i, ent.rsvId))
+        print('No.{} name : {}'.format(i, ent.name))
+        print('No.{} state : {}'.format(i, ent.state))
+        print('No.{} options : {}'.format(i, ent.options))
+        print('No.{} timeWindow : {}'.format(i, ent.timeWindow))
+        print('No.{} numRsvJobs : {}'.format(i, ent.numRsvJobs))
+        print('No.{} numRsvHosts : {}'.format(i, ent.numRsvHosts))
+        
+        for j in range(ent.numRsvHosts):
+            hostInfo_ent = lsf.hostRsvInfoEntArray_getitem(ent.rsvHosts, j)
+            print('   [{}] host : {}'.format(j, hostInfo_ent.host))
+            print('   [{}] numCPUs : {}'.format(j, hostInfo_ent.numCPUs))
+            print('   [{}] numSlots : {}'.format(j, hostInfo_ent.numSlots))
+            print('   [{}] numRsvProcs : {}'.format(j, hostInfo_ent.numRsvProcs))
+            print('   [{}] numusedRsvProcs : {}'.format(j, hostInfo_ent.numusedRsvProcs))
+            print('   [{}] numUsedProcs : {}'.format(j, hostInfo_ent.numUsedProcs))
+
+    return 0
+    
+if __name__ == '__main__':
+    print("LSF Clustername is : {}".format(lsf.ls_getclustername()))
+    printRsvInfo()

examples/job_cputime.py

@@ -0,0 +1,43 @@
+from pythonlsf import lsf
+
+def print_job_cputime(jobid):
+
+    if lsf.lsb_init("job_info") > 0:
+        print("failed to initialise the api")
+        return
+
+    num_jobs_found = lsf.lsb_openjobinfo(jobid, "", "all", "", "", 0x2000)
+    print('num_jobs_found: {}'.format(num_jobs_found))
+
+    if num_jobs_found == -1:
+        print("no job was found")
+        return
+
+    int_ptr = lsf.new_intp()
+    lsf.intp_assign(int_ptr, num_jobs_found)
+
+    job_info = lsf.lsb_readjobinfo(int_ptr)
+
+    lsf.lsb_closejobinfo()
+
+    print('jobId: {}'.format(job_info.jobId))
+    print('jobStatus: {}'.format(job_info.status))
+
+    # Get CPU time from run Rusage
+    time_sum = float(job_info.runRusage.utime) + float(job_info.runRusage.stime)
+    if time_sum > 0:
+        print('The CPU time used is: {} seconds'.format(time_sum))
+
+    # Get CPU time from host Rusage of each execution host
+    if job_info.numhRusages > 0:
+        hRusages = job_info.hostRusage
+        for i in range(0,job_info.numhRusages):
+            hRusage = lsf.hRusageArray_getitem(hRusages, i)
+            cpu_time = float(hRusage.utime) + float(hRusage.stime)
+            print('HOST: {}, CPU_TIME: {} seconds'.format(hRusage.name, cpu_time))
+
+    return
+
+if __name__ == "__main__":
+    id = input("Enter a running job id:\n")
+    print_job_cputime(int(id))

examples/job_info.py

@@ -0,0 +1,36 @@
+from pythonlsf import lsf
+
+
+def get_single_job_info(jobid: int) -> lsf.jobInfoEnt:
+    """
+    Get the jobInfoEnt struct for a single lsf job using its jobId
+
+    This function does not work for array jobs, only single jobs
+    """
+    if lsf.lsb_init("job_info") > 0:
+        print("could not initialise the api")
+        return
+
+    # openjobinfo opens a connection to mbatchd and returns the amount of
+    # jobs in the connection.
+    num_jobs_found = lsf.lsb_openjobinfo(jobid, "", "all", "", "", 0x2000)
+
+    # make and assign an int pointer to the record of the jobs found
+    int_ptr = lsf.new_intp()
+    lsf.intp_assign(int_ptr, num_jobs_found)
+
+    # read the info at int_ptr and assign to a python object so we can read it.
+    job_info = lsf.lsb_readjobinfo(int_ptr)
+
+    # close the connection to avoid a memory leak
+    lsf.lsb_closejobinfo()
+
+    return job_info
+
+
+if __name__ == "__main__":
+    id = input("enter a job id:\n")
+    job_info = get_single_job_info(int(id))
+
+    print(f"job id: {job_info.jobId}\njob name: {job_info.jName}\n"
+          f"status: {job_info.status:#x}")

pythonlsf/lsf.i

@@ -50,6 +50,9 @@ int fclose(FILE *f);
 #endif
 %array_functions(LS_LONG_INT, LS_LONG_INTArray)
 %array_functions(guaranteedResourcePoolEnt, guaranteedResourcePoolEntArray)
+%array_functions(struct rsvInfoEnt, rsvInfoEntArray)
+%array_functions(struct hostRsvInfoEnt, hostRsvInfoEntArray)
+%array_functions(struct hRusage, hRusageArray)
 
 //helper function for transforming char** to python list
 %inline %{
@@ -710,7 +713,7 @@ PyObject * get_pids_from_stream(struct jRusage * jrusage) {
         PyList_SetItem(result, i, o);
     }
     return result;
-}      
+}
 
 long * buildQueryColIndexs() {
     long * colIndexs = NULL;
@@ -726,4 +729,23 @@ long * buildQueryColIndexs() {
     return colIndexs;
 }
 
+PyObject * get_host_info_all() {
+    struct hostInfoEnt *hostinfo;
+    char **hosts = NULL;
+    int numhosts = 0;
+
+    // Return queries as C hostInfoEnt*
+    hostinfo = lsb_hostinfo(hosts, &numhosts);
+
+    PyObject *result = PyList_New(numhosts);     // Create PyObject * to get C returns
+    int i;
+    for (i = 0; i < numhosts; i++) {             // Save queries in a loop to result
+        PyObject *o = SWIG_NewPointerObj(SWIG_as_voidptr(&hostinfo[i]),
+                                         SWIGTYPE_p_hostInfoEnt, 0 | 0 );
+        PyList_SetItem(result,i,o);
+    }
+
+    return result;
+}
+      
 %}

read_JOB_FINISH_submitExt.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+from pythonlsf import lsf
+import sys
+
+# Define function to get key and pair values from the eventrec.eventLog.jobFinishLog.submitExt
+def get_pair_from_submit_ext(submit_ext, id):
+    if submit_ext is None:
+        return None
+
+
+    values_as_list = lsf.string_array_to_pylist(submit_ext.values, submit_ext.num)
+    for i in range(submit_ext.num):
+        key = lsf.intArray_getitem(submit_ext.keys, i)
+        if key  == id:
+            return values_as_list[i]
+
+    return None
+
+
+def display(eventrec):
+    """
+    display event record, this example to get the user group (-G) value from the JOB_FINISH record
+    """
+    if eventrec.type == lsf.EVENT_JOB_FINISH:
+        jobid = eventrec.eventLog.jobFinishLog.jobId
+        fromHost = eventrec.eventLog.jobFinishLog.fromHost
+        submit_ext=eventrec.eventLog.jobFinishLog.submitExt
+        userGroup = get_pair_from_submit_ext(submit_ext,lsf.JDATA_EXT_USRGROUP)
+#        jobgroup = eventrec.eventLog.jobFinishLog.jgroup
+#        jobgroup = eventrec.eventLog.jobFinishLog.submitExt.values[1075]
+        print("EVENT_JOB_FINISH jobid<%d>, fromHost<%s>, to jobgroup <%s>" %(jobid, fromHost, userGroup))
+    else:
+        print("event type is %d" %(eventrec.type))
+
+def read_eventrec(path):
+    """
+    read lsb.events
+    """
+    lineNum = lsf.new_intp()
+    lsf.intp_assign(lineNum, 0)
+    fp = lsf.fopen(path, "r")
+    if fp is None:
+        print("The file %s does not exist." % path)
+        sys.exit(1)
+
+    flag = 1
+
+    if lsf.lsb_init("test") > 0:
+        exit(1)
+
+    while flag > 0:
+        log = lsf.lsb_geteventrec(fp, lineNum)
+        if log:
+            display(log)
+        else:
+            flag = 0
+
+
+if __name__ == '__main__':
+    if len(sys.argv) == 1:
+        print("Usage: %s full_path_lsb.events_file" % (sys.argv[0]))
+        sys.exit(0)
+
+    print("LSF Clustername is :", lsf.ls_getclustername())
+    #read_eventrec("/opt/lsf8.0.1/work/cluster1/logdir/lsb.events")
+    read_eventrec(sys.argv[1])