Merge branch 'develop' into feature/plugins

mpawlik · mpawlik · commit 5321d7a73756 · 2016-04-08T17:08:03.000+02:00
diff --git a/bin/hflow b/bin/hflow
@@ -1,10 +1,14 @@
 #!/usr/bin/env node
+
+var redisURL = process.env.REDIS_URL ? {url: process.env.REDIS_URL} : {};
+
+
 var docopt = require('docopt').docopt,
     spawn = require('child_process').spawn,
     fs = require('fs'),
     pathtool = require('path'),
     redis = require('redis'),
-    rcl = redis.createClient(),
+    rcl = redis.createClient(redisURL),
     wflib = require('../wflib').init(rcl),
     Engine = require('../engine2'),
     async = require('async'),
@@ -92,11 +96,11 @@ function hflow_run() {
 
   var createWf = function(cb) {
     rcl.select(dbId, function(err, rep) {
-      rcl.flushdb(function(err, rep) {
+      //rcl.flushdb(function(err, rep) { // flushing db here deletes the global 'hfid' entry (created earlier)
         wflib.createInstanceFromFile(wffile, '', function(err, id) {
           cb(err, id);
         });
-      });
+      //});
     });
   }
 
diff --git a/engine2/process.js b/engine2/process.js
@@ -306,9 +306,10 @@ var ProcLogic = function() {
             }
         }
 
-        proc.wflib.invokeTaskFunction2(
+        proc.wflib.invokeProcFunction(
                 proc.appId,
                 proc.procId,
+		proc.firingId, 
                 funcIns,
                 proc.sigValues,
                 funcOuts, emul,
diff --git a/examples/MolecularDynamicsParameterStudy/make-movie.sh b/examples/MolecularDynamicsParameterStudy/make-movie.sh
@@ -17,6 +17,9 @@ if [ -n "$2" ]; then
     MOVIE=$2
 fi
 
+BASEDIR=$(dirname $BASH_SOURCE)
+cp -vf $BASEDIR/pov-template.inc pov-template.inc
+
 DIR=$(mktemp -d --tmpdir=.)
 
 
diff --git a/examples/MolecularDynamicsParameterStudy/md_amqp.json b/examples/MolecularDynamicsParameterStudy/md_amqp.json
@@ -0,0 +1,113 @@
+{
+  "processes": [
+    {
+      "name": "run-cmd-openmp-0.5",
+      "function": "amqpCommand",
+      "type": "dataflow",
+      "config": {
+        "executor": {
+          "executable": "/home/ubuntu/MD_v4_OMP/run-cmd-openmp.sh",
+          "args": [
+            1000,
+            0.5,
+            0.5
+          ]
+        }
+      },
+      "ins": [
+        "start"
+      ],
+      "outs": [
+        "psp-output-0.5.tgz"
+      ]
+    },
+    {
+      "name": "run-cmd-openmp-0.6",
+      "function": "amqpCommand",
+      "type": "dataflow",
+      "config": {
+        "executor": {
+          "executable": "/home/ubuntu/MD_v4_OMP/run-cmd-openmp.sh",
+          "args": [
+            1000,
+            0.5,
+            0.6
+          ]
+        }
+      },
+      "ins": [
+        "start"
+      ],
+      "outs": [
+        "psp-output-0.6.tgz"
+      ]
+    },
+    {
+      "name": "make-movie-0.5",
+      "function": "amqpCommand",
+      "type": "dataflow",
+      "config": {
+        "executor": {
+          "executable": "/home/ubuntu/MD_v4_OMP/make-movie.sh",
+          "args": [
+            "psp-output-0.5.tgz",
+            "psp-output-0.5.avi"
+          ]
+        }
+      },
+      "ins": [
+        "psp-output-0.5.tgz"
+      ],
+      "outs": [
+        "psp-output-0.5.avi"
+      ]
+    },
+    {
+      "name": "make-movie-0.6",
+      "function": "amqpCommand",
+      "type": "dataflow",
+      "config": {
+        "executor": {
+          "executable": "/home/ubuntu/MD_v4_OMP/make-movie.sh",
+          "args": [
+            "psp-output-0.6.tgz",
+            "psp-output-0.6.avi"
+          ]
+        }
+      },
+      "ins": [
+        "psp-output-0.6.tgz"
+      ],
+      "outs": [
+        "psp-output-0.6.avi"
+      ]
+    }
+  ],
+  "signals": [
+    {
+      "name": "start",
+      "data": [
+        "start"
+      ]
+    },
+    {
+      "name": "psp-output-0.5.tgz"
+    },
+    {
+      "name": "psp-output-0.5.avi"
+    },
+    {
+      "name": "psp-output-0.6.tgz"
+    },
+    {
+      "name": "psp-output-0.6.avi"
+    }
+  ],
+  "ins": [
+    "start"
+  ],
+  "outs": [
+    "psp-output-0.5.avi",
+    "psp-output-0.6.avi"
+  ]
+}
diff --git a/examples/MolecularDynamicsParameterStudy/run-cmd-openmp.sh b/examples/MolecularDynamicsParameterStudy/run-cmd-openmp.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Copyright 2013 University of Stuttgart, Germany
 # Author: Anthony Sulistio (HLRS)
 # Modified by Maciej Malawski (AGH)
@@ -10,11 +10,15 @@
 # ./run.sh 1000 2 0.8
 
 
+BASEDIR=$(dirname $BASH_SOURCE)
+
+
 MYDIR=$(mktemp -d --tmpdir=.)
 
-cp -vf pov-template.inc $MYDIR/psp-header.inc
+cp -vf $BASEDIR/pov-template.inc $MYDIR/psp-header.inc
+
+cp $BASEDIR/src/main $MYDIR
 
-cp src/main $MYDIR
 
 cd $MYDIR
 rm -f *.pov *.dat *.xyz
@@ -57,4 +61,4 @@ OUTPUT_FILE=psp-output-$TEMPERATURE.tgz
 
 tar zcvf $OUTPUT_FILE psp-*
 
-mv $OUTPUT_FILE ..
+mv $OUTPUT_FILE ..
diff --git a/functions/amqpCommand.js b/functions/amqpCommand.js
@@ -15,6 +15,10 @@ function connect() {
 
     connection.then(function(conn) {
         console.log("[AMQP] Connected!");
+
+        return when(conn.createChannel().then(function(ch) {
+          var ok = ch.assertQueue('hyperflow.jobs', {durable: true}).then(function(qok) { return qok.queue; });
+        }));
     }, function(err) {
         console.error('[AMQP] Connect failed: %s', err);
     })
@@ -23,17 +27,27 @@ var taskCount = 0;
 
 function amqpCommand(ins, outs, config, cb) {
   if(!connection) connect();
-  
+
   connection.then(function(connection) {
     return when(connection.createChannel().then(function(ch) {
+      var options = executor_config.options;
+      if(config.executor.hasOwnProperty('options')) {
+          var executorOptions = config.executor.options;
+          for (var opt in executorOptions) {
+              if(executorOptions.hasOwnProperty(opt)) {
+                  options[opt] = executorOptions[opt];
+              }
+          }
+      }
       var jobMessage = {
         "executable": config.executor.executable,
-        "args": config.executor.args,
-        "inputs": ins.map(identity),
-        "outputs": outs.map(identity),
-        "options": executor_config.options
+        "args":       config.executor.args,
+        "env":        (config.executor.env || {}),
+        "inputs":     ins.map(identity),
+        "outputs":    outs.map(identity),
+        "options":    options
       };
-      
+
       var answer = defer();
       var corrId = uuid.v4();
       function maybeAnswer(msg) {
@@ -55,7 +69,7 @@ function amqpCommand(ins, outs, config, cb) {
         // console.log("[AMQP][" + corrId + "][" + taskCount + "] Publishing job " + JSON.stringify(jobMessage));
         ch.sendToQueue('hyperflow.jobs', new Buffer(JSON.stringify(jobMessage)), {replyTo: queue, contentType: 'application/json', correlationId: corrId});
         return answer.promise;
-      }); 
+      });
 
       return ok.then(function(message) {
         var parsed = JSON.parse(message);
@@ -71,7 +85,7 @@ function amqpCommand(ins, outs, config, cb) {
       });
     }))
   }).then(null, function(err) { console.trace(err.stack); });
-} 
+}
 
 
 exports.amqpCommand = amqpCommand;
diff --git a/functions/index.js b/functions/index.js
@@ -26,6 +26,8 @@ function print2(ins, outs, config, cb) {
             console.log(input.data);
         }
     });
+    console.log("CONFIG");
+    console.log(config);
     cb(null, outs);
 }
 
diff --git a/package.json b/package.json
@@ -10,12 +10,12 @@
   },
   "dependencies": {
     "express": "3.x",
-    "consolidate": "0.9.1",
+    "consolidate": "0.10.x",
     "cradle": "0.6.6",
     "ejs": "0.8.4",
     "eyes": "0.1.8",
     "async": "0.2.x",
-    "redis": "0.9.1",
+    "redis": "2.4.2",
     "xml2js": "0.2.7",
     "underscore": "1.x",
     "uuid": "",
diff --git a/schemas/workflow.schema.json b/schemas/workflow.schema.json
diff --git a/server/hyperflow-server.js b/server/hyperflow-server.js
diff --git a/wflib/index.js b/wflib/index.js

Original file line number	Diff line number	Diff line change
`@@ -306,9 +306,10 @@ var ProcLogic = function() {`
`306`	`306`	`}`
`307`	`307`	`}`
`308`	`308`
`309`		`- proc.wflib.invokeTaskFunction2(`
	`309`	`+ proc.wflib.invokeProcFunction(`
`310`	`310`	`proc.appId,`
`311`	`311`	`proc.procId,`
	`312`	`+ proc.firingId,`
`312`	`313`	`funcIns,`
`313`	`314`	`proc.sigValues,`
`314`	`315`	`funcOuts, emul,`
Original file line number	Diff line number	Diff line change
`@@ -26,6 +26,8 @@ function print2(ins, outs, config, cb) {`
`26`	`26`	`console.log(input.data);`
`27`	`27`	`}`
`28`	`28`	`});`
	`29`	`+ console.log("CONFIG");`
	`30`	`+ console.log(config);`
`29`	`31`	`cb(null, outs);`
`30`	`32`	`}`
`31`	`33`