Merge branch 'release/1.1.0'

Author: balis

README.md

@@ -8,16 +8,14 @@ Browse the [wiki pages](https://github.com/balis/hyperflow/wiki) to learn more a
 
 ##Getting started
 
-The latest release of HyperFlow is 1.0.0
+The latest release of HyperFlow is 1.1.0
 
 ###Installation
-* Download the package: https://github.com/dice-cyfronet/hyperflow/archive/1.0.0.zip
-* Install the latest node.js (http://nodejs.org)
-* Install dependencies (in `hyperflow` directory): `npm install -d`
-* Install the Redis server 2.6.x or higher (http://redis.io) (tested with version 2.6.x)
-* Set an environment variable `HFLOW_PATH` to point to your hyperflow root directory.
-* Add `$HFLOW_PATH/bin` to your `PATH`.
+* Install the latest Node.js (http://nodejs.org)
+* Install HyperFlow: `npm install https://github.com/dice-cyfronet/hyperflow/archive/1.1.0.tar.gz`
+* Install dependencies: <br>`cd hyperflow`<br>`npm install -d`
+* Install Redis server (http://redis.io) 
 
 ###Running
-* Start the redis server
-* Run example workflows from the `examples` directory as follows: <br>```hflow run <wf_directory>```
+* Start the Redis server: `redis-server`
+* Run example workflows from the `examples` directory as follows: <br>```hflow run examples/<wf_directory>```

bin/hflow

@@ -1,56 +1,91 @@
 #!/usr/bin/env node
+
+var redisURL = process.env.REDIS_URL ? {url: process.env.REDIS_URL} : {};
+
+
 var docopt = require('docopt').docopt,
     spawn = require('child_process').spawn,
-    fs = require('fs'), 
+    fs = require('fs'),
     pathtool = require('path'),
     redis = require('redis'),
-    rcl = redis.createClient(),
+    rcl = redis.createClient(redisURL),
     wflib = require('../wflib').init(rcl),
     Engine = require('../engine2'),
     async = require('async'),
-    dbId = 0;
+    dbId = 0,
+    plugins = [];
 
 var doc = "\
 Usage:\n\
-  hflow run <workflow_dir_or_file> [-s]\n\
-  hflow send <wf_id> ( <signal_file> | -d <signal_data> )\n\
+  hflow run <workflow_dir_or_file> [-s] [--with-server] [-p <plugin_module_name> ...]\n\
+  hflow start-server [-p <plugin_module_name> ...]\n\
+  hflow send <wf_id> ( <signal_file> | -d <signal_data> ) [-p <plugin_module_name> ...]\n\
   hflow -h | --help | --version";
 
 var opts = docopt(doc);
+var hfroot = pathtool.join(require('path').dirname(require.main.filename), "..");
 
-var hfroot = process.env.HFLOW_PATH;
+if (opts['-p']) {
+    opts['<plugin_module_name>'].forEach(load_plugin);
+}
 
 if (opts.run) {
   hflow_run();
 } else if (opts.send) {
   hflow_send();
+} else if (opts['start-server']) {
+  hflow_start();
+}
+
+function load_plugin(plugin_name) {
+    try {
+        var Plugin = require(plugin_name);
+        plugins.push(new Plugin());
+    } catch (err) {
+        console.log("Plugin module:", plugin_name, "not found!");
+        console.log(err);
+        process.exit(1);
+    }
+}
+
+function hflow_start() {
+    var server = require('../server/hyperflow-server.js')(rcl, wflib, plugins);
+    server.listen(process.env.PORT, function() { });
+    console.log("HyperFlow server started, app factory URI: http://%s:%d/apps", server.address().address, server.address().port);
 }
 
 function hflow_run() {
   var wfpath = opts['<workflow_dir_or_file>'],
       wfstats = fs.lstatSync(wfpath),
       wffile;
-  
+
+  if (opts['--with-server']) {
+      hflow_start(); // start the HTTP server
+  }
+
   if (wfstats.isDirectory()) {
     wffile = pathtool.join(wfpath, "workflow.json");
   } else if (wfstats.isFile()) {
     wffile = wfpath;
     wfpath = pathtool.dirname(wfpath);
   }
 
-  var runWf = function(wfId) { 
+  var runWf = function(wfId) {
     var config = {"emulate":"false", "workdir": pathtool.resolve(wfpath)};
     var engine = new Engine(config, wflib, wfId, function(err) {
       // This represent custom plugin listening on event from available eventServer
       // engine.eventServer.on('trace.*', function(exec, args) {
       //   console.log('Event captured: ' + exec + ' ' + args + ' job done');
       // });
+      plugins.forEach(function(plugin) {
+          plugin.init(rcl, wflib, engine);
+      });
       engine.runInstance(function(err) {
         console.log("Wf id="+wfId);
         if (opts['-s']) {
           // Flag -s is present: send all input signals to the workflow -> start execution
           wflib.getWfIns(wfId, false, function(err, wfIns) {
-            engine.wflib.getSignalInfo(wfId, wfIns, function(err, sigs) { 
+            engine.wflib.getSignalInfo(wfId, wfIns, function(err, sigs) {
               engine.emitSignals(sigs);
             });
           });
@@ -61,11 +96,11 @@ function hflow_run() {
 
   var createWf = function(cb) {
     rcl.select(dbId, function(err, rep) {
-      rcl.flushdb(function(err, rep) {
+      //rcl.flushdb(function(err, rep) { // flushing db here deletes the global 'hfid' entry (created earlier)
         wflib.createInstanceFromFile(wffile, '', function(err, id) {
-          cb(err, id); 
+          cb(err, id);
         });
-      });
+      //});
     });
   }
 

bin/hyperflow-convert-dax bin/hflow-convert-daxbin/hyperflow-convert-dax renamed to bin/hflow-convert-dax

@@ -13,7 +13,8 @@
  *   - @cb        - callback function (err, wfJson)
  */
 var fs = require('fs'),
-    xml2js = require('xml2js');
+    xml2js = require('xml2js'),
+    parse = require('shell-quote').parse;
 
 // Pegasus DAX converter constructor, accepts optional name of function, used to execute tasks
 var PegasusConverter = function(functionName) {
@@ -57,7 +58,7 @@ function parseDax(filename, cb) {
                 if (err) {
                     cb(new Error("File parse error."));
                 } else {
-			//console.log(JSON.stringify(result, null, 2));
+                    //console.log(JSON.stringify(result, null, 2));
                     cb(null, result);
                 }
             });
@@ -77,7 +78,7 @@ function createWorkflow(dax, functionName, cb) {
 
     dax.adag.job.forEach(function(job) {
         ++nextTaskId;
-	var args = job.argument[0];
+        var args = parse(job.argument[0]);
         wfOut.tasks.push({
             "name": job['$'].name,
             "function": functionName,
@@ -103,11 +104,11 @@ function createWorkflow(dax, functionName, cb) {
 
         var dataId, dataName;
         job.uses.forEach(function(job_data) {
-	    if (job_data['$'].name) {
+        if (job_data['$'].name) {
                 dataName = job_data['$'].name; // dax v3.3
-	    } else {
+        } else {
                 dataName = job_data['$'].file; // dax v2.1
-	    }
+        }
             if (!dataNames[dataName]) {
                 ++nextDataId;
                 wfOut.data.push({

bin/hyperflow-create-workflow

@@ -1,5 +1,5 @@
 /** Hypermedia workflow execution engine. 
- ** Author: Bartosz Balis (2013)
+ ** Author: Bartosz Balis (2013-2015)
  */
 
 /*

converters/pegasus_dax.js

@@ -306,15 +306,19 @@ var ProcLogic = function() {
             }
         }
 
-        proc.wflib.invokeTaskFunction2(
+        proc.wflib.invokeProcFunction(
                 proc.appId,
                 proc.procId,
+		proc.firingId, 
                 funcIns,
                 proc.sigValues,
                 funcOuts, emul,
                 proc.engine.eventServer,
                 proc.engine.config, 
                 function(err, outs, options) {
+		    //console.log("FUNC INVOKED");
+		    //console.log("INS: ", JSON.stringify(proc.sigValues, null, 2));
+		    //console.log("OUTS: ", outs);
                     err ? cb(err): cb(null, outs, asyncInvocation, funcIns, funcOuts);
                 }
         );

engine2/index.js

@@ -0,0 +1,3 @@
+This is the Comet workflow, originally developed in the Kepler system.
+
+The workflow requires "R" tool.

engine2/process.js

@@ -0,0 +1,3 @@
+This example demonstrates a distributed workflow running under the control of two HyperFlow engines. 
+
+See comments in "run.sh" for running instructions.

examples/Comet/README

@@ -4,8 +4,8 @@ if [ "$#" -ne 2 ]; then
   echo "Distributed and decentralized workflow execution test using the HyperFlow REST API."
   echo
   echo "Usage:"
-  echo "1. Run the 1st HyperFlow server:    node app.js    ( -> runs on port1)"
-  echo "2. Run the 2nd HyperFlow server:    node app.js    ( -> runs on port2)"
+  echo "1. Run the 1st HyperFlow server: hflow start-server ( -> runs on port1)"
+  echo "2. Run the 2nd HyperFlow server: hflow start-server ( -> runs on port2)"
   echo "3. Then run this script:"
   echo "       cd examples/DistributedPingPong"
   echo "       ./run.sh <port1> <port2>"