Merge pull request #38 from knix-microfunctions/release/0.8.4

Release/0.8.4

Author: Istemi Ekin Akkus

DataLayerService/Makefile

@@ -14,26 +14,33 @@
 
 SHELL := /bin/bash
 -include ../proxy.mk
+include ../build_env.mk
 UID:=`id -u`
 GID:=`id -g`
-all: \
+
+all: build_thrift \
 	target/datalayerservice.jar
 
-src/main/java/org/microfunctions/data_layer/DataLayerService.java: \
-		thrift/DataLayerMessage.thrift \
-		thrift/DataLayerService.thrift
+build_thrift:
+	cd ../build_tools/thrift/$(THRIFT_VERSION)/; ./build_thrift.sh; cd -
+
+src/main/java/org/microfunctions/data_layer/DataLayerService.java: ../build_env.mk \
+	thrift/DataLayerMessage.thrift \
+	thrift/DataLayerService.thrift
 	docker run --rm -it --name datalayer-thrift \
 		--user $(UID):$(GID) \
 		-v $(CURDIR):/usr/src/datalayer -w /usr/src/datalayer \
-		thrift:0.12.0 \
+		$(THRIFT_IMAGE_NAME):$(THRIFT_VERSION) \
 		bash -c 'thrift --gen java -out ./src/main/java/ ./thrift/DataLayerMessage.thrift; thrift --gen java -out ./src/main/java/ ./thrift/DataLayerService.thrift'
 	touch src/main/java/org/microfunctions/data_layer/DataLayerService.java
 
-PythonClient/data_layer:
+PythonClient/data_layer: ../build_env.mk \
+	thrift/DataLayerMessage.thrift \
+	thrift/DataLayerService.thrift
 	docker run --rm -it --name datalayer-thrift \
 		--user $(UID):$(GID) \
 		-v $(CURDIR):/usr/src/datalayer -w /usr/src/datalayer \
-		thrift:0.12.0 \
+		$(THRIFT_IMAGE_NAME):$(THRIFT_VERSION) \
 		bash -c 'thrift --gen py -out ./PythonClient ./thrift/DataLayerMessage.thrift; thrift --gen py -out ./PythonClient ./thrift/DataLayerService.thrift'
 
 # In case the user doesn't have maven configurations in the home directory, we need to create it
@@ -55,17 +62,15 @@ PythonClient/data_layer:
 JAVA_SOURCES=$(shell find src/ -type f|grep -v src/main/java/org/microfunctions/data_layer/DataLayerService.java)
 GIT_REVISION:=`git describe --tags --always --dirty=-dirty`
 target/datalayerservice.jar: $(JAVA_SOURCES) \
-		src/main/java/org/microfunctions/data_layer/DataLayerService.java \
-		thrift/DataLayerMessage.thrift \
-		thrift/DataLayerService.thrift \
-		~/.m2/settings-docker.xml
+	src/main/java/org/microfunctions/data_layer/DataLayerService.java \
+	~/.m2/settings-docker.xml
 	if [ ! -e ~/.m2/repository ]; then mkdir -p ~/.m2/repository; fi
 	docker run --rm -it --name datalayer-maven \
 		--user $(UID):$(GID) \
 		-v ~/.m2:/usr/share/maven/ref \
 		-v $(CURDIR):/usr/src/datalayer -w /usr/src/datalayer \
 		maven:3.6.1-jdk-8 \
-		mvn -Drevision=$(GIT_REVISION) -DargLine="-Duser.home=/tmp" -s /usr/share/maven/ref/settings-docker.xml package
+		mvn -Drevision=$(GIT_REVISION) -Dlibthrift.version=$(MVN_LIBTHRIFT_VERSION) -DargLine="-Duser.home=/tmp" -s /usr/share/maven/ref/settings-docker.xml package
 
 clean:
 	rm -rf ./target

DataLayerService/pom.xml

@@ -26,7 +26,7 @@
     <dependency>
       <groupId>org.apache.thrift</groupId>
       <artifactId>libthrift</artifactId>
-      <version>0.12.0</version>
+      <version>${libthrift.version}</version>
     </dependency>
 	<dependency>
 	    <groupId>org.apache.logging.log4j</groupId>
@@ -51,7 +51,7 @@
         <artifactId>maven-assembly-plugin</artifactId>
         <version>3.1.1</version>
         <configuration>
-					<finalName>datalayerservice</finalName>
+        <finalName>datalayerservice</finalName>
           <appendAssemblyId>false</appendAssemblyId>
           <descriptorRefs>
             <descriptorRef>jar-with-dependencies</descriptorRef>

FunctionWorker/python/DataLayerClient.py

@@ -27,7 +27,7 @@
 
 class DataLayerClient:
 
-    def __init__(self, locality=1, sid=None, wid=None, suid=None, is_wf_private=False, for_mfn=False, connect="127.0.0.1:4998", init_tables=False):
+    def __init__(self, locality=1, sid=None, wid=None, suid=None, is_wf_private=False, for_mfn=False, connect="127.0.0.1:4998", init_tables=False, drop_keyspace=False):
         self.dladdress = connect
 
         if for_mfn:
@@ -68,6 +68,9 @@ def __init__(self, locality=1, sid=None, wid=None, suid=None, is_wf_private=Fals
         if init_tables:
             self._initialize_tables()
 
+        if drop_keyspace:
+            self._drop_keyspace()
+
     def _initialize_tables(self):
         replication_factor = 3
         # just need the local instance
@@ -87,6 +90,12 @@ def _initialize_tables(self):
             print("Could not initialize tables: " + str(exc))
             raise
 
+    def _drop_keyspace(self):
+        try:
+            self.datalayer.dropKeyspace(self.keyspace, self.locality)
+        except Thrift.TException as exc:
+            print("Could not drop keyspace: " + str(exc))
+            raise
 
     def connect(self):
         retry = 0.5 #s

FunctionWorker/python/LocalQueueClient.py

@@ -36,10 +36,10 @@ def __init__(self, connect="127.0.0.1:4999"):
         self.connect()
 
     def connect(self):
+        host, port = self.qaddress.split(':')
         retry = 0.5 #s
         while True:
             try:
-                host, port = self.qaddress.split(':')
                 self.socket = TSocket.TSocket(host, int(port))
                 self.transport = TTransport.TFramedTransport(self.socket)
                 self.transport.open()
@@ -101,15 +101,8 @@ def getMultipleMessages(self, topic, max_count, timeout):
         return lqm_list
 
     def shutdown(self):
-        try:
-            self.socket.handle.shutdown(socket.SHUT_RDWR)
-            self.transport.close()
-        except Thrift.TException as exc:
-            print("[LocalQueueClient] failed shutdown: " + str(exc))
-        except Exception as exc:
-            print("[LocalQueueClient] failed shutdown: " + str(exc))
-            raise
-        finally:
+        if self.transport.isOpen():
+            #self.socket.handle.shutdown(socket.SHUT_RDWR)
             self.transport.close()
 
     def addTopic(self, topic):

FunctionWorker/python/MicroFunctionsAPI.py

@@ -82,25 +82,16 @@ def __init__(self, uid, sid, wid, funcstatename, key, publication_utils, is_sess
         #self._logger.debug("[MicroFunctionsAPI] init done.")
 
     def ping(self, num):
-        self._logger.info("ping:" + str(num))
+        self._logger.info("ping: " + str(num))
         output = num
         return 'pong ' + str(output)
 
-    def get_privileged_data_layer_client(self, suid=None, keyspace=None, tablename=None, maptablename=None, settablename=None, countertablename=None, init_tables=False):
+    def get_privileged_data_layer_client(self, suid=None, sid=None, init_tables=False, drop_keyspace=False):
         if self._is_privileged:
             if suid is not None:
-                return DataLayerClient(locality=1, suid=suid, connect=self._datalayer, init_tables=init_tables)
-            elif keyspace is not None and tablename is not None:
-                dlc = DataLayerClient(locality=1, for_mfn=True, connect=self._datalayer)
-                dlc.keyspace = keyspace
-                dlc.tablename = tablename
-                if maptablename is not None:
-                    dlc.maptablename = maptablename
-                if settablename is not None:
-                    dlc.settablename = settablename
-                if countertablename is not None:
-                    dlc.countertablename = countertablename
-                return dlc
+                return DataLayerClient(locality=1, suid=suid, connect=self._datalayer, init_tables=init_tables, drop_keyspace=drop_keyspace)
+            elif sid is not None:
+                return DataLayerClient(locality=1, for_mfn=True, sid=sid, connect=self._datalayer, drop_keyspace=drop_keyspace)
         return None
 
     def update_metadata(self, metadata_name, metadata_value, is_privileged_metadata=False):

FunctionWorker/python/StateUtils.py

@@ -442,23 +442,7 @@ def evaluateMapState(self, function_input, key, metadata, sapi):
 
         #assert total_branch_count == len(self.parsedfunctionstateinfo["Branches"])
 
-        k_list = []
-
-        if "WaitForNumBranches" in self.parsedfunctionstateinfo:
-            k_list = self.parsedfunctionstateinfo["WaitForNumBranches"]
-            if not isinstance(k_list, list):
-                self._logger.debug("[StateUtils] WaitForNumBranches must be a sorted list with 1 or more integers")
-                raise Exception("[StateUtils] WaitForNumBranches must be a sorted list with 1 or more integers")
-            k_list.sort()
-            for k in k_list:
-                if not isinstance(k, int):
-                    self._logger.debug("[StateUtils] Values inside WaitForNumBranches must be integers")
-                    raise Exception("[StateUtils] Values inside WaitForNumBranches must be integers")
-                if k > total_branch_count:
-                    self._logger.debug("[StateUtils] Values inside WaitForNumBranches list cannot be greater than the number of branches in the map state")
-                    raise Exception("[StateUtils] Values inside WaitForNumBranches list cannot be greater than the number of branches in the map state")
-        else:
-            k_list.append(total_branch_count)
+        k_list = [total_branch_count]
 
         counter_name_trigger_metadata = {"k-list": k_list, "total-branches": total_branch_count}
 

GUI/Makefile

@@ -12,6 +12,8 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
+SHELL := /bin/bash
+
 default: app/pages/docs/sdk/cli.md app/pages/docs/sdk/sdk.md
 
 app/pages/docs/sdk/sdk.md: ../mfn_sdk/README.md

GUI/app/app.js

@@ -45,12 +45,9 @@ var app = angular.module('MfnWebConsole', [
     var objectKey = '';
     var codeError = '';
 
-    var urlPath = managementServiceEndpoint; // from app/endpoint.js
+    var urlPath = "/management";
     var dataPrefix = "value=";
 
-    
-
-
     return {
 
         getDataPrefix: function() {

GUI/app/pages/pages.module.js

@@ -25,6 +25,7 @@
     'MfnWebConsole.pages.functions',
     'MfnWebConsole.pages.workflows',
     'MfnWebConsole.pages.storage',
+    'MfnWebConsole.pages.profile',
     'MfnWebConsole.pages.docs'