Merge pull request #44 from CSDLLab/2020_05_07_parallel

2020_5_7 parallal optimize

Author: HanZhaang

config/fake_reconstruction_configs/test.json

@@ -1 +1,20 @@
-{"trace": {"tag": "trace configuration", "default": {"minimalScoreAuto": 0.909970122661639, "minimalScoreManual": 1.0, "minimalScoreSeed": 0.9494785696149678, "minimalScore2d": 0.5304879686303995, "refit": false, "spTest": false, "crossoverTest": false, "tuneEnd": true, "edgePath": false, "enhanceMask": true, "seedMethod": 1, "recover": 1, "maxEucDist": 10}}}
+{
+  "trace": {
+    "tag": "trace configuration",
+    "default": {
+      "minimalScoreAuto": 0.3,
+      "minimalScoreManual": 0.3,
+      "minimalScoreSeed": 0.35,
+      "minimalScore2d": 0.5,
+      "refit": false,
+      "spTest": false,
+      "crossoverTest": false,
+      "tuneEnd": true,
+      "edgePath": false,
+      "enhanceMask": true,
+      "seedMethod": 1,
+      "recover": 1,
+      "maxEucDist": 10
+    }
+  }
+}

config/fake_reconstruction_configs/test1best.json

@@ -0,0 +1,20 @@
+{
+  "trace": {
+    "tag": "trace configuration",
+    "default": {
+      "minimalScoreAuto": 0.42039409,
+      "minimalScoreManual": 1,
+      "minimalScoreSeed": 1,
+      "minimalScore2d": 0.59725383,
+      "refit": false,
+      "spTest": false,
+      "crossoverTest": false,
+      "tuneEnd": true,
+      "edgePath": false,
+      "enhanceMask": true,
+      "seedMethod": 1,
+      "recover": 1,
+      "maxEucDist": 10
+    }
+  }
+}

config/fake_reconstruction_configs/test3best.json

@@ -0,0 +1,20 @@
+{
+  "trace": {
+    "tag": "trace configuration",
+    "default": {
+      "minimalScoreAuto": 0.39791192,
+      "minimalScoreManual": 1,
+      "minimalScoreSeed": 98459854,
+      "minimalScore2d": 0.263612,
+      "refit": false,
+      "spTest": false,
+      "crossoverTest": false,
+      "tuneEnd": true,
+      "edgePath": false,
+      "enhanceMask": true,
+      "seedMethod": 1,
+      "recover": 1,
+      "maxEucDist": 10
+    }
+  }
+}

config/fake_reconstruction_configs/test4best.json

@@ -0,0 +1,20 @@
+{
+  "trace": {
+    "tag": "trace configuration",
+    "default": {
+      "minimalScoreAuto": 0.45898798,
+      "minimalScoreManual": 0.74510809,
+      "minimalScoreSeed": 0.82922591,
+      "minimalScore2d": 0.0,
+      "refit": false,
+      "spTest": false,
+      "crossoverTest": false,
+      "tuneEnd": true,
+      "edgePath": false,
+      "enhanceMask": true,
+      "seedMethod": 1,
+      "recover": 1,
+      "maxEucDist": 10
+    }
+  }
+}

pyneval/tools/optimize/SA.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 # @Time    : 2019/8/17
 # @Author  : github.com/guofei9987
+import copy
 
 import numpy as np
 from sko.base import SkoBase
@@ -52,7 +53,7 @@ def __init__(self, func, x0, T_max=100, T_min=1e-7, L=300, max_stay_counter=150,
         self.n_dims = len(x0)
 
         self.best_x = np.array(x0)  # initial solution
-        self.best_y = self.func(self.best_x)
+        self.best_y = self.func(self.best_x, "test_init.swc")[1]
         self.T = self.T_max
         self.iter_cycle = 0
         self.generation_best_X, self.generation_best_Y = [self.best_x], [self.best_y]
@@ -76,18 +77,40 @@ def run(self):
         while True:
             # loop L times under the same Temperature
             for i in range(self.L):
-                x_new = self.get_new_x(x_current)
-                y_new = self.func(x_new)
-                print("[Info: ]i/L = {}/{}".format(
-                    i, self.L
-                ))
-
-                # Metropolis
-                df = y_new - y_current
-                if df < 0 or np.exp(-df / self.T) > np.random.rand():
-                    x_current, y_current = x_new, y_new
-                    if y_new < self.best_y:
-                        self.best_x, self.best_y = x_new, y_new
+                pool = mp.Pool(processes=CPU_CORE_NUM)
+                res_y = []
+                res_x = []
+                lock = mp.Manager().Lock()
+                for j in range(CPU_CORE_NUM):
+                    x_new = self.get_new_x(x_current)
+                    for k in range(len(x_new)):
+                        x_new[k] = max(x_new[k], 0)
+                        x_new[k] = min(x_new[k], 1)
+                    res_y.append(
+                        pool.apply_async(self.func, args=tuple([x_new, "test256_{}".format(j), lock]))
+                    )
+                    res_x.append(x_new)
+
+                print("[Info: ]i/L = {}/{}".format(i, self.L))
+                pool.close()
+                pool.join()
+                for it in range(len(res_x)):
+                    x_new, y_new = res_y[it].get()
+                    print(x_new)
+                    print(y_new)
+                    # Metropolis
+                    df = y_new - y_current
+                    if df < 0 or np.exp(-df / self.T) > np.random.rand():
+                        x_current, y_current = x_new, y_new
+                        print("[Info: ] Jump success")
+                        if y_new < self.best_y:
+                            print("[Info: ] Update success")
+                            self.best_x = copy.deepcopy(x_new)
+                            self.best_y = y_new
+                        break
+                    print("[Info: ] best x = {}".format(self.best_x))
+                    print("[Info: ] best y = {}".format(self.best_y))
+
             print("[Info: ] iter_cycle = {} T = {} stay_counter = {}".format(
                 self.iter_cycle, self.T, stay_counter
             ))

pyneval/tools/optimize/optimize.py

@@ -16,14 +16,15 @@
 g_gold_tree = None
 g_tar_tree = None
 g_rcn_method = None
+g_rcn_config = None
 g_metric_method = None
 g_metric_configs = None
 
 NEUTU_PATH = "../../../../../00_program_file/00_neutu/bin/neutu"
 ORIGIN_PATH = "../../../data/optimation/test1/test1_test.tif"
 GOLD_PATH = "../../../data/optimation/test1/test1_gold.swc"
-TEST_PATH = "../../../data/optimation/output/test1_test.swc"
-CONFIG_PATH = "../../../config/fake_reconstruction_configs/test.json"
+TEST_PATH = "../../../data/optimation/output/"
+CONFIG_PATH = "../../../config/fake_reconstruction_configs/"
 METRIC_CONFIG_PATH = "../../../config/ssd_metric.json"
 LOG_PATH = "../../../output/optimization/neutu_log.txt"
 
@@ -148,35 +149,25 @@
 #     print(X.shape)
 
 
-def SA_optimize(configs, lock=None):
-    global g_gold_tree
+def SA_optimize(configs=None, test_name=None, lock=None):
     global g_metric_method
     global g_metric_configs
+    global g_rcn_config
 
-    # # check the value of configs
-    # res = fake_gaussian(pos=configs)
+    LOC_CONFIG_PATH = os.path.join(CONFIG_PATH, test_name+".json")
+    LOC_TEST_PATH = os.path.join(TEST_PATH, test_name+".swc")
+    rec_config = copy.deepcopy(g_rcn_config)
 
-    rec_config = copy.deepcopy(g_metric_configs)
+    if configs is not None:
+        rec_config["trace"]["default"]["minimalScoreAuto"] = configs[0]
+        rec_config["trace"]["default"]["minimalScoreManual"] = configs[1]
+        rec_config["trace"]["default"]["minimalScoreSeed"] = configs[2]
+        rec_config["trace"]["default"]["minimalScore2d"] = configs[3]
 
-    for i in range(len(configs)):
-        configs[i] = max(configs[i], 0)
-        configs[i] = min(configs[i], 1)
+        read_json.save_json(LOC_CONFIG_PATH, rec_config)
 
-    rec_config["trace"]["default"]["minimalScoreAuto"] = configs[0]
-    rec_config["trace"]["default"]["minimalScoreManual"] = configs[1]
-    rec_config["trace"]["default"]["minimalScoreSeed"] = configs[2]
-    rec_config["trace"]["default"]["minimalScore2d"] = configs[3]
-
-    # save new configs
-    if lock is not None:
-        lock.acquire()
-    try:
-        read_json.save_json(json_file_path=CONFIG_PATH, data=rec_config)
-    finally:
-        if lock is not None:
-            lock.release()
     REC_CMD = "{} --command --trace {} -o {} --config {} > {}".format(
-        NEUTU_PATH, ORIGIN_PATH, TEST_PATH, CONFIG_PATH, LOG_PATH
+        NEUTU_PATH, ORIGIN_PATH, LOC_TEST_PATH, LOC_CONFIG_PATH, LOG_PATH
     )
     try:
         os.system(REC_CMD)
@@ -185,7 +176,7 @@ def SA_optimize(configs, lock=None):
 
     res_tree = swc_node.SwcTree()
     gold_tree = swc_node.SwcTree()
-    res_tree.load(TEST_PATH)
+    res_tree.load(os.path.join(TEST_PATH, test_name+".swc"))
     gold_tree.load(GOLD_PATH)
 
     if lock is not None:
@@ -195,25 +186,26 @@ def SA_optimize(configs, lock=None):
     finally:
         if lock is not None:
             lock.release()
+    score = (main_score["recall"] + main_score["precision"])/2
+    print("[Info: ] ssd loss = {}".format(score))
 
-    print("[Info: ] ssd loss = {}".format(
-        (main_score["recall"] + main_score["precision"])/2)
-    )
-
-    return -(main_score["recall"] + main_score["precision"])/2
+    return configs, -score
 
 
 def main():
     global g_metric_configs
     global g_metric_method
+    global g_rcn_config
     g_metric_method = ssd_metric.ssd_metric
     g_metric_configs = read_json.read_json(METRIC_CONFIG_PATH)
+    g_rcn_config = read_json.read_json(os.path.join(CONFIG_PATH, "test.json"))
+
     # optimize with SA
     # configs here is the config of the reconstruction
     configs = (0.3, 0.3, 0.35, 0.5)
     start = time.time()
     sa_fast = SAFast(func=SA_optimize,
-                     x0=configs, T_max=0.01, T_min=1e-5, q=0.96, L=30, max_stay_counter=20, upper=1, lower=0)
+                     x0=configs, T_max=0.01, T_min=1e-5, q=0.96, L=20, max_stay_counter=30, upper=1, lower=0)
     best_configs, best_value = sa_fast.run()
     print("[Info: ]best configs:\n"
           "        origin minimalScoreAuto = {}\n"
@@ -224,6 +216,11 @@ def main():
           "        time = {}\n" .format(
         best_configs[0], best_configs[1], best_configs[2], best_configs[3], best_value, time.time() - start
     ))
+    g_rcn_config["trace"]["default"]["minimalScoreAuto"] = best_configs[0]
+    g_rcn_config["trace"]["default"]["minimalScoreManual"] = best_configs[1]
+    g_rcn_config["trace"]["default"]["minimalScoreSeed"] = best_configs[2]
+    g_rcn_config["trace"]["default"]["minimalScore2d"] = best_configs[3]
+    read_json.save_json(os.path.join(CONFIG_PATH, "best_x_{}.json".format(time.time())), g_rcn_config)
     # plot the result.
     plt.plot(pd.DataFrame(sa_fast.best_y_history).cummin(axis=0))
     plt.xlabel("iterations")
@@ -234,3 +231,9 @@ def main():
 
 if __name__ == "__main__":
     main()
+
+    # g_metric_method = ssd_metric.ssd_metric
+    # g_metric_configs = read_json.read_json(METRIC_CONFIG_PATH)
+    # g_rcn_config = read_json.read_json(os.path.join(CONFIG_PATH, "test3best.json"))
+    #
+    # SA_optimize(test_name="test3best")

pyneval/tools/swc_cut.py

@@ -3,7 +3,7 @@
 from pyneval.io import swc_writer
 
 SWC_PATH = "../../data/optimation/gold.swc"
-OUTPUT_PATH = "../../data/optimation/test3_gold.swc"
+OUTPUT_PATH = "../../data/optimation/test4_gold.swc"
 
 
 def cut_swc_rectangle(swc_tree, LFD_pos, RBT_pos):
@@ -34,7 +34,11 @@ def cut_swc_rectangle(swc_tree, LFD_pos, RBT_pos):
     # load origin swc file
     swc_tree.load(SWC_PATH)
 
-    cut_swc_rectangle(swc_tree, (0, 0, 0), (256, 256, 256))
+    cut_swc_rectangle(swc_tree, (260, 30, 125), (324, 94, 189))
     swc_tree.get_node_list(update=True)
+    for node in swc_tree.get_node_list():
+        node.set_x(node.get_x()-260)
+        node.set_y(node.get_y()-30)
+        node.set_z(node.get_z()-125)
     # the result will be saved in:
     swc_writer.swc_save(swc_tree=swc_tree, out_path=OUTPUT_PATH)

pyneval/tools/tiff_cut.py

@@ -6,17 +6,17 @@
 from pyneval.metric.utils.tiff_utils import front_expend_step
 from pyneval.model.swc_node import SwcTree
 
-TIFF_PATH = "../../data/optimation/6656_gold.tif"
+TIFF_PATH = "../../data/optimation/6656_test.tif"
 
 
-def tiff_cut(tiff_data, LDF = (0, 0, 0), len = (256, 256, 256)):
+def tiff_cut(tiff_data, LDF = (0, 0, 0), len = (64, 64, 64)):
     res = tiff_data[LDF[0]:LDF[0]+len[0], LDF[1]:LDF[1]+len[1], LDF[2]:LDF[2]+len[2]]
     return res
 
 
 if __name__ == "__main__":
     tiff_file = TiffFile.imread(TIFF_PATH)
-    data = tiff_cut(tiff_file, LDF=(0, 0, 0))
-    TiffFile.imsave("../../data/optimation/test3_gold.tif", data)
+    data = tiff_cut(tiff_file, LDF=(125, 30, 260))
+    TiffFile.imsave("../../data/optimation/test4_test.tif", data)
     # print("??")