Merge pull request #45 from CSDLLab/2020_05_07_down_sample

2020_05_07 add sample rate adjust in down sample

Author: HanZhaang

pyneval/tools/optimize/SA.py

@@ -76,7 +76,8 @@ def run(self):
         stay_counter = 0
         while True:
             # loop L times under the same Temperature
-            for i in range(self.L):
+            i = 0
+            while i < self.L:
                 pool = mp.Pool(processes=CPU_CORE_NUM)
                 res_y = []
                 res_x = []
@@ -95,6 +96,7 @@ def run(self):
                 pool.close()
                 pool.join()
                 for it in range(len(res_x)):
+                    i += 1
                     x_new, y_new = res_y[it].get()
                     print(x_new)
                     print(y_new)

pyneval/tools/optimize/optimize.py

@@ -28,126 +28,6 @@
 METRIC_CONFIG_PATH = "../../../config/ssd_metric.json"
 LOG_PATH = "../../../output/optimization/neutu_log.txt"
 
-# def fake_gaussian(mean=[0.0, 0.0], cov=[[0.0, 1.0], [1.0, 0.0]], pos=(0,0)):
-#     rv = st.multivariate_normal(mean=[1.5, -1], cov=[[0.5, 0],[0, 0.5]])
-#     rv2 = st.multivariate_normal(mean=[-0.1, 0.5], cov=[[1, 0],[0, 1]])
-#     x = np.empty(shape=(1, 1, 2))
-#     x[0,0,0] = pos[0]
-#     x[0,0,1] = pos[1]
-#     return rv.pdf(x) + rv2.pdf(x)
-
-
-# def naive_optimize(gold_tree, rcn_configs, current_config, metric_method, metric_configs):
-#     """
-#     A naive optimization, test every combination of test parameters
-#     recursive parameter searching is used for traversing all the combinations.
-#     """
-#     clen = len(current_config)
-#     if clen == len(rcn_configs):
-#         # metric and get results
-#         # # check the value of configs
-#         # res = fake_gaussian(pos=configs)
-#         rec_config = read_json.read_json(json_file_path=CONFIG_PATH)
-#         current_config[0] = max(current_config[0], 0)
-#         current_config[1] = max(current_config[1], 0)
-#         print("[Info: ] minimalScoreAuto = {} minimalScoreSeed = {}".format(
-#             current_config[0], current_config[1])
-#         )
-#         rec_config["trace"]["default"]["minimalScoreAuto"] = rcn_configs[0]
-#         rec_config["trace"]["default"]["minimalScoreSeed"] = rcn_configs[1]
-#
-#         # save new configs
-#         read_json.save_json(json_file_path=CONFIG_PATH, data=rec_config)
-#         REC_CMD = "{} --command --trace {} -o {} --config {}".format(
-#             NEUTU_PATH, ORIGIN_PATH, TEST_PATH, CONFIG_PATH
-#         )
-#         print(REC_CMD)
-#         try:
-#             print("[Info: ] start tracing")
-#             os.system(REC_CMD)
-#             print("[Info: ] end tracing")
-#         except:
-#             raise Exception("[Error: ] error executing reconstruction")
-#
-#         res_tree = swc_node.SwcTree()
-#         gold_tree = swc_node.SwcTree()
-#         res_tree.load(TEST_PATH)
-#         gold_tree.load(GOLD_PATH)
-#
-#         main_score = g_metric_method(gold_tree, res_tree, g_metric_configs)
-#         # main_score = fake_gaussian(mean=[0.5, -0.3], cov=[[1, 0],[0, 1]], pos=current_config)
-#         global g_score
-#         if main_score > g_score:
-#             g_score = main_score
-#             print("max_score = {}, current_score = {}, x = {}, y = {}".format(
-#                 g_score, main_score, current_config[0], current_config[1])
-#             )
-#         return None
-#
-#     for item in rcn_configs[clen]:
-#         current_config.append(item)
-#         naive_optimize(gold_tree=gold_tree,
-#                        rcn_method=rcn_method,
-#                        rcn_configs=rcn_configs,
-#                        current_config=current_config,
-#                        metric_method=metric_method,
-#                        metric_configs=metric_configs)
-#         current_config.pop()
-#     return None
-
-
-# def naive_main():
-#     global g_metric_configs
-#     global g_metric_method
-#     g_metric_method = ssd_metric.ssd_metric
-#     g_metric_configs = read_json.read_json(METRIC_CONFIG_PATH)
-#
-#     z = np.zeros(shape=(50, 50))
-#     for i in range(50):
-#         for j in range(50):
-#             rec_config = read_json.read_json(json_file_path=CONFIG_PATH)
-#             rec_config["trace"]["default"]["minimalScoreAuto"] = 0.02 * i
-#             rec_config["trace"]["default"]["minimalScoreSeed"] = 0.02 * j
-#             read_json.save_json(json_file_path=CONFIG_PATH, data=rec_config)
-#             REC_CMD = "{} --command --trace {} -o {} --config {}".format(
-#                 NEUTU_PATH, ORIGIN_PATH, TEST_PATH, CONFIG_PATH
-#             )
-#             print(REC_CMD)
-#             try:
-#                 print("[Info: ] start tracing")
-#                 os.system(REC_CMD)
-#                 print("[Info: ] end tracing")
-#             except:
-#                 raise Exception("[Error: ] error executing reconstruction")
-#
-#             res_tree = swc_node.SwcTree()
-#             gold_tree = swc_node.SwcTree()
-#             res_tree.load(TEST_PATH)
-#             gold_tree.load(GOLD_PATH)
-#
-#             main_score = g_metric_method(gold_tree, res_tree, g_metric_configs)
-#             print("[Info: ] call = {} minimalScoreAuto = {} minimalScoreSeed = {}".format(
-#                 main_score["recall"], 0.02 * i, 0.02 * j
-#             ))
-#             z[i][j] = main_score["recall"]
-#
-#     # print(z)
-#     x = np.linspace(0, 0.48, 50)
-#     y = np.linspace(0, 0.48, 50)
-#     X, Y = np.meshgrid(x, y)
-#
-#     fig = plt.figure()
-#     # 创建一个三维坐标轴
-#     ax = plt.axes(projection='3d')
-#     ax.contour3D(X, Y, z, 50, cmap='binary')
-#     ax.set_xlabel('x')
-#     ax.set_ylabel('y')
-#     ax.set_zlabel('z')
-#     ax.plot_surface(X, Y, z, rstride=1, cstride=1, cmap='viridis', edgecolor='none')
-#     ax.set_title('surface')
-#     plt.show()
-#     print(X.shape)
-
 
 def SA_optimize(configs=None, test_name=None, lock=None):
     global g_metric_method
@@ -205,7 +85,7 @@ def main():
     configs = (0.3, 0.3, 0.35, 0.5)
     start = time.time()
     sa_fast = SAFast(func=SA_optimize,
-                     x0=configs, T_max=0.01, T_min=1e-5, q=0.96, L=20, max_stay_counter=30, upper=1, lower=0)
+                     x0=configs, T_max=0.01, T_min=1e-5, q=0.96, L=20, max_stay_counter=50, upper=1, lower=0)
     best_configs, best_value = sa_fast.run()
     print("[Info: ]best configs:\n"
           "        origin minimalScoreAuto = {}\n"

pyneval/tools/re_sample.py

@@ -35,7 +35,7 @@ def itp_ok(node=None, son=None, pa=None,
     return True
 
 
-def down_sample(swc_tree=None, rad_mul=1.50, center_dis=None, stage=0):
+def down_sample(swc_tree=None, rad_mul=1.50, center_dis=None, stage=0, k=1.0):
     stack = queue.LifoQueue()
     stack.put(swc_tree.root())
     down_pa = {}
@@ -62,9 +62,9 @@ def down_sample(swc_tree=None, rad_mul=1.50, center_dis=None, stage=0):
         son_dis, pa_dis, grand_dis = son.parent_distance(), node.distance(down_pa[node]), son.distance(pa)
 
         # 确保针对采样率高的情况
-        if stage == 0 and (son_dis > son.radius() + node.radius() or pa_dis > pa.radius() + node.radius()):
+        if stage == 0 and (son_dis > k*(son.radius() + node.radius()) or pa_dis > k*(pa.radius() + node.radius())):
             continue
-        if stage == 1 and (son_dis > son.radius() + node.radius() and pa_dis > pa.radius() + node.radius()):
+        if stage == 1 and (son_dis > k*(son.radius() + node.radius()) and pa_dis > k*(pa.radius() + node.radius())):
             continue
         if itp_ok(node=node, son=son, pa=pa, rad_mul=rad_mul, center_dis=center_dis):
             is_active[node.get_id()] = False
@@ -77,18 +77,18 @@ def down_sample_swc_tree_command_line(swc_tree, config=None):
     rad_mul = config['rad_mul']
     center_dis = config['center_dis']
     stage = config['stage']
-    return down_sample_swc_tree(swc_tree=swc_tree, rad_mul=rad_mul, center_dis=center_dis, stage=stage)
+    return down_sample_swc_tree_stage(swc_tree=swc_tree, rad_mul=rad_mul, center_dis=center_dis, stage=stage)
 
 
-def down_sample_swc_tree(swc_tree, rad_mul=1.50, center_dis=None, stage=0):
+def down_sample_swc_tree_stage(swc_tree, rad_mul=1.50, center_dis=None, stage=0, k=1.0):
     '''
     :param swc_tree: the tree need to delete node
     :param rad_mul: defult=1.5
     :param center_dis: defult=None
-    :param stage: 0: for 2 degree node, delete if one side is two close, 1: for 2 degree node, delete if two sides are two close
+    :param stage: 0: for 2 degree node, delete if one side is too close, 1: for 2 degree node, delete if two sides are two close
     :return: swc_tree has changed in this function
     '''
-    down_pa, is_activate = down_sample(swc_tree=swc_tree, rad_mul=rad_mul, center_dis=center_dis, stage=stage)
+    down_pa, is_activate = down_sample(swc_tree=swc_tree, rad_mul=rad_mul, center_dis=center_dis, stage=stage, k=k)
     new_swc_tree = SwcTree()
     node_list = [node for node in PreOrderIter(swc_tree.root())]
     id_node_map = {-1: new_swc_tree.root()}
@@ -108,6 +108,12 @@ def down_sample_swc_tree(swc_tree, rad_mul=1.50, center_dis=None, stage=0):
     return new_swc_tree
 
 
+def down_sample_swc_tree(swc_tree, rad_mul=1.50, center_dis=None, k=1.0):
+    res1 = down_sample_swc_tree_stage(swc_tree, rad_mul=rad_mul, center_dis=center_dis, stage=1, k=k)
+    res2 = down_sample_swc_tree_stage(res1, rad_mul=rad_mul, center_dis=center_dis, stage=0, k=k)
+    return res2
+
+
 def re_sample(swc_tree, son, pa, length_threshold):
     '''
     self recursive function, add node on the middle of edge son, pa
@@ -168,9 +174,8 @@ def up_sample_swc_tree(swc_tree, length_threshold=1.0):
 
 
 if __name__ == "__main__":
-    file_name = "6144_12288_17664"
     swc_tree = SwcTree()
-    swc_tree.load("D:\\02_project\\00_neural_tracing\\01_project\PyNeval\data\swc_cut_data\\{}.swc".format(file_name))
-    up_sampled_swc_tree = up_sample_swc_tree(swc_tree=swc_tree, length_threshold=1.0)
-    # up_sampled_swc_tree = down_sample_swc_tree(swc_tree=swc_tree)
-    swc_save(up_sampled_swc_tree, "D:\\02_project\\00_neural_tracing\\01_project\PyNeval\output\\resample\\{}.swc".format(file_name))
+    swc_tree.load("/home/zhanghan/01_project/Pyneval/data/raw/1-8.swc")
+
+    res_swc = down_sample_swc_tree(swc_tree=swc_tree, k=3)
+    swc_save(res_swc, out_path="/home/zhanghan/01_project/Pyneval/output/down_sample.swc")