Paths is picklable

fix in permutations computations for dag extraction

Author: verginer

pathpy/classes/paths.py

@@ -28,6 +28,8 @@
 import numpy as np
 from pathpy.utils import Log, Severity
 from pathpy.utils.exceptions import PathpyError
+from pathpy.utils.default_containers import nested_zero_default as _nested_zero_default
+from pathpy.utils.default_containers import zero_array_default as _zero_array_default
 
 
 class Paths:
@@ -54,8 +56,7 @@ def __init__(self, separator=','):
         #    subpath of a longer path, and j refers to the number of times p
         #    occurs as a *real* or *longest* path (i.e. not being a subpath
         #    of a longer path)
-        self.paths = defaultdict(
-            lambda: defaultdict(lambda: np.array([0.0, 0.0])))
+        self.paths = _nested_zero_default()
 
         # The character used to separate nodes on paths
         self.separator = separator
@@ -145,7 +146,7 @@ def path_lengths(self):
 
 
         """
-        lengths = defaultdict(lambda: np.array([0., 0.]))
+        lengths = _zero_array_default()
 
         for k in self.paths:
             for p in self.paths[k]:

pathpy/path_extraction/dag_paths.py

@@ -24,7 +24,6 @@
 import sys
 import itertools as it
 import functools as ft
-import random
 from collections import defaultdict
 
 from tqdm import tqdm
@@ -54,9 +53,11 @@ def paths_from_dag(dag, node_mapping=None, max_subpath_length=None, separator=',
 
     """
     # Try to topologically sort the graph if not already sorted
-
-    test_key = list(node_mapping.keys())[0]
-    ONE_TO_MANY = isinstance(node_mapping[test_key], set)
+    if node_mapping:
+        test_key = list(node_mapping.keys())[0]
+        ONE_TO_MANY = isinstance(node_mapping[test_key], set)
+    else:
+        ONE_TO_MANY = False
 
     if dag.is_acyclic is None:
         dag.topsort()
@@ -84,8 +85,7 @@ def paths_from_dag(dag, node_mapping=None, max_subpath_length=None, separator=',
             path_counter = defaultdict(lambda: 0)
             for root in tqdm(dag.roots, unit='roots', desc='count unique paths'):
                 for set_path in dag.routes_from_node(root, node_mapping):
-                    blown_up_paths = blowup_set_paths(set_path)
-                    for blown_up_path in blown_up_paths:
+                    for blown_up_path in expand_set_paths(set_path):
                         path_counter[blown_up_path] += 1
 
             for path, count in tqdm(path_counter.items(), unit='path', desc='add paths'):
@@ -97,7 +97,7 @@ def paths_from_dag(dag, node_mapping=None, max_subpath_length=None, separator=',
         return p
 
 
-def blowup_set_paths(set_path):
+def expand_set_paths(set_path):
     """returns all possible paths which are consistent with the sequence of sets
 
     Parameters
@@ -107,29 +107,36 @@ def blowup_set_paths(set_path):
 
     Examples
     -------
-    >>> node_path = [{'320', '324'}, {'324'}, {'324', '429'}]
-    >>> blowup_set_paths(node_path)
-    [('324', '324', '324'),
-    ('320', '324', '429'),
-    ('324', '324', '324'),
-    ('320', '324', '429')]
-
-
-
-    Returns
-    -------
-    list
-        a list of paths
+    >>> node_path = [{1, 2}, {2, 5}, {1, 2}]
+    >>> list(expand_set_paths(node_path))
+    [(1, 2, 1), (2, 2, 1), (1, 5, 1), (2, 5, 1), (1, 2, 2), (2, 2, 2), (1, 5, 2), (2, 5, 2)]
+    >>> node_path = [{1, 2}, {5}, {2, 5}]
+    >>> list(expand_set_paths(node_path))
+    [(1, 5, 2), (2, 5, 2), (1, 5, 5), (2, 5, 5)]
+
+
+    Yields
+    ------
+    tuple
+        a possible path
     """
     # how many possible combinations are there
-    node_sizes = (len(n) for n in set_path)
+    node_sizes = [len(n) for n in set_path]
     num_possibilities = ft.reduce(lambda x, y: x * y, node_sizes, 1)
 
-    all_paths = []
-    iterator = [it.cycle(node_set) for node_set in set_path]
+    # create a list of lists such that each iterator is repeated the number of times
+    # his predecessors have completed their cycle
+    all_periodics = []
+    current_length = 1
+    for node_set in set_path:
+        periodic_num = []
+        for num in node_set:
+            periodic_num.extend([num] * current_length)
+        current_length *= len(node_set)
+        all_periodics.append(periodic_num)
+
+    iterator = [it.cycle(periodic) for periodic in all_periodics]
     for i, elements in enumerate(zip(*iterator)):
         if i >= num_possibilities:
             break
-        all_paths.append(elements)
-    return all_paths
-
+        yield elements

pathpy/utils/default_containers.py

@@ -0,0 +1,27 @@
+"""This module contains the containers for the various classes
+which are used to store nodes, edges and similar
+
+To make the various classes pickle-able the defaultdicts need to be publicly addressable
+function names, this means that no lambda functions are allowed.
+
+All Pathpy classes which required a default value as a container, should use these here.
+"""
+from collections import defaultdict
+import numpy as np
+
+
+def nested_zero_default():
+    """returns a nested default dict (2 levels)
+    with a numpy zero array of length 0 as default
+    """
+    return defaultdict(zero_array_default)
+
+
+def _zero_array():
+    """returns a zero numpy array of length 2"""
+    return np.array([0.0, 0.0])
+
+
+def zero_array_default():
+    """returns a default dict with numpy zero array af length 2 as default"""
+    return defaultdict(_zero_array)

tests/test_DAG.py

@@ -83,6 +83,26 @@ def test_dag_path_mapping(dag_object):
     assert paths_mapped2.observation_count == 7
 
 
+def test_dag_path_mapping_to_many(dag_object):
+    dag = dag_object
+    dag.topsort()
+
+    mapping = {
+        'a': {1, 2},
+        'b': {2, 5},
+        'c': {5},
+        'e': {1, 2},
+        'f': {2, 3},
+        'g': {2, 5},
+        'h': {1},
+        'i': {1, 5},
+        'j': {4}
+    }
+    paths_mapped2 = pp.path_extraction.paths_from_dag(dag, node_mapping=mapping)
+
+    assert paths_mapped2.observation_count is None
+
+
 edges1, types1 = [(1, 2), (1, 3), (2, 3)], ({1}, {2}, {3})
 edges2 = [(1, 2), (1, 3), (2, 3), (3, 7), (4, 2), (4, 5), (4, 6), (5, 7), (6, 5)]
 types2 = ({1, 4}, {2, 6, 3, 5}, {7})
@@ -128,3 +148,29 @@ def test_remove_edge(dag_object: pp.DAG):
     assert 'e' in dag_object.roots
     assert 'e' in dag_object.leafs
     assert 'e' in dag_object.isolate_nodes()
+
+
+def test_set_path_expansion(dag_object: pp.DAG):
+    from collections import defaultdict
+    mapping = {'a': {1, 2},
+               'b': {2, 5},
+               'c': {5},
+               'e': {1, 2},
+               'f': {2, 3},
+               'g': {2, 5},
+               'h': {1},
+               'i': {1, 5},
+               'j': {4}
+               }
+
+    path_count = defaultdict(lambda: 0)
+    # for root in dag_object.roots:
+        # for set_path in dag_object.routes_from_node(root):
+
+
+
+
+
+
+
+

tests/test_Path.py

@@ -230,3 +230,25 @@ def test_path_multiplication(random_paths, factor):
     mult_inplace *= factor
 
     assert sum(mult_paths.paths[2][TEST_PATH]) == sum(mult_inplace.paths[2][TEST_PATH])
+
+
+def test_pickle(random_paths, tmpdir):
+    import pickle
+    from pathpy import Paths
+
+    dir_path = tmpdir.join("test_path.pkl")
+    paths = random_paths(90, 0, 20)
+
+    with open(str(dir_path), 'wb') as f:
+        pickle.dump(paths, f)
+
+    with open(str(dir_path), 'rb') as f:
+        paths_back = pickle.load(f)  # type: Paths
+
+    assert paths.diameter() == paths_back.diameter()
+    assert paths.paths.keys() == paths_back.paths.keys()
+    assert paths.observation_count == paths.observation_count
+
+
+
+