Merge pull request #121 from whiteroses/fix-out-of-memory-issue-119

Improve memory usage in the generation of tasks (to fix issue #119)

Author: prodigysml

Interlace/interlace.py

@@ -7,13 +7,18 @@
 from Interlace.lib.threader import Pool
 
 
-def build_queue(arguments, output, repeat):
-    task_list = InputHelper.process_commands(arguments)
-    for task in task_list:
-        output.terminal(Level.THREAD, task.name(), "Added to Queue")
-    print('Generated {} commands in total'.format(len(task_list)))
+def task_queue_generator_func(arguments, output, repeat):
+    tasks_data = InputHelper.process_data_for_tasks_iterator(arguments)
+    tasks_count = tasks_data["tasks_count"]
+    yield tasks_count
+    tasks_generator_func = InputHelper.make_tasks_generator_func(tasks_data)
+    for i in range(repeat):
+        tasks_iterator = tasks_generator_func()
+        for task in tasks_iterator:
+            output.terminal(Level.THREAD, task.name(), "Added to Queue")
+            yield task
+    print('Generated {} commands in total'.format(tasks_count))
     print('Repeat set to {}'.format(repeat))
-    return task_list * repeat
 
 
 def main():
@@ -27,9 +32,14 @@ def main():
         repeat = int(arguments.repeat)
     else:
         repeat = 1
-    
-    
-    pool = Pool(arguments.threads, build_queue(arguments, output, repeat), arguments.timeout, output, arguments.sober)
+
+    pool = Pool(
+        arguments.threads,
+        task_queue_generator_func(arguments, output, repeat),
+        arguments.timeout,
+        output,
+        arguments.sober,
+    )
     pool.run()
 
 

Interlace/lib/core/__version__.py

@@ -1 +1 @@
-__version__ = '1.8.2'
+__version__ = '1.9.0'