Add deploy control group (#56)

* fix singularity check command. If validation fails exception was being thrown instead of printing error message
* add support for instance replicas
* fix test_no_circular_dependency test
* reuse same sif image for replicas

Signed-off-by: Paulo Miguel Almeida <paulo.miguel.almeida.rodenas@gmail.com>

Author: PauloMigAlmeida

CHANGELOG.md

@@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pypi.
 
 ## [0.1.x](https://github.com/singularityhub/singularity-compose/tree/master) (0.1.x)
+ - add support for instance replicas (0.1.17)
  - fix check command validation (0.1.16)
  - fix a bug triggered when using startoptions in conjunction with network=false (0.1.15)
  - bind volumes can handle tilde expansion (0.1.14)

docs/spec/spec-2.0.md

@@ -168,6 +168,22 @@ for `--fakeroot`:
 
 You could also add "args" here within the start group to provide arguments to the start script.
 
+## Deploy Group
+
+### Replicas
+
+By default `singularity-compose` will launch a single replica of each instance listed in the compose file.
+If your use-case requires multiple instances of the exact same configuration, you can use `deploy->replicas`
+option.
+
+The example below will run 2 container instances with the same instance configuration. 
+
+```yaml
+  instance:
+    ...
+    deploy:      
+      replicas: 2    
+```
 
 ## Environment
 

scompose/config/schema.py

@@ -92,6 +92,13 @@ def validate_config(filepath):
     ],
 }
 
+instance_deploy = {
+    "type": "object",
+    "properties": {
+        "replicas": {"type": "number", "minimum": 1},
+    },
+}
+
 # A single instance
 instance = {
     "type": "object",
@@ -107,6 +114,7 @@ def validate_config(filepath):
         "exec": instance_exec,
         "run": {"oneOf": [instance_run, {"type": "array"}]},
         "post": instance_post,
+        "deploy": instance_deploy,
     },
 }
 

scompose/project/instance.py

@@ -31,7 +31,7 @@ class Instance(object):
     params: all of the parameters defined in the configuration.
     """
 
-    def __init__(self, name, working_dir, sudo=False, params=None):
+    def __init__(self, name, replica_number, working_dir, sudo=False, params=None):
 
         if not params:
             params = {}
@@ -41,6 +41,7 @@ def __init__(self, name, working_dir, sudo=False, params=None):
         self.instance = None
         self.sudo = sudo
         self.set_name(name, params)
+        self.replica_number = replica_number
 
         # Start includes networking args and command
         self.set_start(params)
@@ -61,7 +62,7 @@ def __init__(self, name, working_dir, sudo=False, params=None):
         self.get()
 
     def __str__(self):
-        return "(instance:%s)" % self.name
+        return "(instance:%s)" % self.get_replica_name()
 
     def __repr__(self):
         return self.__str__()
@@ -77,9 +78,12 @@ def set_name(self, name, params):
         """
         self.name = params.get("name", name)
 
+    def get_replica_name(self):
+        return f"{self.name}{self.replica_number}"
+
     @property
     def uri(self):
-        return "instance://%s" % self.name
+        return "instance://%s" % self.get_replica_name()
 
     def set_context(self, params):
         """set and validate parameters from the singularity-compose.yml,
@@ -385,24 +389,23 @@ def get_build_options(self):
         return options
 
     # State
-
     def exists(self):
         """return boolean if an instance exists. We do this by way of listing
         instances, and so the calling user is important.
         """
         instances = [x.name for x in self.client.instances(quiet=True, sudo=self.sudo)]
-        return self.name in instances
+        return self.get_replica_name() in instances
 
     def get(self):
         """If an instance exists, add to self.instance"""
         for instance in self.client.instances(quiet=True, sudo=self.sudo):
-            if instance.name == self.name:
+            if instance.name == self.get_replica_name():
                 self.instance = instance
                 break
 
     def stop(self, timeout=None):
         """delete the instance, if it exists. Singularity doesn't have delete
-        or remove commands, everyting is a stop.
+        or remove commands, everything is a stop.
         """
         if self.instance:
             bot.info("Stopping %s" % self)
@@ -454,7 +457,9 @@ def clear_logs(self):
         log_folder = self._get_log_folder()
 
         for ext in ["out", "err"]:
-            logfile = os.path.join(log_folder, "%s.%s" % (self.name, ext.lower()))
+            logfile = os.path.join(
+                log_folder, "%s.%s" % (self.get_replica_name(), ext.lower())
+            )
 
             # Use Try/catch to account for not existing.
             try:
@@ -486,7 +491,9 @@ def logs(self, tail=0):
         log_folder = self._get_log_folder()
 
         for ext in ["OUT", "ERR"]:
-            logfile = os.path.join(log_folder, "%s.%s" % (self.name, ext.lower()))
+            logfile = os.path.join(
+                log_folder, "%s.%s" % (self.get_replica_name(), ext.lower())
+            )
 
             # Use Try/catch to account for not existing.
             try:
@@ -497,7 +504,9 @@ def logs(self, tail=0):
                     # If the user only wants to see certain number
                     if tail > 0:
                         result = "\n".join(result.split("\n")[-tail:])
-                    bot.custom(prefix=self.name, message=ext, color="CYAN")
+                    bot.custom(
+                        prefix=self.get_replica_name(), message=ext, color="CYAN"
+                    )
                     print(result)
                     bot.newline()
 
@@ -534,7 +543,7 @@ def create(self, ip_address=None, sudo=False, writable_tmpfs=False):
         # Finally, create the instance
         if not self.exists():
 
-            bot.info("Creating %s" % self.name)
+            bot.info("Creating %s" % self.get_replica_name())
 
             # Command options
             options = []
@@ -550,18 +559,23 @@ def create(self, ip_address=None, sudo=False, writable_tmpfs=False):
             options += self.start_opts
 
             # Hostname
-            options += ["--hostname", self.name]
+            options += ["--hostname", self.get_replica_name()]
 
             # Writable Temporary Directory
             if writable_tmpfs:
                 options += ["--writable-tmpfs"]
 
             # Show the command to the user
-            commands = "%s %s %s %s" % (" ".join(options), image, self.name, self.args)
+            commands = "%s %s %s %s" % (
+                " ".join(options),
+                image,
+                self.get_replica_name(),
+                self.args,
+            )
             bot.debug("singularity instance start %s" % commands)
 
             self.instance = self.client.instance(
-                name=self.name,
+                name=self.get_replica_name(),
                 sudo=self.sudo,
                 options=options,
                 image=image,

scompose/project/project.py

@@ -19,6 +19,7 @@
 import os
 import re
 import subprocess
+from copy import deepcopy
 
 
 class Project(object):
@@ -193,14 +194,20 @@ def parse(self):
             # Create each instance object
             for name in self.config.get("instances", []):
                 params = self.config["instances"][name]
-
-                # Validates params
-                self.instances[name] = Instance(
-                    name=name,
-                    params=params,
-                    sudo=self.sudo,
-                    working_dir=self.working_dir,
-                )
+                replicas = params.get("deploy", {"replicas": 1})["replicas"]
+
+                # 1-indexed to mimic docker-compose behaviour
+                for idx in range(1, replicas + 1):
+                    tmp_inst = Instance(
+                        name=name,
+                        replica_number=idx,
+                        # deepcopy is required otherwise changes to one replica would reflect on
+                        # others since they point to the same memory reference
+                        params=deepcopy(params),
+                        sudo=self.sudo,
+                        working_dir=self.working_dir,
+                    )
+                    self.instances[tmp_inst.get_replica_name()] = tmp_inst
 
             self.instances = self._sort_instances(self.instances)
 
@@ -223,8 +230,9 @@ def _sort_instances(self, instances):
                 index = sorted_instances.index(instance)
 
             for dep in depends_on:
-                if not dep in sorted_instances:
-                    sorted_instances.insert(index, dep)
+                for inst in instances.values():
+                    if dep == inst.name:
+                        sorted_instances.insert(index, inst.get_replica_name())
 
         return {k: self.instances[k] for k in sorted_instances}
 
@@ -251,7 +259,7 @@ def get_ip_lookup(self, names, bridge="10.22.0.0/16"):
         next(host_iter)
 
         # If an instance is already running, we want to include it
-        all_names = set(self.config["instances"].keys())
+        all_names = set(self.get_instance_names())
         skip_addresses = [x["ip"] for name, x in self.running.items() if x["ip"]]
 
         # Only use addresses not currently in use
@@ -500,7 +508,7 @@ def _create(
         names = names or self.get_instance_names()
 
         # Keep track of created instances to determine if we have circular dependency structure
-        created = []
+        created = set()
         circular_dep = False
 
         # Generate ip addresses for each
@@ -529,10 +537,10 @@ def _create(
             create_func(
                 working_dir=self.working_dir,
                 writable_tmpfs=writable_tmpfs,
-                ip_address=lookup[instance.name],
+                ip_address=lookup[instance.get_replica_name()],
             )
 
-            created.append(instance.name)
+            created.add(instance.name)
 
             # Run post create commands
             instance.run_post()

scompose/tests/test_client.py

@@ -51,17 +51,17 @@ def test_commands(tmp_path):
     sleep(10)
 
     print("Testing logs")
-    project.logs(["httpd"], tail=20)
+    project.logs(["httpd1"], tail=20)
 
     print("Clearing logs")
-    project.clear_logs(["httpd"])
-    project.logs(["httpd"], tail=20)
+    project.clear_logs(["httpd1"])
+    project.logs(["httpd1"], tail=20)
 
     print("Testing ps")
     project.ps()
 
     print("Testing exec")
-    project.execute("httpd", ["echo", "MarsBar"])
+    project.execute("httpd1", ["echo", "MarsBar"])
 
     # Ensure running
     print(requests.get("http://127.0.0.1").status_code)
@@ -70,6 +70,6 @@ def test_commands(tmp_path):
     project.down()
 
     print("Testing ip lookup")
-    lookup = project.get_ip_lookup(["httpd"])
-    assert "httpd" in lookup
-    assert lookup["httpd"] == "10.22.0.2"
+    lookup = project.get_ip_lookup(["httpd1"])
+    assert "httpd1" in lookup
+    assert lookup["httpd1"] == "10.22.0.2"

scompose/tests/test_command_args.py

@@ -56,4 +56,4 @@ def test_command_args(tmp_path):
     project.down()
 
     log = bot.get_logs()
-    assert "echo arg0 arg1 arg2" in log
+    assert "arg0 arg1 arg2" in log

scompose/tests/test_depends_on.py

@@ -89,7 +89,7 @@ def test_no_circular_dependency(tmp_path):
 
     # Test depends_on DAG order
     keys = list(project.instances.keys())
-    assert keys == ["first", "second", "third"]
+    assert keys == ["first1", "second1", "third1"]
 
     print("Testing up")
     project.up()

scompose/version.py

@@ -8,7 +8,7 @@
 
 """
 
-__version__ = "0.1.16"
+__version__ = "0.1.17"
 AUTHOR = "Vanessa Sochat"
 AUTHOR_EMAIL = "vsoch@users.noreply.github.com"
 NAME = "singularity-compose"