Merge pull request #165 from input-output-hk/clients-happy-path

Author: Shay Bergmann

modules/terraform.nix

@@ -113,13 +113,15 @@
     # For available packages, see or modify /profiles/slim.nix
   in ''
     #!
+    cat <<'EOF' > /etc/finish-bootstrap.sh
+    #!/bin/sh
     export NIX_CONFIG="${nixConf}"
     export PATH="/run/current-system/sw/bin:$PATH"
     set -exuo pipefail
     pushd /run/keys
     err_code=0
     aws s3 cp \
-      "s3://${cfg.s3Bucket}/infra/secrets/${cfg.name}/${cfg.kms}/source/${awsAsg}-source.tar.xz" \
+      "s3://${cfg.s3Bucket}/${var "aws_s3_bucket_object.${awsAsg}-flake.id"}" \
       source.tar.xz || err_code=$?
     if test $err_code -eq 0
     then # automated provisioning
@@ -128,6 +130,9 @@
       nix build ./source#nixosConfigurations.${cfg.name}-${awsAsg}.config.system.build.toplevel
       nixos-rebuild --flake ./source#${cfg.name}-${awsAsg} switch
     fi # manual provisioning
+    EOF
+    chmod +x /etc/finish-bootstrap.sh
+    systemd-run --unit=nixos-init $_
   '';
 
   sshArgs = "-C -oConnectTimeout=5 -oUserKnownHostsFile=/dev/null -oNumberOfPasswordPrompts=0 -oServerAliveInterval=60 -oControlPersist=600 -oStrictHostKeyChecking=no -i ./secrets/ssh-${cfg.name}";
@@ -412,7 +417,7 @@
                   cidr = net.cidr.subnet 8 idx cidr;
                   availabilityZone =
                     var
-                    "module.instance_types_to_azs.availability_zones[${toString idx}]";
+                      "element(module.instance_types_to_azs.availability_zones, ${toString idx})";
                 }))
               lib.listToAttrs
             ];
@@ -1115,7 +1120,7 @@
                   cidr = net.cidr.subnet 2 idx cidr;
                   availabilityZone =
                     var
-                    "module.instance_types_to_azs_${region}.availability_zones[${toString idx}]";
+                    "element(module.instance_types_to_azs_${region}.availability_zones, ${toString idx})";
                 }))
               lib.listToAttrs
             ];

modules/terraform/clients.nix

@@ -192,6 +192,16 @@ in {
             id "aws_vpc_peering_connection.${vpc.region}";
         }));
 
+    resource.aws_s3_bucket_object = lib.flip lib.mapAttrs' config.cluster.awsAutoScalingGroups (name: group:
+      lib.nameValuePair "${name}-flake" rec {
+        bucket = config.cluster.s3Bucket;
+        key    = with config; "infra/secrets/${cluster.name}/${cluster.kms}/source/${name}-source.tar.xz";
+        etag  = var ''filemd5("${source}")'';
+        source = "${pkgs.runCommand "source.tar.xz" {} ''
+          tar cvf $out -C ${config.cluster.flakePath} .
+        ''}";
+      });
+
     resource.aws_subnet = mapAwsAsgVpcs (vpc:
       lib.flip lib.mapAttrsToList vpc.subnets (suffix: subnet:
         lib.nameValuePair "${vpc.region}-${suffix}" {
@@ -316,7 +326,7 @@ in {
         lib.nameValuePair vpc.region {
           provider = awsProviderFor vpc.region;
           vpc_peering_connection_id =
-            id "aws_vpc_peering_connection.${vpc.region}";
+            id "aws_vpc_peering_connection_accepter.${vpc.region}";
 
           requester = {allow_remote_vpc_dns_resolution = true;};
         });
@@ -334,9 +344,8 @@ in {
       requesterMeshPeeringOptions = mapAwsAsgVpcPeers (link:
         lib.nameValuePair "${link.connector}-connect-${link.accepter}" {
           provider = awsProviderFor link.connector;
-          vpc_peering_connection_id =
-            id
-            "aws_vpc_peering_connection.${link.connector}-connect-${link.accepter}";
+          vpc_peering_connection_id = id
+            "aws_vpc_peering_connection_accepter.${link.accepter}-accept-${link.connector}";
 
           requester = {allow_remote_vpc_dns_resolution = true;};
         });
@@ -378,10 +387,17 @@ in {
       lib.nameValuePair group.uid {
         name = group.uid;
         inherit (group.iam.instanceProfile) path;
-        role = group.iam.instanceProfile.role.tfName;
-        lifecycle = [{create_before_destroy = true;}];
+        role = var "data.aws_iam_role.${config.cluster.iam.roles.client.uid}.name";
+        lifecycle = [{ create_before_destroy = true; }];
       });
 
+    data.aws_iam_role = let
+      # deploy for core role
+      inherit (config.cluster.iam.roles.client) uid;
+    in {
+      "${uid}".name = "core-${uid}";
+    };
+
     data.aws_iam_policy_document = let
       # deploy for client role
       role = config.cluster.iam.roles.client;
@@ -395,31 +411,18 @@ in {
               inherit (policy) condition;
             });
         };
-    in
-      lib.listToAttrs (lib.mapAttrsToList op role.policies);
-
-    resource.aws_iam_role = let
-      # deploy for client role
-      role = config.cluster.iam.roles.client;
-    in {
-      "${role.uid}" = {
-        name = role.uid;
-        assume_role_policy = role.assumePolicy.tfJson;
-        lifecycle = [{create_before_destroy = true;}];
-      };
-    };
+    in lib.mapAttrs' op role.policies;
 
     resource.aws_iam_role_policy = let
       # deploy for client role
       role = config.cluster.iam.roles.client;
       op = policyName: policy:
         lib.nameValuePair policy.uid {
           name = policy.uid;
-          role = role.id;
+          role = id "data.aws_iam_role.${role.uid}";
           policy = var "data.aws_iam_policy_document.${policy.uid}.json";
         };
-    in
-      lib.listToAttrs (lib.mapAttrsToList op role.policies);
+    in lib.mapAttrs' op role.policies;
 
     resource.aws_security_group =
       lib.flip lib.mapAttrsToList config.cluster.awsAutoScalingGroups

modules/terraform/core.nix

@@ -225,12 +225,17 @@ in {
 
     resource.aws_iam_role = let
       # deploy for core role
-      role = config.cluster.iam.roles.core;
+      inherit (config.cluster.iam.roles) client core;
     in {
-      "${role.uid}" = {
-        name = role.uid;
-        assume_role_policy = role.assumePolicy.tfJson;
-        lifecycle = [{create_before_destroy = true;}];
+      "${client.uid}" = {
+        name = "core-${client.uid}";
+        assume_role_policy = client.assumePolicy.tfJson;
+        lifecycle = [{ create_before_destroy = true; }];
+      };
+      "${core.uid}" = {
+        name = core.uid;
+        assume_role_policy = core.assumePolicy.tfJson;
+        lifecycle = [{ create_before_destroy = true; }];
       };
     };
 

profiles/client.nix

@@ -19,7 +19,6 @@ in {
     ./auxiliaries/reaper.nix
   ];
 
-  services.s3-upload-flake.enable = deployType == "aws";
   services.zfs-client-options.enable = deployType == "aws";
 
   services.telegraf.extraConfig.global_tags.role = "consul-client";

profiles/glusterfs/client.nix

@@ -14,15 +14,15 @@ in {
     path = with pkgs; [nettools];
   };
 
-  fileSystems."/mnt/gv0" = lib.mkIf cfg.enable {
-    device = "glusterd.service.consul:/gv0";
-    fsType = "glusterfs";
-  };
-
-  systemd.services."mnt-gv0.mount" = lib.mkIf cfg.enable {
-    after = ["consul.service"];
-    wants = ["consul.service"];
-  };
+  systemd.mounts = [
+    (lib.mkIf cfg.enable {
+      after = [ "consul.service" "dnsmasq.service" ];
+      wants = [ "consul.service" "dnsmasq.service" ];
+      what = "glusterd.service.consul:/gv0";
+      where = "/mnt/gv0";
+      type = "glusterfs";
+    })
+  ];
 
   systemd.services.nomad = lib.mkIf cfg.enable {
     after = ["mnt-gv0.mount"];