update and cleanup ansible installation scripts and readme

Istemi Ekin Akkus · Istemi Ekin Akkus · commit d0e3bf27210c · 2020-05-18T00:12:12.000+02:00
diff --git a/ManagementService/management_init.py b/ManagementService/management_init.py
@@ -403,8 +403,6 @@ def printUsage():
                 f.write("docker run -d --ulimit nofile=262144:262144 --name Management --env-file .env --log-opt max-size=500m --log-opt max-file=5 -P -p " + str(MANAGEMENT_SERVICE_EXPOSED_PORT) + ":8080 microfn/sandbox\n")
             os.chmod("start_management.sh", 0o0775)
 
-            print("Please run: start_management.sh  to start the Management container")
-
         DLCLIENT.shutdown()
         DLCLIENT_MANAGEMENT.shutdown()
     elif action == "print":
diff --git a/deploy/ansible/Makefile b/deploy/ansible/Makefile
@@ -101,6 +101,7 @@ GUI_SRC:=../../GUI
 GUIFILES:=$(shell find $(GUI_SRC) | grep -v "knix_gui_deployment_package.tar.gz")
 ../../GUI/knix_gui_deployment_package.tar.gz: $(GUIFILES)
 	touch $@
+	cd ../../GUI; make; cd -
 	tar --exclude=knix_gui_deployment_package.tar.gz -czf $@ -C $(GUI_SRC) .
 
 nginx.yaml: ../../GUI/knix_gui_deployment_package.tar.gz
diff --git a/deploy/ansible/README.md b/deploy/ansible/README.md
@@ -16,7 +16,7 @@
 
 # KNIX ansible installation
 
-To setup KNIX on localhost, or a single remote host, or a cluster of hosts  
+To setup KNIX on localhost, or a single remote host, or a cluster of hosts
 (Tested on: Ubuntu 18.04 and Debian 9 as target machines)
 
 ## Prerequisites: on host machine
@@ -27,6 +27,11 @@ To setup KNIX on localhost, or a single remote host, or a cluster of hosts
 
 ## Prerequisites: on target machines
 
+Note: The following steps should be handled by the init_once.yaml during the installation process.
+If you find something wrong or missing, please consider opening an issue on [GitHub](https://github.com/knix-microfunctions/knix)
+and/or letting us know in our [Slack workspace](https://knix.slack.com).
+Thank you!
+
 1. You can ssh into the target machines/VMs without a password. If not then run (on host machine):
 
     ```bash
@@ -38,7 +43,7 @@ To setup KNIX on localhost, or a single remote host, or a cluster of hosts
 
 3. Appropriate proxies (/etc/profile, /etc/bash.bashrc, and /etc/apt/apt.conf) are set on target machines
 
-4. `python`, `python-dev`, `python-pip` are installed on each of the target machines
+4. `python3`, `python3-dev`, `python3-pip` are installed on each of the target machines
 
     ```bash
     sudo apt-get update
diff --git a/deploy/ansible/init_once.yaml b/deploy/ansible/init_once.yaml
@@ -33,7 +33,7 @@
       https_proxy: "{{ https_proxy.stdout }}"
 
   - debug:
-      msg: 
+      msg:
       - inventory_hostname = {{ inventory_hostname }}
       - ansible_default_ipv4.address = {{ ansible_default_ipv4.address }}
       - mfn_server_installation_folder = {{ mfn_server_installation_folder }}
@@ -52,56 +52,150 @@
         - python3-pip
         - netcat
         - rsync
-        - python-dev 
-        - libffi-dev 
-        - libssl-dev 
-        - libxml2-dev 
+        - python3-dev
+        - libffi-dev
+        - libssl-dev
+        - libxml2-dev
         - libxslt1-dev
         - zlib1g-dev
       state: latest
 
   - name: pip install pyOpenSSL
     pip:
-      name: pyOpenSSL 
+      name: pyOpenSSL
       executable: pip3
     environment:
       http_proxy: "{{ http_proxy }}"
       https_proxy: "{{ https_proxy }}"
 
-  - name: Ensure mfn root folder exists
+  - name: configure /etc/hosts - disable 127.0.1.1
+    lineinfile:
+      path: "/etc/hosts"
+      regexp: "^127.0.1.1"
+      line: "#127.0.1.1 {{ inventory_hostname }}"
+
+  - name: configure /etc/hosts - add ip address hostname mapping
+    lineinfile:
+      path: "/etc/hosts"
+      insertafter: EOF
+      line: "{{ ansible_default_ipv4.address }} {{ inventory_hostname }} {{ inventory_hostname }}"
+
+  - name: ensure docker is at the latest version
+    apt: name=docker-ce update_cache=yes
+    register: dockerupdate
+    ignore_errors: True
+
+  - name: Remove old docker
+    apt:
+      name:
+        - docker
+        - docker-engine
+        - docker.io
+        - runc
+      state: absent
+    when: dockerupdate.failed
+
+  - name: get docker key
+    get_url:
+      url: https://download.docker.com/linux/ubuntu/gpg
+      dest: /etc/apt/docker.gpg.key
+    environment:
+      http_proxy: "{{ http_proxy }}"
+      https_proxy: "{{ https_proxy }}"
+    when: dockerupdate.failed
+
+  - name: install docker key
+    shell: "apt-key add /etc/apt/docker.gpg.key"
+    when: dockerupdate.failed
+
+  - name: add docker repo
+    apt_repository:
+      repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_lsb.codename }} stable"
+      state: present
+    when: dockerupdate.failed
+
+  - name: ensure docker is at the latest version
+    apt: name=docker-ce update_cache=yes
+    when: dockerupdate.failed
+
+  # The below approach risks when updating the docker using regular software updates.
+  # a better way probably is just to override the configuration with a new file
+  #- name: fix docker service command line
+  #  lineinfile:
+  #    path: /lib/systemd/system/docker.service
+  #    backrefs: yes
+  #    regex: "^ExecStart=(.*)-H fd://(.*)$"
+  #    line: "ExecStart=\\1\\2"
+  #  when: dockerupdate.failed
+
+  #- name: configure docker daemon
+  #  copy:
+  #    dest: /etc/docker/daemon.json
+  #    content: |
+  #      {
+  #        "hosts": ["fd://","unix:///var/run/docker.sock","tcp://0.0.0.0"]
+  #      }
+  #################
+
+  # see the following links why this approach might be better
+  # https://stackoverflow.com/questions/44052054/unable-to-start-docker-after-configuring-hosts-in-daemon-json
+  # https://success.docker.com/article/how-do-i-enable-the-remote-api-for-dockerd
+  - name: docker service configuration directory
     file:
-      path: "{{ mfn_server_installation_folder }}"
+      path: /etc/systemd/system/docker.service.d
       state: directory
-      mode: '775'
+
+  - name: add override configuration for docker start
+    copy:
+      dest: /etc/systemd/system/docker.service.d/startup_options.conf
+      content: |
+        [Service]
+        ExecStart=
+        ExecStart=/usr/bin/dockerd -H unix:// -H tcp://0.0.0.0:2375
+
+  - name: override configuration for docker daemon to disable tls
+    copy:
+      dest: /etc/docker/daemon.json
+      content: |
+        {
+          "tls": false
+        }
+
+  - name: configure docker proxy
+    copy:
+      dest: /etc/systemd/system/docker.service.d/proxy.conf
+      content: |
+        [Service]
+        Environment=HTTP_PROXY={{ http_proxy }}
+        Environment=HTTPS_PROXY={{ https_proxy }}
+        Environment=NO_PROXY={{ no_proxy }}
+    when: http_proxy != "" and dockerupdate.failed
+
+  - name: systemd restart docker
+    systemd:
+      daemon_reload: yes
+      name: docker
+      enabled: true
+      state: restarted
+    when: dockerupdate.failed
 
   - name: ensure java is at the latest version
-    apt: 
+    apt:
       name: openjdk-8-jre-headless
       state: latest
       update_cache: yes
 
-  - name: get http_proxy
-    shell: "su - -c 'echo $http_proxy'"
-    register: http_proxy
-
-  - name: get https_proxy
-    shell: "su - -c 'echo $https_proxy'"
-    register: https_proxy
-
-  - set_fact:
-      http_proxy: "{{ http_proxy.stdout }}"
-      https_proxy: "{{ https_proxy.stdout }}"
-
-  - debug:
-      msg:
-        - http_proxy = {{ http_proxy }}
-        - https_proxy = {{ https_proxy }}
+  - name: Ensure knix mfn root folder exists
+    file:
+      path: "{{ mfn_server_installation_folder }}"
+      state: directory
+      mode: '775'
 
   - name: Copy start-all.sh script
     copy:
       mode: '775'
       src: "./scripts/start-all.sh"
-      dest: "{{ mfn_server_installation_folder }}/"      
+      dest: "{{ mfn_server_installation_folder }}/"
 
   - name: Copy stop-all.sh script
     copy:
@@ -110,25 +204,25 @@
       dest: "{{ mfn_server_installation_folder }}/"
 
   - name: copy wait-for-it.sh script
-    copy: 
+    copy:
       mode: '775'
       src: "./scripts/wait-for-it.sh"
       dest: "{{ mfn_server_installation_folder }}/"
 
   - name: copy tail-service-logs.sh script
-    copy: 
+    copy:
       mode: '775'
       src: "./scripts/tail-service-logs.sh"
       dest: "{{ mfn_server_installation_folder }}/"
 
   - name: copy purge-riak.sh script
-    copy: 
+    copy:
       mode: '775'
       src: "./scripts/purge-riak.sh"
       dest: "{{ mfn_server_installation_folder }}/"
 
    #https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html
-  - name: configure /etc/sysctl.conf  vm.max_map_count=262144 
-    sysctl: 
+  - name: configure /etc/sysctl.conf  vm.max_map_count=262144
+    sysctl:
       name: "vm.max_map_count"
       value: "262144"
diff --git a/deploy/ansible/management.yaml b/deploy/ansible/management.yaml
@@ -24,7 +24,6 @@
 
     install_dir: "{{ mfn_server_installation_folder }}/management"
     datalayer_connect: "{{ ansible_default_ipv4.address }}:4998" # same host
-    riak_connect: "{{ hostvars[groups['riak'][0]].ansible_default_ipv4.address }}:8087"
     elasticsearch_connect: "{{ hostvars[groups['elasticsearch'][0]].ansible_default_ipv4.address }}:9200"
     nginx_http_listen_port: "{{ nginx_http_listen_port }}"
     nginx_connect: "{{ hostvars[groups['nginx'][0]].ansible_default_ipv4.address }}:{{ nginx_http_listen_port }}"
@@ -57,28 +56,13 @@
       - management_archive_filename = {{ management_archive_filename }}
       - install_dir = {{ install_dir }}
       - datalayer connect string = {{ datalayer_connect }}
-      - riak_connect = {{ riak_connect }}
       - elasticsearch_connect = {{ elasticsearch_connect }}
       - nginx_connect = {{ nginx_connect }}
       - http_proxy = {{ http_proxy }}
       - https_proxy = {{ https_proxy }}
       - no_proxy = {{ no_proxy }}
 
 
-  # Prerequisites to run the management init script
-  - name: pip install docker
-    pip:
-      name:
-      - docker
-      - riak
-      executable: pip3
-    environment:
-      http_proxy: "{{ http_proxy }}"
-      https_proxy: "{{ https_proxy }}"
-      no_proxy: "{{ no_proxy }}"
-      HTTP_PROXY: "{{ http_proxy }}"
-      HTTPS_PROXY: "{{ https_proxy }}"
-
   - name: remove old management folder
     file:
       state: absent
@@ -137,6 +121,7 @@
         set -x
         cat <<END >>dockerrun.sh
         set -x
+        pip3 install --upgrade pip
         pip3 install docker thrift
         python3 management_init.py start
         END
@@ -175,7 +160,6 @@
     environment:
       MFN_HOSTNAME: "{{ inventory_hostname }}"
       DATALAYER_CONNECT: "{{ datalayer_connect }}"
-      RIAK_CONNECT: "{{ riak_connect }}"
       ELASTICSEARCH_CONNECT: "{{ elasticsearch_connect }}"
       NGINX_CONNECT: "{{ nginx_connect }}"
       http_proxy: "{{ http_proxy }}"
diff --git a/deploy/ansible/riak.yaml b/deploy/ansible/riak.yaml
@@ -277,13 +277,13 @@
       regexp: "^bitcask.max_file_size = "
       line: "bitcask.max_file_size = 500MB"
 
-  - name: configure /etc/security/limits.conf  riak soft nofile 65536
+  - name: configure /etc/security/limits.conf riak soft nofile 65536
     lineinfile:
       path: "/etc/security/limits.conf"
       regexp: "^riak soft nofile"
       line: "riak soft nofile 65536"
 
-  - name: configure /etc/security/limits.conf  riak hard nofile 200000
+  - name: configure /etc/security/limits.conf riak hard nofile 200000
     lineinfile:
       path: "/etc/security/limits.conf"
       regexp: "^riak hard nofile"
diff --git a/deploy/ansible/sandbox.yaml b/deploy/ansible/sandbox.yaml
