Adding loop_control into hdfs_scale

Signed-off-by: Dherendra Singh<dhersing@in.ibm.com>

Author: dheren-git

roles/scale_hdfs/cluster/tasks/append_dict.yml

@@ -1,6 +1,8 @@
 - name : configure | include task for nameservice
   set_fact:
-    hdfs_site_conf_dict: "{{ hdfs_site_conf_dict | default({}) | combine ({ item.key : item.value }) }}"
+    hdfs_site_conf_dict: "{{ hdfs_site_conf_dict | default({}) | combine ({ hdfs_conf_list.key : hdfs_conf_list.value }) }}"
   with_items:
      - { 'key': "dfs.namenode.rpc-address.{{ scale_hdfs_cluster.name }}.{{ map_var.0 }}", 'value': "{{ map_var.1 }}:8020" }
      - { 'key': "dfs.namenode.http-address.{{ scale_hdfs_cluster.name }}.{{ map_var.0 }}", 'value': "{{ map_var.1 }}:50070" }
+  loop_control:
+     loop_var: hdfs_conf_list

roles/scale_hdfs/cluster/tasks/configure.yml

@@ -65,9 +65,11 @@
 
 - name: check | calculate namenodes services 
   set_fact:
-     namenodes_service: "{{ namenodes_service }}nn{{ item }},"
+     namenodes_service: "{{ namenodes_service }}nn{{ namenode_list }},"
   with_sequence: count={{ namenodes_length }}
   when: scale_hdfs_namenodes_list|length > 1
+  loop_control:
+    loop_var: namenode_list
   delegate_to: localhost
   run_once: true
 
@@ -111,14 +113,17 @@
    - name: check | fetch mountpoint
      shell: /usr/lpp/mmfs/bin/mmlsfs "{{ scale_hdfs_filesystem }}" -T -Y | grep defaultMountPoint | awk -F':' '{print $9}'
      register: hdfs_mountpoint
+
    - name: check| decode mountpoint
      command: /usr/lpp/mmfs/bin/mmclidecode "{{ hdfs_mountpoint.stdout }}"
      register: hdfs_mountpoint_decode
      when: hdfs_mountpoint.rc == 0
+
    - name: check| get mountpoint
      set_fact:
        scale_hdfs_mountpoint: "{{ hdfs_mountpoint_decode.stdout }}"
      when: hdfs_mountpoint_decode.rc == 0
+
   run_once: true
   delegate_to: "{{ scale_server }}"
 
@@ -175,7 +180,7 @@
 - block:
    - name: configure | Configuare core-site.xml
      set_fact:
-         core_site_conf_dict: "{{ core_site_conf_dict | default({}) | combine ({ item.key : item.value }) }}"
+         core_site_conf_dict: "{{ core_site_conf_dict | default({}) | combine ({ core_list.key : core_list.value }) }}"
      with_items:
         - { 'key': "fs.defaultFS", 'value': "hdfs://{{scale_hdfs_cluster.name}}" }
         - { 'key': "hadoop.proxyuser.livy.hosts", 'value': "*" }
@@ -184,10 +189,12 @@
         - { 'key': "hadoop.proxyuser.hive.groups", 'value': "*" }
         - { 'key': "hadoop.proxyuser.oozie.hosts", 'value': "*" }
         - { 'key': "hadoop.proxyuser.oozie.groups", 'value': "*" }
+     loop_control:
+       loop_var: core_list
 
    - name: configure | Configuare hdfs-site.xml
      set_fact:
-         hdfs_site_conf_dict: "{{ hdfs_site_conf_dict | default({}) | combine ({ item.key : item.value }) }}"
+         hdfs_site_conf_dict: "{{ hdfs_site_conf_dict | default({}) | combine ({ hdfs_list.key : hdfs_list.value }) }}"
      with_items:
         - { 'key': "dfs.ha.namenodes.{{ scale_hdfs_cluster.name }}", 'value': "{{ namenodes_service }}" }
         - { 'key': "dfs.client.failover.proxy.provider.{{ scale_hdfs_cluster.name }}", 'value': 'org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider' }
@@ -196,6 +203,8 @@
         - { 'key': "dfs.namenode.servicerpc-bind-host", 'value': "0.0.0.0" }
         - { 'key': "dfs.namenode.lifeline.rpc-bind-host", 'value': "0.0.0.0" }
         - { 'key': "dfs.namenode.http-bind-host", 'value': "0.0.0.0" }
+     loop_control:
+       loop_var: hdfs_list
 
    - name: configure | include append_dict.yml
      include_tasks: append_dict.yml
@@ -207,33 +216,42 @@
 
    - name: configure | Configuare hdfs-site.xml for HA
      set_fact:
-        hdfs_site_conf_dict: "{{ hdfs_site_conf_dict | default({}) | combine ({ item.key : item.value }) }}"
+        hdfs_site_conf_dict: "{{ hdfs_site_conf_dict | default({}) | combine ({ hdfs_site_ha_conf.key : hdfs_site_ha_conf.value }) }}"
      with_items:
        - { 'key': "dfs.namenode.shared.edits.dir", 'value': "file:///{{ scale_hdfs_mountpoint }}/HA-{{ scale_hdfs_cluster.name }}" }
      when: ha_enabled|bool
+     loop_control:
+        loop_var: hdfs_site_ha_conf
 
    - name: configure | Configuare gpfs-site.xml
      set_fact:
-         gpfs_site_conf_dict: "{{ gpfs_site_conf_dict | default({}) | combine ({ item.key : item.value }) }}"
+         gpfs_site_conf_dict: "{{ gpfs_site_conf_dict | default({}) | combine ({ hdfs_site_list.key : hdfs_site_list.value }) }}"
      with_items:
        - { 'key': "gpfs.mnt.dir", 'value': "{{ scale_hdfs_mountpoint }}" }
        - { 'key': "gpfs.data.dir", 'value': "{{ scale_hdfs_datadir }}" }
-
+     loop_control:
+        loop_var: hdfs_site_list
   run_once: true
   delegate_to: localhost
 
 - block:
    - name: configure | Set configuration for core-site.xml
-     shell: /usr/lpp/mmfs/hadoop/sbin/mmhdfs config set core-site.xml -k "{{ item.key }}"="{{ item.value }}"
+     shell: /usr/lpp/mmfs/hadoop/sbin/mmhdfs config set core-site.xml -k "{{ core_site_conf.key }}"="{{ core_site_conf.value }}"
      loop:  "{{ core_site_conf_dict | dict2items }}"
+     loop_control:
+        loop_var: core_site_conf
 
    - name: configure | Set configuration for gpfs-site.xml
-     shell: /usr/lpp/mmfs/hadoop/sbin/mmhdfs config set gpfs-site.xml -k "{{ item.key }}"="{{ item.value }}"
+     shell: /usr/lpp/mmfs/hadoop/sbin/mmhdfs config set gpfs-site.xml -k "{{ gpfs_site_conf.key }}"="{{ gpfs_site_conf.value }}"
      loop:  "{{ gpfs_site_conf_dict | dict2items }}"
+     loop_control:
+        loop_var: gpfs_site_conf
 
    - name: configure | Set configuration for hdfs-site.xml
-     shell: /usr/lpp/mmfs/hadoop/sbin/mmhdfs config set hdfs-site.xml -k "{{ item.key }}"="{{ item.value }}"
+     shell: /usr/lpp/mmfs/hadoop/sbin/mmhdfs config set hdfs-site.xml -k "{{ hdfs_site_conf.key }}"="{{ hdfs_site_conf.value }}"
      loop:  "{{ hdfs_site_conf_dict | dict2items }}"
+     loop_control:
+        loop_var: hdfs_site_conf
 
    - name: configure | Remove localhost
      shell: /usr/lpp/mmfs/hadoop/sbin/mmhdfs worker remove localhost

roles/scale_hdfs/node/tasks/apt/install.yml

@@ -8,8 +8,10 @@
 
 - name: install| Install GPFS HDFS deb
   apt:
-   deb: "{{ item }}"
+   deb: "{{ all_packages }}"
    state: present
   when: scale_install_repository_url is not defined and ansible_fqdn in scale_hdfs_nodes_list
   with_items:
-  - "{{ scale_install_all_packages }}"
+  - "{{ scale_install_all_packages }}"
+  loop_control:
+     loop_var: all_packages

roles/scale_hdfs/node/tasks/install.yml

@@ -111,9 +111,6 @@
   run_once: true
   delegate_to: localhost
 
-- debug:
-    var: hdfs_rpm_path_rhel
-
 - include_tasks: install_{{ scale_installmethod }}.yml
 
 - import_tasks: apt/install.yml

roles/scale_hdfs/node/tasks/install_dir_pkg.yml

@@ -67,11 +67,13 @@
 
   - name: install | Add GPFS hdfs package to list
     vars:
-     current_package:  "{{ item.path }}"
+     current_package:  "{{ files.path }}"
     set_fact:
      scale_install_all_packages: "{{ scale_install_all_packages + [ current_package ] }}"
     with_items:
     - "{{ scale_install_gpfs_hdfs.files }}"
+    loop_control:
+      loop_var: files
 
 - block:
   - name: initialize
@@ -80,9 +82,11 @@
 
   - name: install | Add GPFS package to list
     set_fact:
-     debuginfo_package: "{{ debuginfo_package + [ item.path ] }}"
+     debuginfo_package: "{{ debuginfo_package + [ files.path ] }}"
     with_items:
     - "{{ scale_install_gpfs_hdfs_debuginfo.files }}"
+    loop_control:
+      loop_var: files
 
   - name: remove debuginfo from packages
     set_fact:

roles/scale_hdfs/node/tasks/install_local_pkg.yml

@@ -106,9 +106,7 @@
    path: "{{ scale_install_localpkg_tmpdir_path + '/' + scale_install_localpkg_path | basename }}"
    state: absent
 
-- debug:
-    var: hdfs_rpm_path_rhel
-    
+
 - name: install | hdfs path on rhel7
   set_fact:
    scale_hdfs_url: "{{ hdfs_rpm_path_rhel }}"
@@ -149,11 +147,13 @@
 
   - name: install | Add GPFS hdfs package to list
     vars:
-     current_package:  "{{ item.path }}"
+     current_package:  "{{ files.path }}"
     set_fact:
      scale_install_all_packages: "{{ scale_install_all_packages + [ current_package ] }}"
     with_items:
     - "{{ scale_install_gpfs_hdfs.files }}"
+    loop_control:
+       loop_var: files
 
 - block:
   - name: initialize
@@ -162,9 +162,11 @@
 
   - name: install | Add GPFS package to list
     set_fact:
-     debuginfo_package: "{{ debuginfo_package + [ item.path ] }}"
+     debuginfo_package: "{{ debuginfo_package + [ files.path ] }}"
     with_items:
     - "{{ scale_install_gpfs_hdfs_debuginfo.files }}"
+    loop_control:
+       loop_var: files
 
   - name: remove debuginfo from packages
     set_fact:

roles/scale_hdfs/node/tasks/install_remote_pkg.yml

@@ -106,9 +106,11 @@
 
   - name: install | Add GPFS hdfs package to list
     vars:
-     current_package:  "{{ item.path }}"
+     current_package:  "{{ files.path }}"
     set_fact:
      scale_install_all_packages: "{{ scale_install_all_packages + [ current_package ] }}"
     with_items:
     - "{{ scale_install_gpfs_hdfs.files }}"
+    loop_control:
+       loop_var: files
 

roles/scale_hdfs/node/tasks/install_repository.yml

@@ -30,9 +30,6 @@
   when: ansible_distribution in scale_ubuntu_distribution and ansible_distribution_major_version == '20'
 
 
-- debug:
-   var: scale_hdfs_url
-
 - name: install | Configure hdfs YUM repository
   yum_repository:
     name: spectrum-scale-hdfs

roles/scale_hdfs/postcheck/tasks/check.yml

@@ -93,13 +93,20 @@
   register: force_active
   when: ha_enabled|bool and active_status.stdout_lines | length == 0 and ceshost.stdout|length != 0
   delegate_to: "{{ ceshost.stdout }}"
+  failed_when: false
   run_once: true
 
+- name: postcheck | Sleep for 20 seconds
+  wait_for:
+    timeout: 20
+  delegate_to: localhost
+
 - name: postcheck | Check Namenodes active status again
   shell: "{{ scale_hdfs_command_path }}hdfs haadmin -getAllServiceState | grep -i 'active'"
   register: active_status_again
-  when: ha_enabled|bool
+  when: ha_enabled|bool and force_active.rc == 0
   delegate_to: "{{ scale_server }}"
+  failed_when: false
   run_once: true
 
 - fail:

roles/scale_hdfs/precheck/tasks/check.yml

@@ -12,10 +12,12 @@
 
 - name: check | Collect all protocol nodes
   set_fact:
-   scale_protocol_nodes_list: "{{ scale_protocol_nodes_list + [hostvars[item]['ansible_fqdn']] }}"
-  when: hostvars[item]['is_protocol_node'] is defined and hostvars[item]['is_protocol_node']|bool
+   scale_protocol_nodes_list: "{{ scale_protocol_nodes_list + [hostvars[hosts]['ansible_fqdn']] }}"
+  when: hostvars[hosts]['is_protocol_node'] is defined and hostvars[hosts]['is_protocol_node']|bool
   with_items:
    - "{{ ansible_play_hosts }}"
+  loop_control:
+     loop_var: hosts
   delegate_to: localhost
   run_once: true
 
@@ -74,28 +76,32 @@
 - name: check |  Check if HDFS required information has been supplied.
   assert:
    that:
-   - item is defined
-   - item| length > 0
-   - item.name is defined
-   - item.name| length > 0
-   - item.filesystem is defined
-   - item.filesystem| length > 0
-   - item.namenodes is defined
-   - item.namenodes| length > 0
-   - item.datanodes is defined
-   - item.datanodes| length > 0
-   - item.datadir is defined
-   - item.datadir| length > 0
+   - hdfs_cluster is defined
+   - hdfs_cluster| length > 0
+   - hdfs_cluster.name is defined
+   - hdfs_cluster.name| length > 0
+   - hdfs_cluster.filesystem is defined
+   - hdfs_cluster.filesystem| length > 0
+   - hdfs_cluster.namenodes is defined
+   - hdfs_cluster.namenodes| length > 0
+   - hdfs_cluster.datanodes is defined
+   - hdfs_cluster.datanodes| length > 0
+   - hdfs_cluster.datadir is defined
+   - hdfs_cluster.datadir| length > 0
    fail_msg: "HDFS required parameter information is not defined."
    success_msg: "HDFS required information is defined."
   with_items:
    - "{{ scale_hdfs_cluster }}"
+  loop_control:
+     loop_var: hdfs_cluster
   run_once: true
   delegate_to: localhost
 
 - name: check | Verify JAVA_HOME
   include_tasks: java_home.yml
   loop: "{{ scale_hdfs_clusters }}"
+  loop_control:
+     loop_var: hdfs_clusters
 
 - debug:
     msg: "HDFS Precheck ok on {{ scale_hdfs_cluster.name }} cluster."