Merge pull request #557 from dheren-git/scale_hdfs_dev_master_512

Scale HDFS dev to master for 512

Author: rajan-mis

roles/scale_hdfs/README.md

@@ -6,6 +6,15 @@ Role Definition
   - It allows computers connected to the same network or domain to access files from other local computers as easily as if they 
   were on the computer's local hard drive.
 
+OS/Arch Support:
+---------------------------
+- Arch : 
+   -1: x86
+   -2: PowerLe
+- OS   :
+   -1: RHEL7
+   -2: RHEL8
+
 Prerequisite
 ----------------------------
 - CES should be enabled and configured.
@@ -61,11 +70,11 @@ Implementation
     -	If  ‘scale_install_repository_url’ is undefined and ‘scale_install_remotepkg_path’ is defined, then the installation method is remote.
     -	If  ‘scale_install_repository_url’ is undefined and ‘scale_install_remotepkg_path’ is undefined and ‘scale_install_localpkg_path’ is defined, then the installation method is local.
   - Depending on the installation method, appropriate playbook is called for collecting the rpms and rpms are installed on all the nodes on which hdfs is enabled. 
-  
+
 - `Cluster`
   - This role setup all hdfs required configurations.
   - To enable hdfs protocol, we run command `mmces service enable hdfs` on all the nodes where hdfs is enabled.
-  
+
 - `Postcheck`
   - This role verify namenodes and datanodes status.
   - This role uses command `mmces service list` to check if hdfs service is up and running.

roles/scale_hdfs/cluster/tasks/configure.yml

@@ -96,7 +96,7 @@
 
 - name: check | Check free ces ip
   shell:
-   cmd: "mmces address list -Y|grep -v -e hdfs -e object|awk -F: '{print $7}'|grep -v cesAddress"
+   cmd: "mmces address list -Y|awk -F: '(length($9) == 0){print $7}'"
   register: scale_free_ces
   delegate_to: "{{ scale_server }}"
   run_once: true
@@ -108,6 +108,8 @@
   when:
     - scale_existing_ces_ip.stdout | length == 0
     - scale_free_ces.stdout_lines | length > 0
+  delegate_to: "{{ scale_server }}"
+  run_once: true
 
 - block:
    - name: check | fetch mountpoint
@@ -140,7 +142,7 @@
   delegate_to: "{{ scale_server }}"
 
 - name: configure | Check HDFS ces_group
-  shell: "/usr/lpp/mmfs/bin/mmces address list|grep {{ scale_hdfs_ces_group_name }}"
+  shell: "/usr/lpp/mmfs/bin/mmces address list|grep -w {{ scale_hdfs_ces_group_name }}"
   register: check_hdfs_ces_group
   run_once: true
   delegate_to: "{{ scale_server }}"

roles/scale_hdfs/node/defaults/main.yml

@@ -15,13 +15,13 @@ scale_install_localpkg_tmpdir_path: /tmp
 install_debuginfo: true
 
 # Directory to install 3.1.1.x hdfs package
-hdfs_rhel_version_path: 'hdfs_rpms/rhel7/hdfs_3.1.1.x/'
+hdfs_rhel_version_path: 'hdfs_rpms/rhel/hdfs_3.1.1.x/'
 
 # Directory to install 3.3.x hdfs package
-hdfs_rhel_version_path_33: 'hdfs_rpms/rhel7/hdfs_3.3.x/'
+hdfs_rhel_version_path_33: 'hdfs_rpms/rhel/hdfs_3.3.x/'
 
 # Directory to install 3.1.1.x hdfs package
-hdfs_sles_version_path: 'hdfs_rpms/rhel7/hdfs_3.1.1.x/'
+hdfs_sles_version_path: 'hdfs_rpms/rhel/hdfs_3.1.1.x/'
 
-# Directory to install 3.3.x hdfs package
+# Directory to install 3.1.1.x hdfs package
 hdfs_ubuntu_version_path: 'hdfs_debs/ubuntu/hdfs_3.1.1.x/'

roles/scale_hdfs/node/tasks/apt/install.yml

@@ -90,35 +90,32 @@
 - include_tasks: prepare_env.yml
 
 - block:
-  - name:
+  - name: install | Fetch hdfs version 
     set_fact:
-       hdfs_version_path_selection_rhel: "{{ hdfs_rhel_version_path_33 }}"
+      hdfs_version_path_selection_rhel: "{{ hdfs_rhel_version_path_33 }}"
     when: transparency_33_enabled|bool
 
   - name: install | Fetch hdfs rpm dir path for rhel
     set_fact:
-        hdfs_rpm_path_rhel: "{{ hdfs_version_path_selection_rhel }}" 
- 
+      hdfs_rpm_path_rhel: "{{ hdfs_version_path_selection_rhel }}" 
+
+  - name: install | Set correct hdfs rpm dir path for scale release lower 5.1.2
+    set_fact:
+      hdfs_rpm_path_rhel: "{{ hdfs_rpm_path_rhel | replace('/rhel/','/rhel7/') }}" 
+    when: scale_version is version_compare('5.1.2', '<')
+
   - name: install | Fetch hdfs rpm dir path for sles
     set_fact:
-        hdfs_rpm_path_sles: "{{ hdfs_sles_version_path }}" 
+      hdfs_rpm_path_sles: "{{ hdfs_sles_version_path }}" 
 
   - name: install | Fetch hdfs rpm dir path for ubuntu
     set_fact:
-        hdfs_rpm_path_ubuntu: "{{ hdfs_ubuntu_version_path }}" 
+      hdfs_rpm_path_ubuntu: "{{ hdfs_ubuntu_version_path }}" 
 
-  
   run_once: true
   delegate_to: localhost
 
 - include_tasks: install_{{ scale_installmethod }}.yml
 
-- import_tasks: apt/install.yml
-  when: ansible_distribution in scale_ubuntu_distribution
-
 - import_tasks: yum/install.yml
   when: ansible_distribution in scale_rhel_distribution
-
-- import_tasks: zypper/install.yml
-  when: ansible_distribution in scale_sles_distribution
-

roles/scale_hdfs/node/tasks/install.yml

@@ -117,43 +117,27 @@
    scale_hdfs_url: "{{ hdfs_rpm_path_rhel }}"
   when: ansible_distribution in scale_rhel_distribution and ansible_distribution_major_version == '8'
 
-- name: install | hdfs path on sles15
-  set_fact:
-   scale_hdfs_url: "{{ hdfs_rpm_path_sles }}"
-  when: ansible_distribution in scale_sles_distribution and ansible_distribution_major_version >= '15'
-
-- name: install | hdfs path on ubuntu20
-  set_fact:
-   scale_hdfs_url: "{{ hdfs_rpm_path_ubuntu }}"
-  when: ansible_distribution in scale_ubuntu_distribution and ansible_distribution_major_version == '20'
-
-#
-
-# Find hdfs rpms
-
-- block:  ## when: host is defined as a protocol node
-
+- block:
   - name: install | Find gpfs.hdfs-protocol package
     find:
      paths:  "{{ hdfs_extracted_path }}/{{ scale_hdfs_url }}"
      patterns: gpfs.hdfs-protocol*
     register: scale_install_gpfs_hdfs
 
-
   - name: install | Check valid GPFS hdfs(gpfs.hdfs-protocol) package
     assert:
      that: scale_install_gpfs_hdfs.matched > 0
      msg: "No GPFS hdfs(gpfs.hdfs-protocol) package found {{ hdfs_extracted_path }}/{{ scale_hdfs_url }}"
 
   - name: install | Add GPFS hdfs package to list
     vars:
-     current_package:  "{{ files.path }}"
+     current_package:  "{{ filelist.path }}"
     set_fact:
      scale_install_all_packages: "{{ scale_install_all_packages + [ current_package ] }}"
     with_items:
     - "{{ scale_install_gpfs_hdfs.files }}"
     loop_control:
-       loop_var: files
+       loop_var: filelist
 
 - block:
   - name: initialize
@@ -162,14 +146,13 @@
 
   - name: install | Add GPFS package to list
     set_fact:
-     debuginfo_package: "{{ debuginfo_package + [ files.path ] }}"
+     debuginfo_package: "{{ debuginfo_package + [ filepacklist.path ] }}"
     with_items:
     - "{{ scale_install_gpfs_hdfs_debuginfo.files }}"
     loop_control:
-       loop_var: files
+       loop_var: filepacklist
 
   - name: remove debuginfo from packages
     set_fact:
      scale_install_all_packages: "{{ scale_install_all_packages | difference(debuginfo_package)}}"
   when: not install_debuginfo|bool and ansible_distribution in scale_rhel_distribution
-

roles/scale_hdfs/node/tasks/install_local_pkg.yml

@@ -81,16 +81,6 @@
    scale_hdfs_url: "{{ hdfs_rpm_path_rhel }}"
   when: ansible_distribution in scale_rhel_distribution and ansible_distribution_major_version == '8'
 
-- name: install | hdfs path on sles15
-  set_fact:
-   scale_hdfs_url: "{{ hdfs_rpm_path_sles }}"
-  when: ansible_distribution in scale_sles_distribution and ansible_distribution_major_version >= '15'
-
-- name: install | hdfs path on ubuntu20
-  set_fact:
-   scale_hdfs_url: "{{ hdfs_rpm_path_ubuntu }}"
-  when: ansible_distribution in scale_ubuntu_distribution and ansible_distribution_major_version == '20'
-
 - block:  ## when: host is defined as a protocol node
 
   - name: install | Find gpfs.hdfs (gpfs.hdfs-protocol) package

roles/scale_hdfs/node/tasks/install_remote_pkg.yml

@@ -1,14 +1,4 @@
 ---
-- name: install | Initialize
-  set_fact:
-   scale_hdfs_url: ""
-   gpgcheck: "yes"
-
-- name: install | Disable gpg check
-  set_fact:
-   gpgcheck: "no"
-  when: scale_version < '5.0.4.0'
-
 - name: install | hdfs path on rhel7
   set_fact:
    scale_hdfs_url: "{{ hdfs_rpm_path_rhel }}"
@@ -19,23 +9,12 @@
    scale_hdfs_url: "{{ hdfs_rpm_path_rhel }}"
   when: ansible_distribution in scale_rhel_distribution and ansible_distribution_major_version == '8'
 
-- name: install | hdfs path on sles15
-  set_fact:
-   scale_hdfs_url: "{{ hdfs_rpm_path_sles }}"
-  when: ansible_distribution in scale_sles_distribution and ansible_distribution_major_version >= '15'
-
-- name: install | hdfs path on ubuntu20
-  set_fact:
-   scale_hdfs_url: "{{ hdfs_rpm_path_ubuntu }}"
-  when: ansible_distribution in scale_ubuntu_distribution and ansible_distribution_major_version == '20'
-
-
 - name: install | Configure hdfs YUM repository
   yum_repository:
     name: spectrum-scale-hdfs
     description: IBM Spectrum Scale (hdfs)
     baseurl: "{{ scale_install_repository_url }}{{ scale_hdfs_url }}"
-    gpgcheck: "{{ gpgcheck }}"
+    gpgcheck: "{{ scale_install_gpgcheck }}"
     repo_gpgcheck: no
     sslverify: no
     state: present
@@ -44,30 +23,6 @@
     - ansible_pkg_mgr == 'yum' or ansible_pkg_mgr == 'dnf'
     - scale_install_repository_url != 'existing'
 
-- name: install | Configure hdfs zypper repository
-  zypper_repository:
-    name: spectrum-scale-hdfs-rpms
-    repo: "{{ scale_install_repository_url }}{{ scale_hdfs_url }}"
-    runrefresh: yes
-    state: present
-    disable_gpg_check: yes
-  when:
-    - ansible_pkg_mgr == 'zypper'
-    - scale_install_repository_url != 'existing'
-
-- name: install | Configure hdfs APT repository
-  apt_repository:
-    filename: spectrum-scale-hdfs-debs
-    repo: "deb [trusted=yes] {{ scale_install_repository_url }}{{ scale_hdfs_url }} ./"
-    validate_certs: no
-    state: present
-    update_cache: yes
-    codename: IBM Spectrum Scale (HDFS debs)
-    mode: 0777
-  when:
-    - ansible_pkg_mgr == 'apt'
-    - scale_install_repository_url != 'existing'
-
 - name: install | Add GPFS hdfs packages to list
   set_fact:
     scale_install_all_packages: "{{ scale_install_all_packages + [ hdfs_packages ] }}"

roles/scale_hdfs/node/tasks/install_repository.yml

@@ -1,17 +1,8 @@
 ---
-- name: Initialize
-  set_fact:
-    disable_gpgcheck: "no"
-
-- name: Disable gpg check
-  set_fact:
-    disable_gpgcheck: "yes"
-  when: scale_version < '5.0.4.4'
-
 - name: install | Install GPFS HDFS packages
   yum:
    name: "{{ scale_install_all_packages }}"
    state: present
-   disable_gpg_check: "{{ disable_gpgcheck }}"
+   disable_gpg_check: "{{ scale_disable_gpgcheck }}"
   when: ansible_fqdn in scale_hdfs_nodes_list or ansible_fqdn in scale_protocol_nodes_list
- 
+