Merge pull request #264 from KosukeOkamoto/update-discovery-scripts-for-5.0.0

Update script for Watson Discovery 5.0.0 on CP4D

Author: kvineetnair

discovery-data/latest/all-backup-restore.sh

@@ -42,6 +42,8 @@ Basically, you don't need these advanced options.
     --datastore-archive-option="<tar_option>"  Tar options for compression used on archiving the backup files of ElasticSearch, MinIO and internal configuration. Default "-z".
     --postgresql-archive-option="<tar_option>" Tar options for compression used on archiving the backup files of postgres. Note that the backup files of postgresql are archived on its pod by default. Default "-z".
     --etcd-archive-option="<tar_option>"       Tar options used on archiving the backup files of etcd. Note that the backup files of etcd are archived on its pod by default. Default "-z".
+    --reindex-old-index                        Reindex elastic search indices from version 6 to 7 before making the backup. If existing elastic search indices version is already 7, this flag does nothing. They have to be reindexed if you want to restore to 4.8.x. Reindex actually won't change your data, but it takes time. If you want to make the backup without reindex, use '--ignore-old-index' instead.
+    --ignore-old-index                         Whether to skip reindexing old (version 6) elastic search indices before making the backup. You can proceed backup without reindex, but you can't restore the backup data to 4.8.x.
     --skip-verify-archive                      Skip the all verifying process of the archive.
     --skip-verify-backup                       Skip verifying the backup file.
     --skip-verify-datastore-archive            Skip verifying the archive of datastores.
@@ -224,6 +226,12 @@ do
     --skip-quiesce)
       export SKIP_QUIESCE=true
       ;;
+    --reindex-old-index)
+      export REINDEX_OLD_INDEX=true
+      ;;
+    --ignore-old-index)
+      export IGNORE_OLD_INDEX=true
+      ;;
     --skip-quiesce=*)
       export SKIP_QUIESCE="${1#--skip-quiesce=}"
       ;;
@@ -300,6 +308,8 @@ export TENANT_NAME=${TENANT_NAME}
 export SCRIPT_DIR=${SCRIPT_DIR}
 export OC_ARGS="${EXTRA_OC_ARGS}"
 export CLEAN=${CLEAN:-false}
+export REINDEX_OLD_INDEX=${REINDEX_OLD_INDEX:-false}
+export IGNORE_OLD_INDEX=${IGNORE_OLD_INDEX:-false}
 
 ###############
 # Function
@@ -354,6 +364,7 @@ mkdir -p ${TMP_WORK_DIR}
 
 mkdir -p "${BACKUP_RESTORE_LOG_DIR}"
 brlog "INFO" "Component log directory: ${BACKUP_RESTORE_LOG_DIR}"
+
 CURRENT_COMPONENT="pre-process"
 check_datastore_available
 create_elastic_shared_pvc
@@ -369,8 +380,9 @@ if ! "${BACKUP_RESTORE_IN_POD:-false}" ; then
   export MC_COMMAND=${MC_COMMAND}
 fi
 
-
 if [ ${COMMAND} = 'backup' ] ; then
+  ELASTIC_VERSION=$(get_elastic_version)
+  validate_elastic_version "$ELASTIC_VERSION"
   if [ $(compare_version "${WD_VERSION}" "4.0.6") -ge 0 ] ; then
     create_backup_instance_mappings
   fi
@@ -379,11 +391,15 @@ if [ ${COMMAND} = 'backup' ] ; then
       quiesce
     fi
   fi
+  if [[ ${ELASTIC_VERSION} = "ES6" ]] && [[ ${REINDEX_OLD_INDEX} = "true" ]]; then
+    reindex_elastic_es6
+  fi 
   if [ $(compare_version "${WD_VERSION}" "2.1.3") -ge 0 ] ; then
     ALL_COMPONENT=("wddata" "etcd" "postgresql" "elastic" "minio")
   else
     ALL_COMPONENT=("wddata" "etcd" "hdp" "postgresql" "elastic")
   fi
+
   export ALL_COMPONENT
   export VERIFY_COMPONENT=( "${ALL_COMPONENT[@]}")
   BACKUP_FILE=${BACKUP_FILE:-"watson-discovery_$(date "+%Y%m%d_%H%M%S").backup"}

discovery-data/latest/elastic-backup-restore.sh

@@ -239,10 +239,14 @@ function clean_up(){
       else\
         break;\
       fi;\
-    done && \
-    curl -XPUT -s -k -u ${ELASTIC_USER}:${ELASTIC_PASSWORD} "${ELASTIC_ENDPOINT}/_cluster/settings" -H "Content-Type: application/json" -d"{\"transient\": {\"discovery.zen.commit_timeout\": null, \"discovery.zen.publish_timeout\": null}}"' ${OC_ARGS} -c elasticsearch
+    done ' ${OC_ARGS} -c elasticsearch
     echo
 
+    if [ $(compare_version ${WD_VERSION} "5.0.0") -lt 0 ]; then
+      run_cmd_in_pod ${ELASTIC_POD} 'export ELASTIC_ENDPOINT=https://localhost:9200 && \
+      curl -XPUT -s -k -u ${ELASTIC_USER}:${ELASTIC_PASSWORD} "${ELASTIC_ENDPOINT}/_cluster/settings" -H "Content-Type: application/json" -d"{\"transient\": {\"discovery.zen.commit_timeout\": null, \"discovery.zen.publish_timeout\": null}}"' ${OC_ARGS} -c elasticsearch
+    fi
+
     if [ $(compare_version ${WD_VERSION} "4.7.0") -lt 0 ] ; then
       start_minio_port_forward
       ${MC} "${MC_OPTS[@]}" rm --recursive --force --dangerous wdminio/${ELASTIC_BACKUP_BUCKET}/ > /dev/null
@@ -409,13 +413,19 @@ EOF
     run_cmd_in_pod ${ELASTIC_POD} "tar ${ELASTIC_TAR_OPTIONS[*]} -xmpf /workdir/shared_storage/${ELASTIC_BACKUP} -C /workdir/shared_storage && rm -f /workdir/shared_storage/${ELASTIC_BACKUP}" ${OC_ARGS} -c elasticsearch
   fi
   brlog "INFO" "Start Restoring snapshot"
+
   run_cmd_in_pod ${ELASTIC_POD} 'export S3_HOST='${S3_SVC}' && export S3_PORT='${S3_PORT}' && export S3_ELASTIC_BACKUP_BUCKET='${ELASTIC_BACKUP_BUCKET}' && export ELASTIC_ENDPOINT=https://localhost:9200 && \
   S3_IP=$(curl -kv "https://$S3_HOST:$S3_PORT/minio/health/ready" 2>&1 | grep Connected | sed -E "s/.*\(([0-9.]+)\).*/\1/g") && \
-  curl -XPUT --fail -s -k -u ${ELASTIC_USER}:${ELASTIC_PASSWORD} "${ELASTIC_ENDPOINT}/_cluster/settings" -H "Content-Type: application/json" -d"{\"transient\": {\"discovery.zen.commit_timeout\": \"'${ELASTIC_REQUEST_TIMEOUT}'\", \"discovery.zen.publish_timeout\": \"'${ELASTIC_REQUEST_TIMEOUT}'\"}}" && \
   curl -XDELETE --fail -s -k -u ${ELASTIC_USER}:${ELASTIC_PASSWORD} "${ELASTIC_ENDPOINT}/_all?master_timeout='${ELASTIC_REQUEST_TIMEOUT}'" && \
   curl -XDELETE --fail -s -k -u ${ELASTIC_USER}:${ELASTIC_PASSWORD} "${ELASTIC_ENDPOINT}/.*?master_timeout='${ELASTIC_REQUEST_TIMEOUT}'" && \
   curl -XPUT --fail -s -k -u ${ELASTIC_USER}:${ELASTIC_PASSWORD} "${ELASTIC_ENDPOINT}/_snapshot/'${ELASTIC_REPO}'?master_timeout='${ELASTIC_REQUEST_TIMEOUT}'" -H "Content-Type: application/json" '"${REPO_CONFIGURATION}"' && \
   curl -XPOST --fail -s -k -u ${ELASTIC_USER}:${ELASTIC_PASSWORD} "${ELASTIC_ENDPOINT}/_snapshot/'${ELASTIC_REPO}'/'${ELASTIC_SNAPSHOT}'/_restore?master_timeout='${ELASTIC_REQUEST_TIMEOUT}'" -H "Content-Type: application/json" -d"{\"indices\": \"*,-application_logs-*\", \"expand_wildcards\": \"all\", \"allow_no_indices\": \"true\"}" | grep accepted && echo ' ${OC_ARGS} -c elasticsearch
+  
+  if [ $(compare_version ${WD_VERSION} "5.0.0") -lt 0 ]; then
+    run_cmd_in_pod ${ELASTIC_POD} 'export ELASTIC_ENDPOINT=https://localhost:9200 && \
+    curl -XPUT --fail -s -k -u ${ELASTIC_USER}:${ELASTIC_PASSWORD} "${ELASTIC_ENDPOINT}/_cluster/settings" -H "Content-Type: application/json" -d"{\"transient\": {\"discovery.zen.commit_timeout\": \"'${ELASTIC_REQUEST_TIMEOUT}'\", \"discovery.zen.publish_timeout\": \"'${ELASTIC_REQUEST_TIMEOUT}'\"}}" '  ${OC_ARGS} -c elasticsearch
+  fi
+  
   brlog "INFO" "Sent restore request"
   waited_seconds=0
   total_shards=0

discovery-data/latest/lib/function.bash

@@ -5,6 +5,8 @@ export BACKUP_VERSION_FILE="tmp/version.txt"
 export DATASTORE_ARCHIVE_OPTION="${DATASTORE_ARCHIVE_OPTION--z}"
 export BACKUP_RESTORE_LOG_DIR="${BACKUP_RESTORE_LOG_DIR:-wd-backup-restore-logs-$(date "+%Y%m%d_%H%M%S")}"
 export BACKUP_RESTORE_SA="${BACKUP_RESTORE_SA:-wd-discovery-backup-restore-sa}"
+export BACKUP_RESTORE_RBAC_LABEL_KEY="${BACKUP_RESTORE_RBAC_LABEL_KEY:-discovery.ibm.com/component}"
+export BACKUP_RESTORE_RABC_LABEL_VALUE="${BACKUP_RESTORE_RBAC_LABEL_VALUE:-backup-restore-scripts}"
 
 case "${BACKUP_RESTORE_LOG_LEVEL}" in
   "ERROR") export LOG_LEVEL_NUM=0;;
@@ -41,8 +43,17 @@ trap_add(){
 }
 
 trap_remove(){
+  # Remove $1 command from 'trap_commands'.
   trap_commands=( "${trap_commands[@]/$1}" )
-  trap "${trap_commands}" 0 1 2 3 15
+
+  # NOTE: if 'trap_commands' is empty, "${trap_commands}" cause unbound variable error when set -u.
+  if [[ -z "${trap_commands[@]}" ]]; then
+    cmd=""
+  else
+    printf -v joined '%s;' "${trap_commands[@]}"
+    cmd="$(echo -n "${joined%;}")"
+  fi
+  trap "${cmd}" 0 1 2 3 15
 }
 
 disable_trap(){
@@ -340,6 +351,7 @@ wait_cmd(){
   done
 }
 
+# NOTE: This does not detect error exit code.
 fetch_cmd_result(){
   set +e
   local pod=$1
@@ -734,6 +746,11 @@ run_core_init_db_job(){
   done
 }
 
+# Get the file path where it only stores the last result of 'run_cmd_in_pod'.
+get_last_cmd_result_file() {
+  echo "${BACKUP_RESTORE_LOG_DIR}/last_cmd_result.log"
+}
+
 run_cmd_in_pod(){
   local pod="$1"
   shift
@@ -759,7 +776,12 @@ EOF
   _oc_cp ${TMP_WORK_DIR}/${WD_CMD_FILE} ${pod}:/tmp/${WD_CMD_FILE} $@
   oc exec $@ ${pod} -- bash -c "rm -rf /tmp/${WD_CMD_COMPLETION_TOKEN} && /tmp/${WD_CMD_FILE} &"
   wait_cmd ${pod} $@
-  oc exec $@ ${pod} -- bash -c "cat /tmp/${WD_CMD_LOG}; rm -rf /tmp/${WD_CMD_FILE} /tmp/${WD_CMD_LOG} /tmp/${WD_CMD_COMPLETION_TOKEN}" >> "${BACKUP_RESTORE_LOG_DIR}/${CURRENT_COMPONENT}.log"
+  
+  # Create an intermediate log file which only stores the last command result.
+  LAST_CMD_RESULT_FILE=$(get_last_cmd_result_file)
+  oc exec $@ ${pod} -- bash -c "cat /tmp/${WD_CMD_LOG}; rm -rf /tmp/${WD_CMD_FILE} /tmp/${WD_CMD_LOG} /tmp/${WD_CMD_COMPLETION_TOKEN}" > "$LAST_CMD_RESULT_FILE"
+  cat "$LAST_CMD_RESULT_FILE" >> "${BACKUP_RESTORE_LOG_DIR}/${CURRENT_COMPONENT}.log"
+
   files=$(fetch_cmd_result ${pod} "ls /tmp" $@)
   if echo "${files}" | grep "${WD_CMD_FAILED_TOKEN}" > /dev/null ; then
     oc exec $@ ${pod} -- bash -c "rm -f /tmp/${WD_CMD_FAILED_TOKEN}"
@@ -768,6 +790,16 @@ EOF
   fi
 }
 
+# Get the last result (stdout) of 'run_cmd_in_pod'
+get_last_cmd_result_in_pod(){
+  LAST_CMD_RESULT_FILE=$(get_last_cmd_result_file)
+  if [ ! -f "$LAST_CMD_RESULT_FILE" ]; then
+    echo ""
+  else
+    cat "$LAST_CMD_RESULT_FILE"
+  fi
+}
+
 run_script_in_pod(){
   local pod="$1"
   shift
@@ -1026,7 +1058,9 @@ require_mt_mt_migration(){
 
 get_primary_pg_pod(){
   local wd_version=${WD_VERSION:-$(get_version)}
-  if [ $(compare_version "${wd_version}" "4.0.0") -ge 0 ] ; then
+  if [ $(compare_version "${wd_version}" "5.0.0") -ge 0 ] ; then
+    echo "$(oc get pod ${OC_ARGS} -l "icpdsupport/podSelector=discovery-cn-postgres,role=primary" -o jsonpath='{.items[0].metadata.name}')"
+  elif [ $(compare_version "${wd_version}" "4.0.0") -ge 0 ] ; then
     echo "$(oc get pod ${OC_ARGS} -l "postgresql=${TENANT_NAME}-discovery-cn-postgres,role=primary" -o jsonpath='{.items[0].metadata.name}')"
   else
     for POD in $(oc get pods ${OC_ARGS} -o jsonpath='{.items[*].metadata.name}' -l tenant=${TENANT_NAME},component=stolon-keeper) ; do
@@ -1133,6 +1167,52 @@ check_datastore_available(){
   brlog "INFO" "All data store service are available"
 }
 
+# Get elastic indices and return its version as a string. Return the oldest index version. 
+# Assume there are only version 6 or 7 indices in the cluster.
+get_elastic_version() {
+  ELASTIC_POD=$(get_elastic_pod)
+  run_script_in_pod ${ELASTIC_POD} "${SCRIPT_DIR}/reindex_es6_indices.sh" "--dry-run" -c elasticsearch ${OC_ARGS}
+  result=$(get_last_cmd_result_in_pod)
+
+  if echo "$result" | grep -q "Skip ElasticSearch 6 index"; then
+    echo "ES6"
+  elif echo "$result" | grep -q "Skip ElasticSearch 7 index"; then
+    echo "ES7"
+  elif echo "$result" | grep -q "ElasticSearch has no index"; then
+    echo "NoIndex"
+  else
+    echo "Unknown"
+  fi  
+}
+
+validate_elastic_version() {
+  ELASTIC_VERSION="${1:-$(get_elastic_version)}"
+
+  if [[ ${ELASTIC_VERSION} = "ES6" ]]; then
+    brlog "WARN" "Elasticsearch 6 detected."
+    if [[ ${IGNORE_OLD_INDEX} = "true" ]]; then
+      brlog "WARN" "Ignoring old index."
+    elif [[ ${REINDEX_OLD_INDEX} = "true" ]]; then
+      brlog "INFO" "Elasticsearch 6 will be reindexed after quiescing."
+    else
+      brlog "ERROR" "Detected old index created in ElasticSearch 6 that is not supported Watson Discovery 4.8.x. They have to be reindexed if you restore to 4.8.x or higher. Reindex actually won't change your data, but it takes time. Run backup script with '--reindex-old-index' option if you reindex them. You can proceed backup without reindex, but you can't restore the backup data to 4.8.x. If you would like to ignore the error, run backup script with '--ignore-old-index' option."
+      exit 1
+    fi
+  fi 
+}
+
+# Reindex elastic search indices from version 6 to 7.
+reindex_elastic_es6() {
+  brlog "INFO" "Reindexing ..."
+  run_script_in_pod ${ELASTIC_POD} "${SCRIPT_DIR}/reindex_es6_indices.sh" "" -c elasticsearch ${OC_ARGS}
+  brlog "INFO" "Reindex completed. Waiting for ElasticSearch status to be green ..."
+  # Wait a few moments for elastic search to be ready after the reindex.
+  # TODO: implement waiting logic inside of reindex_es6_indices.sh, and remove the sleep below from this function.
+  sleep 60
+  check_elastic_available || { brlog "ERROR" "ElasticSearch is unavailable"; exit 1; }
+}
+
+
 check_etcd_available(){
   setup_etcd_env
   ETCD_POD=$(oc get pods ${OC_ARGS} -o jsonpath="{.items[0].metadata.name}" -l etcd_cluster=${TENANT_NAME}-discovery-etcd)
@@ -1400,15 +1480,17 @@ create_service_instance(){
 
 create_service_account(){
   local service_account="$1"
-  if [ -n "$(oc get sa ${service_account})" ] ; then
+  if [ -n "$(oc get sa "${service_account}")" ] ; then
     brlog "INFO" "Service Account ${service_account} already exists"
   else
     brlog "INFO" "Create Service Account for scripts: ${service_account}"
     local namespace="${NAMESPACE:-$(oc config view --minify --output 'jsonpath={..namespace}')}"
-    oc ${OC_ARGS} create sa ${service_account}
-    oc ${OC_ARGS} policy add-role-to-user admin system:serviceaccount:${namespace}:${service_account}
+    oc ${OC_ARGS} create sa "${service_account}"
+    oc ${OC_ARGS} label sa "${service_account}" "${BACKUP_RESTORE_RBAC_LABEL_KEY}"="${BACKUP_RESTORE_RABC_LABEL_VALUE}"
+    oc ${OC_ARGS} policy add-role-to-user admin "system:serviceaccount:${namespace}:${service_account}"
     oc delete clusterrolebinding "${service_account}-cluster-rb" --ignore-not-found
-    oc ${OC_ARGS} create clusterrolebinding "${service_account}-cluster-rb" --clusterrole cluster-admin --serviceaccount ${namespace}:${service_account} 2>&1
+    oc ${OC_ARGS} create clusterrolebinding "${service_account}-cluster-rb" --clusterrole cluster-admin --serviceaccount "${namespace}:${service_account}" 2>&1
+    oc ${OC_ARGS} label clusterrolebinding "${service_account}-cluster-rb" "${BACKUP_RESTORE_RBAC_LABEL_KEY}"="${BACKUP_RESTORE_RABC_LABEL_VALUE}"
     if [ -n "$(oc ${OC_ARGS} get role discovery-operator-role --ignore-not-found)" ] ; then
       # This is 2.2.1 install. Link operator role to this service account to get permission of discovery resources
       cat <<EOF | oc ${OC_ARGS} apply -f -
@@ -1417,6 +1499,8 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: ${service_account}-rb
+  labels:
+    ${BACKUP_RESTORE_RBAC_LABEL_KEY}: ${BACKUP_RESTORE_RABC_LABEL_VALUE}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
@@ -1448,9 +1532,7 @@ get_oc_token(){
 
 delete_service_account(){
   local service_account="$1"
-  oc ${OC_ARGS} delete rolebinding ${service_account}-rb --ignore-not-found
-  oc ${OC_ARGS} delete sa ${service_account} --ignore-not-found
-  oc ${OC_ARGS} delete clusterrolebinding ${service_account}-cluster-rb --ignore-not-found
+  oc ${OC_ARGS} delete rolebinding,clusterrolebinding,sa -l "${BACKUP_RESTORE_RBAC_LABEL_KEY}"="${BACKUP_RESTORE_RABC_LABEL_VALUE}" --ignore-not-found
   trap_remove "brlog 'INFO' 'You currently oc login as a scripts ServiceAccount. You can rerun scripts with this. Please delete ServiceAccount ${service_account} and clusterrolebinding ${service_account}-cluster-rb when you complete backup or restore'"
   trap_remove "brlog 'INFO' 'brlog 'INFO' 'Please delete rolebinding ${service_account}-rb when you delete ServiceAccount'"
   brlog "INFO" "Deleted scripts service account: ${service_account}"

discovery-data/latest/reindex_es6_indices.sh

@@ -3,6 +3,22 @@
 set -euo pipefail
 
 TMP_WORK_DIR="/tmp/elastic-workdir"
+DRY_RUN=false # If this is true, this script will not reindex any indices. It just print indices versions.
+
+function show_help() {
+    cat <<EOS
+Reindex elastic search indices from version 6 to version 7.
+If version 7 indices are detected, this script does nothing.
+If version 6 indices are detected, this script will perform reindexing to update the indices as version 7.
+
+usage: $0 [options]
+
+options:
+  -h, --help                           Print help info.
+  --dry-run                            [Optional] If specified, this script does not perform reindexing. It will just print detected indices versions to stdout.
+                                       Use this option if you only want to check your elastic search indices versions.
+EOS
+}
 
 function generate_new_settings() {
     local index="$1"
@@ -141,6 +157,7 @@ function clone_index() {
     clone_body_json=clone_body.json
     echo "${clone_body}" > "${clone_body_json}"
 
+    # TODO validate clone result.
     curl -sSfk -u "${ELASTIC_USER}:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" -XPOST "${ELASTIC_ENDPOINT}/${src_index}/_clone/${dst_index}" -d@"${TMP_WORK_DIR}/${clone_body_json}"
     echo ""
 
@@ -178,7 +195,39 @@ function remove_index_if_exists(){
     fi
 }
 
+# Parse argument.
+while [[ $# -gt 0 ]]; do
+    OPT=$1
+    case $OPT in
+    -h | --help)
+        show_help
+        exit 0
+        ;;
+    --dry-run)
+        DRY_RUN=true
+        shift 1
+        ;;
+    -- | -)
+        shift 1
+        param+=("$@")
+        break
+        ;;
+    -*)
+        echo "illegal option: $1"
+        exit 1
+        ;;
+    *)
+        echo "illegal argument: $1"
+        exit 1
+        ;;
+    esac
+done
+
 # Main logic start from here
+if [[ ${DRY_RUN} = "true" ]]; then
+    echo "'--dry-run' specified. Running script in dry running mode. No reindex will be performed."
+fi
+
 trap 'if [ $? -ne 0 ] ; then echo "Error: Please contact support. Do not run this scripts again."; fi' 0 1 2 3 15
 rm -rf "${TMP_WORK_DIR}"
 mkdir -p "${TMP_WORK_DIR}"
@@ -208,6 +257,12 @@ if [ "${index_count}" = "0" ] ; then
     exit 0
 fi
 
+distribution="$(curl -sSkf -u "${ELASTIC_USER}:${ELASTIC_PASSWORD}" "${ELASTIC_ENDPOINT}" | jq -r .version.distribution)"
+if [ "$distribution" = "opensearch" ] ; then
+    echo "Opensearch distribution is being used. There is no need for reindexing."
+    exit 0
+fi
+
 echo "Getting index list"
 indices="$(curl -sSkf -u "${ELASTIC_USER}:${ELASTIC_PASSWORD}" "${ELASTIC_ENDPOINT}/_cat/indices?h=index")"
 
@@ -228,6 +283,8 @@ for index in ${indices} ; do
     version="$(curl -sSkf -u "${ELASTIC_USER}:${ELASTIC_PASSWORD}" "${ELASTIC_ENDPOINT}/${index}/_settings" | jq -r .[].settings.index.version.created)"
     if [[ ${version} = 7* ]]; then
         echo "[${count} / ${total}] Skip ElasticSearch 7 index: ${index}"
+    elif [[ ${version} = 6* && ${DRY_RUN} = "true" ]]; then
+        echo "[${count} / ${total}] Skip ElasticSearch 6 index: ${index}"
     elif [[ ${version} = 6* ]]; then
         echo "[${count} / ${total}] ElasticSearch 6 index found: ${index}"
         echo "----------------------------"

discovery-data/latest/version.txt

@@ -1,2 +1,2 @@
 The Backup and Restore Scripts for the Watson Discovery on CP4D.
-Scripts Version: 4.8.5
+Scripts Version: 5.0.0