feat: ability to download multiple artifacts

Modify `download_ci_artifacts` to accept artifacts paths
as input and download by passing these paths to
`ensure_ci_artifacts`. `ensure_ci_artifacts` will default
to latest s3 artifacts if no args were provided.

Signed-off-by: Egor Lazarchuk <yegorlz@amazon.co.uk>

Author: ShadowCurse

tests/conftest.py

@@ -32,7 +32,7 @@
 import host_tools.cargo_build as build_tools
 from framework import defs, utils
 from framework.artifacts import disks, kernel_params
-from framework.defs import DEFAULT_BINARY_DIR, ARTIFACT_DIR
+from framework.defs import ARTIFACT_DIR, DEFAULT_BINARY_DIR
 from framework.microvm import HugePagesConfig, MicroVMFactory, SnapshotType
 from framework.properties import global_props
 from framework.utils_cpu_templates import (

tools/devtool

@@ -440,8 +440,13 @@ cmd_help() {
     echo "        Builds the rootfs and guest kernel artifacts we use for our CI."
     echo "        Run './tools/devtool build_ci_artifacts help' for more details about the available commands."
     echo ""
-    echo "    download_ci_artifacts"
-    echo "        Downloads the CI artifacts used for testing from our S3 bucket."
+    echo "    download_ci_artifacts [s3_path_1, s3_path_2 ...]"
+    echo "        Downloads artifacts from provided S3 paths and runs ./tools/setup-ci-artifacts.sh. for each of them."
+    echo "        If no arguments are provided, pulls newest artifacts from $DEFAULT_ARTIFACTS_S3_BUCKET"
+    echo "    set_current_artifacts s3_path_1"
+    echo "        Sets the $LOCAL_ARTIFACTS_CURRENT_DIR_FILE to contain a local path where the artifacts should be."
+    echo "        It accepts argument in the same format as `download_ci_artifacts`."
+    echo "        This command does not download artifacts."
     echo ""
 
     cat <<EOF
@@ -590,10 +595,25 @@ cmd_distclean() {
 }
 
 cmd_download_ci_artifacts() {
-    if [ "$1" = "--force" ]; then
-        rm -rf $LOCAL_ARTIFACTS_DIR
+    local artifacts=$1
+
+    if [ ${#artifacts[@]} -eq 0 ]; then
+      download_ci_artifacts
     fi
-    download_ci_artifacts
+
+    for artifact in "${artifacts[@]}"; do
+      download_ci_artifacts $artifact
+    done
+}
+
+cmd_set_current_artifacts() {
+    local artifacts=$1
+    if [ -z $artifacts ]; then
+      say "No artifacts were specified"
+    fi
+    local local_artifacts_path=$(get_local_artifacts_path $artifacts)/$(uname -m)
+    echo $local_artifacts_path > $LOCAL_ARTIFACTS_CURRENT_DIR_FILE
+    say "Current artifacts setup at " $local_artifacts_path
 }
 
 ensure_current_artifacts_are_set_up() {
@@ -607,23 +627,26 @@ ensure_current_artifacts_are_set_up() {
 }
 
 download_ci_artifacts() {
-    if ! command -v aws >/dev/null; then
-      die "AWS CLI not installed, which is required for downloading artifacts for integration tests."
+    local artifacts=$1
+
+    if [ -z $artifacts ]; then
+        local default_artifacts=$(get_newest_s3_artifacts)
+        say "No specific artifacts are defined. Using default artifacts: " $default_artifacts
+        artifacts=$default_artifacts
     fi
 
     # Fetch all the artifacts so they are local
-    local artifacts_s3_url=$(get_newest_s3_artifacts)
-    local artifacts_s3_url_arch=$artifacts_s3_url/$(uname -m)
-    local local_artifacts_path=$(get_local_artifacts_path $artifacts_s3_url)/$(uname -m)
+    local artifacts_arch=$artifacts/$(uname -m)
+    local local_artifacts_path=$(get_local_artifacts_path $artifacts)/$(uname -m)
 
     if [ ! -d "$local_artifacts_path" ]; then
-        say "Fetching artifacts from S3: " $artifacts_s3_url_arch " into: " $local_artifacts_path
+        say "Fetching artifacts from S3: " $artifacts_arch " into: " $local_artifacts_path
         mkdir -pv $local_artifacts_path
-        aws s3 sync --no-sign-request "$artifacts_s3_url_arch" "$local_artifacts_path"
+        aws s3 sync --no-sign-request "$artifacts_arch" "$local_artifacts_path"
         ok_or_die "Failed to download artifacts using awscli!"
         cmd_sh "./tools/setup-ci-artifacts.sh" $local_artifacts_path
     else
-        say "Found existing artifacts " $artifacts_s3_url_arch " at: " $local_artifacts_path
+        say "Found existing artifacts " $artifacts_arch " at: " $local_artifacts_path
     fi
 
     LOCAL_ARTIFACTS_PATH=$local_artifacts_path