chore(ci): login with docker (#1218)

Ultimate goal: login with docker to get higher pull limits

What it involved:
- Rework how the `buildkite-agent` vm is built
  - sourcegraph/infrastructure#6848
  - sourcegraph/infrastructure#6849
  - sourcegraph/infrastructure#6850
  - sourcegraph/infrastructure#6851
  - sourcegraph/infrastructure#6852
  - sourcegraph/infrastructure#6854
  - sourcegraph/infrastructure#6855
  - sourcegraph/infrastructure#6856
- Transition to mise
- Use a specific version of vagrant (2.4.1) otherwise the an older
incompatible version of `vagrant-google` (2.2.0) gets installed, 2.7.0
is the latest and correct one
- Fix vagrant issues

### Test plan
CI

Author: burmudar

.buildkite/ci-checkov.sh

@@ -1,11 +1,7 @@
 #!/usr/bin/env bash
-# Set this to fail on the install 
+# Set this to fail on the install
 set -euxo pipefail
 
-# Install and run the plugin for checkov
-# Use the full path to run pip3.10
-pip3 install checkov
-
 # List of checks we do not want to run here
 # This is a living list and will see additions and mostly removals over time.
 SKIP_CHECKS="CKV_GCP_22,CKV_GCP_66,CKV_GCP_13,CKV_GCP_71,CKV_GCP_61,CKV_GCP_21,CKV_GCP_65,CKV_GCP_67,CKV_GCP_20,CKV_GCP_69,CKV_GCP_12,CKV_GCP_24,CKV_GCP_25,CKV_GCP_64,CKV_GCP_68,CKV2_AWS_5,CKV2_GCP_3,CKV2_GCP_5,CKV_AWS_23,CKV_GCP_70,CKV_GCP_62,CKV_GCP_62,CKV_GCP_62,CKV_GCP_62,CKV_GCP_29,CKV_GCP_39"
@@ -19,7 +15,7 @@ echo "==========================================================================
 # Set not to fail on non-zero exit code
 set +e
 # Run checkov
-python3 -m checkov.main --skip-check $SKIP_CHECKS --quiet --framework terraform --compact -d .
+checkov --skip-check $SKIP_CHECKS --quiet --framework terraform --compact -d .
 
 # Options
 # --quiet: Only show failing tests

.buildkite/hooks/pre-command

@@ -27,4 +27,9 @@ trap remove_pidfile EXIT
 echo $$ > "$PIDFILE"
 
 echo "Installing asdf dependencies as defined in '${WORKDIR}/.tool-versions':"
-asdf install
+if [ ! -f ".use_mise" ]; then
+  asdf install
+else
+  mise install
+  eval "$(mise activate)"
+fi

.buildkite/vagrant-run.sh

@@ -13,14 +13,20 @@ cleanup() {
 }
 
 echo --- ":vagrant: installing plugins"
-plugins=(vagrant-google vagrant-env vagrant-scp)
-for i in "${plugins[@]}"; do
-	if ! vagrant plugin list --no-tty | grep "$i"; then
-		vagrant plugin install "$i"
-	fi
-done
+vagrant --version
+vagrant plugin install vagrant-google --plugin-version '2.7.0'
+vagrant plugin install vagrant-env
+vagrant plugin install vagrant-scp
 
 trap cleanup EXIT
+
+echo --- ":lock: builder account key"
+KEY_PATH="/tmp/e2e-builder.json"
+if [ ! -f ${KEY_PATH} ]; then
+  gcloud secrets versions access latest --secret=e2e-builder-sa-key --quiet --project=sourcegraph-ci > "${KEY_PATH}"
+fi
+export GOOGLE_JSON_KEY_LOCATION="${KEY_PATH}"
+
 echo --- ":vagrant: starting box $box"
 vagrant up "$box" --provider=google || exit_code=$?
 

.tool-versions

@@ -1,6 +1,6 @@
-nodejs 16.7.0
-yarn 1.22.4
-shellcheck 0.7.1
-golang 1.19.8
-github-cli 2.46.0
-python system
+nodejs                   16.7.0
+yarn                     1.22.4
+shellcheck               0.7.1
+golang                   1.19.8
+github-cli               2.46.0
+asdf:bosmak/asdf-checkov latest

.use_mise

@@ -0,0 +1,6 @@
+Buildkite Agent CI use the presence of this file to determine whether it should install tools with mise or install them with ASDF.
+Thus if you delete this file, CI will use ASDF to install tools and not mise.
+
+The file is only meant to be here while we transition to using mise completely.
+
+For more information you can reach out to the dev-infra team on #discuss-dev-infra.

test/Vagrantfile

@@ -62,6 +62,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
         cat << EOF >> /root/.profile
 export GIT_BRANCH=#{ENV['BUILDKITE_BRANCH']}
 export TEST_TYPE=#{ENV['TEST_TYPE']}
+export DOCKER_USERNAME=#{ENV['DOCKER_USERNAME']}
+export DOCKER_PASSWORD=#{ENV['DOCKER_PASSWORD']}
 EOF
         SHELL
 

test/smoke-test.sh

@@ -2,6 +2,9 @@
 set -euxfo pipefail
 
 configure_docker() {
+  if [ -n "${DOCKER_USERNAME}" ] && [ -n "${DOCKER_PASSWORD}" ]; then
+    docker login -u "${DOCKER_USERNAME}" --password-stdin <<<"$DOCKER_PASSWORD"
+  fi
   gcloud auth configure-docker
   gcloud auth configure-docker us-central1-docker.pkg.dev
 }