Merge pull request #9 from DataDog/enhance-imds-testing

Enhance IMDS access detection to support cases where IMDSv2 is enforced (closes #8)

Author: christophetd

README.md

@@ -104,7 +104,9 @@ $ mkat eks find-secrets
 ### Test if pods can access the AWS Instance Metadata Service (IMDS)
 
 Pods accessing the EKS nodes Instance Metadata Service is a [common and dangerous attack vector](https://blog.christophetd.fr/privilege-escalation-in-aws-elastic-kubernetes-service-eks-by-compromising-the-instance-role-of-worker-nodes/) 
-that can be used to escalate privileges. MKAT can test if pods can access the IMDS. It tests it by creating a temporary pod that tries to access the IMDS, and then deletes it.
+that can be used to escalate privileges. MKAT can test if pods can access the IMDS, both through IMDSv1 and IMDSv2. 
+
+It tests this by creating two temporary pods (one for IMDSv1, one for IMDSv2) that try to access the IMDS, and are then deleted.
 
 ```bash
 $ mkat eks test-imds-access
@@ -114,9 +116,10 @@ $ mkat eks test-imds-access
  | | | | | | |   <  | (_| | | |_
  |_| |_| |_| |_|\_\  \__,_|  \__|
 
-2023/04/12 00:35:10 Connected to EKS cluster mkat-cluster
-2023/04/12 00:35:10 Testing if IMDS is accessible to pods by creating a pod that attempts to access it
-2023/04/12 00:35:15 IMDS is accessible and allows any pod to retrieve credentials for the AWS role eksctl-mkat-cluster-nodegroup-ng-NodeInstanceRole-AXWUFF35602Z
+2023/07/11 21:56:19 Connected to EKS cluster mkat-cluster
+2023/07/11 21:56:19 Testing if IMDSv1 and IMDSv2 are accessible from pods by creating a pod that attempts to access it
+2023/07/11 21:56:23 IMDSv2 is accessible: any pod can retrieve credentials for the AWS role eksctl-mkat-cluster-nodegroup-ng-NodeInstanceRole-AXWUFF35602Z
+2023/07/11 21:56:23 IMDSv1 is not accessible to pods in your cluster: able to establish a network connection to the IMDS, but no credentials were returned
 ```
 
 ## FAQ 

cmd/managed-kubernetes-auditing-toolkit/eks/imds.go

@@ -2,39 +2,82 @@ package eks
 
 import (
 	"log"
+	"sync"
 
 	"github.com/datadog/managed-kubernetes-auditing-toolkit/internal/utils"
 	"github.com/datadog/managed-kubernetes-auditing-toolkit/pkg/managed-kubernetes-auditing-toolkit/eks/imds"
 	"github.com/fatih/color"
 	"github.com/spf13/cobra"
 )
 
+var successColor = color.New(color.BgBlack, color.FgGreen, color.Bold)
+var warningColor = color.New(color.BgRed, color.FgWhite, color.Bold)
+
 func buildTestImdsAccessCommand() *cobra.Command {
 	eksFindSecretsCommand := &cobra.Command{
 		Use:                   "test-imds-access",
 		Example:               "mkat eks test-imds-access",
 		Short:                 "Test if your EKS cluster allows pod access to the IMDS",
 		Long:                  "test-imds-access will check if your EKS cluster allows pods to access the IMDS by running a pod and executing a curl command hitting the IMDS",
 		DisableFlagsInUseLine: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			return doTestImdsAccessCommand()
+		Run: func(cmd *cobra.Command, args []string) {
+			doTestImdsAccessCommand()
 		},
 	}
 
 	return eksFindSecretsCommand
 }
 
-func doTestImdsAccessCommand() error {
+func doTestImdsAccessCommand() {
 	tester := imds.ImdsTester{K8sClient: utils.K8sClient(), Namespace: "default"}
-	result, err := tester.TestImdsAccessible()
+	log.Println("Testing if IMDSv1 and IMDSv2 are accessible from pods by creating a pod that attempts to access it")
+
+	// We run the test for IMDSv1 and IMDSv2 in parallel
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go doTestImdsAccess(IMDSv1, &tester, &wg)
+	go doTestImdsAccess(IMDSv2, &tester, &wg)
+	wg.Wait()
+}
+
+type ImdsVersion string
+
+const (
+	IMDSv1 ImdsVersion = "IMDSv1"
+	IMDSv2 ImdsVersion = "IMDSv2"
+)
+
+func doTestImdsAccess(imdsVersion ImdsVersion, tester *imds.ImdsTester, wg *sync.WaitGroup) {
+	var result *imds.ImdsTestResult
+	var err error
+
+	defer wg.Done()
+
+	switch imdsVersion {
+	case IMDSv1:
+		result, err = tester.TestImdsV1Accessible()
+	case IMDSv2:
+		result, err = tester.TestImdsV2Accessible()
+	default:
+		panic("invalid IMDS version")
+	}
+
 	if err != nil {
-		return err
+		log.Printf("Unable to determine if %s is accessible in your cluster: %s\n", imdsVersion, err.Error())
+		return
 	}
+
 	if result.IsImdsAccessible {
-		warningColor := color.New(color.BgRed, color.FgWhite, color.Bold)
-		log.Println(warningColor.Sprint("IMDS is accessible") + " and allows any pod to retrieve credentials for the AWS role " + result.NodeRoleName)
+		log.Printf("%s: %s\n", warningColor.Sprintf("%s is accessible", imdsVersion), result.ResultDescription)
 	} else {
-		log.Println("IMDS is not accessible in your cluster")
+		description := ""
+		if result.ResultDescription != "" {
+			description = ": " + result.ResultDescription
+		}
+		log.Printf("%s %s%s\n",
+			successColor.Sprintf("%s is not accessible", imdsVersion),
+			"to pods in your cluster",
+			description,
+		)
 	}
-	return nil
 }

pkg/managed-kubernetes-auditing-toolkit/eks/imds/imds_tester.go

@@ -10,7 +10,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	typedv1 "k8s.io/client-go/kubernetes/typed/core/v1"
-	"log"
 	"os"
 	"os/signal"
 	"strings"
@@ -24,51 +23,110 @@ type ImdsTester struct {
 }
 
 type ImdsTestResult struct {
-	IsImdsAccessible bool
-	NodeRoleName     string
+	IsImdsAccessible  bool
+	ResultDescription string
 }
 
-const ImdsTesterPodName = "mkat-imds-tester"
+const ImdsTesterV1PodName = "mkat-imds-tester"
+const ImdsTesterV2PodName = "mkat-imds-v2-tester"
 
-func (m *ImdsTester) TestImdsAccessible() (*ImdsTestResult, error) {
+func (m *ImdsTester) TestImdsV1Accessible() (*ImdsTestResult, error) {
+	commandToRun := []string{
+		"sh",
+		"-c",
+		"(curl --silent --show-error --connect-timeout 2 169.254.169.254/latest/meta-data/iam/security-credentials/ || true)",
+	}
+	podLogs, err := m.runCommandInPodAndGetLogs(ImdsTesterV1PodName, commandToRun)
+	if err != nil {
+		return nil, fmt.Errorf("unable to retrieve logs from IMDS tester pod: %v", err)
+	}
+
+	// Case 1: no network connection (e.g. NetworkPolicy in place)
+	if strings.Contains(podLogs, "Failed to connect") {
+		return &ImdsTestResult{
+			IsImdsAccessible:  false,
+			ResultDescription: "unable to establish a network connection to the IMDS",
+		}, nil
+	}
+
+	// Case 2: IMDSv2 enforced, IMDSv1 is accessible at the network level but returns a 401 error
+	if strings.TrimSpace(podLogs) == "" {
+		return &ImdsTestResult{
+			IsImdsAccessible:  false,
+			ResultDescription: "able to establish a network connection to the IMDS, but no credentials were returned",
+		}, nil
+	}
+
+	// Case 3: IMDSv1 is accessible and returns credentials
+	return &ImdsTestResult{
+		IsImdsAccessible:  true,
+		ResultDescription: fmt.Sprintf("any pod can retrieve credentials for the AWS role %s", podLogs),
+	}, nil
+}
+
+func (m *ImdsTester) TestImdsV2Accessible() (*ImdsTestResult, error) {
+	commandToRun := []string{
+		"sh",
+		"-c",
+		// We use "--max-time" because when the IMDS max-response-hop is set to 1, the TCP connection succeeds initially but hangs indefinitely when calling /latest/api/token
+		`TOKEN=$(curl --show-error --max-time 2 --silent -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
+		(curl --silent --show-error --max-time 2 -H "X-aws-ec2-metadata-token: $TOKEN" 169.254.169.254/latest/meta-data/iam/security-credentials/ || true)`,
+	}
+	podLogs, err := m.runCommandInPodAndGetLogs(ImdsTesterV2PodName, commandToRun)
+	if err != nil {
+		return nil, fmt.Errorf("unable to retrieve logs from IMDS tester pod: %v", err)
+	}
+
+	// Case 1: no network connection (e.g. NetworkPolicy in place)
+	if strings.Contains(podLogs, "Failed to connect") || strings.Contains(podLogs, "timed out") {
+		return &ImdsTestResult{
+			IsImdsAccessible:  false,
+			ResultDescription: "unable to establish a network connection to the IMDS",
+		}, nil
+	}
+
+	// Case 3: IMDSv2 is accessible and returns credentials
+	return &ImdsTestResult{
+		IsImdsAccessible:  true,
+		ResultDescription: fmt.Sprintf("any pod can retrieve credentials for the AWS role %s", podLogs),
+	}, nil
+}
+
+func (m *ImdsTester) runCommandInPodAndGetLogs(podName string, command []string) (string, error) {
+	podsClient := m.K8sClient.CoreV1().Pods(m.Namespace)
 	podDefinition := &v1.Pod{
-		ObjectMeta: metav1.ObjectMeta{Name: ImdsTesterPodName, Namespace: m.Namespace},
+		ObjectMeta: metav1.ObjectMeta{Name: podName, Namespace: m.Namespace},
 		Spec: v1.PodSpec{
 			Containers: []v1.Container{{
-				Name:    ImdsTesterPodName,
+				Name:    podName,
 				Image:   "curlimages/curl:8.00.1",
-				Command: []string{"sh", "-c", "(curl --silent --show-error --connect-timeout 2 169.254.169.254/latest/meta-data/iam/security-credentials/ || true)"},
+				Command: command,
 			}},
 			RestartPolicy: v1.RestartPolicyNever, // don't restart the pod once the command has been executed
 		},
 	}
-	podsClient := m.K8sClient.CoreV1().Pods(m.Namespace)
-
-	log.Println("Testing if IMDS is accessible to pods by creating a pod that attempts to access it")
 	_, err := podsClient.Create(context.Background(), podDefinition, metav1.CreateOptions{})
 	if err != nil {
-		return nil, fmt.Errorf("unable to create IMDS tester pod: %v", err)
+		return "", fmt.Errorf("unable to create IMDS tester pod: %v", err)
 	}
 	m.handleCtrlC()
-	defer removePod(podsClient, ImdsTesterPodName)
+	defer removePod(podsClient, podName)
 
 	err = wait.PollImmediate(1*time.Second, 120*time.Second, func() (bool, error) {
-		return podHasSuccessfullyCompleted(podsClient, ImdsTesterPodName)
+		return podHasSuccessfullyCompleted(podsClient, podName)
 	})
 
 	if err != nil {
-		return nil, fmt.Errorf("unable to wait for IMDS tester pod to complete: %v", err)
+		return "", fmt.Errorf("unable to wait for IMDS tester pod to complete: %v", err)
 	}
 
 	// Retrieve command output
-	podLogs, err := getPodLogs(podsClient, ImdsTesterPodName)
+	podLogs, err := getPodLogs(podsClient, podName)
 	if err != nil {
-		return nil, fmt.Errorf("unable to retrieve logs from IMDS tester pod: %v", err)
+		return "", fmt.Errorf("unable to retrieve logs from IMDS tester pod: %v", err)
 	}
-	if strings.Contains(podLogs, "Failed to connect") {
-		return &ImdsTestResult{IsImdsAccessible: false}, nil
-	}
-	return &ImdsTestResult{IsImdsAccessible: true, NodeRoleName: podLogs}, nil
+
+	return podLogs, nil
 }
 
 func (m *ImdsTester) handleCtrlC() {
@@ -77,8 +135,10 @@ func (m *ImdsTester) handleCtrlC() {
 	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
 	go func() {
 		<-c
-		println("Received SIGINT, cleaning up IMDS tester pod")
-		removePod(m.K8sClient.CoreV1().Pods(m.Namespace), ImdsTesterPodName)
+		println("Received SIGINT, cleaning up IMDS tester pods")
+		podsClient := m.K8sClient.CoreV1().Pods(m.Namespace)
+		removePod(podsClient, ImdsTesterV1PodName)
+		removePod(podsClient, ImdsTesterV2PodName)
 		os.Exit(1)
 	}()
 }
@@ -99,7 +159,7 @@ func podHasSuccessfullyCompleted(podsClient typedv1.PodInterface, podName string
 }
 
 func getPodLogs(podsClient typedv1.PodInterface, podName string) (string, error) {
-	podLogsRequest := podsClient.GetLogs(ImdsTesterPodName, &v1.PodLogOptions{})
+	podLogsRequest := podsClient.GetLogs(podName, &v1.PodLogOptions{})
 	podLogs, err := podLogsRequest.Stream(context.Background())
 	if err != nil {
 		return "", fmt.Errorf("unable to get logs for pod %s: %v", podName, err)