feat(terraform): use .terraform cache for remote modules in plan scanning (aquasecurity#9277)

Signed-off-by: nikpivkin <nikita.pivkin@smartforce.io>

Author: nikpivkin

pkg/iac/scanners/terraform/parser/evaluator.go

@@ -31,7 +31,7 @@ type evaluator struct {
 	ctx               *tfcontext.Context
 	blocks            terraform.Blocks
 	inputVars         map[string]cty.Value
-	moduleMetadata    *modulesMetadata
+	moduleMetadata    *ModulesMetadata
 	projectRootPath   string // root of the current scan
 	modulePath        string
 	moduleName        string
@@ -50,7 +50,7 @@ func newEvaluator(
 	moduleName string,
 	blocks terraform.Blocks,
 	inputVars map[string]cty.Value,
-	moduleMetadata *modulesMetadata,
+	moduleMetadata *ModulesMetadata,
 	workspace string,
 	ignores ignore.Rules,
 	logger *log.Logger,

pkg/iac/scanners/terraform/parser/load_module_metadata.go

@@ -6,27 +6,29 @@ import (
 	"path"
 )
 
-const manifestSnapshotFile = ".terraform/modules/modules.json"
-
-type modulesMetadata struct {
-	Modules []struct {
-		Key     string `json:"Key"`
-		Source  string `json:"Source"`
-		Version string `json:"Version"`
-		Dir     string `json:"Dir"`
-	} `json:"Modules"`
+const ManifestSnapshotFile = ".terraform/modules/modules.json"
+
+type ModulesMetadata struct {
+	Modules []ModuleMetadata `json:"Modules"`
+}
+
+type ModuleMetadata struct {
+	Key     string `json:"Key"`
+	Source  string `json:"Source"`
+	Version string `json:"Version"`
+	Dir     string `json:"Dir"`
 }
 
-func loadModuleMetadata(target fs.FS, fullPath string) (*modulesMetadata, string, error) {
-	metadataPath := path.Join(fullPath, manifestSnapshotFile)
+func loadModuleMetadata(target fs.FS, fullPath string) (*ModulesMetadata, string, error) {
+	metadataPath := path.Join(fullPath, ManifestSnapshotFile)
 
 	f, err := target.Open(metadataPath)
 	if err != nil {
 		return nil, metadataPath, err
 	}
 	defer f.Close()
 
-	var metadata modulesMetadata
+	var metadata ModulesMetadata
 	if err := json.NewDecoder(f).Decode(&metadata); err != nil {
 		return nil, metadataPath, err
 	}

pkg/iac/scanners/terraformplan/snapshot/scanner.go

@@ -78,6 +78,7 @@ func (s *Scanner) scan(ctx context.Context, reader io.Reader) (scan.Results, err
 
 	s.inner.AddParserOptions(
 		tfparser.OptionsWithTfVars(snap.inputVariables),
+		tfparser.OptionWithDownloads(false),
 	)
 	return s.inner.ScanFS(ctx, fsys, ".")
 }

pkg/iac/scanners/terraformplan/snapshot/snapshot.go

@@ -15,6 +15,8 @@ import (
 	"github.com/liamg/memoryfs"
 	"github.com/zclconf/go-cty/cty"
 
+	"github.com/aquasecurity/trivy/pkg/iac/scanners/terraform/parser"
+	"github.com/aquasecurity/trivy/pkg/log"
 	iox "github.com/aquasecurity/trivy/pkg/x/io"
 )
 
@@ -27,13 +29,7 @@ const (
 )
 
 type (
-	configSnapshotModuleRecord struct {
-		Key        string `json:"Key"`
-		SourceAddr string `json:"Source,omitempty"`
-		Dir        string `json:"Dir"`
-	}
-
-	configSnapshotModuleManifest []configSnapshotModuleRecord
+	configSnapshotModuleManifest []parser.ModuleMetadata
 )
 
 var errNoTerraformPlan = errors.New("no terraform plan file")
@@ -79,13 +75,11 @@ func parseSnapshot(r io.Reader) (*snapshot, error) {
 		inputVariables: make(map[string]cty.Value),
 	}
 
-	var moduleManifest configSnapshotModuleManifest
-
 	for _, file := range zr.File {
 		switch {
 		case file.Name == configSnapshotManifestFile:
 			var err error
-			moduleManifest, err = readModuleManifest(file)
+			snap.moduleManifest, err = readModuleManifest(file)
 			if err != nil {
 				return nil, err
 			}
@@ -113,12 +107,7 @@ func parseSnapshot(r io.Reader) (*snapshot, error) {
 		}
 	}
 
-	for _, record := range moduleManifest {
-		// skip non-local modules
-		if record.Dir != "." && !strings.HasPrefix(record.SourceAddr, ".") {
-			delete(snap.modules, record.Key)
-			continue
-		}
+	for _, record := range snap.moduleManifest {
 		modSnap := snap.getOrCreateModuleSnapshot(record.Key)
 		modSnap.dir = record.Dir
 	}
@@ -159,6 +148,7 @@ type (
 	}
 
 	snapshot struct {
+		moduleManifest configSnapshotModuleManifest
 		modules        map[string]*snapshotModule
 		inputVariables map[string]cty.Value
 	}
@@ -202,6 +192,10 @@ func (s *snapshot) getOrCreateModuleSnapshot(key string) *snapshotModule {
 func (s *snapshot) toFS() (fs.FS, error) {
 	fsys := memoryfs.New()
 
+	if err := s.writeManifest(fsys); err != nil {
+		log.WithPrefix(log.PrefixMisconfiguration).Error("Failed to write manifest file", log.Err(err))
+	}
+
 	for _, module := range s.modules {
 		if err := fsys.MkdirAll(module.dir, fs.ModePerm); err != nil && !errors.Is(err, os.ErrExist) {
 			return nil, err
@@ -218,3 +212,19 @@ func (s *snapshot) toFS() (fs.FS, error) {
 	}
 	return fsys, nil
 }
+
+func (s *snapshot) writeManifest(fsys *memoryfs.FS) error {
+	if err := fsys.MkdirAll(path.Dir(parser.ManifestSnapshotFile), fs.ModePerm); err != nil {
+		return fmt.Errorf("create manifest directory: %w", err)
+	}
+
+	b, err := json.Marshal(parser.ModulesMetadata{Modules: s.moduleManifest})
+	if err != nil {
+		return fmt.Errorf("marshal manifest snapshot: %w", err)
+	}
+
+	if err := fsys.WriteFile(parser.ManifestSnapshotFile, b, fs.ModePerm); err != nil {
+		return fmt.Errorf("write manifest snapshot: %w", err)
+	}
+	return nil
+}

pkg/iac/scanners/terraformplan/snapshot/snapshot_test.go

@@ -6,7 +6,6 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
-	"sort"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -21,23 +20,46 @@ func TestReadSnapshot(t *testing.T) {
 		expectedFiles []string
 	}{
 		{
-			name:          "just resource",
-			dir:           "just-resource",
-			expectedFiles: []string{"main.tf", "terraform.tf"},
+			name: "just resource",
+			dir:  "just-resource",
+			expectedFiles: []string{
+				"main.tf",
+				"terraform.tf",
+				".terraform/modules/modules.json",
+			},
 		},
 		{
-			name:          "with local module",
-			dir:           "with-local-module",
-			expectedFiles: []string{"main.tf", "modules/ec2/main.tf", "terraform.tf"},
+			name: "with local module",
+			dir:  "with-local-module",
+			expectedFiles: []string{
+				"main.tf",
+				"terraform.tf",
+				"modules/ec2/main.tf",
+				".terraform/modules/modules.json",
+			},
 		},
 		{
 			name: "with nested modules",
 			dir:  "nested-modules",
 			expectedFiles: []string{
 				"main.tf",
+				"terraform.tf",
 				"modules/s3/main.tf",
 				"modules/s3/modules/logging/main.tf",
+				".terraform/modules/modules.json",
+			},
+		},
+		{
+			name: "with remote module",
+			dir:  "with-remote-module",
+			expectedFiles: []string{
+				"main.tf",
 				"terraform.tf",
+				".terraform/modules/modules.json",
+				".terraform/modules/s3_bucket/main.tf",
+				".terraform/modules/s3_bucket/outputs.tf",
+				".terraform/modules/s3_bucket/variables.tf",
+				".terraform/modules/s3_bucket/versions.tf",
 			},
 		},
 	}
@@ -58,7 +80,7 @@ func TestReadSnapshot(t *testing.T) {
 			files, err := getAllfiles(fsys)
 			require.NoError(t, err)
 
-			assert.Equal(t, tt.expectedFiles, files)
+			assert.ElementsMatch(t, tt.expectedFiles, files)
 		})
 	}
 }
@@ -80,8 +102,6 @@ func getAllfiles(fsys fs.FS) ([]string, error) {
 	if err := fs.WalkDir(fsys, ".", walkFn); err != nil {
 		return nil, err
 	}
-
-	sort.Strings(files)
 	return files, nil
 }