Merge pull request #661 from adborden/bugfix/mod-include

Fix handling of include patterns in kpm pkg

Author: Peefy

pkg/utils/utils.go

@@ -48,7 +48,7 @@ func HashDir(dir string) (string, error) {
 
 		// files in the ".git "directory will cause the same repository, cloned at different times,
 		// has different checksum.
-		for _, ignore := range ignores {
+		for _, ignore := range defaultIgnores {
 			if strings.Contains(path, ignore) {
 				return nil
 			}
@@ -120,7 +120,7 @@ func Exists(path string) (bool, error) {
 }
 
 // todo: Consider using the OCI tarball as the standard tar format.
-var ignores = []string{".git", ".tar"}
+var defaultIgnores = []string{".git", ".tar"}
 
 func TarDir(srcDir string, tarPath string, include []string, exclude []string) error {
 	fw, err := os.Create(tarPath)
@@ -132,11 +132,19 @@ func TarDir(srcDir string, tarPath string, include []string, exclude []string) e
 	tw := tar.NewWriter(fw)
 	defer tw.Close()
 
+	// In case the tarPath is within the current working directory, exclude it from the tar itself.
+	ignores := append(defaultIgnores, filepath.Join(srcDir, filepath.Dir(tarPath)))
+
 	err = filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
 
+		// Ignore the current directory root "."
+		if path == srcDir {
+			return nil
+		}
+
 		for _, ignore := range ignores {
 			if strings.Contains(path, ignore) {
 				return nil
@@ -145,7 +153,8 @@ func TarDir(srcDir string, tarPath string, include []string, exclude []string) e
 
 		getNewPattern := func(ex string) string {
 			newPath := ex
-			if !strings.HasPrefix(ex, srcDir+string(filepath.Separator)) {
+			prefix := srcDir + string(filepath.Separator)
+			if !strings.HasPrefix(ex, prefix) {
 				newPath = filepath.Join(srcDir, ex)
 			}
 			return newPath
@@ -157,12 +166,20 @@ func TarDir(srcDir string, tarPath string, include []string, exclude []string) e
 			}
 		}
 
+		// If the include list is empty, all files are included by default.
+		matchedInclude := len(include) == 0
 		for _, inc := range include {
-			if matched, _ := filepath.Match(getNewPattern(inc), path); !matched {
-				return nil
+			if matched, _ := filepath.Match(getNewPattern(inc), path); matched {
+				matchedInclude = true
+				break
 			}
 		}
 
+		// As long as _one_ of the include patterns matches, the file will be included.
+		if !matchedInclude {
+			return nil
+		}
+
 		relPath, _ := filepath.Rel(srcDir, path)
 		relPath = filepath.ToSlash(relPath)
 

pkg/utils/utils_test.go

@@ -2,6 +2,7 @@ package utils
 
 import (
 	"archive/tar"
+	"fmt"
 	"io"
 	"os"
 	"path"
@@ -126,19 +127,22 @@ func TestTarDir(t *testing.T) {
 
 	_ = TarDir(testSrcDir, tarPath, []string{}, []string{"*.mod"})
 	fileNames, _ := getTarFileNames(tarPath)
+	assert.Greater(t, len(fileNames), 0, "expected tar to have greater than 0 files")
 	for _, fileName := range fileNames {
 		flag, _ := filepath.Match(getNewPattern("*.mod"), fileName)
-		assert.Equal(t, flag, false)
+		assert.Equal(t, flag, false, "expected *.mod to be excluded")
 	}
 	_, err = os.Stat(tarPath)
 	assert.Equal(t, err, nil)
 	os.Remove(tarPath)
 
 	_ = TarDir(testSrcDir, tarPath, []string{"*/*.lock", "*.mod"}, []string{})
 	fileNames, _ = getTarFileNames(tarPath)
+	assert.Greater(t, len(fileNames), 0, "expected tar to have greater than 0 files")
 	for _, fileName := range fileNames {
-		flag, _ := filepath.Match(getNewPattern("*/*.lock"), fileName)
-		assert.Equal(t, flag, true)
+		matchedLockPattern, _ := filepath.Match(getNewPattern("*/*.lock"), fileName)
+		matchedModPattern, _ := filepath.Match(getNewPattern("*.mod"), fileName)
+		assert.True(t, matchedLockPattern || matchedModPattern, fmt.Sprintf("expected \"%s\" to match one of */*.lock or *.mod", fileName))
 	}
 	_, err = os.Stat(tarPath)
 	assert.Equal(t, err, nil)

test/e2e/kpm_test.go

@@ -1,11 +1,14 @@
 package e2e
 
 import (
+	"archive/tar"
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	"github.com/onsi/ginkgo/v2"
@@ -22,6 +25,59 @@ import (
 )
 
 var _ = ginkgo.Describe("Kpm CLI Testing", func() {
+	ginkgo.Context("testing 'kpm pkg'", func() {
+		testSuitesRoot := filepath.Join(filepath.Join(filepath.Join(GetWorkDir(), TEST_SUITES_DIR), "kpm"), "kpm_pkg")
+		testSuites := LoadAllTestSuites(testSuitesRoot)
+		testDataRoot := filepath.Join(filepath.Join(GetWorkDir(), TEST_SUITES_DIR), "test_data")
+		for _, ts := range testSuites {
+			ts := ts
+			ginkgo.It(ts.GetTestSuiteInfo(), func() {
+				workspace := GetWorkspace()
+
+				// Copy the test data to the workspace
+				CopyDir(filepath.Join(testDataRoot, ts.Name), filepath.Join(workspace, ts.Name))
+
+				// Run the command from test_suite.input
+				stdout, stderr, _ := ExecKpmWithWorkDir(ts.Input, filepath.Join(workspace, ts.Name))
+
+				// kpm pkg always produces no output
+				gomega.Expect(stdout).To(gomega.BeEmpty(), "unexpected stdout from kpm pkg")
+				gomega.Expect(stderr).To(gomega.BeEmpty(), "unexpected stderr from kpm pkg")
+				//time.Sleep(300 * time.Second)
+
+				// Assert the contents of the tarball without extracting
+				expectedTarContentFilepath := filepath.Join(testDataRoot, ts.Name, "expected_tar_content.txt")
+				exist, _ := utils.Exists(expectedTarContentFilepath)
+				gomega.Expect(exist).To(gomega.BeTrue(), "expected_tar_content.txt not found in test data")
+        expectedTarContentBytes, err := os.ReadFile(expectedTarContentFilepath)
+        gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "failed to read from expected_tar_content.txt")
+        expectedFiles := strings.Split(strings.TrimSpace(string(expectedTarContentBytes)), "\n")
+				sort.Strings(expectedFiles)
+
+				tarballPath := filepath.Join(workspace, ts.Name, "dist", fmt.Sprintf("%s_0.0.1.tar", ts.Name))
+				fmt.Printf("tarballPath: %s\n", tarballPath)
+				exist, _ = utils.Exists(tarballPath)
+				gomega.Expect(exist).To(gomega.BeTrue(), "pkg tarball not found, check kcl.mod name should match test suite")
+				f, err := os.Open(tarballPath)
+				gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "failed to open pkg tarball")
+				defer f.Close()
+
+				tr := tar.NewReader(f)
+				var actualFiles []string
+				for {
+					hdr, err := tr.Next()
+					if err == io.EOF {
+						break
+					}
+					gomega.Expect(err).ShouldNot(gomega.HaveOccurred(), "error reading pkg tarball")
+					actualFiles = append(actualFiles, hdr.Name)
+				}
+				sort.Strings(actualFiles)
+				gomega.Expect(actualFiles).To(gomega.Equal(expectedFiles), "unexpected pkg tarball content")
+			})
+		}
+	})
+
 	ginkgo.Context("testing 'exec kpm outside a kcl package'", func() {
 		testSuites := LoadAllTestSuites(filepath.Join(filepath.Join(filepath.Join(GetWorkDir(), TEST_SUITES_DIR), "kpm"), "exec_outside_pkg"))
 		testDataRoot := filepath.Join(filepath.Join(GetWorkDir(), TEST_SUITES_DIR), "test_data")

test/e2e/test_suite.go

@@ -63,11 +63,13 @@ func LoadTestSuite(testSuitePath, name string) TestSuite {
 	reporter.Report("kpm_e2e: loading '", name, "' from ", testSuitePath)
 	CheckTestSuite(testSuitePath, name)
 	return TestSuite{
-		Name:         name,
+		Name:             name,
 		ExpectStdout: LoadFirstFileWithExt(testSuitePath, STDOUT_EXT),
 		ExpectStderr: LoadFirstFileWithExt(testSuitePath, STDERR_EXT),
-		Input:        LoadFirstFileWithExt(testSuitePath, INPUT_EXT),
-		Envs:         LoadFirstFileWithExt(testSuitePath, ENV_EXT),
+
+		// Strip whitespace to ignore the leading and trailing new lines.
+		Input: strings.TrimSpace(LoadFirstFileWithExt(testSuitePath, INPUT_EXT)),
+		Envs:  LoadFirstFileWithExt(testSuitePath, ENV_EXT),
 	}
 }
 

test/e2e/test_suites/kpm/kpm_pkg/pkg_default/test_suite.env

@@ -0,0 +1,2 @@
+KPM_HOME=""
+KCLVM_VENDOR_HOME=""

test/e2e/test_suites/kpm/kpm_pkg/pkg_default/test_suite.input

@@ -0,0 +1 @@
+kpm pkg --target=dist

test/e2e/test_suites/kpm/kpm_pkg/pkg_default/test_suite.stderr

@@ -0,0 +1,2 @@
+KPM_HOME=""
+KCLVM_VENDOR_HOME=""

test/e2e/test_suites/kpm/kpm_pkg/pkg_default/test_suite.stdout

@@ -0,0 +1 @@
+kpm pkg --target=dist