From 3cca6ed148666f84aa07e2a2cdbdcf9d6f43d375 Mon Sep 17 00:00:00 2001 From: Rajeev Date: Thu, 14 Nov 2024 19:12:31 +0530 Subject: [PATCH 1/9] copied the code from helm --- kubelink/internals/resolver/resolver.go | 262 +++++ kubelink/internals/resolver/resolver_test.go | 310 ++++++ .../testdata/chartpath/base/Chart.yaml | 3 + .../charts/localdependency/Chart.yaml | 3 + .../repository/kubernetes-charts-index.yaml | 49 + kubelink/internals/thirdParty/dep/fs/fs.go | 372 +++++++ .../internals/thirdParty/dep/fs/fs_test.go | 642 +++++++++++++ .../internals/thirdParty/dep/fs/rename.go | 58 ++ .../thirdParty/dep/fs/rename_windows.go | 69 ++ .../dep/fs/testdata/symlinks/file-symlink | 1 + .../dep/fs/testdata/symlinks/invalid-symlink | 1 + .../fs/testdata/symlinks/windows-file-symlink | 1 + .../thirdParty/dep/fs/testdata/test.file | 0 kubelink/internals/urlutil/urlutil.go | 73 ++ kubelink/internals/urlutil/urlutil_test.go | 81 ++ kubelink/pkg/downloader/chart_downloader.go | 408 ++++++++ .../pkg/downloader/chart_downloader_test.go | 346 +++++++ kubelink/pkg/downloader/doc.go | 24 + kubelink/pkg/downloader/manager.go | 905 ++++++++++++++++++ kubelink/pkg/downloader/manager_test.go | 600 ++++++++++++ .../pkg/downloader/testdata/helm-test-key.pub | Bin 0 -> 1243 bytes .../downloader/testdata/helm-test-key.secret | Bin 0 -> 2545 bytes .../testdata/local-subchart-0.1.0.tgz | Bin 0 -> 259 bytes .../testdata/local-subchart/Chart.yaml | 3 + .../pkg/downloader/testdata/repositories.yaml | 28 + .../repository/encoded-url-index.yaml | 15 + .../repository/kubernetes-charts-index.yaml | 49 + .../testdata/repository/malformed-index.yaml | 16 + .../repository/testing-basicauth-index.yaml | 15 + .../repository/testing-ca-file-index.yaml | 15 + .../repository/testing-https-index.yaml | 15 + ...g-https-insecureskip-tls-verify-index.yaml | 14 + .../testdata/repository/testing-index.yaml | 43 + .../repository/testing-querystring-index.yaml | 16 + .../repository/testing-relative-index.yaml | 28 + ...testing-relative-trailing-slash-index.yaml | 28 + .../downloader/testdata/signtest-0.1.0.tgz | Bin 0 -> 973 bytes .../testdata/signtest-0.1.0.tgz.prov | 21 + .../downloader/testdata/signtest/.helmignore | 5 + .../downloader/testdata/signtest/Chart.yaml | 4 + .../testdata/signtest/alpine/Chart.yaml | 7 + .../testdata/signtest/alpine/README.md | 9 + .../signtest/alpine/templates/alpine-pod.yaml | 14 + .../testdata/signtest/alpine/values.yaml | 2 + .../testdata/signtest/templates/pod.yaml | 10 + .../downloader/testdata/signtest/values.yaml | 0 kubelink/wire_gen.go | 2 +- 47 files changed, 4566 insertions(+), 1 deletion(-) create mode 100644 kubelink/internals/resolver/resolver.go create mode 100644 kubelink/internals/resolver/resolver_test.go create mode 100644 kubelink/internals/resolver/testdata/chartpath/base/Chart.yaml create mode 100644 kubelink/internals/resolver/testdata/chartpath/charts/localdependency/Chart.yaml create mode 100644 kubelink/internals/resolver/testdata/repository/kubernetes-charts-index.yaml create mode 100644 kubelink/internals/thirdParty/dep/fs/fs.go create mode 100644 kubelink/internals/thirdParty/dep/fs/fs_test.go create mode 100644 kubelink/internals/thirdParty/dep/fs/rename.go create mode 100644 kubelink/internals/thirdParty/dep/fs/rename_windows.go create mode 120000 kubelink/internals/thirdParty/dep/fs/testdata/symlinks/file-symlink create mode 120000 kubelink/internals/thirdParty/dep/fs/testdata/symlinks/invalid-symlink create mode 120000 kubelink/internals/thirdParty/dep/fs/testdata/symlinks/windows-file-symlink create mode 100644 kubelink/internals/thirdParty/dep/fs/testdata/test.file create mode 100644 kubelink/internals/urlutil/urlutil.go create mode 100644 kubelink/internals/urlutil/urlutil_test.go create mode 100644 kubelink/pkg/downloader/chart_downloader.go create mode 100644 kubelink/pkg/downloader/chart_downloader_test.go create mode 100644 kubelink/pkg/downloader/doc.go create mode 100644 kubelink/pkg/downloader/manager.go create mode 100644 kubelink/pkg/downloader/manager_test.go create mode 100644 kubelink/pkg/downloader/testdata/helm-test-key.pub create mode 100644 kubelink/pkg/downloader/testdata/helm-test-key.secret create mode 100644 kubelink/pkg/downloader/testdata/local-subchart-0.1.0.tgz create mode 100644 kubelink/pkg/downloader/testdata/local-subchart/Chart.yaml create mode 100644 kubelink/pkg/downloader/testdata/repositories.yaml create mode 100644 kubelink/pkg/downloader/testdata/repository/encoded-url-index.yaml create mode 100644 kubelink/pkg/downloader/testdata/repository/kubernetes-charts-index.yaml create mode 100644 kubelink/pkg/downloader/testdata/repository/malformed-index.yaml create mode 100644 kubelink/pkg/downloader/testdata/repository/testing-basicauth-index.yaml create mode 100644 kubelink/pkg/downloader/testdata/repository/testing-ca-file-index.yaml create mode 100644 kubelink/pkg/downloader/testdata/repository/testing-https-index.yaml create mode 100644 kubelink/pkg/downloader/testdata/repository/testing-https-insecureskip-tls-verify-index.yaml create mode 100644 kubelink/pkg/downloader/testdata/repository/testing-index.yaml create mode 100644 kubelink/pkg/downloader/testdata/repository/testing-querystring-index.yaml create mode 100644 kubelink/pkg/downloader/testdata/repository/testing-relative-index.yaml create mode 100644 kubelink/pkg/downloader/testdata/repository/testing-relative-trailing-slash-index.yaml create mode 100644 kubelink/pkg/downloader/testdata/signtest-0.1.0.tgz create mode 100644 kubelink/pkg/downloader/testdata/signtest-0.1.0.tgz.prov create mode 100644 kubelink/pkg/downloader/testdata/signtest/.helmignore create mode 100644 kubelink/pkg/downloader/testdata/signtest/Chart.yaml create mode 100644 kubelink/pkg/downloader/testdata/signtest/alpine/Chart.yaml create mode 100644 kubelink/pkg/downloader/testdata/signtest/alpine/README.md create mode 100644 kubelink/pkg/downloader/testdata/signtest/alpine/templates/alpine-pod.yaml create mode 100644 kubelink/pkg/downloader/testdata/signtest/alpine/values.yaml create mode 100644 kubelink/pkg/downloader/testdata/signtest/templates/pod.yaml create mode 100644 kubelink/pkg/downloader/testdata/signtest/values.yaml diff --git a/kubelink/internals/resolver/resolver.go b/kubelink/internals/resolver/resolver.go new file mode 100644 index 000000000..b6f45da9e --- /dev/null +++ b/kubelink/internals/resolver/resolver.go @@ -0,0 +1,262 @@ +/* +Copyright The Helm Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resolver + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/Masterminds/semver/v3" + "github.com/pkg/errors" + + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/helmpath" + "helm.sh/helm/v3/pkg/provenance" + "helm.sh/helm/v3/pkg/registry" + "helm.sh/helm/v3/pkg/repo" +) + +// Resolver resolves dependencies from semantic version ranges to a particular version. +type Resolver struct { + chartpath string + cachepath string + registryClient *registry.Client +} + +// New creates a new resolver for a given chart, helm home and registry client. +func New(chartpath, cachepath string, registryClient *registry.Client) *Resolver { + return &Resolver{ + chartpath: chartpath, + cachepath: cachepath, + registryClient: registryClient, + } +} + +// Resolve resolves dependencies and returns a lock file with the resolution. +func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string) (*chart.Lock, error) { + + // Now we clone the dependencies, locking as we go. + locked := make([]*chart.Dependency, len(reqs)) + missing := []string{} + for i, d := range reqs { + constraint, err := semver.NewConstraint(d.Version) + if err != nil { + return nil, errors.Wrapf(err, "dependency %q has an invalid version/constraint format", d.Name) + } + + if d.Repository == "" { + // Local chart subfolder + if _, err := GetLocalPath(filepath.Join("charts", d.Name), r.chartpath); err != nil { + return nil, err + } + + locked[i] = &chart.Dependency{ + Name: d.Name, + Repository: "", + Version: d.Version, + } + continue + } + if strings.HasPrefix(d.Repository, "file://") { + chartpath, err := GetLocalPath(d.Repository, r.chartpath) + if err != nil { + return nil, err + } + + ch, err := loader.LoadDir(chartpath) + if err != nil { + return nil, err + } + + v, err := semver.NewVersion(ch.Metadata.Version) + if err != nil { + // Not a legit entry. + continue + } + + if !constraint.Check(v) { + missing = append(missing, fmt.Sprintf("%q (repository %q, version %q)", d.Name, d.Repository, d.Version)) + continue + } + + locked[i] = &chart.Dependency{ + Name: d.Name, + Repository: d.Repository, + Version: ch.Metadata.Version, + } + continue + } + + repoName := repoNames[d.Name] + // if the repository was not defined, but the dependency defines a repository url, bypass the cache + if repoName == "" && d.Repository != "" { + locked[i] = &chart.Dependency{ + Name: d.Name, + Repository: d.Repository, + Version: d.Version, + } + continue + } + + var vs repo.ChartVersions + var version string + var ok bool + found := true + if !registry.IsOCI(d.Repository) { + repoIndex, err := repo.LoadIndexFile(filepath.Join(r.cachepath, helmpath.CacheIndexFile(repoName))) + if err != nil { + return nil, errors.Wrapf(err, "no cached repository for %s found. (try 'helm repo update')", repoName) + } + + vs, ok = repoIndex.Entries[d.Name] + if !ok { + return nil, errors.Errorf("%s chart not found in repo %s", d.Name, d.Repository) + } + found = false + } else { + version = d.Version + + // Check to see if an explicit version has been provided + _, err := semver.NewVersion(version) + + // Use an explicit version, otherwise search for tags + if err == nil { + vs = []*repo.ChartVersion{{ + Metadata: &chart.Metadata{ + Version: version, + }, + }} + + } else { + // Retrieve list of tags for repository + ref := fmt.Sprintf("%s/%s", strings.TrimPrefix(d.Repository, fmt.Sprintf("%s://", registry.OCIScheme)), d.Name) + tags, err := r.registryClient.Tags(ref) + if err != nil { + return nil, errors.Wrapf(err, "could not retrieve list of tags for repository %s", d.Repository) + } + + vs = make(repo.ChartVersions, len(tags)) + for ti, t := range tags { + // Mock chart version objects + version := &repo.ChartVersion{ + Metadata: &chart.Metadata{ + Version: t, + }, + } + vs[ti] = version + } + } + } + + locked[i] = &chart.Dependency{ + Name: d.Name, + Repository: d.Repository, + Version: version, + } + // The versions are already sorted and hence the first one to satisfy the constraint is used + for _, ver := range vs { + v, err := semver.NewVersion(ver.Version) + // OCI does not need URLs + if err != nil || (!registry.IsOCI(d.Repository) && len(ver.URLs) == 0) { + // Not a legit entry. + continue + } + if constraint.Check(v) { + found = true + locked[i].Version = v.Original() + break + } + } + + if !found { + missing = append(missing, fmt.Sprintf("%q (repository %q, version %q)", d.Name, d.Repository, d.Version)) + } + } + if len(missing) > 0 { + return nil, errors.Errorf("can't get a valid version for %d subchart(s): %s. Make sure a matching chart version exists in the repo, or change the version constraint in Chart.yaml", len(missing), strings.Join(missing, ", ")) + } + + digest, err := HashReq(reqs, locked) + if err != nil { + return nil, err + } + + return &chart.Lock{ + Generated: time.Now(), + Digest: digest, + Dependencies: locked, + }, nil +} + +// HashReq generates a hash of the dependencies. +// +// This should be used only to compare against another hash generated by this +// function. +func HashReq(req, lock []*chart.Dependency) (string, error) { + data, err := json.Marshal([2][]*chart.Dependency{req, lock}) + if err != nil { + return "", err + } + s, err := provenance.Digest(bytes.NewBuffer(data)) + return "sha256:" + s, err +} + +// HashV2Req generates a hash of requirements generated in Helm v2. +// +// This should be used only to compare against another hash generated by the +// Helm v2 hash function. It is to handle issue: +// https://github.com/helm/helm/issues/7233 +func HashV2Req(req []*chart.Dependency) (string, error) { + dep := make(map[string][]*chart.Dependency) + dep["dependencies"] = req + data, err := json.Marshal(dep) + if err != nil { + return "", err + } + s, err := provenance.Digest(bytes.NewBuffer(data)) + return "sha256:" + s, err +} + +// GetLocalPath generates absolute local path when use +// "file://" in repository of dependencies +func GetLocalPath(repo, chartpath string) (string, error) { + var depPath string + var err error + p := strings.TrimPrefix(repo, "file://") + + // root path is absolute + if strings.HasPrefix(p, "/") { + if depPath, err = filepath.Abs(p); err != nil { + return "", err + } + } else { + depPath = filepath.Join(chartpath, p) + } + + if _, err = os.Stat(depPath); os.IsNotExist(err) { + return "", errors.Errorf("directory %s not found", depPath) + } else if err != nil { + return "", err + } + + return depPath, nil +} diff --git a/kubelink/internals/resolver/resolver_test.go b/kubelink/internals/resolver/resolver_test.go new file mode 100644 index 000000000..a79852175 --- /dev/null +++ b/kubelink/internals/resolver/resolver_test.go @@ -0,0 +1,310 @@ +/* +Copyright The Helm Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resolver + +import ( + "runtime" + "testing" + + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/registry" +) + +func TestResolve(t *testing.T) { + tests := []struct { + name string + req []*chart.Dependency + expect *chart.Lock + err bool + }{ + { + name: "repo from invalid version", + req: []*chart.Dependency{ + {Name: "base", Repository: "file://base", Version: "1.1.0"}, + }, + expect: &chart.Lock{ + Dependencies: []*chart.Dependency{ + {Name: "base", Repository: "file://base", Version: "0.1.0"}, + }, + }, + err: true, + }, + { + name: "version failure", + req: []*chart.Dependency{ + {Name: "oedipus-rex", Repository: "http://example.com", Version: ">a1"}, + }, + err: true, + }, + { + name: "cache index failure", + req: []*chart.Dependency{ + {Name: "oedipus-rex", Repository: "http://example.com", Version: "1.0.0"}, + }, + expect: &chart.Lock{ + Dependencies: []*chart.Dependency{ + {Name: "oedipus-rex", Repository: "http://example.com", Version: "1.0.0"}, + }, + }, + }, + { + name: "chart not found failure", + req: []*chart.Dependency{ + {Name: "redis", Repository: "http://example.com", Version: "1.0.0"}, + }, + err: true, + }, + { + name: "constraint not satisfied failure", + req: []*chart.Dependency{ + {Name: "alpine", Repository: "http://example.com", Version: ">=1.0.0"}, + }, + err: true, + }, + { + name: "valid lock", + req: []*chart.Dependency{ + {Name: "alpine", Repository: "http://example.com", Version: ">=0.1.0"}, + }, + expect: &chart.Lock{ + Dependencies: []*chart.Dependency{ + {Name: "alpine", Repository: "http://example.com", Version: "0.2.0"}, + }, + }, + }, + { + name: "repo from valid local path", + req: []*chart.Dependency{ + {Name: "base", Repository: "file://base", Version: "0.1.0"}, + }, + expect: &chart.Lock{ + Dependencies: []*chart.Dependency{ + {Name: "base", Repository: "file://base", Version: "0.1.0"}, + }, + }, + }, + { + name: "repo from valid local path with range resolution", + req: []*chart.Dependency{ + {Name: "base", Repository: "file://base", Version: "^0.1.0"}, + }, + expect: &chart.Lock{ + Dependencies: []*chart.Dependency{ + {Name: "base", Repository: "file://base", Version: "0.1.0"}, + }, + }, + }, + { + name: "repo from invalid local path", + req: []*chart.Dependency{ + {Name: "nonexistent", Repository: "file://testdata/nonexistent", Version: "0.1.0"}, + }, + err: true, + }, + { + name: "repo from valid path under charts path", + req: []*chart.Dependency{ + {Name: "localdependency", Repository: "", Version: "0.1.0"}, + }, + expect: &chart.Lock{ + Dependencies: []*chart.Dependency{ + {Name: "localdependency", Repository: "", Version: "0.1.0"}, + }, + }, + }, + { + name: "repo from invalid path under charts path", + req: []*chart.Dependency{ + {Name: "nonexistentdependency", Repository: "", Version: "0.1.0"}, + }, + expect: &chart.Lock{ + Dependencies: []*chart.Dependency{ + {Name: "nonexistentlocaldependency", Repository: "", Version: "0.1.0"}, + }, + }, + err: true, + }, + } + + repoNames := map[string]string{"alpine": "kubernetes-charts", "redis": "kubernetes-charts"} + registryClient, _ := registry.NewClient() + r := New("testdata/chartpath", "testdata/repository", registryClient) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + l, err := r.Resolve(tt.req, repoNames) + if err != nil { + if tt.err { + return + } + t.Fatal(err) + } + + if tt.err { + t.Fatalf("Expected error in test %q", tt.name) + } + + if h, err := HashReq(tt.req, tt.expect.Dependencies); err != nil { + t.Fatal(err) + } else if h != l.Digest { + t.Errorf("%q: hashes don't match.", tt.name) + } + + // Check fields. + if len(l.Dependencies) != len(tt.req) { + t.Errorf("%s: wrong number of dependencies in lock", tt.name) + } + d0 := l.Dependencies[0] + e0 := tt.expect.Dependencies[0] + if d0.Name != e0.Name { + t.Errorf("%s: expected name %s, got %s", tt.name, e0.Name, d0.Name) + } + if d0.Repository != e0.Repository { + t.Errorf("%s: expected repo %s, got %s", tt.name, e0.Repository, d0.Repository) + } + if d0.Version != e0.Version { + t.Errorf("%s: expected version %s, got %s", tt.name, e0.Version, d0.Version) + } + }) + } +} + +func TestHashReq(t *testing.T) { + expect := "sha256:fb239e836325c5fa14b29d1540a13b7d3ba13151b67fe719f820e0ef6d66aaaf" + + tests := []struct { + name string + chartVersion string + lockVersion string + wantError bool + }{ + { + name: "chart with the expected digest", + chartVersion: "0.1.0", + lockVersion: "0.1.0", + wantError: false, + }, + { + name: "ranged version but same resolved lock version", + chartVersion: "^0.1.0", + lockVersion: "0.1.0", + wantError: true, + }, + { + name: "ranged version resolved as higher version", + chartVersion: "^0.1.0", + lockVersion: "0.1.2", + wantError: true, + }, + { + name: "different version", + chartVersion: "0.1.2", + lockVersion: "0.1.2", + wantError: true, + }, + { + name: "different version with a range", + chartVersion: "^0.1.2", + lockVersion: "0.1.2", + wantError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := []*chart.Dependency{ + {Name: "alpine", Version: tt.chartVersion, Repository: "http://localhost:8879/charts"}, + } + lock := []*chart.Dependency{ + {Name: "alpine", Version: tt.lockVersion, Repository: "http://localhost:8879/charts"}, + } + h, err := HashReq(req, lock) + if err != nil { + t.Fatal(err) + } + if !tt.wantError && expect != h { + t.Errorf("Expected %q, got %q", expect, h) + } else if tt.wantError && expect == h { + t.Errorf("Expected not %q, but same", expect) + } + }) + } +} + +func TestGetLocalPath(t *testing.T) { + tests := []struct { + name string + repo string + chartpath string + expect string + winExpect string + err bool + }{ + { + name: "absolute path", + repo: "file:////", + expect: "/", + winExpect: "\\", + }, + { + name: "relative path", + repo: "file://../../testdata/chartpath/base", + chartpath: "foo/bar", + expect: "testdata/chartpath/base", + winExpect: "testdata\\chartpath\\base", + }, + { + name: "current directory path", + repo: "../charts/localdependency", + chartpath: "testdata/chartpath/charts", + expect: "testdata/chartpath/charts/localdependency", + winExpect: "testdata\\chartpath\\charts\\localdependency", + }, + { + name: "invalid local path", + repo: "file://testdata/nonexistent", + chartpath: "testdata/chartpath", + err: true, + }, + { + name: "invalid path under current directory", + repo: "charts/nonexistentdependency", + chartpath: "testdata/chartpath/charts", + err: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p, err := GetLocalPath(tt.repo, tt.chartpath) + if err != nil { + if tt.err { + return + } + t.Fatal(err) + } + if tt.err { + t.Fatalf("Expected error in test %q", tt.name) + } + expect := tt.expect + if runtime.GOOS == "windows" { + expect = tt.winExpect + } + if p != expect { + t.Errorf("%q: expected %q, got %q", tt.name, expect, p) + } + }) + } +} diff --git a/kubelink/internals/resolver/testdata/chartpath/base/Chart.yaml b/kubelink/internals/resolver/testdata/chartpath/base/Chart.yaml new file mode 100644 index 000000000..860b09091 --- /dev/null +++ b/kubelink/internals/resolver/testdata/chartpath/base/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: base +version: 0.1.0 diff --git a/kubelink/internals/resolver/testdata/chartpath/charts/localdependency/Chart.yaml b/kubelink/internals/resolver/testdata/chartpath/charts/localdependency/Chart.yaml new file mode 100644 index 000000000..083c51ee5 --- /dev/null +++ b/kubelink/internals/resolver/testdata/chartpath/charts/localdependency/Chart.yaml @@ -0,0 +1,3 @@ +description: A Helm chart for Kubernetes +name: localdependency +version: 0.1.0 diff --git a/kubelink/internals/resolver/testdata/repository/kubernetes-charts-index.yaml b/kubelink/internals/resolver/testdata/repository/kubernetes-charts-index.yaml new file mode 100644 index 000000000..c6b7962a1 --- /dev/null +++ b/kubelink/internals/resolver/testdata/repository/kubernetes-charts-index.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +entries: + alpine: + - name: alpine + urls: + - https://charts.helm.sh/stable/alpine-0.1.0.tgz + checksum: 0e6661f193211d7a5206918d42f5c2a9470b737d + home: https://helm.sh/helm + sources: + - https://github.com/helm/helm + version: 0.2.0 + description: Deploy a basic Alpine Linux pod + keywords: [] + maintainers: [] + icon: "" + apiVersion: v2 + - name: alpine + urls: + - https://charts.helm.sh/stable/alpine-0.2.0.tgz + checksum: 0e6661f193211d7a5206918d42f5c2a9470b737d + home: https://helm.sh/helm + sources: + - https://github.com/helm/helm + version: 0.1.0 + description: Deploy a basic Alpine Linux pod + keywords: [] + maintainers: [] + icon: "" + apiVersion: v2 + mariadb: + - name: mariadb + urls: + - https://charts.helm.sh/stable/mariadb-0.3.0.tgz + checksum: 65229f6de44a2be9f215d11dbff311673fc8ba56 + home: https://mariadb.org + sources: + - https://github.com/bitnami/bitnami-docker-mariadb + version: 0.3.0 + description: Chart for MariaDB + keywords: + - mariadb + - mysql + - database + - sql + maintainers: + - name: Bitnami + email: containers@bitnami.com + icon: "" + apiVersion: v2 diff --git a/kubelink/internals/thirdParty/dep/fs/fs.go b/kubelink/internals/thirdParty/dep/fs/fs.go new file mode 100644 index 000000000..d29bb5f87 --- /dev/null +++ b/kubelink/internals/thirdParty/dep/fs/fs.go @@ -0,0 +1,372 @@ +/* +Copyright (c) for portions of fs.go are held by The Go Authors, 2016 and are provided under +the BSD license. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +package fs + +import ( + "io" + "os" + "path/filepath" + "runtime" + "syscall" + + "github.com/pkg/errors" +) + +// fs contains a copy of a few functions from dep tool code to avoid a dependency on golang/dep. +// This code is copied from https://github.com/golang/dep/blob/37d6c560cdf407be7b6cd035b23dba89df9275cf/internal/fs/fs.go +// No changes to the code were made other than removing some unused functions + +// RenameWithFallback attempts to rename a file or directory, but falls back to +// copying in the event of a cross-device link error. If the fallback copy +// succeeds, src is still removed, emulating normal rename behavior. +func RenameWithFallback(src, dst string) error { + _, err := os.Stat(src) + if err != nil { + return errors.Wrapf(err, "cannot stat %s", src) + } + + err = os.Rename(src, dst) + if err == nil { + return nil + } + + return renameFallback(err, src, dst) +} + +// renameByCopy attempts to rename a file or directory by copying it to the +// destination and then removing the src thus emulating the rename behavior. +func renameByCopy(src, dst string) error { + var cerr error + if dir, _ := IsDir(src); dir { + cerr = CopyDir(src, dst) + if cerr != nil { + cerr = errors.Wrap(cerr, "copying directory failed") + } + } else { + cerr = copyFile(src, dst) + if cerr != nil { + cerr = errors.Wrap(cerr, "copying file failed") + } + } + + if cerr != nil { + return errors.Wrapf(cerr, "rename fallback failed: cannot rename %s to %s", src, dst) + } + + return errors.Wrapf(os.RemoveAll(src), "cannot delete %s", src) +} + +var ( + errSrcNotDir = errors.New("source is not a directory") + errDstExist = errors.New("destination already exists") +) + +// CopyDir recursively copies a directory tree, attempting to preserve permissions. +// Source directory must exist, destination directory must *not* exist. +func CopyDir(src, dst string) error { + src = filepath.Clean(src) + dst = filepath.Clean(dst) + + // We use os.Lstat() here to ensure we don't fall in a loop where a symlink + // actually links to a one of its parent directories. + fi, err := os.Lstat(src) + if err != nil { + return err + } + if !fi.IsDir() { + return errSrcNotDir + } + + _, err = os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + return err + } + if err == nil { + return errDstExist + } + + if err = os.MkdirAll(dst, fi.Mode()); err != nil { + return errors.Wrapf(err, "cannot mkdir %s", dst) + } + + entries, err := os.ReadDir(src) + if err != nil { + return errors.Wrapf(err, "cannot read directory %s", dst) + } + + for _, entry := range entries { + srcPath := filepath.Join(src, entry.Name()) + dstPath := filepath.Join(dst, entry.Name()) + + if entry.IsDir() { + if err = CopyDir(srcPath, dstPath); err != nil { + return errors.Wrap(err, "copying directory failed") + } + } else { + // This will include symlinks, which is what we want when + // copying things. + if err = copyFile(srcPath, dstPath); err != nil { + return errors.Wrap(err, "copying file failed") + } + } + } + + return nil +} + +// copyFile copies the contents of the file named src to the file named +// by dst. The file will be created if it does not already exist. If the +// destination file exists, all its contents will be replaced by the contents +// of the source file. The file mode will be copied from the source. +func copyFile(src, dst string) (err error) { + if sym, err := IsSymlink(src); err != nil { + return errors.Wrap(err, "symlink check failed") + } else if sym { + if err := cloneSymlink(src, dst); err != nil { + if runtime.GOOS == "windows" { + // If cloning the symlink fails on Windows because the user + // does not have the required privileges, ignore the error and + // fall back to copying the file contents. + // + // ERROR_PRIVILEGE_NOT_HELD is 1314 (0x522): + // https://msdn.microsoft.com/en-us/library/windows/desktop/ms681385(v=vs.85).aspx + if lerr, ok := err.(*os.LinkError); ok && lerr.Err != syscall.Errno(1314) { + return err + } + } else { + return err + } + } else { + return nil + } + } + + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return + } + + if _, err = io.Copy(out, in); err != nil { + out.Close() + return + } + + // Check for write errors on Close + if err = out.Close(); err != nil { + return + } + + si, err := os.Stat(src) + if err != nil { + return + } + + // Temporary fix for Go < 1.9 + // + // See: https://github.com/golang/dep/issues/774 + // and https://github.com/golang/go/issues/20829 + if runtime.GOOS == "windows" { + dst = fixLongPath(dst) + } + err = os.Chmod(dst, si.Mode()) + + return +} + +// cloneSymlink will create a new symlink that points to the resolved path of sl. +// If sl is a relative symlink, dst will also be a relative symlink. +func cloneSymlink(sl, dst string) error { + resolved, err := os.Readlink(sl) + if err != nil { + return err + } + + return os.Symlink(resolved, dst) +} + +// IsDir determines is the path given is a directory or not. +func IsDir(name string) (bool, error) { + fi, err := os.Stat(name) + if err != nil { + return false, err + } + if !fi.IsDir() { + return false, errors.Errorf("%q is not a directory", name) + } + return true, nil +} + +// IsSymlink determines if the given path is a symbolic link. +func IsSymlink(path string) (bool, error) { + l, err := os.Lstat(path) + if err != nil { + return false, err + } + + return l.Mode()&os.ModeSymlink == os.ModeSymlink, nil +} + +// fixLongPath returns the extended-length (\\?\-prefixed) form of +// path when needed, in order to avoid the default 260 character file +// path limit imposed by Windows. If path is not easily converted to +// the extended-length form (for example, if path is a relative path +// or contains .. elements), or is short enough, fixLongPath returns +// path unmodified. +// +// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath +func fixLongPath(path string) string { + // Do nothing (and don't allocate) if the path is "short". + // Empirically (at least on the Windows Server 2013 builder), + // the kernel is arbitrarily okay with < 248 bytes. That + // matches what the docs above say: + // "When using an API to create a directory, the specified + // path cannot be so long that you cannot append an 8.3 file + // name (that is, the directory name cannot exceed MAX_PATH + // minus 12)." Since MAX_PATH is 260, 260 - 12 = 248. + // + // The MSDN docs appear to say that a normal path that is 248 bytes long + // will work; empirically the path must be less than 248 bytes long. + if len(path) < 248 { + // Don't fix. (This is how Go 1.7 and earlier worked, + // not automatically generating the \\?\ form) + return path + } + + // The extended form begins with \\?\, as in + // \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt. + // The extended form disables evaluation of . and .. path + // elements and disables the interpretation of / as equivalent + // to \. The conversion here rewrites / to \ and elides + // . elements as well as trailing or duplicate separators. For + // simplicity it avoids the conversion entirely for relative + // paths or paths containing .. elements. For now, + // \\server\share paths are not converted to + // \\?\UNC\server\share paths because the rules for doing so + // are less well-specified. + if len(path) >= 2 && path[:2] == `\\` { + // Don't canonicalize UNC paths. + return path + } + if !isAbs(path) { + // Relative path + return path + } + + const prefix = `\\?` + + pathbuf := make([]byte, len(prefix)+len(path)+len(`\`)) + copy(pathbuf, prefix) + n := len(path) + r, w := 0, len(prefix) + for r < n { + switch { + case os.IsPathSeparator(path[r]): + // empty block + r++ + case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])): + // /./ + r++ + case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])): + // /../ is currently unhandled + return path + default: + pathbuf[w] = '\\' + w++ + for ; r < n && !os.IsPathSeparator(path[r]); r++ { + pathbuf[w] = path[r] + w++ + } + } + } + // A drive's root directory needs a trailing \ + if w == len(`\\?\c:`) { + pathbuf[w] = '\\' + w++ + } + return string(pathbuf[:w]) +} + +func isAbs(path string) (b bool) { + v := volumeName(path) + if v == "" { + return false + } + path = path[len(v):] + if path == "" { + return false + } + return os.IsPathSeparator(path[0]) +} + +func volumeName(path string) (v string) { + if len(path) < 2 { + return "" + } + // with drive letter + c := path[0] + if path[1] == ':' && + ('0' <= c && c <= '9' || 'a' <= c && c <= 'z' || + 'A' <= c && c <= 'Z') { + return path[:2] + } + // is it UNC + if l := len(path); l >= 5 && os.IsPathSeparator(path[0]) && os.IsPathSeparator(path[1]) && + !os.IsPathSeparator(path[2]) && path[2] != '.' { + // first, leading `\\` and next shouldn't be `\`. its server name. + for n := 3; n < l-1; n++ { + // second, next '\' shouldn't be repeated. + if os.IsPathSeparator(path[n]) { + n++ + // third, following something characters. its share name. + if !os.IsPathSeparator(path[n]) { + if path[n] == '.' { + break + } + for ; n < l; n++ { + if os.IsPathSeparator(path[n]) { + break + } + } + return path[:n] + } + break + } + } + } + return "" +} diff --git a/kubelink/internals/thirdParty/dep/fs/fs_test.go b/kubelink/internals/thirdParty/dep/fs/fs_test.go new file mode 100644 index 000000000..d42c3f110 --- /dev/null +++ b/kubelink/internals/thirdParty/dep/fs/fs_test.go @@ -0,0 +1,642 @@ +/* +Copyright (c) for portions of fs_test.go are held by The Go Authors, 2016 and are provided under +the BSD license. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +package fs + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "sync" + "testing" +) + +var ( + mu sync.Mutex +) + +func TestRenameWithFallback(t *testing.T) { + dir := t.TempDir() + + if err := RenameWithFallback(filepath.Join(dir, "does_not_exists"), filepath.Join(dir, "dst")); err == nil { + t.Fatal("expected an error for non existing file, but got nil") + } + + srcpath := filepath.Join(dir, "src") + + if srcf, err := os.Create(srcpath); err != nil { + t.Fatal(err) + } else { + srcf.Close() + } + + if err := RenameWithFallback(srcpath, filepath.Join(dir, "dst")); err != nil { + t.Fatal(err) + } + + srcpath = filepath.Join(dir, "a") + if err := os.MkdirAll(srcpath, 0777); err != nil { + t.Fatal(err) + } + + dstpath := filepath.Join(dir, "b") + if err := os.MkdirAll(dstpath, 0777); err != nil { + t.Fatal(err) + } + + if err := RenameWithFallback(srcpath, dstpath); err == nil { + t.Fatal("expected an error if dst is an existing directory, but got nil") + } +} + +func TestCopyDir(t *testing.T) { + dir := t.TempDir() + + srcdir := filepath.Join(dir, "src") + if err := os.MkdirAll(srcdir, 0755); err != nil { + t.Fatal(err) + } + + files := []struct { + path string + contents string + fi os.FileInfo + }{ + {path: "myfile", contents: "hello world"}, + {path: filepath.Join("subdir", "file"), contents: "subdir file"}, + } + + // Create structure indicated in 'files' + for i, file := range files { + fn := filepath.Join(srcdir, file.path) + dn := filepath.Dir(fn) + if err := os.MkdirAll(dn, 0755); err != nil { + t.Fatal(err) + } + + fh, err := os.Create(fn) + if err != nil { + t.Fatal(err) + } + + if _, err = fh.Write([]byte(file.contents)); err != nil { + t.Fatal(err) + } + fh.Close() + + files[i].fi, err = os.Stat(fn) + if err != nil { + t.Fatal(err) + } + } + + destdir := filepath.Join(dir, "dest") + if err := CopyDir(srcdir, destdir); err != nil { + t.Fatal(err) + } + + // Compare copy against structure indicated in 'files' + for _, file := range files { + fn := filepath.Join(srcdir, file.path) + dn := filepath.Dir(fn) + dirOK, err := IsDir(dn) + if err != nil { + t.Fatal(err) + } + if !dirOK { + t.Fatalf("expected %s to be a directory", dn) + } + + got, err := os.ReadFile(fn) + if err != nil { + t.Fatal(err) + } + + if file.contents != string(got) { + t.Fatalf("expected: %s, got: %s", file.contents, string(got)) + } + + gotinfo, err := os.Stat(fn) + if err != nil { + t.Fatal(err) + } + + if file.fi.Mode() != gotinfo.Mode() { + t.Fatalf("expected %s: %#v\n to be the same mode as %s: %#v", + file.path, file.fi.Mode(), fn, gotinfo.Mode()) + } + } +} + +func TestCopyDirFail_SrcInaccessible(t *testing.T) { + if runtime.GOOS == "windows" { + // XXX: setting permissions works differently in + // Microsoft Windows. Skipping this until a + // compatible implementation is provided. + t.Skip("skipping on windows") + } + + var currentUID = os.Getuid() + + if currentUID == 0 { + // Skipping if root, because all files are accessible + t.Skip("Skipping for root user") + } + + var srcdir, dstdir string + + cleanup := setupInaccessibleDir(t, func(dir string) error { + srcdir = filepath.Join(dir, "src") + return os.MkdirAll(srcdir, 0755) + }) + defer cleanup() + + dir := t.TempDir() + + dstdir = filepath.Join(dir, "dst") + if err := CopyDir(srcdir, dstdir); err == nil { + t.Fatalf("expected error for CopyDir(%s, %s), got none", srcdir, dstdir) + } +} + +func TestCopyDirFail_DstInaccessible(t *testing.T) { + if runtime.GOOS == "windows" { + // XXX: setting permissions works differently in + // Microsoft Windows. Skipping this until a + // compatible implementation is provided. + t.Skip("skipping on windows") + } + + var currentUID = os.Getuid() + + if currentUID == 0 { + // Skipping if root, because all files are accessible + t.Skip("Skipping for root user") + } + + var srcdir, dstdir string + + dir := t.TempDir() + + srcdir = filepath.Join(dir, "src") + if err := os.MkdirAll(srcdir, 0755); err != nil { + t.Fatal(err) + } + + cleanup := setupInaccessibleDir(t, func(dir string) error { + dstdir = filepath.Join(dir, "dst") + return nil + }) + defer cleanup() + + if err := CopyDir(srcdir, dstdir); err == nil { + t.Fatalf("expected error for CopyDir(%s, %s), got none", srcdir, dstdir) + } +} + +func TestCopyDirFail_SrcIsNotDir(t *testing.T) { + var srcdir, dstdir string + var err error + + dir := t.TempDir() + + srcdir = filepath.Join(dir, "src") + if _, err = os.Create(srcdir); err != nil { + t.Fatal(err) + } + + dstdir = filepath.Join(dir, "dst") + + if err = CopyDir(srcdir, dstdir); err == nil { + t.Fatalf("expected error for CopyDir(%s, %s), got none", srcdir, dstdir) + } + + if err != errSrcNotDir { + t.Fatalf("expected %v error for CopyDir(%s, %s), got %s", errSrcNotDir, srcdir, dstdir, err) + } + +} + +func TestCopyDirFail_DstExists(t *testing.T) { + var srcdir, dstdir string + var err error + + dir := t.TempDir() + + srcdir = filepath.Join(dir, "src") + if err = os.MkdirAll(srcdir, 0755); err != nil { + t.Fatal(err) + } + + dstdir = filepath.Join(dir, "dst") + if err = os.MkdirAll(dstdir, 0755); err != nil { + t.Fatal(err) + } + + if err = CopyDir(srcdir, dstdir); err == nil { + t.Fatalf("expected error for CopyDir(%s, %s), got none", srcdir, dstdir) + } + + if err != errDstExist { + t.Fatalf("expected %v error for CopyDir(%s, %s), got %s", errDstExist, srcdir, dstdir, err) + } +} + +func TestCopyDirFailOpen(t *testing.T) { + if runtime.GOOS == "windows" { + // XXX: setting permissions works differently in + // Microsoft Windows. os.Chmod(..., 0222) below is not + // enough for the file to be readonly, and os.Chmod(..., + // 0000) returns an invalid argument error. Skipping + // this until a compatible implementation is + // provided. + t.Skip("skipping on windows") + } + + var currentUID = os.Getuid() + + if currentUID == 0 { + // Skipping if root, because all files are accessible + t.Skip("Skipping for root user") + } + + var srcdir, dstdir string + + dir := t.TempDir() + + srcdir = filepath.Join(dir, "src") + if err := os.MkdirAll(srcdir, 0755); err != nil { + t.Fatal(err) + } + + srcfn := filepath.Join(srcdir, "file") + srcf, err := os.Create(srcfn) + if err != nil { + t.Fatal(err) + } + srcf.Close() + + // setup source file so that it cannot be read + if err = os.Chmod(srcfn, 0222); err != nil { + t.Fatal(err) + } + + dstdir = filepath.Join(dir, "dst") + + if err = CopyDir(srcdir, dstdir); err == nil { + t.Fatalf("expected error for CopyDir(%s, %s), got none", srcdir, dstdir) + } +} + +func TestCopyFile(t *testing.T) { + dir := t.TempDir() + + srcf, err := os.Create(filepath.Join(dir, "srcfile")) + if err != nil { + t.Fatal(err) + } + + want := "hello world" + if _, err := srcf.Write([]byte(want)); err != nil { + t.Fatal(err) + } + srcf.Close() + + destf := filepath.Join(dir, "destf") + if err := copyFile(srcf.Name(), destf); err != nil { + t.Fatal(err) + } + + got, err := os.ReadFile(destf) + if err != nil { + t.Fatal(err) + } + + if want != string(got) { + t.Fatalf("expected: %s, got: %s", want, string(got)) + } + + wantinfo, err := os.Stat(srcf.Name()) + if err != nil { + t.Fatal(err) + } + + gotinfo, err := os.Stat(destf) + if err != nil { + t.Fatal(err) + } + + if wantinfo.Mode() != gotinfo.Mode() { + t.Fatalf("expected %s: %#v\n to be the same mode as %s: %#v", srcf.Name(), wantinfo.Mode(), destf, gotinfo.Mode()) + } +} + +func cleanUpDir(dir string) { + // NOTE(mattn): It seems that sometimes git.exe is not dead + // when cleanUpDir() is called. But we do not know any way to wait for it. + if runtime.GOOS == "windows" { + mu.Lock() + exec.Command(`taskkill`, `/F`, `/IM`, `git.exe`).Run() + mu.Unlock() + } + if dir != "" { + os.RemoveAll(dir) + } +} + +func TestCopyFileSymlink(t *testing.T) { + tempdir := t.TempDir() + + testcases := map[string]string{ + filepath.Join("./testdata/symlinks/file-symlink"): filepath.Join(tempdir, "dst-file"), + filepath.Join("./testdata/symlinks/windows-file-symlink"): filepath.Join(tempdir, "windows-dst-file"), + filepath.Join("./testdata/symlinks/invalid-symlink"): filepath.Join(tempdir, "invalid-symlink"), + } + + for symlink, dst := range testcases { + t.Run(symlink, func(t *testing.T) { + var err error + if err = copyFile(symlink, dst); err != nil { + t.Fatalf("failed to copy symlink: %s", err) + } + + var want, got string + + if runtime.GOOS == "windows" { + // Creating symlinks on Windows require an additional permission + // regular users aren't granted usually. So we copy the file + // content as a fall back instead of creating a real symlink. + srcb, err := os.ReadFile(symlink) + if err != nil { + t.Fatalf("%+v", err) + } + dstb, err := os.ReadFile(dst) + if err != nil { + t.Fatalf("%+v", err) + } + + want = string(srcb) + got = string(dstb) + } else { + want, err = os.Readlink(symlink) + if err != nil { + t.Fatalf("%+v", err) + } + + got, err = os.Readlink(dst) + if err != nil { + t.Fatalf("could not resolve symlink: %s", err) + } + } + + if want != got { + t.Fatalf("resolved path is incorrect. expected %s, got %s", want, got) + } + }) + } +} + +func TestCopyFileFail(t *testing.T) { + if runtime.GOOS == "windows" { + // XXX: setting permissions works differently in + // Microsoft Windows. Skipping this until a + // compatible implementation is provided. + t.Skip("skipping on windows") + } + + var currentUID = os.Getuid() + + if currentUID == 0 { + // Skipping if root, because all files are accessible + t.Skip("Skipping for root user") + } + + dir := t.TempDir() + + srcf, err := os.Create(filepath.Join(dir, "srcfile")) + if err != nil { + t.Fatal(err) + } + srcf.Close() + + var dstdir string + + cleanup := setupInaccessibleDir(t, func(dir string) error { + dstdir = filepath.Join(dir, "dir") + return os.Mkdir(dstdir, 0777) + }) + defer cleanup() + + fn := filepath.Join(dstdir, "file") + if err := copyFile(srcf.Name(), fn); err == nil { + t.Fatalf("expected error for %s, got none", fn) + } +} + +// setupInaccessibleDir creates a temporary location with a single +// directory in it, in such a way that directory is not accessible +// after this function returns. +// +// op is called with the directory as argument, so that it can create +// files or other test artifacts. +// +// If setupInaccessibleDir fails in its preparation, or op fails, t.Fatal +// will be invoked. +// +// This function returns a cleanup function that removes all the temporary +// files this function creates. It is the caller's responsibility to call +// this function before the test is done running, whether there's an error or not. +func setupInaccessibleDir(t *testing.T, op func(dir string) error) func() { + dir := t.TempDir() + + subdir := filepath.Join(dir, "dir") + + cleanup := func() { + if err := os.Chmod(subdir, 0777); err != nil { + t.Error(err) + } + } + + if err := os.Mkdir(subdir, 0777); err != nil { + cleanup() + t.Fatal(err) + return nil + } + + if err := op(subdir); err != nil { + cleanup() + t.Fatal(err) + return nil + } + + if err := os.Chmod(subdir, 0666); err != nil { + cleanup() + t.Fatal(err) + return nil + } + + return cleanup +} + +func TestIsDir(t *testing.T) { + + var currentUID = os.Getuid() + + if currentUID == 0 { + // Skipping if root, because all files are accessible + t.Skip("Skipping for root user") + } + + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + var dn string + + cleanup := setupInaccessibleDir(t, func(dir string) error { + dn = filepath.Join(dir, "dir") + return os.Mkdir(dn, 0777) + }) + defer cleanup() + + tests := map[string]struct { + exists bool + err bool + }{ + wd: {true, false}, + filepath.Join(wd, "testdata"): {true, false}, + filepath.Join(wd, "main.go"): {false, true}, + filepath.Join(wd, "this_file_does_not_exist.thing"): {false, true}, + dn: {false, true}, + } + + if runtime.GOOS == "windows" { + // This test doesn't work on Microsoft Windows because + // of the differences in how file permissions are + // implemented. For this to work, the directory where + // the directory exists should be inaccessible. + delete(tests, dn) + } + + for f, want := range tests { + got, err := IsDir(f) + if err != nil && !want.err { + t.Fatalf("expected no error, got %v", err) + } + + if got != want.exists { + t.Fatalf("expected %t for %s, got %t", want.exists, f, got) + } + } +} + +func TestIsSymlink(t *testing.T) { + + var currentUID = os.Getuid() + + if currentUID == 0 { + // Skipping if root, because all files are accessible + t.Skip("Skipping for root user") + } + + dir := t.TempDir() + + dirPath := filepath.Join(dir, "directory") + if err := os.MkdirAll(dirPath, 0777); err != nil { + t.Fatal(err) + } + + filePath := filepath.Join(dir, "file") + f, err := os.Create(filePath) + if err != nil { + t.Fatal(err) + } + f.Close() + + dirSymlink := filepath.Join(dir, "dirSymlink") + fileSymlink := filepath.Join(dir, "fileSymlink") + + if err = os.Symlink(dirPath, dirSymlink); err != nil { + t.Fatal(err) + } + if err = os.Symlink(filePath, fileSymlink); err != nil { + t.Fatal(err) + } + + var ( + inaccessibleFile string + inaccessibleSymlink string + ) + + cleanup := setupInaccessibleDir(t, func(dir string) error { + inaccessibleFile = filepath.Join(dir, "file") + if fh, err := os.Create(inaccessibleFile); err != nil { + return err + } else if err = fh.Close(); err != nil { + return err + } + + inaccessibleSymlink = filepath.Join(dir, "symlink") + return os.Symlink(inaccessibleFile, inaccessibleSymlink) + }) + defer cleanup() + + tests := map[string]struct{ expected, err bool }{ + dirPath: {false, false}, + filePath: {false, false}, + dirSymlink: {true, false}, + fileSymlink: {true, false}, + inaccessibleFile: {false, true}, + inaccessibleSymlink: {false, true}, + } + + if runtime.GOOS == "windows" { + // XXX: setting permissions works differently in Windows. Skipping + // these cases until a compatible implementation is provided. + delete(tests, inaccessibleFile) + delete(tests, inaccessibleSymlink) + } + + for path, want := range tests { + got, err := IsSymlink(path) + if err != nil { + if !want.err { + t.Errorf("expected no error, got %v", err) + } + } + + if got != want.expected { + t.Errorf("expected %t for %s, got %t", want.expected, path, got) + } + } +} diff --git a/kubelink/internals/thirdParty/dep/fs/rename.go b/kubelink/internals/thirdParty/dep/fs/rename.go new file mode 100644 index 000000000..a3e5e56a6 --- /dev/null +++ b/kubelink/internals/thirdParty/dep/fs/rename.go @@ -0,0 +1,58 @@ +//go:build !windows + +/* +Copyright (c) for portions of rename.go are held by The Go Authors, 2016 and are provided under +the BSD license. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +package fs + +import ( + "os" + "syscall" + + "github.com/pkg/errors" +) + +// renameFallback attempts to determine the appropriate fallback to failed rename +// operation depending on the resulting error. +func renameFallback(err error, src, dst string) error { + // Rename may fail if src and dst are on different devices; fall back to + // copy if we detect that case. syscall.EXDEV is the common name for the + // cross device link error which has varying output text across different + // operating systems. + terr, ok := err.(*os.LinkError) + if !ok { + return err + } else if terr.Err != syscall.EXDEV { + return errors.Wrapf(terr, "link error: cannot rename %s to %s", src, dst) + } + + return renameByCopy(src, dst) +} diff --git a/kubelink/internals/thirdParty/dep/fs/rename_windows.go b/kubelink/internals/thirdParty/dep/fs/rename_windows.go new file mode 100644 index 000000000..a377720a6 --- /dev/null +++ b/kubelink/internals/thirdParty/dep/fs/rename_windows.go @@ -0,0 +1,69 @@ +//go:build windows + +/* +Copyright (c) for portions of rename_windows.go are held by The Go Authors, 2016 and are provided under +the BSD license. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +package fs + +import ( + "os" + "syscall" + + "github.com/pkg/errors" +) + +// renameFallback attempts to determine the appropriate fallback to failed rename +// operation depending on the resulting error. +func renameFallback(err error, src, dst string) error { + // Rename may fail if src and dst are on different devices; fall back to + // copy if we detect that case. syscall.EXDEV is the common name for the + // cross device link error which has varying output text across different + // operating systems. + terr, ok := err.(*os.LinkError) + if !ok { + return err + } + + if terr.Err != syscall.EXDEV { + // In windows it can drop down to an operating system call that + // returns an operating system error with a different number and + // message. Checking for that as a fall back. + noerr, ok := terr.Err.(syscall.Errno) + + // 0x11 (ERROR_NOT_SAME_DEVICE) is the windows error. + // See https://msdn.microsoft.com/en-us/library/cc231199.aspx + if ok && noerr != 0x11 { + return errors.Wrapf(terr, "link error: cannot rename %s to %s", src, dst) + } + } + + return renameByCopy(src, dst) +} diff --git a/kubelink/internals/thirdParty/dep/fs/testdata/symlinks/file-symlink b/kubelink/internals/thirdParty/dep/fs/testdata/symlinks/file-symlink new file mode 120000 index 000000000..4c52274de --- /dev/null +++ b/kubelink/internals/thirdParty/dep/fs/testdata/symlinks/file-symlink @@ -0,0 +1 @@ +../test.file \ No newline at end of file diff --git a/kubelink/internals/thirdParty/dep/fs/testdata/symlinks/invalid-symlink b/kubelink/internals/thirdParty/dep/fs/testdata/symlinks/invalid-symlink new file mode 120000 index 000000000..0edf4f301 --- /dev/null +++ b/kubelink/internals/thirdParty/dep/fs/testdata/symlinks/invalid-symlink @@ -0,0 +1 @@ +/non/existing/file \ No newline at end of file diff --git a/kubelink/internals/thirdParty/dep/fs/testdata/symlinks/windows-file-symlink b/kubelink/internals/thirdParty/dep/fs/testdata/symlinks/windows-file-symlink new file mode 120000 index 000000000..af1d6c8f5 --- /dev/null +++ b/kubelink/internals/thirdParty/dep/fs/testdata/symlinks/windows-file-symlink @@ -0,0 +1 @@ +C:/Users/ibrahim/go/src/github.com/golang/dep/internal/fs/testdata/test.file \ No newline at end of file diff --git a/kubelink/internals/thirdParty/dep/fs/testdata/test.file b/kubelink/internals/thirdParty/dep/fs/testdata/test.file new file mode 100644 index 000000000..e69de29bb diff --git a/kubelink/internals/urlutil/urlutil.go b/kubelink/internals/urlutil/urlutil.go new file mode 100644 index 000000000..a8cf7398c --- /dev/null +++ b/kubelink/internals/urlutil/urlutil.go @@ -0,0 +1,73 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package urlutil + +import ( + "net/url" + "path" + "path/filepath" +) + +// URLJoin joins a base URL to one or more path components. +// +// It's like filepath.Join for URLs. If the baseURL is pathish, this will still +// perform a join. +// +// If the URL is unparsable, this returns an error. +func URLJoin(baseURL string, paths ...string) (string, error) { + u, err := url.Parse(baseURL) + if err != nil { + return "", err + } + // We want path instead of filepath because path always uses /. + all := []string{u.Path} + all = append(all, paths...) + u.Path = path.Join(all...) + return u.String(), nil +} + +// Equal normalizes two URLs and then compares for equality. +func Equal(a, b string) bool { + au, err := url.Parse(a) + if err != nil { + a = filepath.Clean(a) + b = filepath.Clean(b) + // If urls are paths, return true only if they are an exact match + return a == b + } + bu, err := url.Parse(b) + if err != nil { + return false + } + + for _, u := range []*url.URL{au, bu} { + if u.Path == "" { + u.Path = "/" + } + u.Path = filepath.Clean(u.Path) + } + return au.String() == bu.String() +} + +// ExtractHostname returns hostname from URL +func ExtractHostname(addr string) (string, error) { + u, err := url.Parse(addr) + if err != nil { + return "", err + } + return u.Hostname(), nil +} diff --git a/kubelink/internals/urlutil/urlutil_test.go b/kubelink/internals/urlutil/urlutil_test.go new file mode 100644 index 000000000..82acc40fe --- /dev/null +++ b/kubelink/internals/urlutil/urlutil_test.go @@ -0,0 +1,81 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package urlutil + +import "testing" + +func TestURLJoin(t *testing.T) { + tests := []struct { + name, url, expect string + paths []string + }{ + {name: "URL, one path", url: "http://example.com", paths: []string{"hello"}, expect: "http://example.com/hello"}, + {name: "Long URL, one path", url: "http://example.com/but/first", paths: []string{"slurm"}, expect: "http://example.com/but/first/slurm"}, + {name: "URL, two paths", url: "http://example.com", paths: []string{"hello", "world"}, expect: "http://example.com/hello/world"}, + {name: "URL, no paths", url: "http://example.com", paths: []string{}, expect: "http://example.com"}, + {name: "basepath, two paths", url: "../example.com", paths: []string{"hello", "world"}, expect: "../example.com/hello/world"}, + } + + for _, tt := range tests { + if got, err := URLJoin(tt.url, tt.paths...); err != nil { + t.Errorf("%s: error %q", tt.name, err) + } else if got != tt.expect { + t.Errorf("%s: expected %q, got %q", tt.name, tt.expect, got) + } + } +} + +func TestEqual(t *testing.T) { + for _, tt := range []struct { + a, b string + match bool + }{ + {"http://example.com", "http://example.com", true}, + {"http://example.com", "http://another.example.com", false}, + {"https://example.com", "https://example.com", true}, + {"http://example.com/", "http://example.com", true}, + {"https://example.com", "http://example.com", false}, + {"http://example.com/foo", "http://example.com/foo/", true}, + {"http://example.com/foo//", "http://example.com/foo/", true}, + {"http://example.com/./foo/", "http://example.com/foo/", true}, + {"http://example.com/bar/../foo/", "http://example.com/foo/", true}, + {"/foo", "/foo", true}, + {"/foo", "/foo/", true}, + {"/foo/.", "/foo/", true}, + {"%/1234", "%/1234", true}, + {"%/1234", "%/123", false}, + {"/1234", "%/1234", false}, + } { + if tt.match != Equal(tt.a, tt.b) { + t.Errorf("Expected %q==%q to be %t", tt.a, tt.b, tt.match) + } + } +} + +func TestExtractHostname(t *testing.T) { + tests := map[string]string{ + "http://example.com": "example.com", + "https://example.com/foo": "example.com", + + "https://example.com:31337/not/with/a/bang/but/a/whimper": "example.com", + } + for start, expect := range tests { + if got, _ := ExtractHostname(start); got != expect { + t.Errorf("Got %q, expected %q", got, expect) + } + } +} diff --git a/kubelink/pkg/downloader/chart_downloader.go b/kubelink/pkg/downloader/chart_downloader.go new file mode 100644 index 000000000..dde6a1057 --- /dev/null +++ b/kubelink/pkg/downloader/chart_downloader.go @@ -0,0 +1,408 @@ +/* +Copyright The Helm Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package downloader + +import ( + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/Masterminds/semver/v3" + "github.com/pkg/errors" + + "helm.sh/helm/v3/internal/fileutil" + "helm.sh/helm/v3/internal/urlutil" + "helm.sh/helm/v3/pkg/getter" + "helm.sh/helm/v3/pkg/helmpath" + "helm.sh/helm/v3/pkg/provenance" + "helm.sh/helm/v3/pkg/registry" + "helm.sh/helm/v3/pkg/repo" +) + +// VerificationStrategy describes a strategy for determining whether to verify a chart. +type VerificationStrategy int + +const ( + // VerifyNever will skip all verification of a chart. + VerifyNever VerificationStrategy = iota + // VerifyIfPossible will attempt a verification, it will not error if verification + // data is missing. But it will not stop processing if verification fails. + VerifyIfPossible + // VerifyAlways will always attempt a verification, and will fail if the + // verification fails. + VerifyAlways + // VerifyLater will fetch verification data, but not do any verification. + // This is to accommodate the case where another step of the process will + // perform verification. + VerifyLater +) + +// ErrNoOwnerRepo indicates that a given chart URL can't be found in any repos. +var ErrNoOwnerRepo = errors.New("could not find a repo containing the given URL") + +// ChartDownloader handles downloading a chart. +// +// It is capable of performing verifications on charts as well. +type ChartDownloader struct { + // Out is the location to write warning and info messages. + Out io.Writer + // Verify indicates what verification strategy to use. + Verify VerificationStrategy + // Keyring is the keyring file used for verification. + Keyring string + // Getter collection for the operation + Getters getter.Providers + // Options provide parameters to be passed along to the Getter being initialized. + Options []getter.Option + RegistryClient *registry.Client + RepositoryConfig string + RepositoryCache string +} + +// DownloadTo retrieves a chart. Depending on the settings, it may also download a provenance file. +// +// If Verify is set to VerifyNever, the verification will be nil. +// If Verify is set to VerifyIfPossible, this will return a verification (or nil on failure), and print a warning on failure. +// If Verify is set to VerifyAlways, this will return a verification or an error if the verification fails. +// If Verify is set to VerifyLater, this will download the prov file (if it exists), but not verify it. +// +// For VerifyNever and VerifyIfPossible, the Verification may be empty. +// +// Returns a string path to the location where the file was downloaded and a verification +// (if provenance was verified), or an error if something bad happened. +func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *provenance.Verification, error) { + u, err := c.ResolveChartVersion(ref, version) + if err != nil { + return "", nil, err + } + + g, err := c.Getters.ByScheme(u.Scheme) + if err != nil { + return "", nil, err + } + + c.Options = append(c.Options, getter.WithAcceptHeader("application/gzip,application/octet-stream")) + + data, err := g.Get(u.String(), c.Options...) + if err != nil { + return "", nil, err + } + + name := filepath.Base(u.Path) + if u.Scheme == registry.OCIScheme { + idx := strings.LastIndexByte(name, ':') + name = fmt.Sprintf("%s-%s.tgz", name[:idx], name[idx+1:]) + } + + destfile := filepath.Join(dest, name) + if err := fileutil.AtomicWriteFile(destfile, data, 0644); err != nil { + return destfile, nil, err + } + + // If provenance is requested, verify it. + ver := &provenance.Verification{} + if c.Verify > VerifyNever { + body, err := g.Get(u.String() + ".prov") + if err != nil { + if c.Verify == VerifyAlways { + return destfile, ver, errors.Errorf("failed to fetch provenance %q", u.String()+".prov") + } + fmt.Fprintf(c.Out, "WARNING: Verification not found for %s: %s\n", ref, err) + return destfile, ver, nil + } + provfile := destfile + ".prov" + if err := fileutil.AtomicWriteFile(provfile, body, 0644); err != nil { + return destfile, nil, err + } + + if c.Verify != VerifyLater { + ver, err = VerifyChart(destfile, c.Keyring) + if err != nil { + // Fail always in this case, since it means the verification step + // failed. + return destfile, ver, err + } + } + } + return destfile, ver, nil +} + +func (c *ChartDownloader) getOciURI(ref, version string, u *url.URL) (*url.URL, error) { + var tag string + var err error + + // Evaluate whether an explicit version has been provided. Otherwise, determine version to use + _, errSemVer := semver.NewVersion(version) + if errSemVer == nil { + tag = version + } else { + // Retrieve list of repository tags + tags, err := c.RegistryClient.Tags(strings.TrimPrefix(ref, fmt.Sprintf("%s://", registry.OCIScheme))) + if err != nil { + return nil, err + } + if len(tags) == 0 { + return nil, errors.Errorf("Unable to locate any tags in provided repository: %s", ref) + } + + // Determine if version provided + // If empty, try to get the highest available tag + // If exact version, try to find it + // If semver constraint string, try to find a match + tag, err = registry.GetTagMatchingVersionOrConstraint(tags, version) + if err != nil { + return nil, err + } + } + + u.Path = fmt.Sprintf("%s:%s", u.Path, tag) + + return u, err +} + +// ResolveChartVersion resolves a chart reference to a URL. +// +// It returns the URL and sets the ChartDownloader's Options that can fetch +// the URL using the appropriate Getter. +// +// A reference may be an HTTP URL, an oci reference URL, a 'reponame/chartname' +// reference, or a local path. +// +// A version is a SemVer string (1.2.3-beta.1+f334a6789). +// +// - For fully qualified URLs, the version will be ignored (since URLs aren't versioned) +// - For a chart reference +// - If version is non-empty, this will return the URL for that version +// - If version is empty, this will return the URL for the latest version +// - If no version can be found, an error is returned +func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, error) { + u, err := url.Parse(ref) + if err != nil { + return nil, errors.Errorf("invalid chart URL format: %s", ref) + } + + if registry.IsOCI(u.String()) { + return c.getOciURI(ref, version, u) + } + + rf, err := loadRepoConfig(c.RepositoryConfig) + if err != nil { + return u, err + } + + if u.IsAbs() && len(u.Host) > 0 && len(u.Path) > 0 { + // In this case, we have to find the parent repo that contains this chart + // URL. And this is an unfortunate problem, as it requires actually going + // through each repo cache file and finding a matching URL. But basically + // we want to find the repo in case we have special SSL cert config + // for that repo. + + rc, err := c.scanReposForURL(ref, rf) + if err != nil { + // If there is no special config, return the default HTTP client and + // swallow the error. + if err == ErrNoOwnerRepo { + // Make sure to add the ref URL as the URL for the getter + c.Options = append(c.Options, getter.WithURL(ref)) + return u, nil + } + return u, err + } + + // If we get here, we don't need to go through the next phase of looking + // up the URL. We have it already. So we just set the parameters and return. + c.Options = append( + c.Options, + getter.WithURL(rc.URL), + ) + if rc.CertFile != "" || rc.KeyFile != "" || rc.CAFile != "" { + c.Options = append(c.Options, getter.WithTLSClientConfig(rc.CertFile, rc.KeyFile, rc.CAFile)) + } + if rc.Username != "" && rc.Password != "" { + c.Options = append( + c.Options, + getter.WithBasicAuth(rc.Username, rc.Password), + getter.WithPassCredentialsAll(rc.PassCredentialsAll), + ) + } + return u, nil + } + + // See if it's of the form: repo/path_to_chart + p := strings.SplitN(u.Path, "/", 2) + if len(p) < 2 { + return u, errors.Errorf("non-absolute URLs should be in form of repo_name/path_to_chart, got: %s", u) + } + + repoName := p[0] + chartName := p[1] + rc, err := pickChartRepositoryConfigByName(repoName, rf.Repositories) + + if err != nil { + return u, err + } + + // Now that we have the chart repository information we can use that URL + // to set the URL for the getter. + c.Options = append(c.Options, getter.WithURL(rc.URL)) + + r, err := repo.NewChartRepository(rc, c.Getters) + if err != nil { + return u, err + } + + if r != nil && r.Config != nil { + if r.Config.CertFile != "" || r.Config.KeyFile != "" || r.Config.CAFile != "" { + c.Options = append(c.Options, getter.WithTLSClientConfig(r.Config.CertFile, r.Config.KeyFile, r.Config.CAFile)) + } + if r.Config.Username != "" && r.Config.Password != "" { + c.Options = append(c.Options, + getter.WithBasicAuth(r.Config.Username, r.Config.Password), + getter.WithPassCredentialsAll(r.Config.PassCredentialsAll), + ) + } + } + + // Next, we need to load the index, and actually look up the chart. + idxFile := filepath.Join(c.RepositoryCache, helmpath.CacheIndexFile(r.Config.Name)) + i, err := repo.LoadIndexFile(idxFile) + if err != nil { + return u, errors.Wrap(err, "no cached repo found. (try 'helm repo update')") + } + + cv, err := i.Get(chartName, version) + if err != nil { + return u, errors.Wrapf(err, "chart %q matching %s not found in %s index. (try 'helm repo update')", chartName, version, r.Config.Name) + } + + if len(cv.URLs) == 0 { + return u, errors.Errorf("chart %q has no downloadable URLs", ref) + } + + // TODO: Seems that picking first URL is not fully correct + resolvedURL, err := repo.ResolveReferenceURL(rc.URL, cv.URLs[0]) + + if err != nil { + return u, errors.Errorf("invalid chart URL format: %s", ref) + } + + return url.Parse(resolvedURL) +} + +// VerifyChart takes a path to a chart archive and a keyring, and verifies the chart. +// +// It assumes that a chart archive file is accompanied by a provenance file whose +// name is the archive file name plus the ".prov" extension. +func VerifyChart(path, keyring string) (*provenance.Verification, error) { + // For now, error out if it's not a tar file. + switch fi, err := os.Stat(path); { + case err != nil: + return nil, err + case fi.IsDir(): + return nil, errors.New("unpacked charts cannot be verified") + case !isTar(path): + return nil, errors.New("chart must be a tgz file") + } + + provfile := path + ".prov" + if _, err := os.Stat(provfile); err != nil { + return nil, errors.Wrapf(err, "could not load provenance file %s", provfile) + } + + sig, err := provenance.NewFromKeyring(keyring, "") + if err != nil { + return nil, errors.Wrap(err, "failed to load keyring") + } + return sig.Verify(path, provfile) +} + +// isTar tests whether the given file is a tar file. +// +// Currently, this simply checks extension, since a subsequent function will +// untar the file and validate its binary format. +func isTar(filename string) bool { + return strings.EqualFold(filepath.Ext(filename), ".tgz") +} + +func pickChartRepositoryConfigByName(name string, cfgs []*repo.Entry) (*repo.Entry, error) { + for _, rc := range cfgs { + if rc.Name == name { + if rc.URL == "" { + return nil, errors.Errorf("no URL found for repository %s", name) + } + return rc, nil + } + } + return nil, errors.Errorf("repo %s not found", name) +} + +// scanReposForURL scans all repos to find which repo contains the given URL. +// +// This will attempt to find the given URL in all of the known repositories files. +// +// If the URL is found, this will return the repo entry that contained that URL. +// +// If all of the repos are checked, but the URL is not found, an ErrNoOwnerRepo +// error is returned. +// +// Other errors may be returned when repositories cannot be loaded or searched. +// +// Technically, the fact that a URL is not found in a repo is not a failure indication. +// Charts are not required to be included in an index before they are valid. So +// be mindful of this case. +// +// The same URL can technically exist in two or more repositories. This algorithm +// will return the first one it finds. Order is determined by the order of repositories +// in the repositories.yaml file. +func (c *ChartDownloader) scanReposForURL(u string, rf *repo.File) (*repo.Entry, error) { + // FIXME: This is far from optimal. Larger installations and index files will + // incur a performance hit for this type of scanning. + for _, rc := range rf.Repositories { + r, err := repo.NewChartRepository(rc, c.Getters) + if err != nil { + return nil, err + } + + idxFile := filepath.Join(c.RepositoryCache, helmpath.CacheIndexFile(r.Config.Name)) + i, err := repo.LoadIndexFile(idxFile) + if err != nil { + return nil, errors.Wrap(err, "no cached repo found. (try 'helm repo update')") + } + + for _, entry := range i.Entries { + for _, ver := range entry { + for _, dl := range ver.URLs { + if urlutil.Equal(u, dl) { + return rc, nil + } + } + } + } + } + // This means that there is no repo file for the given URL. + return nil, ErrNoOwnerRepo +} + +func loadRepoConfig(file string) (*repo.File, error) { + r, err := repo.LoadFile(file) + if err != nil && !os.IsNotExist(errors.Cause(err)) { + return nil, err + } + return r, nil +} diff --git a/kubelink/pkg/downloader/chart_downloader_test.go b/kubelink/pkg/downloader/chart_downloader_test.go new file mode 100644 index 000000000..131e21306 --- /dev/null +++ b/kubelink/pkg/downloader/chart_downloader_test.go @@ -0,0 +1,346 @@ +/* +Copyright The Helm Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package downloader + +import ( + "os" + "path/filepath" + "testing" + + "helm.sh/helm/v3/internal/test/ensure" + "helm.sh/helm/v3/pkg/cli" + "helm.sh/helm/v3/pkg/getter" + "helm.sh/helm/v3/pkg/repo" + "helm.sh/helm/v3/pkg/repo/repotest" +) + +const ( + repoConfig = "testdata/repositories.yaml" + repoCache = "testdata/repository" +) + +func TestResolveChartRef(t *testing.T) { + tests := []struct { + name, ref, expect, version string + fail bool + }{ + {name: "full URL", ref: "http://example.com/foo-1.2.3.tgz", expect: "http://example.com/foo-1.2.3.tgz"}, + {name: "full URL, HTTPS", ref: "https://example.com/foo-1.2.3.tgz", expect: "https://example.com/foo-1.2.3.tgz"}, + {name: "full URL, with authentication", ref: "http://username:password@example.com/foo-1.2.3.tgz", expect: "http://username:password@example.com/foo-1.2.3.tgz"}, + {name: "reference, testing repo", ref: "testing/alpine", expect: "http://example.com/alpine-1.2.3.tgz"}, + {name: "reference, version, testing repo", ref: "testing/alpine", version: "0.2.0", expect: "http://example.com/alpine-0.2.0.tgz"}, + {name: "reference, version, malformed repo", ref: "malformed/alpine", version: "1.2.3", expect: "http://dl.example.com/alpine-1.2.3.tgz"}, + {name: "reference, querystring repo", ref: "testing-querystring/alpine", expect: "http://example.com/alpine-1.2.3.tgz?key=value"}, + {name: "reference, testing-relative repo", ref: "testing-relative/foo", expect: "http://example.com/helm/charts/foo-1.2.3.tgz"}, + {name: "reference, testing-relative repo", ref: "testing-relative/bar", expect: "http://example.com/helm/bar-1.2.3.tgz"}, + {name: "reference, testing-relative-trailing-slash repo", ref: "testing-relative-trailing-slash/foo", expect: "http://example.com/helm/charts/foo-1.2.3.tgz"}, + {name: "reference, testing-relative-trailing-slash repo", ref: "testing-relative-trailing-slash/bar", expect: "http://example.com/helm/bar-1.2.3.tgz"}, + {name: "encoded URL", ref: "encoded-url/foobar", expect: "http://example.com/with%2Fslash/charts/foobar-4.2.1.tgz"}, + {name: "full URL, HTTPS, irrelevant version", ref: "https://example.com/foo-1.2.3.tgz", version: "0.1.0", expect: "https://example.com/foo-1.2.3.tgz", fail: true}, + {name: "full URL, file", ref: "file:///foo-1.2.3.tgz", fail: true}, + {name: "invalid", ref: "invalid-1.2.3", fail: true}, + {name: "not found", ref: "nosuchthing/invalid-1.2.3", fail: true}, + } + + c := ChartDownloader{ + Out: os.Stderr, + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + Getters: getter.All(&cli.EnvSettings{ + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + }), + } + + for _, tt := range tests { + u, err := c.ResolveChartVersion(tt.ref, tt.version) + if err != nil { + if tt.fail { + continue + } + t.Errorf("%s: failed with error %q", tt.name, err) + continue + } + if got := u.String(); got != tt.expect { + t.Errorf("%s: expected %s, got %s", tt.name, tt.expect, got) + } + } +} + +func TestResolveChartOpts(t *testing.T) { + tests := []struct { + name, ref, version string + expect []getter.Option + }{ + { + name: "repo with CA-file", + ref: "testing-ca-file/foo", + expect: []getter.Option{ + getter.WithURL("https://example.com/foo-1.2.3.tgz"), + getter.WithTLSClientConfig("cert", "key", "ca"), + }, + }, + } + + c := ChartDownloader{ + Out: os.Stderr, + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + Getters: getter.All(&cli.EnvSettings{ + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + }), + } + + // snapshot options + snapshotOpts := c.Options + + for _, tt := range tests { + // reset chart downloader options for each test case + c.Options = snapshotOpts + + expect, err := getter.NewHTTPGetter(tt.expect...) + if err != nil { + t.Errorf("%s: failed to setup http client: %s", tt.name, err) + continue + } + + u, err := c.ResolveChartVersion(tt.ref, tt.version) + if err != nil { + t.Errorf("%s: failed with error %s", tt.name, err) + continue + } + + got, err := getter.NewHTTPGetter( + append( + c.Options, + getter.WithURL(u.String()), + )..., + ) + if err != nil { + t.Errorf("%s: failed to create http client: %s", tt.name, err) + continue + } + + if *(got.(*getter.HTTPGetter)) != *(expect.(*getter.HTTPGetter)) { + t.Errorf("%s: expected %s, got %s", tt.name, expect, got) + } + } +} + +func TestVerifyChart(t *testing.T) { + v, err := VerifyChart("testdata/signtest-0.1.0.tgz", "testdata/helm-test-key.pub") + if err != nil { + t.Fatal(err) + } + // The verification is tested at length in the provenance package. Here, + // we just want a quick sanity check that the v is not empty. + if len(v.FileHash) == 0 { + t.Error("Digest missing") + } +} + +func TestIsTar(t *testing.T) { + tests := map[string]bool{ + "foo.tgz": true, + "foo/bar/baz.tgz": true, + "foo-1.2.3.4.5.tgz": true, + "foo.tar.gz": false, // for our purposes + "foo.tgz.1": false, + "footgz": false, + } + + for src, expect := range tests { + if isTar(src) != expect { + t.Errorf("%q should be %t", src, expect) + } + } +} + +func TestDownloadTo(t *testing.T) { + srv := repotest.NewTempServerWithCleanupAndBasicAuth(t, "testdata/*.tgz*") + defer srv.Stop() + if err := srv.CreateIndex(); err != nil { + t.Fatal(err) + } + + if err := srv.LinkIndices(); err != nil { + t.Fatal(err) + } + + c := ChartDownloader{ + Out: os.Stderr, + Verify: VerifyAlways, + Keyring: "testdata/helm-test-key.pub", + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + Getters: getter.All(&cli.EnvSettings{ + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + }), + Options: []getter.Option{ + getter.WithBasicAuth("username", "password"), + getter.WithPassCredentialsAll(false), + }, + } + cname := "/signtest-0.1.0.tgz" + dest := srv.Root() + where, v, err := c.DownloadTo(srv.URL()+cname, "", dest) + if err != nil { + t.Fatal(err) + } + + if expect := filepath.Join(dest, cname); where != expect { + t.Errorf("Expected download to %s, got %s", expect, where) + } + + if v.FileHash == "" { + t.Error("File hash was empty, but verification is required.") + } + + if _, err := os.Stat(filepath.Join(dest, cname)); err != nil { + t.Error(err) + } +} + +func TestDownloadTo_TLS(t *testing.T) { + // Set up mock server w/ tls enabled + srv, err := repotest.NewTempServerWithCleanup(t, "testdata/*.tgz*") + srv.Stop() + if err != nil { + t.Fatal(err) + } + srv.StartTLS() + defer srv.Stop() + if err := srv.CreateIndex(); err != nil { + t.Fatal(err) + } + if err := srv.LinkIndices(); err != nil { + t.Fatal(err) + } + + repoConfig := filepath.Join(srv.Root(), "repositories.yaml") + repoCache := srv.Root() + + c := ChartDownloader{ + Out: os.Stderr, + Verify: VerifyAlways, + Keyring: "testdata/helm-test-key.pub", + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + Getters: getter.All(&cli.EnvSettings{ + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + }), + Options: []getter.Option{}, + } + cname := "test/signtest" + dest := srv.Root() + where, v, err := c.DownloadTo(cname, "", dest) + if err != nil { + t.Fatal(err) + } + + target := filepath.Join(dest, "signtest-0.1.0.tgz") + if expect := target; where != expect { + t.Errorf("Expected download to %s, got %s", expect, where) + } + + if v.FileHash == "" { + t.Error("File hash was empty, but verification is required.") + } + + if _, err := os.Stat(target); err != nil { + t.Error(err) + } +} + +func TestDownloadTo_VerifyLater(t *testing.T) { + ensure.HelmHome(t) + + dest := t.TempDir() + + // Set up a fake repo + srv, err := repotest.NewTempServerWithCleanup(t, "testdata/*.tgz*") + if err != nil { + t.Fatal(err) + } + defer srv.Stop() + if err := srv.LinkIndices(); err != nil { + t.Fatal(err) + } + + c := ChartDownloader{ + Out: os.Stderr, + Verify: VerifyLater, + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + Getters: getter.All(&cli.EnvSettings{ + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + }), + } + cname := "/signtest-0.1.0.tgz" + where, _, err := c.DownloadTo(srv.URL()+cname, "", dest) + if err != nil { + t.Fatal(err) + } + + if expect := filepath.Join(dest, cname); where != expect { + t.Errorf("Expected download to %s, got %s", expect, where) + } + + if _, err := os.Stat(filepath.Join(dest, cname)); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dest, cname+".prov")); err != nil { + t.Fatal(err) + } +} + +func TestScanReposForURL(t *testing.T) { + c := ChartDownloader{ + Out: os.Stderr, + Verify: VerifyLater, + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + Getters: getter.All(&cli.EnvSettings{ + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + }), + } + + u := "http://example.com/alpine-0.2.0.tgz" + rf, err := repo.LoadFile(repoConfig) + if err != nil { + t.Fatal(err) + } + + entry, err := c.scanReposForURL(u, rf) + if err != nil { + t.Fatal(err) + } + + if entry.Name != "testing" { + t.Errorf("Unexpected repo %q for URL %q", entry.Name, u) + } + + // A lookup failure should produce an ErrNoOwnerRepo + u = "https://no.such.repo/foo/bar-1.23.4.tgz" + if _, err = c.scanReposForURL(u, rf); err != ErrNoOwnerRepo { + t.Fatalf("expected ErrNoOwnerRepo, got %v", err) + } +} diff --git a/kubelink/pkg/downloader/doc.go b/kubelink/pkg/downloader/doc.go new file mode 100644 index 000000000..848468090 --- /dev/null +++ b/kubelink/pkg/downloader/doc.go @@ -0,0 +1,24 @@ +/* +Copyright The Helm Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Package downloader provides a library for downloading charts. + +This package contains various tools for downloading charts from repository +servers, and then storing them in Helm-specific directory structures. This +library contains many functions that depend on a specific +filesystem layout. +*/ +package downloader diff --git a/kubelink/pkg/downloader/manager.go b/kubelink/pkg/downloader/manager.go new file mode 100644 index 000000000..d7c32241f --- /dev/null +++ b/kubelink/pkg/downloader/manager.go @@ -0,0 +1,905 @@ +/* +Copyright The Helm Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package downloader + +import ( + "crypto" + "encoding/hex" + "fmt" + "io" + "log" + "net/url" + "os" + "path" + "path/filepath" + "regexp" + "strings" + "sync" + + "github.com/Masterminds/semver/v3" + "github.com/pkg/errors" + "sigs.k8s.io/yaml" + + "github.com/devtron-labs/kubelink/internals/resolver" + "github.com/devtron-labs/kubelink/internals/thirdParty/dep/fs" + "github.com/devtron-labs/kubelink/internals/urlutil" + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" + "helm.sh/helm/v3/pkg/getter" + "helm.sh/helm/v3/pkg/helmpath" + "helm.sh/helm/v3/pkg/registry" + "helm.sh/helm/v3/pkg/repo" +) + +// ErrRepoNotFound indicates that chart repositories can't be found in local repo cache. +// The value of Repos is missing repos. +type ErrRepoNotFound struct { + Repos []string +} + +// Error implements the error interface. +func (e ErrRepoNotFound) Error() string { + return fmt.Sprintf("no repository definition for %s", strings.Join(e.Repos, ", ")) +} + +// Manager handles the lifecycle of fetching, resolving, and storing dependencies. +type Manager struct { + // Out is used to print warnings and notifications. + Out io.Writer + // ChartPath is the path to the unpacked base chart upon which this operates. + ChartPath string + // Verification indicates whether the chart should be verified. + Verify VerificationStrategy + // Debug is the global "--debug" flag + Debug bool + // Keyring is the key ring file. + Keyring string + // SkipUpdate indicates that the repository should not be updated first. + SkipUpdate bool + // Getter collection for the operation + Getters []getter.Provider + RegistryClient *registry.Client + RepositoryConfig string + RepositoryCache string +} + +// Build rebuilds a local charts directory from a lockfile. +// +// If the lockfile is not present, this will run a Manager.Update() +// +// If SkipUpdate is set, this will not update the repository. +func (m *Manager) Build() error { + c, err := m.loadChartDir() + if err != nil { + return err + } + + // If a lock file is found, run a build from that. Otherwise, just do + // an update. + lock := c.Lock + if lock == nil { + return m.Update() + } + + // Check that all of the repos we're dependent on actually exist. + req := c.Metadata.Dependencies + + // If using apiVersion v1, calculate the hash before resolve repo names + // because resolveRepoNames will change req if req uses repo alias + // and Helm 2 calculate the digest from the original req + // Fix for: https://github.com/helm/helm/issues/7619 + var v2Sum string + if c.Metadata.APIVersion == chart.APIVersionV1 { + v2Sum, err = resolver.HashV2Req(req) + if err != nil { + return errors.New("the lock file (requirements.lock) is out of sync with the dependencies file (requirements.yaml). Please update the dependencies") + } + } + + if _, err := m.resolveRepoNames(req); err != nil { + return err + } + + if sum, err := resolver.HashReq(req, lock.Dependencies); err != nil || sum != lock.Digest { + // If lock digest differs and chart is apiVersion v1, it maybe because the lock was built + // with Helm 2 and therefore should be checked with Helm v2 hash + // Fix for: https://github.com/helm/helm/issues/7233 + if c.Metadata.APIVersion == chart.APIVersionV1 { + log.Println("warning: a valid Helm v3 hash was not found. Checking against Helm v2 hash...") + if v2Sum != lock.Digest { + return errors.New("the lock file (requirements.lock) is out of sync with the dependencies file (requirements.yaml). Please update the dependencies") + } + } else { + return errors.New("the lock file (Chart.lock) is out of sync with the dependencies file (Chart.yaml). Please update the dependencies") + } + } + + // Check that all of the repos we're dependent on actually exist. + if err := m.hasAllRepos(lock.Dependencies); err != nil { + return err + } + + if !m.SkipUpdate { + // For each repo in the file, update the cached copy of that repo + if err := m.UpdateRepositories(); err != nil { + return err + } + } + + // Now we need to fetch every package here into charts/ + return m.downloadAll(lock.Dependencies) +} + +// Update updates a local charts directory. +// +// It first reads the Chart.yaml file, and then attempts to +// negotiate versions based on that. It will download the versions +// from remote chart repositories unless SkipUpdate is true. +func (m *Manager) Update() error { + c, err := m.loadChartDir() + if err != nil { + return err + } + + // If no dependencies are found, we consider this a successful + // completion. + req := c.Metadata.Dependencies + if req == nil { + return nil + } + + // Get the names of the repositories the dependencies need that Helm is + // configured to know about. + repoNames, err := m.resolveRepoNames(req) + if err != nil { + return err + } + + // For the repositories Helm is not configured to know about, ensure Helm + // has some information about them and, when possible, the index files + // locally. + // TODO(mattfarina): Repositories should be explicitly added by end users + // rather than automatic. In Helm v4 require users to add repositories. They + // should have to add them in order to make sure they are aware of the + // repositories and opt-in to any locations, for security. + repoNames, err = m.ensureMissingRepos(repoNames, req) + if err != nil { + return err + } + + // For each of the repositories Helm is configured to know about, update + // the index information locally. + if !m.SkipUpdate { + if err := m.UpdateRepositories(); err != nil { + return err + } + } + + // Now we need to find out which version of a chart best satisfies the + // dependencies in the Chart.yaml + lock, err := m.resolve(req, repoNames) + if err != nil { + return err + } + + // Now we need to fetch every package here into charts/ + if err := m.downloadAll(lock.Dependencies); err != nil { + return err + } + + // downloadAll might overwrite dependency version, recalculate lock digest + newDigest, err := resolver.HashReq(req, lock.Dependencies) + if err != nil { + return err + } + lock.Digest = newDigest + + // If the lock file hasn't changed, don't write a new one. + oldLock := c.Lock + if oldLock != nil && oldLock.Digest == lock.Digest { + return nil + } + + // Finally, we need to write the lockfile. + return writeLock(m.ChartPath, lock, c.Metadata.APIVersion == chart.APIVersionV1) +} + +func (m *Manager) loadChartDir() (*chart.Chart, error) { + if fi, err := os.Stat(m.ChartPath); err != nil { + return nil, errors.Wrapf(err, "could not find %s", m.ChartPath) + } else if !fi.IsDir() { + return nil, errors.New("only unpacked charts can be updated") + } + return loader.LoadDir(m.ChartPath) +} + +// resolve takes a list of dependencies and translates them into an exact version to download. +// +// This returns a lock file, which has all of the dependencies normalized to a specific version. +func (m *Manager) resolve(req []*chart.Dependency, repoNames map[string]string) (*chart.Lock, error) { + res := resolver.New(m.ChartPath, m.RepositoryCache, m.RegistryClient) + return res.Resolve(req, repoNames) +} + +// downloadAll takes a list of dependencies and downloads them into charts/ +// +// It will delete versions of the chart that exist on disk and might cause +// a conflict. +func (m *Manager) downloadAll(deps []*chart.Dependency) error { + repos, err := m.loadChartRepositories() + if err != nil { + return err + } + + destPath := filepath.Join(m.ChartPath, "charts") + tmpPath := filepath.Join(m.ChartPath, fmt.Sprintf("tmpcharts-%d", os.Getpid())) + + // Check if 'charts' directory is not actually a directory. If it does not exist, create it. + if fi, err := os.Stat(destPath); err == nil { + if !fi.IsDir() { + return errors.Errorf("%q is not a directory", destPath) + } + } else if os.IsNotExist(err) { + if err := os.MkdirAll(destPath, 0755); err != nil { + return err + } + } else { + return fmt.Errorf("unable to retrieve file info for '%s': %v", destPath, err) + } + + // Prepare tmpPath + if err := os.MkdirAll(tmpPath, 0755); err != nil { + return err + } + defer os.RemoveAll(tmpPath) + + fmt.Fprintf(m.Out, "Saving %d charts\n", len(deps)) + var saveError error + churls := make(map[string]struct{}) + for _, dep := range deps { + // No repository means the chart is in charts directory + if dep.Repository == "" { + fmt.Fprintf(m.Out, "Dependency %s did not declare a repository. Assuming it exists in the charts directory\n", dep.Name) + // NOTE: we are only validating the local dependency conforms to the constraints. No copying to tmpPath is necessary. + chartPath := filepath.Join(destPath, dep.Name) + ch, err := loader.LoadDir(chartPath) + if err != nil { + return fmt.Errorf("unable to load chart '%s': %v", chartPath, err) + } + + constraint, err := semver.NewConstraint(dep.Version) + if err != nil { + return fmt.Errorf("dependency %s has an invalid version/constraint format: %s", dep.Name, err) + } + + v, err := semver.NewVersion(ch.Metadata.Version) + if err != nil { + return fmt.Errorf("invalid version %s for dependency %s: %s", dep.Version, dep.Name, err) + } + + if !constraint.Check(v) { + saveError = fmt.Errorf("dependency %s at version %s does not satisfy the constraint %s", dep.Name, ch.Metadata.Version, dep.Version) + break + } + continue + } + if strings.HasPrefix(dep.Repository, "file://") { + if m.Debug { + fmt.Fprintf(m.Out, "Archiving %s from repo %s\n", dep.Name, dep.Repository) + } + ver, err := tarFromLocalDir(m.ChartPath, dep.Name, dep.Repository, dep.Version, tmpPath) + if err != nil { + saveError = err + break + } + dep.Version = ver + continue + } + + // Any failure to resolve/download a chart should fail: + // https://github.com/helm/helm/issues/1439 + churl, username, password, insecureskiptlsverify, passcredentialsall, caFile, certFile, keyFile, err := m.findChartURL(dep.Name, dep.Version, dep.Repository, repos) + if err != nil { + saveError = errors.Wrapf(err, "could not find %s", churl) + break + } + + if _, ok := churls[churl]; ok { + fmt.Fprintf(m.Out, "Already downloaded %s from repo %s\n", dep.Name, dep.Repository) + continue + } + + fmt.Fprintf(m.Out, "Downloading %s from repo %s\n", dep.Name, dep.Repository) + + dl := ChartDownloader{ + Out: m.Out, + Verify: m.Verify, + Keyring: m.Keyring, + RepositoryConfig: m.RepositoryConfig, + RepositoryCache: m.RepositoryCache, + RegistryClient: m.RegistryClient, + Getters: m.Getters, + Options: []getter.Option{ + getter.WithBasicAuth(username, password), + getter.WithPassCredentialsAll(passcredentialsall), + getter.WithInsecureSkipVerifyTLS(insecureskiptlsverify), + getter.WithTLSClientConfig(certFile, keyFile, caFile), + }, + } + + version := "" + if registry.IsOCI(churl) { + churl, version, err = parseOCIRef(churl) + if err != nil { + return errors.Wrapf(err, "could not parse OCI reference") + } + dl.Options = append(dl.Options, + getter.WithRegistryClient(m.RegistryClient), + getter.WithTagName(version)) + } + + if _, _, err = dl.DownloadTo(churl, version, tmpPath); err != nil { + saveError = errors.Wrapf(err, "could not download %s", churl) + break + } + + churls[churl] = struct{}{} + } + + // TODO: this should probably be refactored to be a []error, so we can capture and provide more information rather than "last error wins". + if saveError == nil { + // now we can move all downloaded charts to destPath and delete outdated dependencies + if err := m.safeMoveDeps(deps, tmpPath, destPath); err != nil { + return err + } + } else { + fmt.Fprintln(m.Out, "Save error occurred: ", saveError) + return saveError + } + return nil +} + +func parseOCIRef(chartRef string) (string, string, error) { + refTagRegexp := regexp.MustCompile(`^(oci://[^:]+(:[0-9]{1,5})?[^:]+):(.*)$`) + caps := refTagRegexp.FindStringSubmatch(chartRef) + if len(caps) != 4 { + return "", "", errors.Errorf("improperly formatted oci chart reference: %s", chartRef) + } + chartRef = caps[1] + tag := caps[3] + + return chartRef, tag, nil +} + +// safeMoveDep moves all dependencies in the source and moves them into dest. +// +// It does this by first matching the file name to an expected pattern, then loading +// the file to verify that it is a chart. +// +// Any charts in dest that do not exist in source are removed (barring local dependencies) +// +// Because it requires tar file introspection, it is more intensive than a basic move. +// +// This will only return errors that should stop processing entirely. Other errors +// will emit log messages or be ignored. +func (m *Manager) safeMoveDeps(deps []*chart.Dependency, source, dest string) error { + existsInSourceDirectory := map[string]bool{} + isLocalDependency := map[string]bool{} + sourceFiles, err := os.ReadDir(source) + if err != nil { + return err + } + // attempt to read destFiles; fail fast if we can't + destFiles, err := os.ReadDir(dest) + if err != nil { + return err + } + + for _, dep := range deps { + if dep.Repository == "" { + isLocalDependency[dep.Name] = true + } + } + + for _, file := range sourceFiles { + if file.IsDir() { + continue + } + filename := file.Name() + sourcefile := filepath.Join(source, filename) + destfile := filepath.Join(dest, filename) + existsInSourceDirectory[filename] = true + if _, err := loader.LoadFile(sourcefile); err != nil { + fmt.Fprintf(m.Out, "Could not verify %s for moving: %s (Skipping)", sourcefile, err) + continue + } + // NOTE: no need to delete the dest; os.Rename replaces it. + if err := fs.RenameWithFallback(sourcefile, destfile); err != nil { + fmt.Fprintf(m.Out, "Unable to move %s to charts dir %s (Skipping)", sourcefile, err) + continue + } + } + + fmt.Fprintln(m.Out, "Deleting outdated charts") + // find all files that exist in dest that do not exist in source; delete them (outdated dependencies) + for _, file := range destFiles { + if !file.IsDir() && !existsInSourceDirectory[file.Name()] { + fname := filepath.Join(dest, file.Name()) + ch, err := loader.LoadFile(fname) + if err != nil { + fmt.Fprintf(m.Out, "Could not verify %s for deletion: %s (Skipping)\n", fname, err) + continue + } + // local dependency - skip + if isLocalDependency[ch.Name()] { + continue + } + if err := os.Remove(fname); err != nil { + fmt.Fprintf(m.Out, "Could not delete %s: %s (Skipping)", fname, err) + continue + } + } + } + + return nil +} + +// hasAllRepos ensures that all of the referenced deps are in the local repo cache. +func (m *Manager) hasAllRepos(deps []*chart.Dependency) error { + rf, err := loadRepoConfig(m.RepositoryConfig) + if err != nil { + return err + } + repos := rf.Repositories + + // Verify that all repositories referenced in the deps are actually known + // by Helm. + missing := []string{} +Loop: + for _, dd := range deps { + // If repo is from local path or OCI, continue + if strings.HasPrefix(dd.Repository, "file://") || registry.IsOCI(dd.Repository) { + continue + } + + if dd.Repository == "" { + continue + } + for _, repo := range repos { + if urlutil.Equal(repo.URL, strings.TrimSuffix(dd.Repository, "/")) { + continue Loop + } + } + missing = append(missing, dd.Repository) + } + if len(missing) > 0 { + return ErrRepoNotFound{missing} + } + return nil +} + +// ensureMissingRepos attempts to ensure the repository information for repos +// not managed by Helm is present. This takes in the repoNames Helm is configured +// to work with along with the chart dependencies. It will find the deps not +// in a known repo and attempt to ensure the data is present for steps like +// version resolution. +func (m *Manager) ensureMissingRepos(repoNames map[string]string, deps []*chart.Dependency) (map[string]string, error) { + + var ru []*repo.Entry + + for _, dd := range deps { + + // If the chart is in the local charts directory no repository needs + // to be specified. + if dd.Repository == "" { + continue + } + + // When the repoName for a dependency is known we can skip ensuring + if _, ok := repoNames[dd.Name]; ok { + continue + } + + // The generated repository name, which will result in an index being + // locally cached, has a name pattern of "helm-manager-" followed by a + // sha256 of the repo name. This assumes end users will never create + // repositories with these names pointing to other repositories. Using + // this method of naming allows the existing repository pulling and + // resolution code to do most of the work. + rn, err := key(dd.Repository) + if err != nil { + return repoNames, err + } + rn = managerKeyPrefix + rn + + repoNames[dd.Name] = rn + + // Assuming the repository is generally available. For Helm managed + // access controls the repository needs to be added through the user + // managed system. This path will work for public charts, like those + // supplied by Bitnami, but not for protected charts, like corp ones + // behind a username and pass. + ri := &repo.Entry{ + Name: rn, + URL: dd.Repository, + } + ru = append(ru, ri) + } + + // Calls to UpdateRepositories (a public function) will only update + // repositories configured by the user. Here we update repos found in + // the dependencies that are not known to the user if update skipping + // is not configured. + if !m.SkipUpdate && len(ru) > 0 { + fmt.Fprintln(m.Out, "Getting updates for unmanaged Helm repositories...") + if err := m.parallelRepoUpdate(ru); err != nil { + return repoNames, err + } + } + + return repoNames, nil +} + +// resolveRepoNames returns the repo names of the referenced deps which can be used to fetch the cached index file +// and replaces aliased repository URLs into resolved URLs in dependencies. +func (m *Manager) resolveRepoNames(deps []*chart.Dependency) (map[string]string, error) { + rf, err := loadRepoConfig(m.RepositoryConfig) + if err != nil { + if os.IsNotExist(err) { + return make(map[string]string), nil + } + return nil, err + } + repos := rf.Repositories + + reposMap := make(map[string]string) + + // Verify that all repositories referenced in the deps are actually known + // by Helm. + missing := []string{} + for _, dd := range deps { + // Don't map the repository, we don't need to download chart from charts directory + if dd.Repository == "" { + continue + } + // if dep chart is from local path, verify the path is valid + if strings.HasPrefix(dd.Repository, "file://") { + if _, err := resolver.GetLocalPath(dd.Repository, m.ChartPath); err != nil { + return nil, err + } + + if m.Debug { + fmt.Fprintf(m.Out, "Repository from local path: %s\n", dd.Repository) + } + reposMap[dd.Name] = dd.Repository + continue + } + + if registry.IsOCI(dd.Repository) { + reposMap[dd.Name] = dd.Repository + continue + } + + found := false + + for _, repo := range repos { + if (strings.HasPrefix(dd.Repository, "@") && strings.TrimPrefix(dd.Repository, "@") == repo.Name) || + (strings.HasPrefix(dd.Repository, "alias:") && strings.TrimPrefix(dd.Repository, "alias:") == repo.Name) { + found = true + dd.Repository = repo.URL + reposMap[dd.Name] = repo.Name + break + } else if urlutil.Equal(repo.URL, dd.Repository) { + found = true + reposMap[dd.Name] = repo.Name + break + } + } + if !found { + repository := dd.Repository + // Add if URL + _, err := url.ParseRequestURI(repository) + if err == nil { + reposMap[repository] = repository + continue + } + missing = append(missing, repository) + } + } + if len(missing) > 0 { + errorMessage := fmt.Sprintf("no repository definition for %s. Please add them via 'helm repo add'", strings.Join(missing, ", ")) + // It is common for people to try to enter "stable" as a repository instead of the actual URL. + // For this case, let's give them a suggestion. + containsNonURL := false + for _, repo := range missing { + if !strings.Contains(repo, "//") && !strings.HasPrefix(repo, "@") && !strings.HasPrefix(repo, "alias:") { + containsNonURL = true + } + } + if containsNonURL { + errorMessage += ` +Note that repositories must be URLs or aliases. For example, to refer to the "example" +repository, use "https://charts.example.com/" or "@example" instead of +"example". Don't forget to add the repo, too ('helm repo add').` + } + return nil, errors.New(errorMessage) + } + return reposMap, nil +} + +// UpdateRepositories updates all of the local repos to the latest. +func (m *Manager) UpdateRepositories() error { + rf, err := loadRepoConfig(m.RepositoryConfig) + if err != nil { + return err + } + repos := rf.Repositories + if len(repos) > 0 { + fmt.Fprintln(m.Out, "Hang tight while we grab the latest from your chart repositories...") + // This prints warnings straight to out. + if err := m.parallelRepoUpdate(repos); err != nil { + return err + } + fmt.Fprintln(m.Out, "Update Complete. ⎈Happy Helming!⎈") + } + return nil +} + +func (m *Manager) parallelRepoUpdate(repos []*repo.Entry) error { + + var wg sync.WaitGroup + for _, c := range repos { + r, err := repo.NewChartRepository(c, m.Getters) + if err != nil { + return err + } + r.CachePath = m.RepositoryCache + wg.Add(1) + go func(r *repo.ChartRepository) { + if _, err := r.DownloadIndexFile(); err != nil { + // For those dependencies that are not known to helm and using a + // generated key name we display the repo url. + if strings.HasPrefix(r.Config.Name, managerKeyPrefix) { + fmt.Fprintf(m.Out, "...Unable to get an update from the %q chart repository:\n\t%s\n", r.Config.URL, err) + } else { + fmt.Fprintf(m.Out, "...Unable to get an update from the %q chart repository (%s):\n\t%s\n", r.Config.Name, r.Config.URL, err) + } + } else { + // For those dependencies that are not known to helm and using a + // generated key name we display the repo url. + if strings.HasPrefix(r.Config.Name, managerKeyPrefix) { + fmt.Fprintf(m.Out, "...Successfully got an update from the %q chart repository\n", r.Config.URL) + } else { + fmt.Fprintf(m.Out, "...Successfully got an update from the %q chart repository\n", r.Config.Name) + } + } + wg.Done() + }(r) + } + wg.Wait() + + return nil +} + +// findChartURL searches the cache of repo data for a chart that has the name and the repoURL specified. +// +// 'name' is the name of the chart. Version is an exact semver, or an empty string. If empty, the +// newest version will be returned. +// +// repoURL is the repository to search +// +// If it finds a URL that is "relative", it will prepend the repoURL. +func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository) (url, username, password string, insecureskiptlsverify, passcredentialsall bool, caFile, certFile, keyFile string, err error) { + if registry.IsOCI(repoURL) { + return fmt.Sprintf("%s/%s:%s", repoURL, name, version), "", "", false, false, "", "", "", nil + } + + for _, cr := range repos { + + if urlutil.Equal(repoURL, cr.Config.URL) { + var entry repo.ChartVersions + entry, err = findEntryByName(name, cr) + if err != nil { + // TODO: Where linting is skipped in this function we should + // refactor to remove naked returns while ensuring the same + // behavior + //nolint:nakedret + return + } + var ve *repo.ChartVersion + ve, err = findVersionedEntry(version, entry) + if err != nil { + //nolint:nakedret + return + } + url, err = normalizeURL(repoURL, ve.URLs[0]) + if err != nil { + //nolint:nakedret + return + } + username = cr.Config.Username + password = cr.Config.Password + passcredentialsall = cr.Config.PassCredentialsAll + insecureskiptlsverify = cr.Config.InsecureSkipTLSverify + caFile = cr.Config.CAFile + certFile = cr.Config.CertFile + keyFile = cr.Config.KeyFile + //nolint:nakedret + return + } + } + url, err = repo.FindChartInRepoURL(repoURL, name, version, certFile, keyFile, caFile, m.Getters) + if err == nil { + return url, username, password, false, false, "", "", "", err + } + err = errors.Errorf("chart %s not found in %s: %s", name, repoURL, err) + return url, username, password, false, false, "", "", "", err +} + +// findEntryByName finds an entry in the chart repository whose name matches the given name. +// +// It returns the ChartVersions for that entry. +func findEntryByName(name string, cr *repo.ChartRepository) (repo.ChartVersions, error) { + for ename, entry := range cr.IndexFile.Entries { + if ename == name { + return entry, nil + } + } + return nil, errors.New("entry not found") +} + +// findVersionedEntry takes a ChartVersions list and returns a single chart version that satisfies the version constraints. +// +// If version is empty, the first chart found is returned. +func findVersionedEntry(version string, vers repo.ChartVersions) (*repo.ChartVersion, error) { + for _, verEntry := range vers { + if len(verEntry.URLs) == 0 { + // Not a legit entry. + continue + } + + if version == "" || versionEquals(version, verEntry.Version) { + return verEntry, nil + } + } + return nil, errors.New("no matching version") +} + +func versionEquals(v1, v2 string) bool { + sv1, err := semver.NewVersion(v1) + if err != nil { + // Fallback to string comparison. + return v1 == v2 + } + sv2, err := semver.NewVersion(v2) + if err != nil { + return false + } + return sv1.Equal(sv2) +} + +func normalizeURL(baseURL, urlOrPath string) (string, error) { + u, err := url.Parse(urlOrPath) + if err != nil { + return urlOrPath, err + } + if u.IsAbs() { + return u.String(), nil + } + u2, err := url.Parse(baseURL) + if err != nil { + return urlOrPath, errors.Wrap(err, "base URL failed to parse") + } + + u2.RawPath = path.Join(u2.RawPath, urlOrPath) + u2.Path = path.Join(u2.Path, urlOrPath) + return u2.String(), nil +} + +// loadChartRepositories reads the repositories.yaml, and then builds a map of +// ChartRepositories. +// +// The key is the local name (which is only present in the repositories.yaml). +func (m *Manager) loadChartRepositories() (map[string]*repo.ChartRepository, error) { + indices := map[string]*repo.ChartRepository{} + + // Load repositories.yaml file + rf, err := loadRepoConfig(m.RepositoryConfig) + if err != nil { + return indices, errors.Wrapf(err, "failed to load %s", m.RepositoryConfig) + } + + for _, re := range rf.Repositories { + lname := re.Name + idxFile := filepath.Join(m.RepositoryCache, helmpath.CacheIndexFile(lname)) + index, err := repo.LoadIndexFile(idxFile) + if err != nil { + return indices, err + } + + // TODO: use constructor + cr := &repo.ChartRepository{ + Config: re, + IndexFile: index, + } + indices[lname] = cr + } + return indices, nil +} + +// writeLock writes a lockfile to disk +func writeLock(chartpath string, lock *chart.Lock, legacyLockfile bool) error { + data, err := yaml.Marshal(lock) + if err != nil { + return err + } + lockfileName := "Chart.lock" + if legacyLockfile { + lockfileName = "requirements.lock" + } + dest := filepath.Join(chartpath, lockfileName) + return os.WriteFile(dest, data, 0644) +} + +// archive a dep chart from local directory and save it into destPath +func tarFromLocalDir(chartpath, name, repo, version, destPath string) (string, error) { + if !strings.HasPrefix(repo, "file://") { + return "", errors.Errorf("wrong format: chart %s repository %s", name, repo) + } + + origPath, err := resolver.GetLocalPath(repo, chartpath) + if err != nil { + return "", err + } + + ch, err := loader.LoadDir(origPath) + if err != nil { + return "", err + } + + constraint, err := semver.NewConstraint(version) + if err != nil { + return "", errors.Wrapf(err, "dependency %s has an invalid version/constraint format", name) + } + + v, err := semver.NewVersion(ch.Metadata.Version) + if err != nil { + return "", err + } + + if constraint.Check(v) { + _, err = chartutil.Save(ch, destPath) + return ch.Metadata.Version, err + } + + return "", errors.Errorf("can't get a valid version for dependency %s", name) +} + +// The prefix to use for cache keys created by the manager for repo names +const managerKeyPrefix = "helm-manager-" + +// key is used to turn a name, such as a repository url, into a filesystem +// safe name that is unique for querying. To accomplish this a unique hash of +// the string is used. +func key(name string) (string, error) { + in := strings.NewReader(name) + hash := crypto.SHA256.New() + if _, err := io.Copy(hash, in); err != nil { + return "", nil + } + return hex.EncodeToString(hash.Sum(nil)), nil +} diff --git a/kubelink/pkg/downloader/manager_test.go b/kubelink/pkg/downloader/manager_test.go new file mode 100644 index 000000000..db2487d16 --- /dev/null +++ b/kubelink/pkg/downloader/manager_test.go @@ -0,0 +1,600 @@ +/* +Copyright The Helm Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package downloader + +import ( + "bytes" + "os" + "path/filepath" + "reflect" + "testing" + + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" + "helm.sh/helm/v3/pkg/getter" + "helm.sh/helm/v3/pkg/repo/repotest" +) + +func TestVersionEquals(t *testing.T) { + tests := []struct { + name, v1, v2 string + expect bool + }{ + {name: "semver match", v1: "1.2.3-beta.11", v2: "1.2.3-beta.11", expect: true}, + {name: "semver match, build info", v1: "1.2.3-beta.11+a", v2: "1.2.3-beta.11+b", expect: true}, + {name: "string match", v1: "abcdef123", v2: "abcdef123", expect: true}, + {name: "semver mismatch", v1: "1.2.3-beta.11", v2: "1.2.3-beta.22", expect: false}, + {name: "semver mismatch, invalid semver", v1: "1.2.3-beta.11", v2: "stinkycheese", expect: false}, + } + + for _, tt := range tests { + if versionEquals(tt.v1, tt.v2) != tt.expect { + t.Errorf("%s: failed comparison of %q and %q (expect equal: %t)", tt.name, tt.v1, tt.v2, tt.expect) + } + } +} + +func TestNormalizeURL(t *testing.T) { + tests := []struct { + name, base, path, expect string + }{ + {name: "basic URL", base: "https://example.com", path: "http://helm.sh/foo", expect: "http://helm.sh/foo"}, + {name: "relative path", base: "https://helm.sh/charts", path: "foo", expect: "https://helm.sh/charts/foo"}, + {name: "Encoded path", base: "https://helm.sh/a%2Fb/charts", path: "foo", expect: "https://helm.sh/a%2Fb/charts/foo"}, + } + + for _, tt := range tests { + got, err := normalizeURL(tt.base, tt.path) + if err != nil { + t.Errorf("%s: error %s", tt.name, err) + continue + } else if got != tt.expect { + t.Errorf("%s: expected %q, got %q", tt.name, tt.expect, got) + } + } +} + +func TestFindChartURL(t *testing.T) { + var b bytes.Buffer + m := &Manager{ + Out: &b, + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + } + repos, err := m.loadChartRepositories() + if err != nil { + t.Fatal(err) + } + + name := "alpine" + version := "0.1.0" + repoURL := "http://example.com/charts" + + churl, username, password, insecureSkipTLSVerify, passcredentialsall, _, _, _, err := m.findChartURL(name, version, repoURL, repos) + if err != nil { + t.Fatal(err) + } + + if churl != "https://charts.helm.sh/stable/alpine-0.1.0.tgz" { + t.Errorf("Unexpected URL %q", churl) + } + if username != "" { + t.Errorf("Unexpected username %q", username) + } + if password != "" { + t.Errorf("Unexpected password %q", password) + } + if passcredentialsall != false { + t.Errorf("Unexpected passcredentialsall %t", passcredentialsall) + } + if insecureSkipTLSVerify { + t.Errorf("Unexpected insecureSkipTLSVerify %t", insecureSkipTLSVerify) + } + + name = "tlsfoo" + version = "1.2.3" + repoURL = "https://example-https-insecureskiptlsverify.com" + + churl, username, password, insecureSkipTLSVerify, passcredentialsall, _, _, _, err = m.findChartURL(name, version, repoURL, repos) + if err != nil { + t.Fatal(err) + } + + if !insecureSkipTLSVerify { + t.Errorf("Unexpected insecureSkipTLSVerify %t", insecureSkipTLSVerify) + } + if churl != "https://example.com/tlsfoo-1.2.3.tgz" { + t.Errorf("Unexpected URL %q", churl) + } + if username != "" { + t.Errorf("Unexpected username %q", username) + } + if password != "" { + t.Errorf("Unexpected password %q", password) + } + if passcredentialsall != false { + t.Errorf("Unexpected passcredentialsall %t", passcredentialsall) + } +} + +func TestGetRepoNames(t *testing.T) { + b := bytes.NewBuffer(nil) + m := &Manager{ + Out: b, + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + } + tests := []struct { + name string + req []*chart.Dependency + expect map[string]string + err bool + }{ + { + name: "no repo definition, but references a url", + req: []*chart.Dependency{ + {Name: "oedipus-rex", Repository: "http://example.com/test"}, + }, + expect: map[string]string{"http://example.com/test": "http://example.com/test"}, + }, + { + name: "no repo definition failure -- stable repo", + req: []*chart.Dependency{ + {Name: "oedipus-rex", Repository: "stable"}, + }, + err: true, + }, + { + name: "no repo definition failure", + req: []*chart.Dependency{ + {Name: "oedipus-rex", Repository: "http://example.com"}, + }, + expect: map[string]string{"oedipus-rex": "testing"}, + }, + { + name: "repo from local path", + req: []*chart.Dependency{ + {Name: "local-dep", Repository: "file://./testdata/signtest"}, + }, + expect: map[string]string{"local-dep": "file://./testdata/signtest"}, + }, + { + name: "repo alias (alias:)", + req: []*chart.Dependency{ + {Name: "oedipus-rex", Repository: "alias:testing"}, + }, + expect: map[string]string{"oedipus-rex": "testing"}, + }, + { + name: "repo alias (@)", + req: []*chart.Dependency{ + {Name: "oedipus-rex", Repository: "@testing"}, + }, + expect: map[string]string{"oedipus-rex": "testing"}, + }, + { + name: "repo from local chart under charts path", + req: []*chart.Dependency{ + {Name: "local-subchart", Repository: ""}, + }, + expect: map[string]string{}, + }, + } + + for _, tt := range tests { + l, err := m.resolveRepoNames(tt.req) + if err != nil { + if tt.err { + continue + } + t.Fatal(err) + } + + if tt.err { + t.Fatalf("Expected error in test %q", tt.name) + } + + // m1 and m2 are the maps we want to compare + eq := reflect.DeepEqual(l, tt.expect) + if !eq { + t.Errorf("%s: expected map %v, got %v", tt.name, l, tt.name) + } + } +} + +func TestDownloadAll(t *testing.T) { + chartPath := t.TempDir() + m := &Manager{ + Out: new(bytes.Buffer), + RepositoryConfig: repoConfig, + RepositoryCache: repoCache, + ChartPath: chartPath, + } + signtest, err := loader.LoadDir(filepath.Join("testdata", "signtest")) + if err != nil { + t.Fatal(err) + } + if err := chartutil.SaveDir(signtest, filepath.Join(chartPath, "testdata")); err != nil { + t.Fatal(err) + } + + local, err := loader.LoadDir(filepath.Join("testdata", "local-subchart")) + if err != nil { + t.Fatal(err) + } + if err := chartutil.SaveDir(local, filepath.Join(chartPath, "charts")); err != nil { + t.Fatal(err) + } + + signDep := &chart.Dependency{ + Name: signtest.Name(), + Repository: "file://./testdata/signtest", + Version: signtest.Metadata.Version, + } + localDep := &chart.Dependency{ + Name: local.Name(), + Repository: "", + Version: local.Metadata.Version, + } + + // create a 'tmpcharts' directory to test #5567 + if err := os.MkdirAll(filepath.Join(chartPath, "tmpcharts"), 0755); err != nil { + t.Fatal(err) + } + if err := m.downloadAll([]*chart.Dependency{signDep, localDep}); err != nil { + t.Error(err) + } + + if _, err := os.Stat(filepath.Join(chartPath, "charts", "signtest-0.1.0.tgz")); os.IsNotExist(err) { + t.Error(err) + } + + // A chart with a bad name like this cannot be loaded and saved. Handling in + // the loading and saving will return an error about the invalid name. In + // this case, the chart needs to be created directly. + badchartyaml := `apiVersion: v2 +description: A Helm chart for Kubernetes +name: ../bad-local-subchart +version: 0.1.0` + if err := os.MkdirAll(filepath.Join(chartPath, "testdata", "bad-local-subchart"), 0755); err != nil { + t.Fatal(err) + } + err = os.WriteFile(filepath.Join(chartPath, "testdata", "bad-local-subchart", "Chart.yaml"), []byte(badchartyaml), 0644) + if err != nil { + t.Fatal(err) + } + + badLocalDep := &chart.Dependency{ + Name: "../bad-local-subchart", + Repository: "file://./testdata/bad-local-subchart", + Version: "0.1.0", + } + + err = m.downloadAll([]*chart.Dependency{badLocalDep}) + if err == nil { + t.Fatal("Expected error for bad dependency name") + } +} + +func TestUpdateBeforeBuild(t *testing.T) { + // Set up a fake repo + srv, err := repotest.NewTempServerWithCleanup(t, "testdata/*.tgz*") + if err != nil { + t.Fatal(err) + } + defer srv.Stop() + if err := srv.LinkIndices(); err != nil { + t.Fatal(err) + } + dir := func(p ...string) string { + return filepath.Join(append([]string{srv.Root()}, p...)...) + } + + // Save dep + d := &chart.Chart{ + Metadata: &chart.Metadata{ + Name: "dep-chart", + Version: "0.1.0", + APIVersion: "v1", + }, + } + if err := chartutil.SaveDir(d, dir()); err != nil { + t.Fatal(err) + } + // Save a chart + c := &chart.Chart{ + Metadata: &chart.Metadata{ + Name: "with-dependency", + Version: "0.1.0", + APIVersion: "v2", + Dependencies: []*chart.Dependency{{ + Name: d.Metadata.Name, + Version: ">=0.1.0", + Repository: "file://../dep-chart", + }}, + }, + } + if err := chartutil.SaveDir(c, dir()); err != nil { + t.Fatal(err) + } + + // Set-up a manager + b := bytes.NewBuffer(nil) + g := getter.Providers{getter.Provider{ + Schemes: []string{"http", "https"}, + New: getter.NewHTTPGetter, + }} + m := &Manager{ + ChartPath: dir(c.Metadata.Name), + Out: b, + Getters: g, + RepositoryConfig: dir("repositories.yaml"), + RepositoryCache: dir(), + } + + // Update before Build. see issue: https://github.com/helm/helm/issues/7101 + err = m.Update() + if err != nil { + t.Fatal(err) + } + + err = m.Build() + if err != nil { + t.Fatal(err) + } +} + +// TestUpdateWithNoRepo is for the case of a dependency that has no repo listed. +// This happens when the dependency is in the charts directory and does not need +// to be fetched. +func TestUpdateWithNoRepo(t *testing.T) { + // Set up a fake repo + srv, err := repotest.NewTempServerWithCleanup(t, "testdata/*.tgz*") + if err != nil { + t.Fatal(err) + } + defer srv.Stop() + if err := srv.LinkIndices(); err != nil { + t.Fatal(err) + } + dir := func(p ...string) string { + return filepath.Join(append([]string{srv.Root()}, p...)...) + } + + // Setup the dependent chart + d := &chart.Chart{ + Metadata: &chart.Metadata{ + Name: "dep-chart", + Version: "0.1.0", + APIVersion: "v1", + }, + } + + // Save a chart with the dependency + c := &chart.Chart{ + Metadata: &chart.Metadata{ + Name: "with-dependency", + Version: "0.1.0", + APIVersion: "v2", + Dependencies: []*chart.Dependency{{ + Name: d.Metadata.Name, + Version: "0.1.0", + }}, + }, + } + if err := chartutil.SaveDir(c, dir()); err != nil { + t.Fatal(err) + } + + // Save dependent chart into the parents charts directory. If the chart is + // not in the charts directory Helm will return an error that it is not + // found. + if err := chartutil.SaveDir(d, dir(c.Metadata.Name, "charts")); err != nil { + t.Fatal(err) + } + + // Set-up a manager + b := bytes.NewBuffer(nil) + g := getter.Providers{getter.Provider{ + Schemes: []string{"http", "https"}, + New: getter.NewHTTPGetter, + }} + m := &Manager{ + ChartPath: dir(c.Metadata.Name), + Out: b, + Getters: g, + RepositoryConfig: dir("repositories.yaml"), + RepositoryCache: dir(), + } + + // Test the update + err = m.Update() + if err != nil { + t.Fatal(err) + } +} + +// This function is the skeleton test code of failing tests for #6416 and #6871 and bugs due to #5874. +// +// This function is used by below tests that ensures success of build operation +// with optional fields, alias, condition, tags, and even with ranged version. +// Parent chart includes local-subchart 0.1.0 subchart from a fake repository, by default. +// If each of these main fields (name, version, repository) is not supplied by dep param, default value will be used. +func checkBuildWithOptionalFields(t *testing.T, chartName string, dep chart.Dependency) { + // Set up a fake repo + srv, err := repotest.NewTempServerWithCleanup(t, "testdata/*.tgz*") + if err != nil { + t.Fatal(err) + } + defer srv.Stop() + if err := srv.LinkIndices(); err != nil { + t.Fatal(err) + } + dir := func(p ...string) string { + return filepath.Join(append([]string{srv.Root()}, p...)...) + } + + // Set main fields if not exist + if dep.Name == "" { + dep.Name = "local-subchart" + } + if dep.Version == "" { + dep.Version = "0.1.0" + } + if dep.Repository == "" { + dep.Repository = srv.URL() + } + + // Save a chart + c := &chart.Chart{ + Metadata: &chart.Metadata{ + Name: chartName, + Version: "0.1.0", + APIVersion: "v2", + Dependencies: []*chart.Dependency{&dep}, + }, + } + if err := chartutil.SaveDir(c, dir()); err != nil { + t.Fatal(err) + } + + // Set-up a manager + b := bytes.NewBuffer(nil) + g := getter.Providers{getter.Provider{ + Schemes: []string{"http", "https"}, + New: getter.NewHTTPGetter, + }} + m := &Manager{ + ChartPath: dir(chartName), + Out: b, + Getters: g, + RepositoryConfig: dir("repositories.yaml"), + RepositoryCache: dir(), + } + + // First build will update dependencies and create Chart.lock file. + err = m.Build() + if err != nil { + t.Fatal(err) + } + + // Second build should be passed. See PR #6655. + err = m.Build() + if err != nil { + t.Fatal(err) + } +} + +func TestBuild_WithoutOptionalFields(t *testing.T) { + // Dependency has main fields only (name/version/repository) + checkBuildWithOptionalFields(t, "without-optional-fields", chart.Dependency{}) +} + +func TestBuild_WithSemVerRange(t *testing.T) { + // Dependency version is the form of SemVer range + checkBuildWithOptionalFields(t, "with-semver-range", chart.Dependency{ + Version: ">=0.1.0", + }) +} + +func TestBuild_WithAlias(t *testing.T) { + // Dependency has an alias + checkBuildWithOptionalFields(t, "with-alias", chart.Dependency{ + Alias: "local-subchart-alias", + }) +} + +func TestBuild_WithCondition(t *testing.T) { + // Dependency has a condition + checkBuildWithOptionalFields(t, "with-condition", chart.Dependency{ + Condition: "some.condition", + }) +} + +func TestBuild_WithTags(t *testing.T) { + // Dependency has several tags + checkBuildWithOptionalFields(t, "with-tags", chart.Dependency{ + Tags: []string{"tag1", "tag2"}, + }) +} + +// Failing test for #6871 +func TestBuild_WithRepositoryAlias(t *testing.T) { + // Dependency repository is aliased in Chart.yaml + checkBuildWithOptionalFields(t, "with-repository-alias", chart.Dependency{ + Repository: "@test", + }) +} + +func TestErrRepoNotFound_Error(t *testing.T) { + type fields struct { + Repos []string + } + tests := []struct { + name string + fields fields + want string + }{ + { + name: "OK", + fields: fields{ + Repos: []string{"https://charts1.example.com", "https://charts2.example.com"}, + }, + want: "no repository definition for https://charts1.example.com, https://charts2.example.com", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := ErrRepoNotFound{ + Repos: tt.fields.Repos, + } + if got := e.Error(); got != tt.want { + t.Errorf("Error() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestKey(t *testing.T) { + tests := []struct { + name string + expect string + }{ + { + name: "file:////tmp", + expect: "afeed3459e92a874f6373aca264ce1459bfa91f9c1d6612f10ae3dc2ee955df3", + }, + { + name: "https://example.com/charts", + expect: "7065c57c94b2411ad774638d76823c7ccb56415441f5ab2f5ece2f3845728e5d", + }, + { + name: "foo/bar/baz", + expect: "15c46a4f8a189ae22f36f201048881d6c090c93583bedcf71f5443fdef224c82", + }, + } + + for _, tt := range tests { + o, err := key(tt.name) + if err != nil { + t.Fatalf("unable to generate key for %q with error: %s", tt.name, err) + } + if o != tt.expect { + t.Errorf("wrong key name generated for %q, expected %q but got %q", tt.name, tt.expect, o) + } + } +} diff --git a/kubelink/pkg/downloader/testdata/helm-test-key.pub b/kubelink/pkg/downloader/testdata/helm-test-key.pub new file mode 100644 index 0000000000000000000000000000000000000000..38714f25adaf701b08e11fd559a587074bbde0e4 GIT binary patch literal 1243 zcmV<11SI>J0SyFKmTjH^2mr{k15wFPQdpTAAEclY4oV^-7nTy5x$&xF;PC4il}o-Pk4@#$knlp#(|J0GE?qli_lr;7-o zyY8vBsN_GJe;#w<`JdR7riNL&RJlcS)FG+W=91;dYS6NZ2tY?kZ8Sw9{r=e4|L3E{ zRod|EPC!PWgW&pe&4qiqKQAijj;G~fyjcC^m%0p54Tn{h%YFKd0VC4n=#~SRc@BVd znj66*)Om%*SEQfX{1*Tb0RRECT}WkYZ6H)-b98BLXCNq4XlZjGYh`&Lb7*gMY-AvB zZftoVVr3w8b7f>8W^ZyJbY*jNX>MmOAVg0fPES-IR8mz_R4yqXJZNQXZ7p6uVakV zT7GGV$jaKjyjfI_a~N1!Hk?5C$0wa&4)R=i$v7t&ZMycW#RkavpF%A?>MTT2anNDzOQUm<++zEOykJ9-@&c2QXq3owqf7fek`=L@+7iF zv;IW2Q>Q&r+V@cWDF&hAUUsCKlDinerKgvJUJCl$5gjb7NhM{mBP%!M^mX-iS8xFf zuLB{@MDqvtZzF#Bxd9CXSC(y_0SExW>8~h=U8|!do4*OJj2u#!KDe3v+1T+aVzU5di=Ji2)x37y$|Z2?YXImTjH_8w>yn2@r%kznCAv zhhp0`2mpxVj5j%o&5i)?`r7iES|8dA@p2kk@+XS(tjBGN)6>tm^=gayCn`gTEC*K74Y~{I_PREk) z)PstIMx1RxB@cK8%Mey%;nVnKriAKUk2Ky?dBMG3uXItKL$3N(#3P^pQa*K$l)wUy F^>pMLK0g2e literal 0 HcmV?d00001 diff --git a/kubelink/pkg/downloader/testdata/helm-test-key.secret b/kubelink/pkg/downloader/testdata/helm-test-key.secret new file mode 100644 index 0000000000000000000000000000000000000000..a966aef93ed97d01d764f29940738df6df2d9d24 GIT binary patch literal 2545 zcmVclY4oV^-7nTy5x$&xF;PC4il}o-Pk4@#$knlp#(|J0GE?qli_lr;7-o zyY8vBsN_GJe;#w<`JdR7riNL&RJlcS)FG+W=91;dYS6NZ2tY?kZ8Sw9{r=e4|L3E{ zRod|EPC!PWgW&pe&4qiqKQAijj;G~fyjcC^m%0p54Tn{h%YFKd0VC4n=#~SRc@BVd znj66*)Om%*SEQfX{1*Tb0RRC22mUT~!#(ymA#eaSp1lpODzX${Vf^l{qDyu}xC-Z; zRnH<54GSVm<$?Ua1k#(+mu~3_*CIx=sPuoZB#9t`5)>)SncaZ0<~%)I$~BM-5aP3W z%`ewoaI;P40uHnDeE!9-_o2Lr{wDfL45jGGU-JZ36T9ToJqMX(TnRN-EvGi{o6aI#oT_2HU(J8=theYZsj5h?ml@F2 zqCpxqkdZi=~i+&Z}q^cR< zq>lNT5cnJ5X@K!3vOww0B>@Bg*7x*i59vbegj}$ELl?K2l`+`uY;jn;@-#}^!(c8$ z&Y`@LLxZ_Y>^#gGbxsy-2s=w7cVmR@z_%b#0_e^qDmIrpKw6U7N;6^TN}@&nxKj6i zje++&m}XQA&G8O8FX86?Frxrjmu5ktfDRyHBb|j&n&H#v>T!Mdmk8Y#1OV>P(*gow;}0v-BdsmdUSV3M9tIkRO0OTBw16eYCzxs>OEG!?i}$^8yY+hFlb3GJ~F z@#2Vimrfeb0(o3X?>!tSIROL!mGC1>cHXGVp;VD$oE{N!h=IF(C(PNLd6^nZO^!ix zHnE%@Y*d~bJl_M}WW0D1EM+&xdQI5#y67>-8{P4^*?j9-RL!3>e89fC4fbJFTGXY* zQA`Z&jZ*qV!0N>p>(<2RFPDhHj^h*B*O(i139Dwv{>MY%puY021Or@)I~ufINM&qo zAXH^@bZKs9AShI5X>%ZJWqBZTXm53FWFT*DY^`Z*m}XWpi|CZf7na zL{A`2PgEdOQdLt_E-4^9Xk~0|Ep%mbbZKs9Kxk!bZ7y?YK8XQ01QP)Y03iheSC(y_ z0viJb3ke7Z0|gZd2?z@X76JnS00JHX0vCV)3JDN|JHMD8!G~grMF;@2`RLQ-(ihS@ zk(ZDi8>PUMNBttVp^;f=(#~5ORUCP*V~o^Verbou%G$oXSyYd67+6|1oIv=;C!Jsp z@?3ezI42oxy7sHZ2FUrJLM=V)2rULvozb@g^vZ~+Ui10l{t^9T2DBYydv1DFI?mTjH^2mrz9 zuPBIJtD_~GzX`6498#D*yg_W@HI~u}LQvFZ zjHz2I7O5nm<}d0gU&SbRw}dGu2{gYWzK!Qb2tL4r=Ttf(&pz_gadeY}n}E@spby2h zn?Jq8s}cOpE&?`36cTnn-abrV^*hkY1rlNa6>W(?OAePZXMfE&?IzWku7z=T;E66)b)o_po?XSOFY;U!8IY8l|z)F~<`!sdiAt3+}0RRC2 z2mUKrERA52qzq^qU4-%uqeMA@h`$YTvMnKwO3MFdg819*{h|i5{tcC;Av-jm`%7`? zISDa>*_u$~x5)kpVt_aYB_e`#K)Xd5tcJ05BQ>ps?qeo`#OS{-ilRZ+9`nljqxsy1 zp;Lu#*--$l?6qncfhI%m^w(3lOt}ywL5?%+_Ov|T=-O)O#|1&>=}51a%Sb~KTR2_K z!};{n;NgPO;;v%0;n-j>b-Y|l)x=^&d84lKmr8o*+q*$Sul50u>9%n+e!b~90-}xc znpRXgsh*hBzGXpmnXaxdFnD1FEnbiC?537`DY#mL7&iHNEY4|+!A|s9dFssoYIy_z z+imR1K+cnVPeX&M1X~ed#U~gsS6HR0zgm1NR~u@{BN;*5Gvl)42%Kq{=4gSyFIAOo zw)ZGKn^3RZn+iXfb*zL1mnJJGsvTLnDB5DF8)!;+KX&@(mJ7k5LnTlXYxI(#)c`4{ zo6I4Djv|uTRI{JJ9glvUHq0WkzV7H91OVbY6u%#c1Z-!^cIjhIC)Ek7Hx7cRvtc6M zYLV(#kP^D1#2+7pDzLBFanZFqRw>On{`4qC48A)&{zk{n0CZEKY$SfN1Rk^!_V}?Oa05R<~;U7Vou+rQZcj7^Zr@2q2}K8g2gzsQ|Y$Hp^5`riTL^4T#Q?}_!b9ge@36zaVNe`|(D|D@%b z?q#ETmMPVDW6=SC^ zp>(H_BkTP!*5u$7;(xt$0Z3AJ%*#wE`2MxJYbiqwMV z55Sy@$Oto*a)E^@IG(B#1R_APzVCxJvjDnj!`b?U+KFs4f-?w1(lA6SB!RPKJ5Wx? zlJL}niiAd-Z9pXtcm~T5R%GGR_+_Sq>RpdC-c)(PyDc zVQyr3R8em|NM&qo0PNJuio!4y2H>vq6nTN^{F#IDc zVQyr3R8em|NM&qo0PI;!Z{s!-&Y8brUgz=_Sktr}$Ea^Xvp|8iX|P!oD2ie|mc|mX z6v>j56T{7aFGN{RptN<1*ba)-bCFFAIYWuhe96m92l8R?O^z<`H5TgZ&=5k1>0}bG zLWuTN3`du{-*J36nocjyKpfnXKSAjOx-;==UG2^NM}SuTM9xd2XRsQwlzif(4e|dK zd`qf;q&gX}G!DKi7vwYr@=RkvGiXi^TQzG4KIDSE^{zVnQ|$P^LRFGKiUZike<8*# z{*Onaj{hgY=CLE|my8|%0~JgTD{>4VF*=~sVa28w)d|0wGg8BYv-qqfgS&OPO6ZZHjWOhV=w{qp0jiKm`e}7wAQ%b!RMqDWXdd{ zz>wrpXYas~!XQ@!7DN7Q9CgahK~siRbpijkj+XL)Qn;5PhyQ)W;YY33V04^WnFN*` zD5;4vetq}pE*M9QXEJoI;9%JCzjnq)X#?!#|M*4xA6<+){+|MWSN~s=Rb~wc3-mI9 zt9U}-d#TF@uqI`>sRDZ*g7ve(po$;d=kdC257cLhc~iQC{EYQ?!kG+tx!{Q@qI^B6 zYa*N;ZT^3Fe|7!CdtRgm)Ul8M6ESSZ|H-uD|49%7Iz3=v6~R4v$VijJKq-{IivA&| zCNYP39{YigFwmCVbI#buoM8S`Kh7bQj*?*9x~T;`Agsu(!ON(~nzX7OqF<=vKeEJ> z)h)9Giw+A4i^A#e;`HZiQiyBkB|M$hS&LG{ht9ST#$-&KR`}ShFIx8n|ViWC6ihh>Q4(d&FZbS zwzqfY?IgA%@H_lgnotSGashYlS&90`8}00960pc^bi03-ka*J$OZ literal 0 HcmV?d00001 diff --git a/kubelink/pkg/downloader/testdata/signtest-0.1.0.tgz.prov b/kubelink/pkg/downloader/testdata/signtest-0.1.0.tgz.prov new file mode 100644 index 000000000..d325bb266 --- /dev/null +++ b/kubelink/pkg/downloader/testdata/signtest-0.1.0.tgz.prov @@ -0,0 +1,21 @@ +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA512 + +apiVersion: v1 +description: A Helm chart for Kubernetes +name: signtest +version: 0.1.0 + +... +files: + signtest-0.1.0.tgz: sha256:e5ef611620fb97704d8751c16bab17fedb68883bfb0edc76f78a70e9173f9b55 +-----BEGIN PGP SIGNATURE----- + +wsBcBAEBCgAQBQJcoosfCRCEO7+YH8GHYgAA220IALAs8T8NPgkcLvHu+5109cAN +BOCNPSZDNsqLZW/2Dc9cKoBG7Jen4Qad+i5l9351kqn3D9Gm6eRfAWcjfggRobV/ +9daZ19h0nl4O1muQNAkjvdgZt8MOP3+PB3I3/Tu2QCYjI579SLUmuXlcZR5BCFPR +PJy+e3QpV2PcdeU2KZLG4tjtlrq+3QC9ZHHEJLs+BVN9d46Dwo6CxJdHJrrrAkTw +M8MhA92vbiTTPRSCZI9x5qDAwJYhoq0oxLflpuL2tIlo3qVoCsaTSURwMESEHO32 +XwYG7BaVDMELWhAorBAGBGBwWFbJ1677qQ2gd9CN0COiVhekWlFRcnn60800r84= +=k9Y9 +-----END PGP SIGNATURE----- \ No newline at end of file diff --git a/kubelink/pkg/downloader/testdata/signtest/.helmignore b/kubelink/pkg/downloader/testdata/signtest/.helmignore new file mode 100644 index 000000000..435b756d8 --- /dev/null +++ b/kubelink/pkg/downloader/testdata/signtest/.helmignore @@ -0,0 +1,5 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +.git diff --git a/kubelink/pkg/downloader/testdata/signtest/Chart.yaml b/kubelink/pkg/downloader/testdata/signtest/Chart.yaml new file mode 100644 index 000000000..f1f73723a --- /dev/null +++ b/kubelink/pkg/downloader/testdata/signtest/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: A Helm chart for Kubernetes +name: signtest +version: 0.1.0 diff --git a/kubelink/pkg/downloader/testdata/signtest/alpine/Chart.yaml b/kubelink/pkg/downloader/testdata/signtest/alpine/Chart.yaml new file mode 100644 index 000000000..eec261220 --- /dev/null +++ b/kubelink/pkg/downloader/testdata/signtest/alpine/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +description: Deploy a basic Alpine Linux pod +home: https://helm.sh/helm +name: alpine +sources: +- https://github.com/helm/helm +version: 0.1.0 diff --git a/kubelink/pkg/downloader/testdata/signtest/alpine/README.md b/kubelink/pkg/downloader/testdata/signtest/alpine/README.md new file mode 100644 index 000000000..28bebae07 --- /dev/null +++ b/kubelink/pkg/downloader/testdata/signtest/alpine/README.md @@ -0,0 +1,9 @@ +This example was generated using the command `helm create alpine`. + +The `templates/` directory contains a very simple pod resource with a +couple of parameters. + +The `values.yaml` file contains the default values for the +`alpine-pod.yaml` template. + +You can install this example using `helm install ./alpine`. diff --git a/kubelink/pkg/downloader/testdata/signtest/alpine/templates/alpine-pod.yaml b/kubelink/pkg/downloader/testdata/signtest/alpine/templates/alpine-pod.yaml new file mode 100644 index 000000000..5bbae10af --- /dev/null +++ b/kubelink/pkg/downloader/testdata/signtest/alpine/templates/alpine-pod.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Pod +metadata: + name: {{.Release.Name}}-{{.Chart.Name}} + labels: + app.kubernetes.io/managed-by: {{.Release.Service}} + chartName: {{.Chart.Name}} + chartVersion: {{.Chart.Version | quote}} +spec: + restartPolicy: {{default "Never" .restart_policy}} + containers: + - name: waiter + image: "alpine:3.3" + command: ["/bin/sleep","9000"] diff --git a/kubelink/pkg/downloader/testdata/signtest/alpine/values.yaml b/kubelink/pkg/downloader/testdata/signtest/alpine/values.yaml new file mode 100644 index 000000000..bb6c06ae4 --- /dev/null +++ b/kubelink/pkg/downloader/testdata/signtest/alpine/values.yaml @@ -0,0 +1,2 @@ +# The pod name +name: my-alpine diff --git a/kubelink/pkg/downloader/testdata/signtest/templates/pod.yaml b/kubelink/pkg/downloader/testdata/signtest/templates/pod.yaml new file mode 100644 index 000000000..9b00ccaf7 --- /dev/null +++ b/kubelink/pkg/downloader/testdata/signtest/templates/pod.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Pod +metadata: + name: signtest +spec: + restartPolicy: Never + containers: + - name: waiter + image: "alpine:3.3" + command: ["/bin/sleep","9000"] diff --git a/kubelink/pkg/downloader/testdata/signtest/values.yaml b/kubelink/pkg/downloader/testdata/signtest/values.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/kubelink/wire_gen.go b/kubelink/wire_gen.go index 0eaca950c..ccc1455c3 100644 --- a/kubelink/wire_gen.go +++ b/kubelink/wire_gen.go @@ -1,6 +1,6 @@ // Code generated by Wire. DO NOT EDIT. -//go:generate go run github.com/google/wire/cmd/wire +//go:generate go run -mod=mod github.com/google/wire/cmd/wire //go:build !wireinject // +build !wireinject From 569f8f08e04c20cb408d337d0a352d434819146e Mon Sep 17 00:00:00 2001 From: Rajeev Date: Sun, 17 Nov 2024 14:14:20 +0530 Subject: [PATCH 2/9] poc for the updating external app --- kubelink/internals/fileutil/fileutil.go | 50 ++++++ kubelink/internals/fileutil/fileutil_test.go | 57 +++++++ kubelink/internals/resolver/resolver.go | 8 +- kubelink/pkg/downloader/chart_downloader.go | 6 +- kubelink/pkg/downloader/manager.go | 4 +- kubelink/pkg/helmClient/client.go | 44 +++--- .../helmApplicationService/helmAppService.go | 144 +++++++++++++++++- 7 files changed, 277 insertions(+), 36 deletions(-) create mode 100644 kubelink/internals/fileutil/fileutil.go create mode 100644 kubelink/internals/fileutil/fileutil_test.go diff --git a/kubelink/internals/fileutil/fileutil.go b/kubelink/internals/fileutil/fileutil.go new file mode 100644 index 000000000..a110824a6 --- /dev/null +++ b/kubelink/internals/fileutil/fileutil.go @@ -0,0 +1,50 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fileutil + +import ( + "io" + "os" + "path/filepath" + + "github.com/devtron-labs/kubelink/internals/thirdParty/dep/fs" +) + +// AtomicWriteFile atomically (as atomic as os.Rename allows) writes a file to a +// disk. +func AtomicWriteFile(filename string, reader io.Reader, mode os.FileMode) error { + tempFile, err := os.CreateTemp(filepath.Split(filename)) + if err != nil { + return err + } + tempName := tempFile.Name() + + if _, err := io.Copy(tempFile, reader); err != nil { + tempFile.Close() // return value is ignored as we are already on error path + return err + } + + if err := tempFile.Close(); err != nil { + return err + } + + if err := os.Chmod(tempName, mode); err != nil { + return err + } + + return fs.RenameWithFallback(tempName, filename) +} diff --git a/kubelink/internals/fileutil/fileutil_test.go b/kubelink/internals/fileutil/fileutil_test.go new file mode 100644 index 000000000..92920d3c4 --- /dev/null +++ b/kubelink/internals/fileutil/fileutil_test.go @@ -0,0 +1,57 @@ +/* +Copyright The Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fileutil + +import ( + "bytes" + "os" + "path/filepath" + "testing" +) + +func TestAtomicWriteFile(t *testing.T) { + dir := t.TempDir() + + testpath := filepath.Join(dir, "test") + stringContent := "Test content" + reader := bytes.NewReader([]byte(stringContent)) + mode := os.FileMode(0644) + + err := AtomicWriteFile(testpath, reader, mode) + if err != nil { + t.Errorf("AtomicWriteFile error: %s", err) + } + + got, err := os.ReadFile(testpath) + if err != nil { + t.Fatal(err) + } + + if stringContent != string(got) { + t.Fatalf("expected: %s, got: %s", stringContent, string(got)) + } + + gotinfo, err := os.Stat(testpath) + if err != nil { + t.Fatal(err) + } + + if mode != gotinfo.Mode() { + t.Fatalf("expected %s: to be the same mode as %s", + mode, gotinfo.Mode()) + } +} diff --git a/kubelink/internals/resolver/resolver.go b/kubelink/internals/resolver/resolver.go index b6f45da9e..36603a2e1 100644 --- a/kubelink/internals/resolver/resolver.go +++ b/kubelink/internals/resolver/resolver.go @@ -94,7 +94,7 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string } if !constraint.Check(v) { - missing = append(missing, fmt.Sprintf("%q (repository %q, version %q)", d.Name, d.Repository, d.Version)) + missing = append(missing, d.Name) continue } @@ -172,7 +172,7 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string Repository: d.Repository, Version: version, } - // The versions are already sorted and hence the first one to satisfy the constraint is used + // The version are already sorted and hence the first one to satisfy the constraint is used for _, ver := range vs { v, err := semver.NewVersion(ver.Version) // OCI does not need URLs @@ -188,11 +188,11 @@ func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string } if !found { - missing = append(missing, fmt.Sprintf("%q (repository %q, version %q)", d.Name, d.Repository, d.Version)) + missing = append(missing, d.Name) } } if len(missing) > 0 { - return nil, errors.Errorf("can't get a valid version for %d subchart(s): %s. Make sure a matching chart version exists in the repo, or change the version constraint in Chart.yaml", len(missing), strings.Join(missing, ", ")) + return nil, errors.Errorf("can't get a valid version for repositories %s. Try changing the version constraint in Chart.yaml", strings.Join(missing, ", ")) } digest, err := HashReq(reqs, locked) diff --git a/kubelink/pkg/downloader/chart_downloader.go b/kubelink/pkg/downloader/chart_downloader.go index dde6a1057..b8c0d78e4 100644 --- a/kubelink/pkg/downloader/chart_downloader.go +++ b/kubelink/pkg/downloader/chart_downloader.go @@ -26,8 +26,8 @@ import ( "github.com/Masterminds/semver/v3" "github.com/pkg/errors" - "helm.sh/helm/v3/internal/fileutil" - "helm.sh/helm/v3/internal/urlutil" + "github.com/devtron-labs/kubelink/internals/fileutil" + "github.com/devtron-labs/kubelink/internals/urlutil" "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/helmpath" "helm.sh/helm/v3/pkg/provenance" @@ -97,8 +97,6 @@ func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *proven return "", nil, err } - c.Options = append(c.Options, getter.WithAcceptHeader("application/gzip,application/octet-stream")) - data, err := g.Get(u.String(), c.Options...) if err != nil { return "", nil, err diff --git a/kubelink/pkg/downloader/manager.go b/kubelink/pkg/downloader/manager.go index d7c32241f..ae8e0e737 100644 --- a/kubelink/pkg/downloader/manager.go +++ b/kubelink/pkg/downloader/manager.go @@ -173,7 +173,7 @@ func (m *Manager) Update() error { // has some information about them and, when possible, the index files // locally. // TODO(mattfarina): Repositories should be explicitly added by end users - // rather than automatic. In Helm v4 require users to add repositories. They + // rather than automattic. In Helm v4 require users to add repositories. They // should have to add them in order to make sure they are aware of the // repositories and opt-in to any locations, for security. repoNames, err = m.ensureMissingRepos(repoNames, req) @@ -246,7 +246,7 @@ func (m *Manager) downloadAll(deps []*chart.Dependency) error { } destPath := filepath.Join(m.ChartPath, "charts") - tmpPath := filepath.Join(m.ChartPath, fmt.Sprintf("tmpcharts-%d", os.Getpid())) + tmpPath := filepath.Join(m.ChartPath, "tmpcharts") // Check if 'charts' directory is not actually a directory. If it does not exist, create it. if fi, err := os.Stat(destPath); err == nil { diff --git a/kubelink/pkg/helmClient/client.go b/kubelink/pkg/helmClient/client.go index 9d27445e8..100a3007a 100644 --- a/kubelink/pkg/helmClient/client.go +++ b/kubelink/pkg/helmClient/client.go @@ -46,8 +46,8 @@ var storage = repo.File{} const ( CHART_WORKING_DIR_PATH = "/tmp/charts/" - defaultCachePath = "/home/devtron/devtroncd/.helmcache" - defaultRepositoryConfigPath = "/home/devtron/devtroncd/.helmrepo" + DefaultCachePath = "/home/devtron/devtroncd/.helmcache" + DefaultRepositoryConfigPath = "/home/devtron/devtroncd/.helmrepo" ) // NewClientFromRestConf returns a new Helm client constructed with the provided REST config options @@ -56,7 +56,7 @@ func NewClientFromRestConf(options *RestConfClientOptions) (Client, error) { clientGetter := NewRESTClientGetter(options.Namespace, nil, options.RestConfig) - err := setEnvSettings(options.Options, settings) + err := SetEnvSettings(options.Options, settings) if err != nil { return nil, err } @@ -66,7 +66,7 @@ func NewClientFromRestConf(options *RestConfClientOptions) (Client, error) { // newClient returns a new Helm client via the provided options and REST config func newClient(options *Options, clientGetter genericclioptions.RESTClientGetter, settings *cli.EnvSettings) (Client, error) { - err := setEnvSettings(options, settings) + err := SetEnvSettings(options, settings) if err != nil { return nil, err } @@ -99,12 +99,12 @@ func newClient(options *Options, clientGetter genericclioptions.RESTClientGetter }, nil } -// setEnvSettings sets the client's environment settings based on the provided client configuration -func setEnvSettings(options *Options, settings *cli.EnvSettings) error { +// SetEnvSettings sets the client's environment settings based on the provided client configuration +func SetEnvSettings(options *Options, settings *cli.EnvSettings) error { if options == nil { options = &Options{ - RepositoryConfig: defaultRepositoryConfigPath, - RepositoryCache: defaultCachePath, + RepositoryConfig: DefaultRepositoryConfigPath, + RepositoryCache: DefaultCachePath, Linting: true, } } @@ -121,11 +121,11 @@ func setEnvSettings(options *Options, settings *cli.EnvSettings) error { }*/ if options.RepositoryConfig == "" { - options.RepositoryConfig = defaultRepositoryConfigPath + options.RepositoryConfig = DefaultRepositoryConfigPath } if options.RepositoryCache == "" { - options.RepositoryCache = defaultCachePath + options.RepositoryCache = DefaultCachePath } settings.RepositoryCache = options.RepositoryCache @@ -796,11 +796,24 @@ func updateDependencies(helmChart *chart.Chart, chartPathOptions *action.ChartPa } func GetChartBytes(helmChart *chart.Chart) ([]byte, error) { + + absFilePath, err := GetChartSavedDir(helmChart) + if err != nil { + fmt.Println("error in getting saved chart data directory path", "err", err) + } + chartBytes, err := os.ReadFile(absFilePath) + if err != nil { + fmt.Println("error in reading chartdata from the file ", " filePath : ", absFilePath, " err : ", err) + } + + return chartBytes, nil +} +func GetChartSavedDir(helmChart *chart.Chart) (string, error) { dirPath := CHART_WORKING_DIR_PATH outputChartPathDir := fmt.Sprintf("%s/%s", dirPath, strconv.FormatInt(time.Now().UnixNano(), 16)) err := os.MkdirAll(outputChartPathDir, os.ModePerm) if err != nil { - return nil, err + return "", err } defer func() { @@ -812,13 +825,8 @@ func GetChartBytes(helmChart *chart.Chart) ([]byte, error) { absFilePath, err := chartutil.Save(helmChart, outputChartPathDir) if err != nil { fmt.Println("error in saving chartdata in the destination dir ", " dir : ", outputChartPathDir, " err : ", err) - return nil, err + return "", err } - chartBytes, err := os.ReadFile(absFilePath) - if err != nil { - fmt.Println("error in reading chartdata from the file ", " filePath : ", absFilePath, " err : ", err) - } - - return chartBytes, nil + return absFilePath, nil } diff --git a/kubelink/pkg/service/helmApplicationService/helmAppService.go b/kubelink/pkg/service/helmApplicationService/helmAppService.go index 88408f14a..37dac9f9f 100644 --- a/kubelink/pkg/service/helmApplicationService/helmAppService.go +++ b/kubelink/pkg/service/helmApplicationService/helmAppService.go @@ -28,12 +28,17 @@ import ( "github.com/devtron-labs/kubelink/converter" error2 "github.com/devtron-labs/kubelink/error" repository "github.com/devtron-labs/kubelink/pkg/cluster" + "github.com/devtron-labs/kubelink/pkg/downloader" "github.com/devtron-labs/kubelink/pkg/service/commonHelmService" + chart2 "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/cli" + "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/registry" "helm.sh/helm/v3/pkg/storage/driver" "net/url" "path" + "sigs.k8s.io/yaml" "strings" "time" @@ -558,13 +563,23 @@ func (impl *HelmAppServiceImpl) UpgradeRelease(ctx context.Context, request *cli if err != nil { return nil, err } - registryClient, err := registry.NewClient() + + opts := []registry.ClientOption{ + registry.ClientOptDebug(false), + registry.ClientOptEnableCache(true), + registry.ClientOptWriter(os.Stderr), + } + //if plainHTTP { + // opts = append(opts, registry.ClientOptPlainHTTP()) + //} + + registryClient, err := registry.NewClient(opts...) if err != nil { impl.logger.Errorw(HELM_CLIENT_ERROR, "err", err) return nil, err } - - helmRelease, err := impl.common.GetHelmRelease(releaseIdentifier.ClusterConfig, releaseIdentifier.ReleaseNamespace, releaseIdentifier.ReleaseName) + var helmRelease *release.Release + helmRelease, err = impl.common.GetHelmRelease(releaseIdentifier.ClusterConfig, releaseIdentifier.ReleaseNamespace, releaseIdentifier.ReleaseName) if err != nil { impl.logger.Errorw("Error in getting helm release ", "err", err) internalErr := error2.ConvertHelmErrorToInternalError(err) @@ -574,12 +589,125 @@ func (impl *HelmAppServiceImpl) UpgradeRelease(ctx context.Context, request *cli return nil, err } + dirPath := "/tmp/dir" + outputChartPathDir := fmt.Sprintf("%s", dirPath) + err = os.MkdirAll(outputChartPathDir, os.ModePerm) + if err != nil { + return nil, err + } + + defer func() { + err := os.RemoveAll(outputChartPathDir) + if err != nil { + fmt.Println("error in deleting dir", " dir: ", outputChartPathDir, " err: ", err) + } + }() + abpath, err := chartutil.Save(helmRelease.Chart, outputChartPathDir) + if err != nil { + fmt.Println("error in saving chartdata in the destination dir ", " dir : ", outputChartPathDir, " err : ", err) + return nil, err + } + + // Unpack the .tgz file to a directory + h, err := os.Open(abpath) + if err != nil { + return nil, err + } + if err := chartutil.Expand(dirPath, h); err != nil { + fmt.Println("error unpacking chart", "dir:", abpath, "err:", err) + return nil, err + } + + outputChartPathDir = filepath.Join(dirPath, helmRelease.Chart.Metadata.Name) + + outputBuffer := bytes.NewBuffer(nil) + + settings := cli.New() + err = helmClient.SetEnvSettings(&helmClient.Options{ + RepositoryConfig: helmClient.DefaultRepositoryConfigPath, + RepositoryCache: helmClient.DefaultCachePath, + Linting: true, + }, settings) + + manager := &downloader.Manager{ + ChartPath: outputChartPathDir, + Out: outputBuffer, + Getters: getter.All(settings), + RepositoryConfig: helmClient.DefaultRepositoryConfigPath, + RepositoryCache: helmClient.DefaultCachePath, + RegistryClient: registryClient, + } + // Update dependencies before building the chart + err = manager.Update() + err = manager.Build() + if err != nil { + impl.logger.Errorw("Error updating chart dependencies", "err", err) + return nil, err + } + + // Step 1: Locate the .tgz file in the charts directory + chartsDir := filepath.Join(outputChartPathDir, "charts") + files, err := os.ReadDir(chartsDir) + if err != nil { + impl.logger.Errorw("Error reading charts directory", "dir", chartsDir, "err", err) + return nil, err + } + + for _, file := range files { + if strings.HasSuffix(file.Name(), ".tgz") { + tgzPath := filepath.Join(chartsDir, file.Name()) + + // Step 2: Expand the .tgz file + //expandedChartDir := filepath.Join(chartsDir, strings.TrimSuffix(file.Name(), ".tgz")) + tgzFile, err := os.Open(tgzPath) + if err != nil { + impl.logger.Errorw("Error opening tgz file", "file", tgzPath, "err", err) + return nil, err + } + + err = chartutil.Expand(chartsDir, tgzFile) + //tgzFile.Close() // Close the file after expanding + if err != nil { + impl.logger.Errorw("Error expanding tgz file", "file", tgzPath, "err", err) + return nil, err + } + + chartsDir = filepath.Join(chartsDir, helmRelease.Chart.Metadata.Dependencies[0].Name) + + // Step 3: Load the expanded chart + expandedChart, err := loader.LoadDir(chartsDir) + if err != nil { + impl.logger.Errorw("Error loading expanded chart", "dir", chartsDir, "err", err) + return nil, err + } + + // Step 4: Set the expanded chart as a dependency + helmRelease.Chart.SetDependencies(expandedChart) + } + } + + // Step 5: Update the Chart.Lock file + lockFilePath := filepath.Join(outputChartPathDir, "Chart.lock") + lockFileData, err := os.ReadFile(lockFilePath) + if err != nil { + impl.logger.Errorw("Error reading Chart.lock file", "file", lockFilePath, "err", err) + return nil, err + } + + helmRelease.Chart.Lock = &chart2.Lock{} + err = yaml.Unmarshal(lockFileData, helmRelease.Chart.Lock) + if err != nil { + impl.logger.Errorw("Error unmarshalling Chart.lock data", "file", lockFilePath, "err", err) + return nil, err + } + updateChartSpec := &helmClient.ChartSpec{ - ReleaseName: releaseIdentifier.ReleaseName, - Namespace: releaseIdentifier.ReleaseNamespace, - ValuesYaml: request.ValuesYaml, - MaxHistory: int(request.HistoryMax), - RegistryClient: registryClient, + ReleaseName: releaseIdentifier.ReleaseName, + Namespace: releaseIdentifier.ReleaseNamespace, + ValuesYaml: request.ValuesYaml, + MaxHistory: int(request.HistoryMax), + RegistryClient: registryClient, + DependencyUpdate: true, } impl.logger.Debug("Upgrading release") From a40ec9ab98a1e5947e9440c9bf2d9d2e1c79b3a4 Mon Sep 17 00:00:00 2001 From: Rajeev Date: Sun, 17 Nov 2024 16:07:47 +0530 Subject: [PATCH 3/9] removed extra files and folders --- kubelink/internals/fileutil/fileutil.go | 50 - kubelink/internals/fileutil/fileutil_test.go | 57 -- kubelink/internals/resolver/resolver.go | 262 ----- kubelink/internals/resolver/resolver_test.go | 310 ------ .../testdata/chartpath/base/Chart.yaml | 3 - .../charts/localdependency/Chart.yaml | 3 - .../repository/kubernetes-charts-index.yaml | 49 - kubelink/internals/thirdParty/dep/fs/fs.go | 372 ------- .../internals/thirdParty/dep/fs/fs_test.go | 642 ------------- .../internals/thirdParty/dep/fs/rename.go | 58 -- .../thirdParty/dep/fs/rename_windows.go | 69 -- .../dep/fs/testdata/symlinks/file-symlink | 1 - .../dep/fs/testdata/symlinks/invalid-symlink | 1 - .../fs/testdata/symlinks/windows-file-symlink | 1 - .../thirdParty/dep/fs/testdata/test.file | 0 kubelink/internals/urlutil/urlutil.go | 73 -- kubelink/internals/urlutil/urlutil_test.go | 81 -- kubelink/pkg/downloader/chart_downloader.go | 406 -------- .../pkg/downloader/chart_downloader_test.go | 346 ------- kubelink/pkg/downloader/doc.go | 24 - kubelink/pkg/downloader/manager.go | 905 ------------------ kubelink/pkg/downloader/manager_test.go | 600 ------------ .../pkg/downloader/testdata/helm-test-key.pub | Bin 1243 -> 0 bytes .../downloader/testdata/helm-test-key.secret | Bin 2545 -> 0 bytes .../testdata/local-subchart-0.1.0.tgz | Bin 259 -> 0 bytes .../testdata/local-subchart/Chart.yaml | 3 - .../pkg/downloader/testdata/repositories.yaml | 28 - .../repository/encoded-url-index.yaml | 15 - .../repository/kubernetes-charts-index.yaml | 49 - .../testdata/repository/malformed-index.yaml | 16 - .../repository/testing-basicauth-index.yaml | 15 - .../repository/testing-ca-file-index.yaml | 15 - .../repository/testing-https-index.yaml | 15 - ...g-https-insecureskip-tls-verify-index.yaml | 14 - .../testdata/repository/testing-index.yaml | 43 - .../repository/testing-querystring-index.yaml | 16 - .../repository/testing-relative-index.yaml | 28 - ...testing-relative-trailing-slash-index.yaml | 28 - .../downloader/testdata/signtest-0.1.0.tgz | Bin 973 -> 0 bytes .../testdata/signtest-0.1.0.tgz.prov | 21 - .../downloader/testdata/signtest/.helmignore | 5 - .../downloader/testdata/signtest/Chart.yaml | 4 - .../testdata/signtest/alpine/Chart.yaml | 7 - .../testdata/signtest/alpine/README.md | 9 - .../signtest/alpine/templates/alpine-pod.yaml | 14 - .../testdata/signtest/alpine/values.yaml | 2 - .../testdata/signtest/templates/pod.yaml | 10 - .../downloader/testdata/signtest/values.yaml | 0 .../helmApplicationService/helmAppService.go | 262 ++--- 49 files changed, 141 insertions(+), 4791 deletions(-) delete mode 100644 kubelink/internals/fileutil/fileutil.go delete mode 100644 kubelink/internals/fileutil/fileutil_test.go delete mode 100644 kubelink/internals/resolver/resolver.go delete mode 100644 kubelink/internals/resolver/resolver_test.go delete mode 100644 kubelink/internals/resolver/testdata/chartpath/base/Chart.yaml delete mode 100644 kubelink/internals/resolver/testdata/chartpath/charts/localdependency/Chart.yaml delete mode 100644 kubelink/internals/resolver/testdata/repository/kubernetes-charts-index.yaml delete mode 100644 kubelink/internals/thirdParty/dep/fs/fs.go delete mode 100644 kubelink/internals/thirdParty/dep/fs/fs_test.go delete mode 100644 kubelink/internals/thirdParty/dep/fs/rename.go delete mode 100644 kubelink/internals/thirdParty/dep/fs/rename_windows.go delete mode 120000 kubelink/internals/thirdParty/dep/fs/testdata/symlinks/file-symlink delete mode 120000 kubelink/internals/thirdParty/dep/fs/testdata/symlinks/invalid-symlink delete mode 120000 kubelink/internals/thirdParty/dep/fs/testdata/symlinks/windows-file-symlink delete mode 100644 kubelink/internals/thirdParty/dep/fs/testdata/test.file delete mode 100644 kubelink/internals/urlutil/urlutil.go delete mode 100644 kubelink/internals/urlutil/urlutil_test.go delete mode 100644 kubelink/pkg/downloader/chart_downloader.go delete mode 100644 kubelink/pkg/downloader/chart_downloader_test.go delete mode 100644 kubelink/pkg/downloader/doc.go delete mode 100644 kubelink/pkg/downloader/manager.go delete mode 100644 kubelink/pkg/downloader/manager_test.go delete mode 100644 kubelink/pkg/downloader/testdata/helm-test-key.pub delete mode 100644 kubelink/pkg/downloader/testdata/helm-test-key.secret delete mode 100644 kubelink/pkg/downloader/testdata/local-subchart-0.1.0.tgz delete mode 100644 kubelink/pkg/downloader/testdata/local-subchart/Chart.yaml delete mode 100644 kubelink/pkg/downloader/testdata/repositories.yaml delete mode 100644 kubelink/pkg/downloader/testdata/repository/encoded-url-index.yaml delete mode 100644 kubelink/pkg/downloader/testdata/repository/kubernetes-charts-index.yaml delete mode 100644 kubelink/pkg/downloader/testdata/repository/malformed-index.yaml delete mode 100644 kubelink/pkg/downloader/testdata/repository/testing-basicauth-index.yaml delete mode 100644 kubelink/pkg/downloader/testdata/repository/testing-ca-file-index.yaml delete mode 100644 kubelink/pkg/downloader/testdata/repository/testing-https-index.yaml delete mode 100644 kubelink/pkg/downloader/testdata/repository/testing-https-insecureskip-tls-verify-index.yaml delete mode 100644 kubelink/pkg/downloader/testdata/repository/testing-index.yaml delete mode 100644 kubelink/pkg/downloader/testdata/repository/testing-querystring-index.yaml delete mode 100644 kubelink/pkg/downloader/testdata/repository/testing-relative-index.yaml delete mode 100644 kubelink/pkg/downloader/testdata/repository/testing-relative-trailing-slash-index.yaml delete mode 100644 kubelink/pkg/downloader/testdata/signtest-0.1.0.tgz delete mode 100644 kubelink/pkg/downloader/testdata/signtest-0.1.0.tgz.prov delete mode 100644 kubelink/pkg/downloader/testdata/signtest/.helmignore delete mode 100644 kubelink/pkg/downloader/testdata/signtest/Chart.yaml delete mode 100644 kubelink/pkg/downloader/testdata/signtest/alpine/Chart.yaml delete mode 100644 kubelink/pkg/downloader/testdata/signtest/alpine/README.md delete mode 100644 kubelink/pkg/downloader/testdata/signtest/alpine/templates/alpine-pod.yaml delete mode 100644 kubelink/pkg/downloader/testdata/signtest/alpine/values.yaml delete mode 100644 kubelink/pkg/downloader/testdata/signtest/templates/pod.yaml delete mode 100644 kubelink/pkg/downloader/testdata/signtest/values.yaml diff --git a/kubelink/internals/fileutil/fileutil.go b/kubelink/internals/fileutil/fileutil.go deleted file mode 100644 index a110824a6..000000000 --- a/kubelink/internals/fileutil/fileutil.go +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright The Helm Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package fileutil - -import ( - "io" - "os" - "path/filepath" - - "github.com/devtron-labs/kubelink/internals/thirdParty/dep/fs" -) - -// AtomicWriteFile atomically (as atomic as os.Rename allows) writes a file to a -// disk. -func AtomicWriteFile(filename string, reader io.Reader, mode os.FileMode) error { - tempFile, err := os.CreateTemp(filepath.Split(filename)) - if err != nil { - return err - } - tempName := tempFile.Name() - - if _, err := io.Copy(tempFile, reader); err != nil { - tempFile.Close() // return value is ignored as we are already on error path - return err - } - - if err := tempFile.Close(); err != nil { - return err - } - - if err := os.Chmod(tempName, mode); err != nil { - return err - } - - return fs.RenameWithFallback(tempName, filename) -} diff --git a/kubelink/internals/fileutil/fileutil_test.go b/kubelink/internals/fileutil/fileutil_test.go deleted file mode 100644 index 92920d3c4..000000000 --- a/kubelink/internals/fileutil/fileutil_test.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright The Helm Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package fileutil - -import ( - "bytes" - "os" - "path/filepath" - "testing" -) - -func TestAtomicWriteFile(t *testing.T) { - dir := t.TempDir() - - testpath := filepath.Join(dir, "test") - stringContent := "Test content" - reader := bytes.NewReader([]byte(stringContent)) - mode := os.FileMode(0644) - - err := AtomicWriteFile(testpath, reader, mode) - if err != nil { - t.Errorf("AtomicWriteFile error: %s", err) - } - - got, err := os.ReadFile(testpath) - if err != nil { - t.Fatal(err) - } - - if stringContent != string(got) { - t.Fatalf("expected: %s, got: %s", stringContent, string(got)) - } - - gotinfo, err := os.Stat(testpath) - if err != nil { - t.Fatal(err) - } - - if mode != gotinfo.Mode() { - t.Fatalf("expected %s: to be the same mode as %s", - mode, gotinfo.Mode()) - } -} diff --git a/kubelink/internals/resolver/resolver.go b/kubelink/internals/resolver/resolver.go deleted file mode 100644 index 36603a2e1..000000000 --- a/kubelink/internals/resolver/resolver.go +++ /dev/null @@ -1,262 +0,0 @@ -/* -Copyright The Helm Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package resolver - -import ( - "bytes" - "encoding/json" - "fmt" - "os" - "path/filepath" - "strings" - "time" - - "github.com/Masterminds/semver/v3" - "github.com/pkg/errors" - - "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/chart/loader" - "helm.sh/helm/v3/pkg/helmpath" - "helm.sh/helm/v3/pkg/provenance" - "helm.sh/helm/v3/pkg/registry" - "helm.sh/helm/v3/pkg/repo" -) - -// Resolver resolves dependencies from semantic version ranges to a particular version. -type Resolver struct { - chartpath string - cachepath string - registryClient *registry.Client -} - -// New creates a new resolver for a given chart, helm home and registry client. -func New(chartpath, cachepath string, registryClient *registry.Client) *Resolver { - return &Resolver{ - chartpath: chartpath, - cachepath: cachepath, - registryClient: registryClient, - } -} - -// Resolve resolves dependencies and returns a lock file with the resolution. -func (r *Resolver) Resolve(reqs []*chart.Dependency, repoNames map[string]string) (*chart.Lock, error) { - - // Now we clone the dependencies, locking as we go. - locked := make([]*chart.Dependency, len(reqs)) - missing := []string{} - for i, d := range reqs { - constraint, err := semver.NewConstraint(d.Version) - if err != nil { - return nil, errors.Wrapf(err, "dependency %q has an invalid version/constraint format", d.Name) - } - - if d.Repository == "" { - // Local chart subfolder - if _, err := GetLocalPath(filepath.Join("charts", d.Name), r.chartpath); err != nil { - return nil, err - } - - locked[i] = &chart.Dependency{ - Name: d.Name, - Repository: "", - Version: d.Version, - } - continue - } - if strings.HasPrefix(d.Repository, "file://") { - chartpath, err := GetLocalPath(d.Repository, r.chartpath) - if err != nil { - return nil, err - } - - ch, err := loader.LoadDir(chartpath) - if err != nil { - return nil, err - } - - v, err := semver.NewVersion(ch.Metadata.Version) - if err != nil { - // Not a legit entry. - continue - } - - if !constraint.Check(v) { - missing = append(missing, d.Name) - continue - } - - locked[i] = &chart.Dependency{ - Name: d.Name, - Repository: d.Repository, - Version: ch.Metadata.Version, - } - continue - } - - repoName := repoNames[d.Name] - // if the repository was not defined, but the dependency defines a repository url, bypass the cache - if repoName == "" && d.Repository != "" { - locked[i] = &chart.Dependency{ - Name: d.Name, - Repository: d.Repository, - Version: d.Version, - } - continue - } - - var vs repo.ChartVersions - var version string - var ok bool - found := true - if !registry.IsOCI(d.Repository) { - repoIndex, err := repo.LoadIndexFile(filepath.Join(r.cachepath, helmpath.CacheIndexFile(repoName))) - if err != nil { - return nil, errors.Wrapf(err, "no cached repository for %s found. (try 'helm repo update')", repoName) - } - - vs, ok = repoIndex.Entries[d.Name] - if !ok { - return nil, errors.Errorf("%s chart not found in repo %s", d.Name, d.Repository) - } - found = false - } else { - version = d.Version - - // Check to see if an explicit version has been provided - _, err := semver.NewVersion(version) - - // Use an explicit version, otherwise search for tags - if err == nil { - vs = []*repo.ChartVersion{{ - Metadata: &chart.Metadata{ - Version: version, - }, - }} - - } else { - // Retrieve list of tags for repository - ref := fmt.Sprintf("%s/%s", strings.TrimPrefix(d.Repository, fmt.Sprintf("%s://", registry.OCIScheme)), d.Name) - tags, err := r.registryClient.Tags(ref) - if err != nil { - return nil, errors.Wrapf(err, "could not retrieve list of tags for repository %s", d.Repository) - } - - vs = make(repo.ChartVersions, len(tags)) - for ti, t := range tags { - // Mock chart version objects - version := &repo.ChartVersion{ - Metadata: &chart.Metadata{ - Version: t, - }, - } - vs[ti] = version - } - } - } - - locked[i] = &chart.Dependency{ - Name: d.Name, - Repository: d.Repository, - Version: version, - } - // The version are already sorted and hence the first one to satisfy the constraint is used - for _, ver := range vs { - v, err := semver.NewVersion(ver.Version) - // OCI does not need URLs - if err != nil || (!registry.IsOCI(d.Repository) && len(ver.URLs) == 0) { - // Not a legit entry. - continue - } - if constraint.Check(v) { - found = true - locked[i].Version = v.Original() - break - } - } - - if !found { - missing = append(missing, d.Name) - } - } - if len(missing) > 0 { - return nil, errors.Errorf("can't get a valid version for repositories %s. Try changing the version constraint in Chart.yaml", strings.Join(missing, ", ")) - } - - digest, err := HashReq(reqs, locked) - if err != nil { - return nil, err - } - - return &chart.Lock{ - Generated: time.Now(), - Digest: digest, - Dependencies: locked, - }, nil -} - -// HashReq generates a hash of the dependencies. -// -// This should be used only to compare against another hash generated by this -// function. -func HashReq(req, lock []*chart.Dependency) (string, error) { - data, err := json.Marshal([2][]*chart.Dependency{req, lock}) - if err != nil { - return "", err - } - s, err := provenance.Digest(bytes.NewBuffer(data)) - return "sha256:" + s, err -} - -// HashV2Req generates a hash of requirements generated in Helm v2. -// -// This should be used only to compare against another hash generated by the -// Helm v2 hash function. It is to handle issue: -// https://github.com/helm/helm/issues/7233 -func HashV2Req(req []*chart.Dependency) (string, error) { - dep := make(map[string][]*chart.Dependency) - dep["dependencies"] = req - data, err := json.Marshal(dep) - if err != nil { - return "", err - } - s, err := provenance.Digest(bytes.NewBuffer(data)) - return "sha256:" + s, err -} - -// GetLocalPath generates absolute local path when use -// "file://" in repository of dependencies -func GetLocalPath(repo, chartpath string) (string, error) { - var depPath string - var err error - p := strings.TrimPrefix(repo, "file://") - - // root path is absolute - if strings.HasPrefix(p, "/") { - if depPath, err = filepath.Abs(p); err != nil { - return "", err - } - } else { - depPath = filepath.Join(chartpath, p) - } - - if _, err = os.Stat(depPath); os.IsNotExist(err) { - return "", errors.Errorf("directory %s not found", depPath) - } else if err != nil { - return "", err - } - - return depPath, nil -} diff --git a/kubelink/internals/resolver/resolver_test.go b/kubelink/internals/resolver/resolver_test.go deleted file mode 100644 index a79852175..000000000 --- a/kubelink/internals/resolver/resolver_test.go +++ /dev/null @@ -1,310 +0,0 @@ -/* -Copyright The Helm Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package resolver - -import ( - "runtime" - "testing" - - "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/registry" -) - -func TestResolve(t *testing.T) { - tests := []struct { - name string - req []*chart.Dependency - expect *chart.Lock - err bool - }{ - { - name: "repo from invalid version", - req: []*chart.Dependency{ - {Name: "base", Repository: "file://base", Version: "1.1.0"}, - }, - expect: &chart.Lock{ - Dependencies: []*chart.Dependency{ - {Name: "base", Repository: "file://base", Version: "0.1.0"}, - }, - }, - err: true, - }, - { - name: "version failure", - req: []*chart.Dependency{ - {Name: "oedipus-rex", Repository: "http://example.com", Version: ">a1"}, - }, - err: true, - }, - { - name: "cache index failure", - req: []*chart.Dependency{ - {Name: "oedipus-rex", Repository: "http://example.com", Version: "1.0.0"}, - }, - expect: &chart.Lock{ - Dependencies: []*chart.Dependency{ - {Name: "oedipus-rex", Repository: "http://example.com", Version: "1.0.0"}, - }, - }, - }, - { - name: "chart not found failure", - req: []*chart.Dependency{ - {Name: "redis", Repository: "http://example.com", Version: "1.0.0"}, - }, - err: true, - }, - { - name: "constraint not satisfied failure", - req: []*chart.Dependency{ - {Name: "alpine", Repository: "http://example.com", Version: ">=1.0.0"}, - }, - err: true, - }, - { - name: "valid lock", - req: []*chart.Dependency{ - {Name: "alpine", Repository: "http://example.com", Version: ">=0.1.0"}, - }, - expect: &chart.Lock{ - Dependencies: []*chart.Dependency{ - {Name: "alpine", Repository: "http://example.com", Version: "0.2.0"}, - }, - }, - }, - { - name: "repo from valid local path", - req: []*chart.Dependency{ - {Name: "base", Repository: "file://base", Version: "0.1.0"}, - }, - expect: &chart.Lock{ - Dependencies: []*chart.Dependency{ - {Name: "base", Repository: "file://base", Version: "0.1.0"}, - }, - }, - }, - { - name: "repo from valid local path with range resolution", - req: []*chart.Dependency{ - {Name: "base", Repository: "file://base", Version: "^0.1.0"}, - }, - expect: &chart.Lock{ - Dependencies: []*chart.Dependency{ - {Name: "base", Repository: "file://base", Version: "0.1.0"}, - }, - }, - }, - { - name: "repo from invalid local path", - req: []*chart.Dependency{ - {Name: "nonexistent", Repository: "file://testdata/nonexistent", Version: "0.1.0"}, - }, - err: true, - }, - { - name: "repo from valid path under charts path", - req: []*chart.Dependency{ - {Name: "localdependency", Repository: "", Version: "0.1.0"}, - }, - expect: &chart.Lock{ - Dependencies: []*chart.Dependency{ - {Name: "localdependency", Repository: "", Version: "0.1.0"}, - }, - }, - }, - { - name: "repo from invalid path under charts path", - req: []*chart.Dependency{ - {Name: "nonexistentdependency", Repository: "", Version: "0.1.0"}, - }, - expect: &chart.Lock{ - Dependencies: []*chart.Dependency{ - {Name: "nonexistentlocaldependency", Repository: "", Version: "0.1.0"}, - }, - }, - err: true, - }, - } - - repoNames := map[string]string{"alpine": "kubernetes-charts", "redis": "kubernetes-charts"} - registryClient, _ := registry.NewClient() - r := New("testdata/chartpath", "testdata/repository", registryClient) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - l, err := r.Resolve(tt.req, repoNames) - if err != nil { - if tt.err { - return - } - t.Fatal(err) - } - - if tt.err { - t.Fatalf("Expected error in test %q", tt.name) - } - - if h, err := HashReq(tt.req, tt.expect.Dependencies); err != nil { - t.Fatal(err) - } else if h != l.Digest { - t.Errorf("%q: hashes don't match.", tt.name) - } - - // Check fields. - if len(l.Dependencies) != len(tt.req) { - t.Errorf("%s: wrong number of dependencies in lock", tt.name) - } - d0 := l.Dependencies[0] - e0 := tt.expect.Dependencies[0] - if d0.Name != e0.Name { - t.Errorf("%s: expected name %s, got %s", tt.name, e0.Name, d0.Name) - } - if d0.Repository != e0.Repository { - t.Errorf("%s: expected repo %s, got %s", tt.name, e0.Repository, d0.Repository) - } - if d0.Version != e0.Version { - t.Errorf("%s: expected version %s, got %s", tt.name, e0.Version, d0.Version) - } - }) - } -} - -func TestHashReq(t *testing.T) { - expect := "sha256:fb239e836325c5fa14b29d1540a13b7d3ba13151b67fe719f820e0ef6d66aaaf" - - tests := []struct { - name string - chartVersion string - lockVersion string - wantError bool - }{ - { - name: "chart with the expected digest", - chartVersion: "0.1.0", - lockVersion: "0.1.0", - wantError: false, - }, - { - name: "ranged version but same resolved lock version", - chartVersion: "^0.1.0", - lockVersion: "0.1.0", - wantError: true, - }, - { - name: "ranged version resolved as higher version", - chartVersion: "^0.1.0", - lockVersion: "0.1.2", - wantError: true, - }, - { - name: "different version", - chartVersion: "0.1.2", - lockVersion: "0.1.2", - wantError: true, - }, - { - name: "different version with a range", - chartVersion: "^0.1.2", - lockVersion: "0.1.2", - wantError: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - req := []*chart.Dependency{ - {Name: "alpine", Version: tt.chartVersion, Repository: "http://localhost:8879/charts"}, - } - lock := []*chart.Dependency{ - {Name: "alpine", Version: tt.lockVersion, Repository: "http://localhost:8879/charts"}, - } - h, err := HashReq(req, lock) - if err != nil { - t.Fatal(err) - } - if !tt.wantError && expect != h { - t.Errorf("Expected %q, got %q", expect, h) - } else if tt.wantError && expect == h { - t.Errorf("Expected not %q, but same", expect) - } - }) - } -} - -func TestGetLocalPath(t *testing.T) { - tests := []struct { - name string - repo string - chartpath string - expect string - winExpect string - err bool - }{ - { - name: "absolute path", - repo: "file:////", - expect: "/", - winExpect: "\\", - }, - { - name: "relative path", - repo: "file://../../testdata/chartpath/base", - chartpath: "foo/bar", - expect: "testdata/chartpath/base", - winExpect: "testdata\\chartpath\\base", - }, - { - name: "current directory path", - repo: "../charts/localdependency", - chartpath: "testdata/chartpath/charts", - expect: "testdata/chartpath/charts/localdependency", - winExpect: "testdata\\chartpath\\charts\\localdependency", - }, - { - name: "invalid local path", - repo: "file://testdata/nonexistent", - chartpath: "testdata/chartpath", - err: true, - }, - { - name: "invalid path under current directory", - repo: "charts/nonexistentdependency", - chartpath: "testdata/chartpath/charts", - err: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - p, err := GetLocalPath(tt.repo, tt.chartpath) - if err != nil { - if tt.err { - return - } - t.Fatal(err) - } - if tt.err { - t.Fatalf("Expected error in test %q", tt.name) - } - expect := tt.expect - if runtime.GOOS == "windows" { - expect = tt.winExpect - } - if p != expect { - t.Errorf("%q: expected %q, got %q", tt.name, expect, p) - } - }) - } -} diff --git a/kubelink/internals/resolver/testdata/chartpath/base/Chart.yaml b/kubelink/internals/resolver/testdata/chartpath/base/Chart.yaml deleted file mode 100644 index 860b09091..000000000 --- a/kubelink/internals/resolver/testdata/chartpath/base/Chart.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiVersion: v2 -name: base -version: 0.1.0 diff --git a/kubelink/internals/resolver/testdata/chartpath/charts/localdependency/Chart.yaml b/kubelink/internals/resolver/testdata/chartpath/charts/localdependency/Chart.yaml deleted file mode 100644 index 083c51ee5..000000000 --- a/kubelink/internals/resolver/testdata/chartpath/charts/localdependency/Chart.yaml +++ /dev/null @@ -1,3 +0,0 @@ -description: A Helm chart for Kubernetes -name: localdependency -version: 0.1.0 diff --git a/kubelink/internals/resolver/testdata/repository/kubernetes-charts-index.yaml b/kubelink/internals/resolver/testdata/repository/kubernetes-charts-index.yaml deleted file mode 100644 index c6b7962a1..000000000 --- a/kubelink/internals/resolver/testdata/repository/kubernetes-charts-index.yaml +++ /dev/null @@ -1,49 +0,0 @@ -apiVersion: v1 -entries: - alpine: - - name: alpine - urls: - - https://charts.helm.sh/stable/alpine-0.1.0.tgz - checksum: 0e6661f193211d7a5206918d42f5c2a9470b737d - home: https://helm.sh/helm - sources: - - https://github.com/helm/helm - version: 0.2.0 - description: Deploy a basic Alpine Linux pod - keywords: [] - maintainers: [] - icon: "" - apiVersion: v2 - - name: alpine - urls: - - https://charts.helm.sh/stable/alpine-0.2.0.tgz - checksum: 0e6661f193211d7a5206918d42f5c2a9470b737d - home: https://helm.sh/helm - sources: - - https://github.com/helm/helm - version: 0.1.0 - description: Deploy a basic Alpine Linux pod - keywords: [] - maintainers: [] - icon: "" - apiVersion: v2 - mariadb: - - name: mariadb - urls: - - https://charts.helm.sh/stable/mariadb-0.3.0.tgz - checksum: 65229f6de44a2be9f215d11dbff311673fc8ba56 - home: https://mariadb.org - sources: - - https://github.com/bitnami/bitnami-docker-mariadb - version: 0.3.0 - description: Chart for MariaDB - keywords: - - mariadb - - mysql - - database - - sql - maintainers: - - name: Bitnami - email: containers@bitnami.com - icon: "" - apiVersion: v2 diff --git a/kubelink/internals/thirdParty/dep/fs/fs.go b/kubelink/internals/thirdParty/dep/fs/fs.go deleted file mode 100644 index d29bb5f87..000000000 --- a/kubelink/internals/thirdParty/dep/fs/fs.go +++ /dev/null @@ -1,372 +0,0 @@ -/* -Copyright (c) for portions of fs.go are held by The Go Authors, 2016 and are provided under -the BSD license. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -package fs - -import ( - "io" - "os" - "path/filepath" - "runtime" - "syscall" - - "github.com/pkg/errors" -) - -// fs contains a copy of a few functions from dep tool code to avoid a dependency on golang/dep. -// This code is copied from https://github.com/golang/dep/blob/37d6c560cdf407be7b6cd035b23dba89df9275cf/internal/fs/fs.go -// No changes to the code were made other than removing some unused functions - -// RenameWithFallback attempts to rename a file or directory, but falls back to -// copying in the event of a cross-device link error. If the fallback copy -// succeeds, src is still removed, emulating normal rename behavior. -func RenameWithFallback(src, dst string) error { - _, err := os.Stat(src) - if err != nil { - return errors.Wrapf(err, "cannot stat %s", src) - } - - err = os.Rename(src, dst) - if err == nil { - return nil - } - - return renameFallback(err, src, dst) -} - -// renameByCopy attempts to rename a file or directory by copying it to the -// destination and then removing the src thus emulating the rename behavior. -func renameByCopy(src, dst string) error { - var cerr error - if dir, _ := IsDir(src); dir { - cerr = CopyDir(src, dst) - if cerr != nil { - cerr = errors.Wrap(cerr, "copying directory failed") - } - } else { - cerr = copyFile(src, dst) - if cerr != nil { - cerr = errors.Wrap(cerr, "copying file failed") - } - } - - if cerr != nil { - return errors.Wrapf(cerr, "rename fallback failed: cannot rename %s to %s", src, dst) - } - - return errors.Wrapf(os.RemoveAll(src), "cannot delete %s", src) -} - -var ( - errSrcNotDir = errors.New("source is not a directory") - errDstExist = errors.New("destination already exists") -) - -// CopyDir recursively copies a directory tree, attempting to preserve permissions. -// Source directory must exist, destination directory must *not* exist. -func CopyDir(src, dst string) error { - src = filepath.Clean(src) - dst = filepath.Clean(dst) - - // We use os.Lstat() here to ensure we don't fall in a loop where a symlink - // actually links to a one of its parent directories. - fi, err := os.Lstat(src) - if err != nil { - return err - } - if !fi.IsDir() { - return errSrcNotDir - } - - _, err = os.Stat(dst) - if err != nil && !os.IsNotExist(err) { - return err - } - if err == nil { - return errDstExist - } - - if err = os.MkdirAll(dst, fi.Mode()); err != nil { - return errors.Wrapf(err, "cannot mkdir %s", dst) - } - - entries, err := os.ReadDir(src) - if err != nil { - return errors.Wrapf(err, "cannot read directory %s", dst) - } - - for _, entry := range entries { - srcPath := filepath.Join(src, entry.Name()) - dstPath := filepath.Join(dst, entry.Name()) - - if entry.IsDir() { - if err = CopyDir(srcPath, dstPath); err != nil { - return errors.Wrap(err, "copying directory failed") - } - } else { - // This will include symlinks, which is what we want when - // copying things. - if err = copyFile(srcPath, dstPath); err != nil { - return errors.Wrap(err, "copying file failed") - } - } - } - - return nil -} - -// copyFile copies the contents of the file named src to the file named -// by dst. The file will be created if it does not already exist. If the -// destination file exists, all its contents will be replaced by the contents -// of the source file. The file mode will be copied from the source. -func copyFile(src, dst string) (err error) { - if sym, err := IsSymlink(src); err != nil { - return errors.Wrap(err, "symlink check failed") - } else if sym { - if err := cloneSymlink(src, dst); err != nil { - if runtime.GOOS == "windows" { - // If cloning the symlink fails on Windows because the user - // does not have the required privileges, ignore the error and - // fall back to copying the file contents. - // - // ERROR_PRIVILEGE_NOT_HELD is 1314 (0x522): - // https://msdn.microsoft.com/en-us/library/windows/desktop/ms681385(v=vs.85).aspx - if lerr, ok := err.(*os.LinkError); ok && lerr.Err != syscall.Errno(1314) { - return err - } - } else { - return err - } - } else { - return nil - } - } - - in, err := os.Open(src) - if err != nil { - return - } - defer in.Close() - - out, err := os.Create(dst) - if err != nil { - return - } - - if _, err = io.Copy(out, in); err != nil { - out.Close() - return - } - - // Check for write errors on Close - if err = out.Close(); err != nil { - return - } - - si, err := os.Stat(src) - if err != nil { - return - } - - // Temporary fix for Go < 1.9 - // - // See: https://github.com/golang/dep/issues/774 - // and https://github.com/golang/go/issues/20829 - if runtime.GOOS == "windows" { - dst = fixLongPath(dst) - } - err = os.Chmod(dst, si.Mode()) - - return -} - -// cloneSymlink will create a new symlink that points to the resolved path of sl. -// If sl is a relative symlink, dst will also be a relative symlink. -func cloneSymlink(sl, dst string) error { - resolved, err := os.Readlink(sl) - if err != nil { - return err - } - - return os.Symlink(resolved, dst) -} - -// IsDir determines is the path given is a directory or not. -func IsDir(name string) (bool, error) { - fi, err := os.Stat(name) - if err != nil { - return false, err - } - if !fi.IsDir() { - return false, errors.Errorf("%q is not a directory", name) - } - return true, nil -} - -// IsSymlink determines if the given path is a symbolic link. -func IsSymlink(path string) (bool, error) { - l, err := os.Lstat(path) - if err != nil { - return false, err - } - - return l.Mode()&os.ModeSymlink == os.ModeSymlink, nil -} - -// fixLongPath returns the extended-length (\\?\-prefixed) form of -// path when needed, in order to avoid the default 260 character file -// path limit imposed by Windows. If path is not easily converted to -// the extended-length form (for example, if path is a relative path -// or contains .. elements), or is short enough, fixLongPath returns -// path unmodified. -// -// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath -func fixLongPath(path string) string { - // Do nothing (and don't allocate) if the path is "short". - // Empirically (at least on the Windows Server 2013 builder), - // the kernel is arbitrarily okay with < 248 bytes. That - // matches what the docs above say: - // "When using an API to create a directory, the specified - // path cannot be so long that you cannot append an 8.3 file - // name (that is, the directory name cannot exceed MAX_PATH - // minus 12)." Since MAX_PATH is 260, 260 - 12 = 248. - // - // The MSDN docs appear to say that a normal path that is 248 bytes long - // will work; empirically the path must be less than 248 bytes long. - if len(path) < 248 { - // Don't fix. (This is how Go 1.7 and earlier worked, - // not automatically generating the \\?\ form) - return path - } - - // The extended form begins with \\?\, as in - // \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt. - // The extended form disables evaluation of . and .. path - // elements and disables the interpretation of / as equivalent - // to \. The conversion here rewrites / to \ and elides - // . elements as well as trailing or duplicate separators. For - // simplicity it avoids the conversion entirely for relative - // paths or paths containing .. elements. For now, - // \\server\share paths are not converted to - // \\?\UNC\server\share paths because the rules for doing so - // are less well-specified. - if len(path) >= 2 && path[:2] == `\\` { - // Don't canonicalize UNC paths. - return path - } - if !isAbs(path) { - // Relative path - return path - } - - const prefix = `\\?` - - pathbuf := make([]byte, len(prefix)+len(path)+len(`\`)) - copy(pathbuf, prefix) - n := len(path) - r, w := 0, len(prefix) - for r < n { - switch { - case os.IsPathSeparator(path[r]): - // empty block - r++ - case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])): - // /./ - r++ - case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])): - // /../ is currently unhandled - return path - default: - pathbuf[w] = '\\' - w++ - for ; r < n && !os.IsPathSeparator(path[r]); r++ { - pathbuf[w] = path[r] - w++ - } - } - } - // A drive's root directory needs a trailing \ - if w == len(`\\?\c:`) { - pathbuf[w] = '\\' - w++ - } - return string(pathbuf[:w]) -} - -func isAbs(path string) (b bool) { - v := volumeName(path) - if v == "" { - return false - } - path = path[len(v):] - if path == "" { - return false - } - return os.IsPathSeparator(path[0]) -} - -func volumeName(path string) (v string) { - if len(path) < 2 { - return "" - } - // with drive letter - c := path[0] - if path[1] == ':' && - ('0' <= c && c <= '9' || 'a' <= c && c <= 'z' || - 'A' <= c && c <= 'Z') { - return path[:2] - } - // is it UNC - if l := len(path); l >= 5 && os.IsPathSeparator(path[0]) && os.IsPathSeparator(path[1]) && - !os.IsPathSeparator(path[2]) && path[2] != '.' { - // first, leading `\\` and next shouldn't be `\`. its server name. - for n := 3; n < l-1; n++ { - // second, next '\' shouldn't be repeated. - if os.IsPathSeparator(path[n]) { - n++ - // third, following something characters. its share name. - if !os.IsPathSeparator(path[n]) { - if path[n] == '.' { - break - } - for ; n < l; n++ { - if os.IsPathSeparator(path[n]) { - break - } - } - return path[:n] - } - break - } - } - } - return "" -} diff --git a/kubelink/internals/thirdParty/dep/fs/fs_test.go b/kubelink/internals/thirdParty/dep/fs/fs_test.go deleted file mode 100644 index d42c3f110..000000000 --- a/kubelink/internals/thirdParty/dep/fs/fs_test.go +++ /dev/null @@ -1,642 +0,0 @@ -/* -Copyright (c) for portions of fs_test.go are held by The Go Authors, 2016 and are provided under -the BSD license. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -package fs - -import ( - "os" - "os/exec" - "path/filepath" - "runtime" - "sync" - "testing" -) - -var ( - mu sync.Mutex -) - -func TestRenameWithFallback(t *testing.T) { - dir := t.TempDir() - - if err := RenameWithFallback(filepath.Join(dir, "does_not_exists"), filepath.Join(dir, "dst")); err == nil { - t.Fatal("expected an error for non existing file, but got nil") - } - - srcpath := filepath.Join(dir, "src") - - if srcf, err := os.Create(srcpath); err != nil { - t.Fatal(err) - } else { - srcf.Close() - } - - if err := RenameWithFallback(srcpath, filepath.Join(dir, "dst")); err != nil { - t.Fatal(err) - } - - srcpath = filepath.Join(dir, "a") - if err := os.MkdirAll(srcpath, 0777); err != nil { - t.Fatal(err) - } - - dstpath := filepath.Join(dir, "b") - if err := os.MkdirAll(dstpath, 0777); err != nil { - t.Fatal(err) - } - - if err := RenameWithFallback(srcpath, dstpath); err == nil { - t.Fatal("expected an error if dst is an existing directory, but got nil") - } -} - -func TestCopyDir(t *testing.T) { - dir := t.TempDir() - - srcdir := filepath.Join(dir, "src") - if err := os.MkdirAll(srcdir, 0755); err != nil { - t.Fatal(err) - } - - files := []struct { - path string - contents string - fi os.FileInfo - }{ - {path: "myfile", contents: "hello world"}, - {path: filepath.Join("subdir", "file"), contents: "subdir file"}, - } - - // Create structure indicated in 'files' - for i, file := range files { - fn := filepath.Join(srcdir, file.path) - dn := filepath.Dir(fn) - if err := os.MkdirAll(dn, 0755); err != nil { - t.Fatal(err) - } - - fh, err := os.Create(fn) - if err != nil { - t.Fatal(err) - } - - if _, err = fh.Write([]byte(file.contents)); err != nil { - t.Fatal(err) - } - fh.Close() - - files[i].fi, err = os.Stat(fn) - if err != nil { - t.Fatal(err) - } - } - - destdir := filepath.Join(dir, "dest") - if err := CopyDir(srcdir, destdir); err != nil { - t.Fatal(err) - } - - // Compare copy against structure indicated in 'files' - for _, file := range files { - fn := filepath.Join(srcdir, file.path) - dn := filepath.Dir(fn) - dirOK, err := IsDir(dn) - if err != nil { - t.Fatal(err) - } - if !dirOK { - t.Fatalf("expected %s to be a directory", dn) - } - - got, err := os.ReadFile(fn) - if err != nil { - t.Fatal(err) - } - - if file.contents != string(got) { - t.Fatalf("expected: %s, got: %s", file.contents, string(got)) - } - - gotinfo, err := os.Stat(fn) - if err != nil { - t.Fatal(err) - } - - if file.fi.Mode() != gotinfo.Mode() { - t.Fatalf("expected %s: %#v\n to be the same mode as %s: %#v", - file.path, file.fi.Mode(), fn, gotinfo.Mode()) - } - } -} - -func TestCopyDirFail_SrcInaccessible(t *testing.T) { - if runtime.GOOS == "windows" { - // XXX: setting permissions works differently in - // Microsoft Windows. Skipping this until a - // compatible implementation is provided. - t.Skip("skipping on windows") - } - - var currentUID = os.Getuid() - - if currentUID == 0 { - // Skipping if root, because all files are accessible - t.Skip("Skipping for root user") - } - - var srcdir, dstdir string - - cleanup := setupInaccessibleDir(t, func(dir string) error { - srcdir = filepath.Join(dir, "src") - return os.MkdirAll(srcdir, 0755) - }) - defer cleanup() - - dir := t.TempDir() - - dstdir = filepath.Join(dir, "dst") - if err := CopyDir(srcdir, dstdir); err == nil { - t.Fatalf("expected error for CopyDir(%s, %s), got none", srcdir, dstdir) - } -} - -func TestCopyDirFail_DstInaccessible(t *testing.T) { - if runtime.GOOS == "windows" { - // XXX: setting permissions works differently in - // Microsoft Windows. Skipping this until a - // compatible implementation is provided. - t.Skip("skipping on windows") - } - - var currentUID = os.Getuid() - - if currentUID == 0 { - // Skipping if root, because all files are accessible - t.Skip("Skipping for root user") - } - - var srcdir, dstdir string - - dir := t.TempDir() - - srcdir = filepath.Join(dir, "src") - if err := os.MkdirAll(srcdir, 0755); err != nil { - t.Fatal(err) - } - - cleanup := setupInaccessibleDir(t, func(dir string) error { - dstdir = filepath.Join(dir, "dst") - return nil - }) - defer cleanup() - - if err := CopyDir(srcdir, dstdir); err == nil { - t.Fatalf("expected error for CopyDir(%s, %s), got none", srcdir, dstdir) - } -} - -func TestCopyDirFail_SrcIsNotDir(t *testing.T) { - var srcdir, dstdir string - var err error - - dir := t.TempDir() - - srcdir = filepath.Join(dir, "src") - if _, err = os.Create(srcdir); err != nil { - t.Fatal(err) - } - - dstdir = filepath.Join(dir, "dst") - - if err = CopyDir(srcdir, dstdir); err == nil { - t.Fatalf("expected error for CopyDir(%s, %s), got none", srcdir, dstdir) - } - - if err != errSrcNotDir { - t.Fatalf("expected %v error for CopyDir(%s, %s), got %s", errSrcNotDir, srcdir, dstdir, err) - } - -} - -func TestCopyDirFail_DstExists(t *testing.T) { - var srcdir, dstdir string - var err error - - dir := t.TempDir() - - srcdir = filepath.Join(dir, "src") - if err = os.MkdirAll(srcdir, 0755); err != nil { - t.Fatal(err) - } - - dstdir = filepath.Join(dir, "dst") - if err = os.MkdirAll(dstdir, 0755); err != nil { - t.Fatal(err) - } - - if err = CopyDir(srcdir, dstdir); err == nil { - t.Fatalf("expected error for CopyDir(%s, %s), got none", srcdir, dstdir) - } - - if err != errDstExist { - t.Fatalf("expected %v error for CopyDir(%s, %s), got %s", errDstExist, srcdir, dstdir, err) - } -} - -func TestCopyDirFailOpen(t *testing.T) { - if runtime.GOOS == "windows" { - // XXX: setting permissions works differently in - // Microsoft Windows. os.Chmod(..., 0222) below is not - // enough for the file to be readonly, and os.Chmod(..., - // 0000) returns an invalid argument error. Skipping - // this until a compatible implementation is - // provided. - t.Skip("skipping on windows") - } - - var currentUID = os.Getuid() - - if currentUID == 0 { - // Skipping if root, because all files are accessible - t.Skip("Skipping for root user") - } - - var srcdir, dstdir string - - dir := t.TempDir() - - srcdir = filepath.Join(dir, "src") - if err := os.MkdirAll(srcdir, 0755); err != nil { - t.Fatal(err) - } - - srcfn := filepath.Join(srcdir, "file") - srcf, err := os.Create(srcfn) - if err != nil { - t.Fatal(err) - } - srcf.Close() - - // setup source file so that it cannot be read - if err = os.Chmod(srcfn, 0222); err != nil { - t.Fatal(err) - } - - dstdir = filepath.Join(dir, "dst") - - if err = CopyDir(srcdir, dstdir); err == nil { - t.Fatalf("expected error for CopyDir(%s, %s), got none", srcdir, dstdir) - } -} - -func TestCopyFile(t *testing.T) { - dir := t.TempDir() - - srcf, err := os.Create(filepath.Join(dir, "srcfile")) - if err != nil { - t.Fatal(err) - } - - want := "hello world" - if _, err := srcf.Write([]byte(want)); err != nil { - t.Fatal(err) - } - srcf.Close() - - destf := filepath.Join(dir, "destf") - if err := copyFile(srcf.Name(), destf); err != nil { - t.Fatal(err) - } - - got, err := os.ReadFile(destf) - if err != nil { - t.Fatal(err) - } - - if want != string(got) { - t.Fatalf("expected: %s, got: %s", want, string(got)) - } - - wantinfo, err := os.Stat(srcf.Name()) - if err != nil { - t.Fatal(err) - } - - gotinfo, err := os.Stat(destf) - if err != nil { - t.Fatal(err) - } - - if wantinfo.Mode() != gotinfo.Mode() { - t.Fatalf("expected %s: %#v\n to be the same mode as %s: %#v", srcf.Name(), wantinfo.Mode(), destf, gotinfo.Mode()) - } -} - -func cleanUpDir(dir string) { - // NOTE(mattn): It seems that sometimes git.exe is not dead - // when cleanUpDir() is called. But we do not know any way to wait for it. - if runtime.GOOS == "windows" { - mu.Lock() - exec.Command(`taskkill`, `/F`, `/IM`, `git.exe`).Run() - mu.Unlock() - } - if dir != "" { - os.RemoveAll(dir) - } -} - -func TestCopyFileSymlink(t *testing.T) { - tempdir := t.TempDir() - - testcases := map[string]string{ - filepath.Join("./testdata/symlinks/file-symlink"): filepath.Join(tempdir, "dst-file"), - filepath.Join("./testdata/symlinks/windows-file-symlink"): filepath.Join(tempdir, "windows-dst-file"), - filepath.Join("./testdata/symlinks/invalid-symlink"): filepath.Join(tempdir, "invalid-symlink"), - } - - for symlink, dst := range testcases { - t.Run(symlink, func(t *testing.T) { - var err error - if err = copyFile(symlink, dst); err != nil { - t.Fatalf("failed to copy symlink: %s", err) - } - - var want, got string - - if runtime.GOOS == "windows" { - // Creating symlinks on Windows require an additional permission - // regular users aren't granted usually. So we copy the file - // content as a fall back instead of creating a real symlink. - srcb, err := os.ReadFile(symlink) - if err != nil { - t.Fatalf("%+v", err) - } - dstb, err := os.ReadFile(dst) - if err != nil { - t.Fatalf("%+v", err) - } - - want = string(srcb) - got = string(dstb) - } else { - want, err = os.Readlink(symlink) - if err != nil { - t.Fatalf("%+v", err) - } - - got, err = os.Readlink(dst) - if err != nil { - t.Fatalf("could not resolve symlink: %s", err) - } - } - - if want != got { - t.Fatalf("resolved path is incorrect. expected %s, got %s", want, got) - } - }) - } -} - -func TestCopyFileFail(t *testing.T) { - if runtime.GOOS == "windows" { - // XXX: setting permissions works differently in - // Microsoft Windows. Skipping this until a - // compatible implementation is provided. - t.Skip("skipping on windows") - } - - var currentUID = os.Getuid() - - if currentUID == 0 { - // Skipping if root, because all files are accessible - t.Skip("Skipping for root user") - } - - dir := t.TempDir() - - srcf, err := os.Create(filepath.Join(dir, "srcfile")) - if err != nil { - t.Fatal(err) - } - srcf.Close() - - var dstdir string - - cleanup := setupInaccessibleDir(t, func(dir string) error { - dstdir = filepath.Join(dir, "dir") - return os.Mkdir(dstdir, 0777) - }) - defer cleanup() - - fn := filepath.Join(dstdir, "file") - if err := copyFile(srcf.Name(), fn); err == nil { - t.Fatalf("expected error for %s, got none", fn) - } -} - -// setupInaccessibleDir creates a temporary location with a single -// directory in it, in such a way that directory is not accessible -// after this function returns. -// -// op is called with the directory as argument, so that it can create -// files or other test artifacts. -// -// If setupInaccessibleDir fails in its preparation, or op fails, t.Fatal -// will be invoked. -// -// This function returns a cleanup function that removes all the temporary -// files this function creates. It is the caller's responsibility to call -// this function before the test is done running, whether there's an error or not. -func setupInaccessibleDir(t *testing.T, op func(dir string) error) func() { - dir := t.TempDir() - - subdir := filepath.Join(dir, "dir") - - cleanup := func() { - if err := os.Chmod(subdir, 0777); err != nil { - t.Error(err) - } - } - - if err := os.Mkdir(subdir, 0777); err != nil { - cleanup() - t.Fatal(err) - return nil - } - - if err := op(subdir); err != nil { - cleanup() - t.Fatal(err) - return nil - } - - if err := os.Chmod(subdir, 0666); err != nil { - cleanup() - t.Fatal(err) - return nil - } - - return cleanup -} - -func TestIsDir(t *testing.T) { - - var currentUID = os.Getuid() - - if currentUID == 0 { - // Skipping if root, because all files are accessible - t.Skip("Skipping for root user") - } - - wd, err := os.Getwd() - if err != nil { - t.Fatal(err) - } - - var dn string - - cleanup := setupInaccessibleDir(t, func(dir string) error { - dn = filepath.Join(dir, "dir") - return os.Mkdir(dn, 0777) - }) - defer cleanup() - - tests := map[string]struct { - exists bool - err bool - }{ - wd: {true, false}, - filepath.Join(wd, "testdata"): {true, false}, - filepath.Join(wd, "main.go"): {false, true}, - filepath.Join(wd, "this_file_does_not_exist.thing"): {false, true}, - dn: {false, true}, - } - - if runtime.GOOS == "windows" { - // This test doesn't work on Microsoft Windows because - // of the differences in how file permissions are - // implemented. For this to work, the directory where - // the directory exists should be inaccessible. - delete(tests, dn) - } - - for f, want := range tests { - got, err := IsDir(f) - if err != nil && !want.err { - t.Fatalf("expected no error, got %v", err) - } - - if got != want.exists { - t.Fatalf("expected %t for %s, got %t", want.exists, f, got) - } - } -} - -func TestIsSymlink(t *testing.T) { - - var currentUID = os.Getuid() - - if currentUID == 0 { - // Skipping if root, because all files are accessible - t.Skip("Skipping for root user") - } - - dir := t.TempDir() - - dirPath := filepath.Join(dir, "directory") - if err := os.MkdirAll(dirPath, 0777); err != nil { - t.Fatal(err) - } - - filePath := filepath.Join(dir, "file") - f, err := os.Create(filePath) - if err != nil { - t.Fatal(err) - } - f.Close() - - dirSymlink := filepath.Join(dir, "dirSymlink") - fileSymlink := filepath.Join(dir, "fileSymlink") - - if err = os.Symlink(dirPath, dirSymlink); err != nil { - t.Fatal(err) - } - if err = os.Symlink(filePath, fileSymlink); err != nil { - t.Fatal(err) - } - - var ( - inaccessibleFile string - inaccessibleSymlink string - ) - - cleanup := setupInaccessibleDir(t, func(dir string) error { - inaccessibleFile = filepath.Join(dir, "file") - if fh, err := os.Create(inaccessibleFile); err != nil { - return err - } else if err = fh.Close(); err != nil { - return err - } - - inaccessibleSymlink = filepath.Join(dir, "symlink") - return os.Symlink(inaccessibleFile, inaccessibleSymlink) - }) - defer cleanup() - - tests := map[string]struct{ expected, err bool }{ - dirPath: {false, false}, - filePath: {false, false}, - dirSymlink: {true, false}, - fileSymlink: {true, false}, - inaccessibleFile: {false, true}, - inaccessibleSymlink: {false, true}, - } - - if runtime.GOOS == "windows" { - // XXX: setting permissions works differently in Windows. Skipping - // these cases until a compatible implementation is provided. - delete(tests, inaccessibleFile) - delete(tests, inaccessibleSymlink) - } - - for path, want := range tests { - got, err := IsSymlink(path) - if err != nil { - if !want.err { - t.Errorf("expected no error, got %v", err) - } - } - - if got != want.expected { - t.Errorf("expected %t for %s, got %t", want.expected, path, got) - } - } -} diff --git a/kubelink/internals/thirdParty/dep/fs/rename.go b/kubelink/internals/thirdParty/dep/fs/rename.go deleted file mode 100644 index a3e5e56a6..000000000 --- a/kubelink/internals/thirdParty/dep/fs/rename.go +++ /dev/null @@ -1,58 +0,0 @@ -//go:build !windows - -/* -Copyright (c) for portions of rename.go are held by The Go Authors, 2016 and are provided under -the BSD license. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -package fs - -import ( - "os" - "syscall" - - "github.com/pkg/errors" -) - -// renameFallback attempts to determine the appropriate fallback to failed rename -// operation depending on the resulting error. -func renameFallback(err error, src, dst string) error { - // Rename may fail if src and dst are on different devices; fall back to - // copy if we detect that case. syscall.EXDEV is the common name for the - // cross device link error which has varying output text across different - // operating systems. - terr, ok := err.(*os.LinkError) - if !ok { - return err - } else if terr.Err != syscall.EXDEV { - return errors.Wrapf(terr, "link error: cannot rename %s to %s", src, dst) - } - - return renameByCopy(src, dst) -} diff --git a/kubelink/internals/thirdParty/dep/fs/rename_windows.go b/kubelink/internals/thirdParty/dep/fs/rename_windows.go deleted file mode 100644 index a377720a6..000000000 --- a/kubelink/internals/thirdParty/dep/fs/rename_windows.go +++ /dev/null @@ -1,69 +0,0 @@ -//go:build windows - -/* -Copyright (c) for portions of rename_windows.go are held by The Go Authors, 2016 and are provided under -the BSD license. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -*/ - -package fs - -import ( - "os" - "syscall" - - "github.com/pkg/errors" -) - -// renameFallback attempts to determine the appropriate fallback to failed rename -// operation depending on the resulting error. -func renameFallback(err error, src, dst string) error { - // Rename may fail if src and dst are on different devices; fall back to - // copy if we detect that case. syscall.EXDEV is the common name for the - // cross device link error which has varying output text across different - // operating systems. - terr, ok := err.(*os.LinkError) - if !ok { - return err - } - - if terr.Err != syscall.EXDEV { - // In windows it can drop down to an operating system call that - // returns an operating system error with a different number and - // message. Checking for that as a fall back. - noerr, ok := terr.Err.(syscall.Errno) - - // 0x11 (ERROR_NOT_SAME_DEVICE) is the windows error. - // See https://msdn.microsoft.com/en-us/library/cc231199.aspx - if ok && noerr != 0x11 { - return errors.Wrapf(terr, "link error: cannot rename %s to %s", src, dst) - } - } - - return renameByCopy(src, dst) -} diff --git a/kubelink/internals/thirdParty/dep/fs/testdata/symlinks/file-symlink b/kubelink/internals/thirdParty/dep/fs/testdata/symlinks/file-symlink deleted file mode 120000 index 4c52274de..000000000 --- a/kubelink/internals/thirdParty/dep/fs/testdata/symlinks/file-symlink +++ /dev/null @@ -1 +0,0 @@ -../test.file \ No newline at end of file diff --git a/kubelink/internals/thirdParty/dep/fs/testdata/symlinks/invalid-symlink b/kubelink/internals/thirdParty/dep/fs/testdata/symlinks/invalid-symlink deleted file mode 120000 index 0edf4f301..000000000 --- a/kubelink/internals/thirdParty/dep/fs/testdata/symlinks/invalid-symlink +++ /dev/null @@ -1 +0,0 @@ -/non/existing/file \ No newline at end of file diff --git a/kubelink/internals/thirdParty/dep/fs/testdata/symlinks/windows-file-symlink b/kubelink/internals/thirdParty/dep/fs/testdata/symlinks/windows-file-symlink deleted file mode 120000 index af1d6c8f5..000000000 --- a/kubelink/internals/thirdParty/dep/fs/testdata/symlinks/windows-file-symlink +++ /dev/null @@ -1 +0,0 @@ -C:/Users/ibrahim/go/src/github.com/golang/dep/internal/fs/testdata/test.file \ No newline at end of file diff --git a/kubelink/internals/thirdParty/dep/fs/testdata/test.file b/kubelink/internals/thirdParty/dep/fs/testdata/test.file deleted file mode 100644 index e69de29bb..000000000 diff --git a/kubelink/internals/urlutil/urlutil.go b/kubelink/internals/urlutil/urlutil.go deleted file mode 100644 index a8cf7398c..000000000 --- a/kubelink/internals/urlutil/urlutil.go +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright The Helm Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package urlutil - -import ( - "net/url" - "path" - "path/filepath" -) - -// URLJoin joins a base URL to one or more path components. -// -// It's like filepath.Join for URLs. If the baseURL is pathish, this will still -// perform a join. -// -// If the URL is unparsable, this returns an error. -func URLJoin(baseURL string, paths ...string) (string, error) { - u, err := url.Parse(baseURL) - if err != nil { - return "", err - } - // We want path instead of filepath because path always uses /. - all := []string{u.Path} - all = append(all, paths...) - u.Path = path.Join(all...) - return u.String(), nil -} - -// Equal normalizes two URLs and then compares for equality. -func Equal(a, b string) bool { - au, err := url.Parse(a) - if err != nil { - a = filepath.Clean(a) - b = filepath.Clean(b) - // If urls are paths, return true only if they are an exact match - return a == b - } - bu, err := url.Parse(b) - if err != nil { - return false - } - - for _, u := range []*url.URL{au, bu} { - if u.Path == "" { - u.Path = "/" - } - u.Path = filepath.Clean(u.Path) - } - return au.String() == bu.String() -} - -// ExtractHostname returns hostname from URL -func ExtractHostname(addr string) (string, error) { - u, err := url.Parse(addr) - if err != nil { - return "", err - } - return u.Hostname(), nil -} diff --git a/kubelink/internals/urlutil/urlutil_test.go b/kubelink/internals/urlutil/urlutil_test.go deleted file mode 100644 index 82acc40fe..000000000 --- a/kubelink/internals/urlutil/urlutil_test.go +++ /dev/null @@ -1,81 +0,0 @@ -/* -Copyright The Helm Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package urlutil - -import "testing" - -func TestURLJoin(t *testing.T) { - tests := []struct { - name, url, expect string - paths []string - }{ - {name: "URL, one path", url: "http://example.com", paths: []string{"hello"}, expect: "http://example.com/hello"}, - {name: "Long URL, one path", url: "http://example.com/but/first", paths: []string{"slurm"}, expect: "http://example.com/but/first/slurm"}, - {name: "URL, two paths", url: "http://example.com", paths: []string{"hello", "world"}, expect: "http://example.com/hello/world"}, - {name: "URL, no paths", url: "http://example.com", paths: []string{}, expect: "http://example.com"}, - {name: "basepath, two paths", url: "../example.com", paths: []string{"hello", "world"}, expect: "../example.com/hello/world"}, - } - - for _, tt := range tests { - if got, err := URLJoin(tt.url, tt.paths...); err != nil { - t.Errorf("%s: error %q", tt.name, err) - } else if got != tt.expect { - t.Errorf("%s: expected %q, got %q", tt.name, tt.expect, got) - } - } -} - -func TestEqual(t *testing.T) { - for _, tt := range []struct { - a, b string - match bool - }{ - {"http://example.com", "http://example.com", true}, - {"http://example.com", "http://another.example.com", false}, - {"https://example.com", "https://example.com", true}, - {"http://example.com/", "http://example.com", true}, - {"https://example.com", "http://example.com", false}, - {"http://example.com/foo", "http://example.com/foo/", true}, - {"http://example.com/foo//", "http://example.com/foo/", true}, - {"http://example.com/./foo/", "http://example.com/foo/", true}, - {"http://example.com/bar/../foo/", "http://example.com/foo/", true}, - {"/foo", "/foo", true}, - {"/foo", "/foo/", true}, - {"/foo/.", "/foo/", true}, - {"%/1234", "%/1234", true}, - {"%/1234", "%/123", false}, - {"/1234", "%/1234", false}, - } { - if tt.match != Equal(tt.a, tt.b) { - t.Errorf("Expected %q==%q to be %t", tt.a, tt.b, tt.match) - } - } -} - -func TestExtractHostname(t *testing.T) { - tests := map[string]string{ - "http://example.com": "example.com", - "https://example.com/foo": "example.com", - - "https://example.com:31337/not/with/a/bang/but/a/whimper": "example.com", - } - for start, expect := range tests { - if got, _ := ExtractHostname(start); got != expect { - t.Errorf("Got %q, expected %q", got, expect) - } - } -} diff --git a/kubelink/pkg/downloader/chart_downloader.go b/kubelink/pkg/downloader/chart_downloader.go deleted file mode 100644 index b8c0d78e4..000000000 --- a/kubelink/pkg/downloader/chart_downloader.go +++ /dev/null @@ -1,406 +0,0 @@ -/* -Copyright The Helm Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package downloader - -import ( - "fmt" - "io" - "net/url" - "os" - "path/filepath" - "strings" - - "github.com/Masterminds/semver/v3" - "github.com/pkg/errors" - - "github.com/devtron-labs/kubelink/internals/fileutil" - "github.com/devtron-labs/kubelink/internals/urlutil" - "helm.sh/helm/v3/pkg/getter" - "helm.sh/helm/v3/pkg/helmpath" - "helm.sh/helm/v3/pkg/provenance" - "helm.sh/helm/v3/pkg/registry" - "helm.sh/helm/v3/pkg/repo" -) - -// VerificationStrategy describes a strategy for determining whether to verify a chart. -type VerificationStrategy int - -const ( - // VerifyNever will skip all verification of a chart. - VerifyNever VerificationStrategy = iota - // VerifyIfPossible will attempt a verification, it will not error if verification - // data is missing. But it will not stop processing if verification fails. - VerifyIfPossible - // VerifyAlways will always attempt a verification, and will fail if the - // verification fails. - VerifyAlways - // VerifyLater will fetch verification data, but not do any verification. - // This is to accommodate the case where another step of the process will - // perform verification. - VerifyLater -) - -// ErrNoOwnerRepo indicates that a given chart URL can't be found in any repos. -var ErrNoOwnerRepo = errors.New("could not find a repo containing the given URL") - -// ChartDownloader handles downloading a chart. -// -// It is capable of performing verifications on charts as well. -type ChartDownloader struct { - // Out is the location to write warning and info messages. - Out io.Writer - // Verify indicates what verification strategy to use. - Verify VerificationStrategy - // Keyring is the keyring file used for verification. - Keyring string - // Getter collection for the operation - Getters getter.Providers - // Options provide parameters to be passed along to the Getter being initialized. - Options []getter.Option - RegistryClient *registry.Client - RepositoryConfig string - RepositoryCache string -} - -// DownloadTo retrieves a chart. Depending on the settings, it may also download a provenance file. -// -// If Verify is set to VerifyNever, the verification will be nil. -// If Verify is set to VerifyIfPossible, this will return a verification (or nil on failure), and print a warning on failure. -// If Verify is set to VerifyAlways, this will return a verification or an error if the verification fails. -// If Verify is set to VerifyLater, this will download the prov file (if it exists), but not verify it. -// -// For VerifyNever and VerifyIfPossible, the Verification may be empty. -// -// Returns a string path to the location where the file was downloaded and a verification -// (if provenance was verified), or an error if something bad happened. -func (c *ChartDownloader) DownloadTo(ref, version, dest string) (string, *provenance.Verification, error) { - u, err := c.ResolveChartVersion(ref, version) - if err != nil { - return "", nil, err - } - - g, err := c.Getters.ByScheme(u.Scheme) - if err != nil { - return "", nil, err - } - - data, err := g.Get(u.String(), c.Options...) - if err != nil { - return "", nil, err - } - - name := filepath.Base(u.Path) - if u.Scheme == registry.OCIScheme { - idx := strings.LastIndexByte(name, ':') - name = fmt.Sprintf("%s-%s.tgz", name[:idx], name[idx+1:]) - } - - destfile := filepath.Join(dest, name) - if err := fileutil.AtomicWriteFile(destfile, data, 0644); err != nil { - return destfile, nil, err - } - - // If provenance is requested, verify it. - ver := &provenance.Verification{} - if c.Verify > VerifyNever { - body, err := g.Get(u.String() + ".prov") - if err != nil { - if c.Verify == VerifyAlways { - return destfile, ver, errors.Errorf("failed to fetch provenance %q", u.String()+".prov") - } - fmt.Fprintf(c.Out, "WARNING: Verification not found for %s: %s\n", ref, err) - return destfile, ver, nil - } - provfile := destfile + ".prov" - if err := fileutil.AtomicWriteFile(provfile, body, 0644); err != nil { - return destfile, nil, err - } - - if c.Verify != VerifyLater { - ver, err = VerifyChart(destfile, c.Keyring) - if err != nil { - // Fail always in this case, since it means the verification step - // failed. - return destfile, ver, err - } - } - } - return destfile, ver, nil -} - -func (c *ChartDownloader) getOciURI(ref, version string, u *url.URL) (*url.URL, error) { - var tag string - var err error - - // Evaluate whether an explicit version has been provided. Otherwise, determine version to use - _, errSemVer := semver.NewVersion(version) - if errSemVer == nil { - tag = version - } else { - // Retrieve list of repository tags - tags, err := c.RegistryClient.Tags(strings.TrimPrefix(ref, fmt.Sprintf("%s://", registry.OCIScheme))) - if err != nil { - return nil, err - } - if len(tags) == 0 { - return nil, errors.Errorf("Unable to locate any tags in provided repository: %s", ref) - } - - // Determine if version provided - // If empty, try to get the highest available tag - // If exact version, try to find it - // If semver constraint string, try to find a match - tag, err = registry.GetTagMatchingVersionOrConstraint(tags, version) - if err != nil { - return nil, err - } - } - - u.Path = fmt.Sprintf("%s:%s", u.Path, tag) - - return u, err -} - -// ResolveChartVersion resolves a chart reference to a URL. -// -// It returns the URL and sets the ChartDownloader's Options that can fetch -// the URL using the appropriate Getter. -// -// A reference may be an HTTP URL, an oci reference URL, a 'reponame/chartname' -// reference, or a local path. -// -// A version is a SemVer string (1.2.3-beta.1+f334a6789). -// -// - For fully qualified URLs, the version will be ignored (since URLs aren't versioned) -// - For a chart reference -// - If version is non-empty, this will return the URL for that version -// - If version is empty, this will return the URL for the latest version -// - If no version can be found, an error is returned -func (c *ChartDownloader) ResolveChartVersion(ref, version string) (*url.URL, error) { - u, err := url.Parse(ref) - if err != nil { - return nil, errors.Errorf("invalid chart URL format: %s", ref) - } - - if registry.IsOCI(u.String()) { - return c.getOciURI(ref, version, u) - } - - rf, err := loadRepoConfig(c.RepositoryConfig) - if err != nil { - return u, err - } - - if u.IsAbs() && len(u.Host) > 0 && len(u.Path) > 0 { - // In this case, we have to find the parent repo that contains this chart - // URL. And this is an unfortunate problem, as it requires actually going - // through each repo cache file and finding a matching URL. But basically - // we want to find the repo in case we have special SSL cert config - // for that repo. - - rc, err := c.scanReposForURL(ref, rf) - if err != nil { - // If there is no special config, return the default HTTP client and - // swallow the error. - if err == ErrNoOwnerRepo { - // Make sure to add the ref URL as the URL for the getter - c.Options = append(c.Options, getter.WithURL(ref)) - return u, nil - } - return u, err - } - - // If we get here, we don't need to go through the next phase of looking - // up the URL. We have it already. So we just set the parameters and return. - c.Options = append( - c.Options, - getter.WithURL(rc.URL), - ) - if rc.CertFile != "" || rc.KeyFile != "" || rc.CAFile != "" { - c.Options = append(c.Options, getter.WithTLSClientConfig(rc.CertFile, rc.KeyFile, rc.CAFile)) - } - if rc.Username != "" && rc.Password != "" { - c.Options = append( - c.Options, - getter.WithBasicAuth(rc.Username, rc.Password), - getter.WithPassCredentialsAll(rc.PassCredentialsAll), - ) - } - return u, nil - } - - // See if it's of the form: repo/path_to_chart - p := strings.SplitN(u.Path, "/", 2) - if len(p) < 2 { - return u, errors.Errorf("non-absolute URLs should be in form of repo_name/path_to_chart, got: %s", u) - } - - repoName := p[0] - chartName := p[1] - rc, err := pickChartRepositoryConfigByName(repoName, rf.Repositories) - - if err != nil { - return u, err - } - - // Now that we have the chart repository information we can use that URL - // to set the URL for the getter. - c.Options = append(c.Options, getter.WithURL(rc.URL)) - - r, err := repo.NewChartRepository(rc, c.Getters) - if err != nil { - return u, err - } - - if r != nil && r.Config != nil { - if r.Config.CertFile != "" || r.Config.KeyFile != "" || r.Config.CAFile != "" { - c.Options = append(c.Options, getter.WithTLSClientConfig(r.Config.CertFile, r.Config.KeyFile, r.Config.CAFile)) - } - if r.Config.Username != "" && r.Config.Password != "" { - c.Options = append(c.Options, - getter.WithBasicAuth(r.Config.Username, r.Config.Password), - getter.WithPassCredentialsAll(r.Config.PassCredentialsAll), - ) - } - } - - // Next, we need to load the index, and actually look up the chart. - idxFile := filepath.Join(c.RepositoryCache, helmpath.CacheIndexFile(r.Config.Name)) - i, err := repo.LoadIndexFile(idxFile) - if err != nil { - return u, errors.Wrap(err, "no cached repo found. (try 'helm repo update')") - } - - cv, err := i.Get(chartName, version) - if err != nil { - return u, errors.Wrapf(err, "chart %q matching %s not found in %s index. (try 'helm repo update')", chartName, version, r.Config.Name) - } - - if len(cv.URLs) == 0 { - return u, errors.Errorf("chart %q has no downloadable URLs", ref) - } - - // TODO: Seems that picking first URL is not fully correct - resolvedURL, err := repo.ResolveReferenceURL(rc.URL, cv.URLs[0]) - - if err != nil { - return u, errors.Errorf("invalid chart URL format: %s", ref) - } - - return url.Parse(resolvedURL) -} - -// VerifyChart takes a path to a chart archive and a keyring, and verifies the chart. -// -// It assumes that a chart archive file is accompanied by a provenance file whose -// name is the archive file name plus the ".prov" extension. -func VerifyChart(path, keyring string) (*provenance.Verification, error) { - // For now, error out if it's not a tar file. - switch fi, err := os.Stat(path); { - case err != nil: - return nil, err - case fi.IsDir(): - return nil, errors.New("unpacked charts cannot be verified") - case !isTar(path): - return nil, errors.New("chart must be a tgz file") - } - - provfile := path + ".prov" - if _, err := os.Stat(provfile); err != nil { - return nil, errors.Wrapf(err, "could not load provenance file %s", provfile) - } - - sig, err := provenance.NewFromKeyring(keyring, "") - if err != nil { - return nil, errors.Wrap(err, "failed to load keyring") - } - return sig.Verify(path, provfile) -} - -// isTar tests whether the given file is a tar file. -// -// Currently, this simply checks extension, since a subsequent function will -// untar the file and validate its binary format. -func isTar(filename string) bool { - return strings.EqualFold(filepath.Ext(filename), ".tgz") -} - -func pickChartRepositoryConfigByName(name string, cfgs []*repo.Entry) (*repo.Entry, error) { - for _, rc := range cfgs { - if rc.Name == name { - if rc.URL == "" { - return nil, errors.Errorf("no URL found for repository %s", name) - } - return rc, nil - } - } - return nil, errors.Errorf("repo %s not found", name) -} - -// scanReposForURL scans all repos to find which repo contains the given URL. -// -// This will attempt to find the given URL in all of the known repositories files. -// -// If the URL is found, this will return the repo entry that contained that URL. -// -// If all of the repos are checked, but the URL is not found, an ErrNoOwnerRepo -// error is returned. -// -// Other errors may be returned when repositories cannot be loaded or searched. -// -// Technically, the fact that a URL is not found in a repo is not a failure indication. -// Charts are not required to be included in an index before they are valid. So -// be mindful of this case. -// -// The same URL can technically exist in two or more repositories. This algorithm -// will return the first one it finds. Order is determined by the order of repositories -// in the repositories.yaml file. -func (c *ChartDownloader) scanReposForURL(u string, rf *repo.File) (*repo.Entry, error) { - // FIXME: This is far from optimal. Larger installations and index files will - // incur a performance hit for this type of scanning. - for _, rc := range rf.Repositories { - r, err := repo.NewChartRepository(rc, c.Getters) - if err != nil { - return nil, err - } - - idxFile := filepath.Join(c.RepositoryCache, helmpath.CacheIndexFile(r.Config.Name)) - i, err := repo.LoadIndexFile(idxFile) - if err != nil { - return nil, errors.Wrap(err, "no cached repo found. (try 'helm repo update')") - } - - for _, entry := range i.Entries { - for _, ver := range entry { - for _, dl := range ver.URLs { - if urlutil.Equal(u, dl) { - return rc, nil - } - } - } - } - } - // This means that there is no repo file for the given URL. - return nil, ErrNoOwnerRepo -} - -func loadRepoConfig(file string) (*repo.File, error) { - r, err := repo.LoadFile(file) - if err != nil && !os.IsNotExist(errors.Cause(err)) { - return nil, err - } - return r, nil -} diff --git a/kubelink/pkg/downloader/chart_downloader_test.go b/kubelink/pkg/downloader/chart_downloader_test.go deleted file mode 100644 index 131e21306..000000000 --- a/kubelink/pkg/downloader/chart_downloader_test.go +++ /dev/null @@ -1,346 +0,0 @@ -/* -Copyright The Helm Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package downloader - -import ( - "os" - "path/filepath" - "testing" - - "helm.sh/helm/v3/internal/test/ensure" - "helm.sh/helm/v3/pkg/cli" - "helm.sh/helm/v3/pkg/getter" - "helm.sh/helm/v3/pkg/repo" - "helm.sh/helm/v3/pkg/repo/repotest" -) - -const ( - repoConfig = "testdata/repositories.yaml" - repoCache = "testdata/repository" -) - -func TestResolveChartRef(t *testing.T) { - tests := []struct { - name, ref, expect, version string - fail bool - }{ - {name: "full URL", ref: "http://example.com/foo-1.2.3.tgz", expect: "http://example.com/foo-1.2.3.tgz"}, - {name: "full URL, HTTPS", ref: "https://example.com/foo-1.2.3.tgz", expect: "https://example.com/foo-1.2.3.tgz"}, - {name: "full URL, with authentication", ref: "http://username:password@example.com/foo-1.2.3.tgz", expect: "http://username:password@example.com/foo-1.2.3.tgz"}, - {name: "reference, testing repo", ref: "testing/alpine", expect: "http://example.com/alpine-1.2.3.tgz"}, - {name: "reference, version, testing repo", ref: "testing/alpine", version: "0.2.0", expect: "http://example.com/alpine-0.2.0.tgz"}, - {name: "reference, version, malformed repo", ref: "malformed/alpine", version: "1.2.3", expect: "http://dl.example.com/alpine-1.2.3.tgz"}, - {name: "reference, querystring repo", ref: "testing-querystring/alpine", expect: "http://example.com/alpine-1.2.3.tgz?key=value"}, - {name: "reference, testing-relative repo", ref: "testing-relative/foo", expect: "http://example.com/helm/charts/foo-1.2.3.tgz"}, - {name: "reference, testing-relative repo", ref: "testing-relative/bar", expect: "http://example.com/helm/bar-1.2.3.tgz"}, - {name: "reference, testing-relative-trailing-slash repo", ref: "testing-relative-trailing-slash/foo", expect: "http://example.com/helm/charts/foo-1.2.3.tgz"}, - {name: "reference, testing-relative-trailing-slash repo", ref: "testing-relative-trailing-slash/bar", expect: "http://example.com/helm/bar-1.2.3.tgz"}, - {name: "encoded URL", ref: "encoded-url/foobar", expect: "http://example.com/with%2Fslash/charts/foobar-4.2.1.tgz"}, - {name: "full URL, HTTPS, irrelevant version", ref: "https://example.com/foo-1.2.3.tgz", version: "0.1.0", expect: "https://example.com/foo-1.2.3.tgz", fail: true}, - {name: "full URL, file", ref: "file:///foo-1.2.3.tgz", fail: true}, - {name: "invalid", ref: "invalid-1.2.3", fail: true}, - {name: "not found", ref: "nosuchthing/invalid-1.2.3", fail: true}, - } - - c := ChartDownloader{ - Out: os.Stderr, - RepositoryConfig: repoConfig, - RepositoryCache: repoCache, - Getters: getter.All(&cli.EnvSettings{ - RepositoryConfig: repoConfig, - RepositoryCache: repoCache, - }), - } - - for _, tt := range tests { - u, err := c.ResolveChartVersion(tt.ref, tt.version) - if err != nil { - if tt.fail { - continue - } - t.Errorf("%s: failed with error %q", tt.name, err) - continue - } - if got := u.String(); got != tt.expect { - t.Errorf("%s: expected %s, got %s", tt.name, tt.expect, got) - } - } -} - -func TestResolveChartOpts(t *testing.T) { - tests := []struct { - name, ref, version string - expect []getter.Option - }{ - { - name: "repo with CA-file", - ref: "testing-ca-file/foo", - expect: []getter.Option{ - getter.WithURL("https://example.com/foo-1.2.3.tgz"), - getter.WithTLSClientConfig("cert", "key", "ca"), - }, - }, - } - - c := ChartDownloader{ - Out: os.Stderr, - RepositoryConfig: repoConfig, - RepositoryCache: repoCache, - Getters: getter.All(&cli.EnvSettings{ - RepositoryConfig: repoConfig, - RepositoryCache: repoCache, - }), - } - - // snapshot options - snapshotOpts := c.Options - - for _, tt := range tests { - // reset chart downloader options for each test case - c.Options = snapshotOpts - - expect, err := getter.NewHTTPGetter(tt.expect...) - if err != nil { - t.Errorf("%s: failed to setup http client: %s", tt.name, err) - continue - } - - u, err := c.ResolveChartVersion(tt.ref, tt.version) - if err != nil { - t.Errorf("%s: failed with error %s", tt.name, err) - continue - } - - got, err := getter.NewHTTPGetter( - append( - c.Options, - getter.WithURL(u.String()), - )..., - ) - if err != nil { - t.Errorf("%s: failed to create http client: %s", tt.name, err) - continue - } - - if *(got.(*getter.HTTPGetter)) != *(expect.(*getter.HTTPGetter)) { - t.Errorf("%s: expected %s, got %s", tt.name, expect, got) - } - } -} - -func TestVerifyChart(t *testing.T) { - v, err := VerifyChart("testdata/signtest-0.1.0.tgz", "testdata/helm-test-key.pub") - if err != nil { - t.Fatal(err) - } - // The verification is tested at length in the provenance package. Here, - // we just want a quick sanity check that the v is not empty. - if len(v.FileHash) == 0 { - t.Error("Digest missing") - } -} - -func TestIsTar(t *testing.T) { - tests := map[string]bool{ - "foo.tgz": true, - "foo/bar/baz.tgz": true, - "foo-1.2.3.4.5.tgz": true, - "foo.tar.gz": false, // for our purposes - "foo.tgz.1": false, - "footgz": false, - } - - for src, expect := range tests { - if isTar(src) != expect { - t.Errorf("%q should be %t", src, expect) - } - } -} - -func TestDownloadTo(t *testing.T) { - srv := repotest.NewTempServerWithCleanupAndBasicAuth(t, "testdata/*.tgz*") - defer srv.Stop() - if err := srv.CreateIndex(); err != nil { - t.Fatal(err) - } - - if err := srv.LinkIndices(); err != nil { - t.Fatal(err) - } - - c := ChartDownloader{ - Out: os.Stderr, - Verify: VerifyAlways, - Keyring: "testdata/helm-test-key.pub", - RepositoryConfig: repoConfig, - RepositoryCache: repoCache, - Getters: getter.All(&cli.EnvSettings{ - RepositoryConfig: repoConfig, - RepositoryCache: repoCache, - }), - Options: []getter.Option{ - getter.WithBasicAuth("username", "password"), - getter.WithPassCredentialsAll(false), - }, - } - cname := "/signtest-0.1.0.tgz" - dest := srv.Root() - where, v, err := c.DownloadTo(srv.URL()+cname, "", dest) - if err != nil { - t.Fatal(err) - } - - if expect := filepath.Join(dest, cname); where != expect { - t.Errorf("Expected download to %s, got %s", expect, where) - } - - if v.FileHash == "" { - t.Error("File hash was empty, but verification is required.") - } - - if _, err := os.Stat(filepath.Join(dest, cname)); err != nil { - t.Error(err) - } -} - -func TestDownloadTo_TLS(t *testing.T) { - // Set up mock server w/ tls enabled - srv, err := repotest.NewTempServerWithCleanup(t, "testdata/*.tgz*") - srv.Stop() - if err != nil { - t.Fatal(err) - } - srv.StartTLS() - defer srv.Stop() - if err := srv.CreateIndex(); err != nil { - t.Fatal(err) - } - if err := srv.LinkIndices(); err != nil { - t.Fatal(err) - } - - repoConfig := filepath.Join(srv.Root(), "repositories.yaml") - repoCache := srv.Root() - - c := ChartDownloader{ - Out: os.Stderr, - Verify: VerifyAlways, - Keyring: "testdata/helm-test-key.pub", - RepositoryConfig: repoConfig, - RepositoryCache: repoCache, - Getters: getter.All(&cli.EnvSettings{ - RepositoryConfig: repoConfig, - RepositoryCache: repoCache, - }), - Options: []getter.Option{}, - } - cname := "test/signtest" - dest := srv.Root() - where, v, err := c.DownloadTo(cname, "", dest) - if err != nil { - t.Fatal(err) - } - - target := filepath.Join(dest, "signtest-0.1.0.tgz") - if expect := target; where != expect { - t.Errorf("Expected download to %s, got %s", expect, where) - } - - if v.FileHash == "" { - t.Error("File hash was empty, but verification is required.") - } - - if _, err := os.Stat(target); err != nil { - t.Error(err) - } -} - -func TestDownloadTo_VerifyLater(t *testing.T) { - ensure.HelmHome(t) - - dest := t.TempDir() - - // Set up a fake repo - srv, err := repotest.NewTempServerWithCleanup(t, "testdata/*.tgz*") - if err != nil { - t.Fatal(err) - } - defer srv.Stop() - if err := srv.LinkIndices(); err != nil { - t.Fatal(err) - } - - c := ChartDownloader{ - Out: os.Stderr, - Verify: VerifyLater, - RepositoryConfig: repoConfig, - RepositoryCache: repoCache, - Getters: getter.All(&cli.EnvSettings{ - RepositoryConfig: repoConfig, - RepositoryCache: repoCache, - }), - } - cname := "/signtest-0.1.0.tgz" - where, _, err := c.DownloadTo(srv.URL()+cname, "", dest) - if err != nil { - t.Fatal(err) - } - - if expect := filepath.Join(dest, cname); where != expect { - t.Errorf("Expected download to %s, got %s", expect, where) - } - - if _, err := os.Stat(filepath.Join(dest, cname)); err != nil { - t.Fatal(err) - } - if _, err := os.Stat(filepath.Join(dest, cname+".prov")); err != nil { - t.Fatal(err) - } -} - -func TestScanReposForURL(t *testing.T) { - c := ChartDownloader{ - Out: os.Stderr, - Verify: VerifyLater, - RepositoryConfig: repoConfig, - RepositoryCache: repoCache, - Getters: getter.All(&cli.EnvSettings{ - RepositoryConfig: repoConfig, - RepositoryCache: repoCache, - }), - } - - u := "http://example.com/alpine-0.2.0.tgz" - rf, err := repo.LoadFile(repoConfig) - if err != nil { - t.Fatal(err) - } - - entry, err := c.scanReposForURL(u, rf) - if err != nil { - t.Fatal(err) - } - - if entry.Name != "testing" { - t.Errorf("Unexpected repo %q for URL %q", entry.Name, u) - } - - // A lookup failure should produce an ErrNoOwnerRepo - u = "https://no.such.repo/foo/bar-1.23.4.tgz" - if _, err = c.scanReposForURL(u, rf); err != ErrNoOwnerRepo { - t.Fatalf("expected ErrNoOwnerRepo, got %v", err) - } -} diff --git a/kubelink/pkg/downloader/doc.go b/kubelink/pkg/downloader/doc.go deleted file mode 100644 index 848468090..000000000 --- a/kubelink/pkg/downloader/doc.go +++ /dev/null @@ -1,24 +0,0 @@ -/* -Copyright The Helm Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -/* -Package downloader provides a library for downloading charts. - -This package contains various tools for downloading charts from repository -servers, and then storing them in Helm-specific directory structures. This -library contains many functions that depend on a specific -filesystem layout. -*/ -package downloader diff --git a/kubelink/pkg/downloader/manager.go b/kubelink/pkg/downloader/manager.go deleted file mode 100644 index ae8e0e737..000000000 --- a/kubelink/pkg/downloader/manager.go +++ /dev/null @@ -1,905 +0,0 @@ -/* -Copyright The Helm Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package downloader - -import ( - "crypto" - "encoding/hex" - "fmt" - "io" - "log" - "net/url" - "os" - "path" - "path/filepath" - "regexp" - "strings" - "sync" - - "github.com/Masterminds/semver/v3" - "github.com/pkg/errors" - "sigs.k8s.io/yaml" - - "github.com/devtron-labs/kubelink/internals/resolver" - "github.com/devtron-labs/kubelink/internals/thirdParty/dep/fs" - "github.com/devtron-labs/kubelink/internals/urlutil" - "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/chart/loader" - "helm.sh/helm/v3/pkg/chartutil" - "helm.sh/helm/v3/pkg/getter" - "helm.sh/helm/v3/pkg/helmpath" - "helm.sh/helm/v3/pkg/registry" - "helm.sh/helm/v3/pkg/repo" -) - -// ErrRepoNotFound indicates that chart repositories can't be found in local repo cache. -// The value of Repos is missing repos. -type ErrRepoNotFound struct { - Repos []string -} - -// Error implements the error interface. -func (e ErrRepoNotFound) Error() string { - return fmt.Sprintf("no repository definition for %s", strings.Join(e.Repos, ", ")) -} - -// Manager handles the lifecycle of fetching, resolving, and storing dependencies. -type Manager struct { - // Out is used to print warnings and notifications. - Out io.Writer - // ChartPath is the path to the unpacked base chart upon which this operates. - ChartPath string - // Verification indicates whether the chart should be verified. - Verify VerificationStrategy - // Debug is the global "--debug" flag - Debug bool - // Keyring is the key ring file. - Keyring string - // SkipUpdate indicates that the repository should not be updated first. - SkipUpdate bool - // Getter collection for the operation - Getters []getter.Provider - RegistryClient *registry.Client - RepositoryConfig string - RepositoryCache string -} - -// Build rebuilds a local charts directory from a lockfile. -// -// If the lockfile is not present, this will run a Manager.Update() -// -// If SkipUpdate is set, this will not update the repository. -func (m *Manager) Build() error { - c, err := m.loadChartDir() - if err != nil { - return err - } - - // If a lock file is found, run a build from that. Otherwise, just do - // an update. - lock := c.Lock - if lock == nil { - return m.Update() - } - - // Check that all of the repos we're dependent on actually exist. - req := c.Metadata.Dependencies - - // If using apiVersion v1, calculate the hash before resolve repo names - // because resolveRepoNames will change req if req uses repo alias - // and Helm 2 calculate the digest from the original req - // Fix for: https://github.com/helm/helm/issues/7619 - var v2Sum string - if c.Metadata.APIVersion == chart.APIVersionV1 { - v2Sum, err = resolver.HashV2Req(req) - if err != nil { - return errors.New("the lock file (requirements.lock) is out of sync with the dependencies file (requirements.yaml). Please update the dependencies") - } - } - - if _, err := m.resolveRepoNames(req); err != nil { - return err - } - - if sum, err := resolver.HashReq(req, lock.Dependencies); err != nil || sum != lock.Digest { - // If lock digest differs and chart is apiVersion v1, it maybe because the lock was built - // with Helm 2 and therefore should be checked with Helm v2 hash - // Fix for: https://github.com/helm/helm/issues/7233 - if c.Metadata.APIVersion == chart.APIVersionV1 { - log.Println("warning: a valid Helm v3 hash was not found. Checking against Helm v2 hash...") - if v2Sum != lock.Digest { - return errors.New("the lock file (requirements.lock) is out of sync with the dependencies file (requirements.yaml). Please update the dependencies") - } - } else { - return errors.New("the lock file (Chart.lock) is out of sync with the dependencies file (Chart.yaml). Please update the dependencies") - } - } - - // Check that all of the repos we're dependent on actually exist. - if err := m.hasAllRepos(lock.Dependencies); err != nil { - return err - } - - if !m.SkipUpdate { - // For each repo in the file, update the cached copy of that repo - if err := m.UpdateRepositories(); err != nil { - return err - } - } - - // Now we need to fetch every package here into charts/ - return m.downloadAll(lock.Dependencies) -} - -// Update updates a local charts directory. -// -// It first reads the Chart.yaml file, and then attempts to -// negotiate versions based on that. It will download the versions -// from remote chart repositories unless SkipUpdate is true. -func (m *Manager) Update() error { - c, err := m.loadChartDir() - if err != nil { - return err - } - - // If no dependencies are found, we consider this a successful - // completion. - req := c.Metadata.Dependencies - if req == nil { - return nil - } - - // Get the names of the repositories the dependencies need that Helm is - // configured to know about. - repoNames, err := m.resolveRepoNames(req) - if err != nil { - return err - } - - // For the repositories Helm is not configured to know about, ensure Helm - // has some information about them and, when possible, the index files - // locally. - // TODO(mattfarina): Repositories should be explicitly added by end users - // rather than automattic. In Helm v4 require users to add repositories. They - // should have to add them in order to make sure they are aware of the - // repositories and opt-in to any locations, for security. - repoNames, err = m.ensureMissingRepos(repoNames, req) - if err != nil { - return err - } - - // For each of the repositories Helm is configured to know about, update - // the index information locally. - if !m.SkipUpdate { - if err := m.UpdateRepositories(); err != nil { - return err - } - } - - // Now we need to find out which version of a chart best satisfies the - // dependencies in the Chart.yaml - lock, err := m.resolve(req, repoNames) - if err != nil { - return err - } - - // Now we need to fetch every package here into charts/ - if err := m.downloadAll(lock.Dependencies); err != nil { - return err - } - - // downloadAll might overwrite dependency version, recalculate lock digest - newDigest, err := resolver.HashReq(req, lock.Dependencies) - if err != nil { - return err - } - lock.Digest = newDigest - - // If the lock file hasn't changed, don't write a new one. - oldLock := c.Lock - if oldLock != nil && oldLock.Digest == lock.Digest { - return nil - } - - // Finally, we need to write the lockfile. - return writeLock(m.ChartPath, lock, c.Metadata.APIVersion == chart.APIVersionV1) -} - -func (m *Manager) loadChartDir() (*chart.Chart, error) { - if fi, err := os.Stat(m.ChartPath); err != nil { - return nil, errors.Wrapf(err, "could not find %s", m.ChartPath) - } else if !fi.IsDir() { - return nil, errors.New("only unpacked charts can be updated") - } - return loader.LoadDir(m.ChartPath) -} - -// resolve takes a list of dependencies and translates them into an exact version to download. -// -// This returns a lock file, which has all of the dependencies normalized to a specific version. -func (m *Manager) resolve(req []*chart.Dependency, repoNames map[string]string) (*chart.Lock, error) { - res := resolver.New(m.ChartPath, m.RepositoryCache, m.RegistryClient) - return res.Resolve(req, repoNames) -} - -// downloadAll takes a list of dependencies and downloads them into charts/ -// -// It will delete versions of the chart that exist on disk and might cause -// a conflict. -func (m *Manager) downloadAll(deps []*chart.Dependency) error { - repos, err := m.loadChartRepositories() - if err != nil { - return err - } - - destPath := filepath.Join(m.ChartPath, "charts") - tmpPath := filepath.Join(m.ChartPath, "tmpcharts") - - // Check if 'charts' directory is not actually a directory. If it does not exist, create it. - if fi, err := os.Stat(destPath); err == nil { - if !fi.IsDir() { - return errors.Errorf("%q is not a directory", destPath) - } - } else if os.IsNotExist(err) { - if err := os.MkdirAll(destPath, 0755); err != nil { - return err - } - } else { - return fmt.Errorf("unable to retrieve file info for '%s': %v", destPath, err) - } - - // Prepare tmpPath - if err := os.MkdirAll(tmpPath, 0755); err != nil { - return err - } - defer os.RemoveAll(tmpPath) - - fmt.Fprintf(m.Out, "Saving %d charts\n", len(deps)) - var saveError error - churls := make(map[string]struct{}) - for _, dep := range deps { - // No repository means the chart is in charts directory - if dep.Repository == "" { - fmt.Fprintf(m.Out, "Dependency %s did not declare a repository. Assuming it exists in the charts directory\n", dep.Name) - // NOTE: we are only validating the local dependency conforms to the constraints. No copying to tmpPath is necessary. - chartPath := filepath.Join(destPath, dep.Name) - ch, err := loader.LoadDir(chartPath) - if err != nil { - return fmt.Errorf("unable to load chart '%s': %v", chartPath, err) - } - - constraint, err := semver.NewConstraint(dep.Version) - if err != nil { - return fmt.Errorf("dependency %s has an invalid version/constraint format: %s", dep.Name, err) - } - - v, err := semver.NewVersion(ch.Metadata.Version) - if err != nil { - return fmt.Errorf("invalid version %s for dependency %s: %s", dep.Version, dep.Name, err) - } - - if !constraint.Check(v) { - saveError = fmt.Errorf("dependency %s at version %s does not satisfy the constraint %s", dep.Name, ch.Metadata.Version, dep.Version) - break - } - continue - } - if strings.HasPrefix(dep.Repository, "file://") { - if m.Debug { - fmt.Fprintf(m.Out, "Archiving %s from repo %s\n", dep.Name, dep.Repository) - } - ver, err := tarFromLocalDir(m.ChartPath, dep.Name, dep.Repository, dep.Version, tmpPath) - if err != nil { - saveError = err - break - } - dep.Version = ver - continue - } - - // Any failure to resolve/download a chart should fail: - // https://github.com/helm/helm/issues/1439 - churl, username, password, insecureskiptlsverify, passcredentialsall, caFile, certFile, keyFile, err := m.findChartURL(dep.Name, dep.Version, dep.Repository, repos) - if err != nil { - saveError = errors.Wrapf(err, "could not find %s", churl) - break - } - - if _, ok := churls[churl]; ok { - fmt.Fprintf(m.Out, "Already downloaded %s from repo %s\n", dep.Name, dep.Repository) - continue - } - - fmt.Fprintf(m.Out, "Downloading %s from repo %s\n", dep.Name, dep.Repository) - - dl := ChartDownloader{ - Out: m.Out, - Verify: m.Verify, - Keyring: m.Keyring, - RepositoryConfig: m.RepositoryConfig, - RepositoryCache: m.RepositoryCache, - RegistryClient: m.RegistryClient, - Getters: m.Getters, - Options: []getter.Option{ - getter.WithBasicAuth(username, password), - getter.WithPassCredentialsAll(passcredentialsall), - getter.WithInsecureSkipVerifyTLS(insecureskiptlsverify), - getter.WithTLSClientConfig(certFile, keyFile, caFile), - }, - } - - version := "" - if registry.IsOCI(churl) { - churl, version, err = parseOCIRef(churl) - if err != nil { - return errors.Wrapf(err, "could not parse OCI reference") - } - dl.Options = append(dl.Options, - getter.WithRegistryClient(m.RegistryClient), - getter.WithTagName(version)) - } - - if _, _, err = dl.DownloadTo(churl, version, tmpPath); err != nil { - saveError = errors.Wrapf(err, "could not download %s", churl) - break - } - - churls[churl] = struct{}{} - } - - // TODO: this should probably be refactored to be a []error, so we can capture and provide more information rather than "last error wins". - if saveError == nil { - // now we can move all downloaded charts to destPath and delete outdated dependencies - if err := m.safeMoveDeps(deps, tmpPath, destPath); err != nil { - return err - } - } else { - fmt.Fprintln(m.Out, "Save error occurred: ", saveError) - return saveError - } - return nil -} - -func parseOCIRef(chartRef string) (string, string, error) { - refTagRegexp := regexp.MustCompile(`^(oci://[^:]+(:[0-9]{1,5})?[^:]+):(.*)$`) - caps := refTagRegexp.FindStringSubmatch(chartRef) - if len(caps) != 4 { - return "", "", errors.Errorf("improperly formatted oci chart reference: %s", chartRef) - } - chartRef = caps[1] - tag := caps[3] - - return chartRef, tag, nil -} - -// safeMoveDep moves all dependencies in the source and moves them into dest. -// -// It does this by first matching the file name to an expected pattern, then loading -// the file to verify that it is a chart. -// -// Any charts in dest that do not exist in source are removed (barring local dependencies) -// -// Because it requires tar file introspection, it is more intensive than a basic move. -// -// This will only return errors that should stop processing entirely. Other errors -// will emit log messages or be ignored. -func (m *Manager) safeMoveDeps(deps []*chart.Dependency, source, dest string) error { - existsInSourceDirectory := map[string]bool{} - isLocalDependency := map[string]bool{} - sourceFiles, err := os.ReadDir(source) - if err != nil { - return err - } - // attempt to read destFiles; fail fast if we can't - destFiles, err := os.ReadDir(dest) - if err != nil { - return err - } - - for _, dep := range deps { - if dep.Repository == "" { - isLocalDependency[dep.Name] = true - } - } - - for _, file := range sourceFiles { - if file.IsDir() { - continue - } - filename := file.Name() - sourcefile := filepath.Join(source, filename) - destfile := filepath.Join(dest, filename) - existsInSourceDirectory[filename] = true - if _, err := loader.LoadFile(sourcefile); err != nil { - fmt.Fprintf(m.Out, "Could not verify %s for moving: %s (Skipping)", sourcefile, err) - continue - } - // NOTE: no need to delete the dest; os.Rename replaces it. - if err := fs.RenameWithFallback(sourcefile, destfile); err != nil { - fmt.Fprintf(m.Out, "Unable to move %s to charts dir %s (Skipping)", sourcefile, err) - continue - } - } - - fmt.Fprintln(m.Out, "Deleting outdated charts") - // find all files that exist in dest that do not exist in source; delete them (outdated dependencies) - for _, file := range destFiles { - if !file.IsDir() && !existsInSourceDirectory[file.Name()] { - fname := filepath.Join(dest, file.Name()) - ch, err := loader.LoadFile(fname) - if err != nil { - fmt.Fprintf(m.Out, "Could not verify %s for deletion: %s (Skipping)\n", fname, err) - continue - } - // local dependency - skip - if isLocalDependency[ch.Name()] { - continue - } - if err := os.Remove(fname); err != nil { - fmt.Fprintf(m.Out, "Could not delete %s: %s (Skipping)", fname, err) - continue - } - } - } - - return nil -} - -// hasAllRepos ensures that all of the referenced deps are in the local repo cache. -func (m *Manager) hasAllRepos(deps []*chart.Dependency) error { - rf, err := loadRepoConfig(m.RepositoryConfig) - if err != nil { - return err - } - repos := rf.Repositories - - // Verify that all repositories referenced in the deps are actually known - // by Helm. - missing := []string{} -Loop: - for _, dd := range deps { - // If repo is from local path or OCI, continue - if strings.HasPrefix(dd.Repository, "file://") || registry.IsOCI(dd.Repository) { - continue - } - - if dd.Repository == "" { - continue - } - for _, repo := range repos { - if urlutil.Equal(repo.URL, strings.TrimSuffix(dd.Repository, "/")) { - continue Loop - } - } - missing = append(missing, dd.Repository) - } - if len(missing) > 0 { - return ErrRepoNotFound{missing} - } - return nil -} - -// ensureMissingRepos attempts to ensure the repository information for repos -// not managed by Helm is present. This takes in the repoNames Helm is configured -// to work with along with the chart dependencies. It will find the deps not -// in a known repo and attempt to ensure the data is present for steps like -// version resolution. -func (m *Manager) ensureMissingRepos(repoNames map[string]string, deps []*chart.Dependency) (map[string]string, error) { - - var ru []*repo.Entry - - for _, dd := range deps { - - // If the chart is in the local charts directory no repository needs - // to be specified. - if dd.Repository == "" { - continue - } - - // When the repoName for a dependency is known we can skip ensuring - if _, ok := repoNames[dd.Name]; ok { - continue - } - - // The generated repository name, which will result in an index being - // locally cached, has a name pattern of "helm-manager-" followed by a - // sha256 of the repo name. This assumes end users will never create - // repositories with these names pointing to other repositories. Using - // this method of naming allows the existing repository pulling and - // resolution code to do most of the work. - rn, err := key(dd.Repository) - if err != nil { - return repoNames, err - } - rn = managerKeyPrefix + rn - - repoNames[dd.Name] = rn - - // Assuming the repository is generally available. For Helm managed - // access controls the repository needs to be added through the user - // managed system. This path will work for public charts, like those - // supplied by Bitnami, but not for protected charts, like corp ones - // behind a username and pass. - ri := &repo.Entry{ - Name: rn, - URL: dd.Repository, - } - ru = append(ru, ri) - } - - // Calls to UpdateRepositories (a public function) will only update - // repositories configured by the user. Here we update repos found in - // the dependencies that are not known to the user if update skipping - // is not configured. - if !m.SkipUpdate && len(ru) > 0 { - fmt.Fprintln(m.Out, "Getting updates for unmanaged Helm repositories...") - if err := m.parallelRepoUpdate(ru); err != nil { - return repoNames, err - } - } - - return repoNames, nil -} - -// resolveRepoNames returns the repo names of the referenced deps which can be used to fetch the cached index file -// and replaces aliased repository URLs into resolved URLs in dependencies. -func (m *Manager) resolveRepoNames(deps []*chart.Dependency) (map[string]string, error) { - rf, err := loadRepoConfig(m.RepositoryConfig) - if err != nil { - if os.IsNotExist(err) { - return make(map[string]string), nil - } - return nil, err - } - repos := rf.Repositories - - reposMap := make(map[string]string) - - // Verify that all repositories referenced in the deps are actually known - // by Helm. - missing := []string{} - for _, dd := range deps { - // Don't map the repository, we don't need to download chart from charts directory - if dd.Repository == "" { - continue - } - // if dep chart is from local path, verify the path is valid - if strings.HasPrefix(dd.Repository, "file://") { - if _, err := resolver.GetLocalPath(dd.Repository, m.ChartPath); err != nil { - return nil, err - } - - if m.Debug { - fmt.Fprintf(m.Out, "Repository from local path: %s\n", dd.Repository) - } - reposMap[dd.Name] = dd.Repository - continue - } - - if registry.IsOCI(dd.Repository) { - reposMap[dd.Name] = dd.Repository - continue - } - - found := false - - for _, repo := range repos { - if (strings.HasPrefix(dd.Repository, "@") && strings.TrimPrefix(dd.Repository, "@") == repo.Name) || - (strings.HasPrefix(dd.Repository, "alias:") && strings.TrimPrefix(dd.Repository, "alias:") == repo.Name) { - found = true - dd.Repository = repo.URL - reposMap[dd.Name] = repo.Name - break - } else if urlutil.Equal(repo.URL, dd.Repository) { - found = true - reposMap[dd.Name] = repo.Name - break - } - } - if !found { - repository := dd.Repository - // Add if URL - _, err := url.ParseRequestURI(repository) - if err == nil { - reposMap[repository] = repository - continue - } - missing = append(missing, repository) - } - } - if len(missing) > 0 { - errorMessage := fmt.Sprintf("no repository definition for %s. Please add them via 'helm repo add'", strings.Join(missing, ", ")) - // It is common for people to try to enter "stable" as a repository instead of the actual URL. - // For this case, let's give them a suggestion. - containsNonURL := false - for _, repo := range missing { - if !strings.Contains(repo, "//") && !strings.HasPrefix(repo, "@") && !strings.HasPrefix(repo, "alias:") { - containsNonURL = true - } - } - if containsNonURL { - errorMessage += ` -Note that repositories must be URLs or aliases. For example, to refer to the "example" -repository, use "https://charts.example.com/" or "@example" instead of -"example". Don't forget to add the repo, too ('helm repo add').` - } - return nil, errors.New(errorMessage) - } - return reposMap, nil -} - -// UpdateRepositories updates all of the local repos to the latest. -func (m *Manager) UpdateRepositories() error { - rf, err := loadRepoConfig(m.RepositoryConfig) - if err != nil { - return err - } - repos := rf.Repositories - if len(repos) > 0 { - fmt.Fprintln(m.Out, "Hang tight while we grab the latest from your chart repositories...") - // This prints warnings straight to out. - if err := m.parallelRepoUpdate(repos); err != nil { - return err - } - fmt.Fprintln(m.Out, "Update Complete. ⎈Happy Helming!⎈") - } - return nil -} - -func (m *Manager) parallelRepoUpdate(repos []*repo.Entry) error { - - var wg sync.WaitGroup - for _, c := range repos { - r, err := repo.NewChartRepository(c, m.Getters) - if err != nil { - return err - } - r.CachePath = m.RepositoryCache - wg.Add(1) - go func(r *repo.ChartRepository) { - if _, err := r.DownloadIndexFile(); err != nil { - // For those dependencies that are not known to helm and using a - // generated key name we display the repo url. - if strings.HasPrefix(r.Config.Name, managerKeyPrefix) { - fmt.Fprintf(m.Out, "...Unable to get an update from the %q chart repository:\n\t%s\n", r.Config.URL, err) - } else { - fmt.Fprintf(m.Out, "...Unable to get an update from the %q chart repository (%s):\n\t%s\n", r.Config.Name, r.Config.URL, err) - } - } else { - // For those dependencies that are not known to helm and using a - // generated key name we display the repo url. - if strings.HasPrefix(r.Config.Name, managerKeyPrefix) { - fmt.Fprintf(m.Out, "...Successfully got an update from the %q chart repository\n", r.Config.URL) - } else { - fmt.Fprintf(m.Out, "...Successfully got an update from the %q chart repository\n", r.Config.Name) - } - } - wg.Done() - }(r) - } - wg.Wait() - - return nil -} - -// findChartURL searches the cache of repo data for a chart that has the name and the repoURL specified. -// -// 'name' is the name of the chart. Version is an exact semver, or an empty string. If empty, the -// newest version will be returned. -// -// repoURL is the repository to search -// -// If it finds a URL that is "relative", it will prepend the repoURL. -func (m *Manager) findChartURL(name, version, repoURL string, repos map[string]*repo.ChartRepository) (url, username, password string, insecureskiptlsverify, passcredentialsall bool, caFile, certFile, keyFile string, err error) { - if registry.IsOCI(repoURL) { - return fmt.Sprintf("%s/%s:%s", repoURL, name, version), "", "", false, false, "", "", "", nil - } - - for _, cr := range repos { - - if urlutil.Equal(repoURL, cr.Config.URL) { - var entry repo.ChartVersions - entry, err = findEntryByName(name, cr) - if err != nil { - // TODO: Where linting is skipped in this function we should - // refactor to remove naked returns while ensuring the same - // behavior - //nolint:nakedret - return - } - var ve *repo.ChartVersion - ve, err = findVersionedEntry(version, entry) - if err != nil { - //nolint:nakedret - return - } - url, err = normalizeURL(repoURL, ve.URLs[0]) - if err != nil { - //nolint:nakedret - return - } - username = cr.Config.Username - password = cr.Config.Password - passcredentialsall = cr.Config.PassCredentialsAll - insecureskiptlsverify = cr.Config.InsecureSkipTLSverify - caFile = cr.Config.CAFile - certFile = cr.Config.CertFile - keyFile = cr.Config.KeyFile - //nolint:nakedret - return - } - } - url, err = repo.FindChartInRepoURL(repoURL, name, version, certFile, keyFile, caFile, m.Getters) - if err == nil { - return url, username, password, false, false, "", "", "", err - } - err = errors.Errorf("chart %s not found in %s: %s", name, repoURL, err) - return url, username, password, false, false, "", "", "", err -} - -// findEntryByName finds an entry in the chart repository whose name matches the given name. -// -// It returns the ChartVersions for that entry. -func findEntryByName(name string, cr *repo.ChartRepository) (repo.ChartVersions, error) { - for ename, entry := range cr.IndexFile.Entries { - if ename == name { - return entry, nil - } - } - return nil, errors.New("entry not found") -} - -// findVersionedEntry takes a ChartVersions list and returns a single chart version that satisfies the version constraints. -// -// If version is empty, the first chart found is returned. -func findVersionedEntry(version string, vers repo.ChartVersions) (*repo.ChartVersion, error) { - for _, verEntry := range vers { - if len(verEntry.URLs) == 0 { - // Not a legit entry. - continue - } - - if version == "" || versionEquals(version, verEntry.Version) { - return verEntry, nil - } - } - return nil, errors.New("no matching version") -} - -func versionEquals(v1, v2 string) bool { - sv1, err := semver.NewVersion(v1) - if err != nil { - // Fallback to string comparison. - return v1 == v2 - } - sv2, err := semver.NewVersion(v2) - if err != nil { - return false - } - return sv1.Equal(sv2) -} - -func normalizeURL(baseURL, urlOrPath string) (string, error) { - u, err := url.Parse(urlOrPath) - if err != nil { - return urlOrPath, err - } - if u.IsAbs() { - return u.String(), nil - } - u2, err := url.Parse(baseURL) - if err != nil { - return urlOrPath, errors.Wrap(err, "base URL failed to parse") - } - - u2.RawPath = path.Join(u2.RawPath, urlOrPath) - u2.Path = path.Join(u2.Path, urlOrPath) - return u2.String(), nil -} - -// loadChartRepositories reads the repositories.yaml, and then builds a map of -// ChartRepositories. -// -// The key is the local name (which is only present in the repositories.yaml). -func (m *Manager) loadChartRepositories() (map[string]*repo.ChartRepository, error) { - indices := map[string]*repo.ChartRepository{} - - // Load repositories.yaml file - rf, err := loadRepoConfig(m.RepositoryConfig) - if err != nil { - return indices, errors.Wrapf(err, "failed to load %s", m.RepositoryConfig) - } - - for _, re := range rf.Repositories { - lname := re.Name - idxFile := filepath.Join(m.RepositoryCache, helmpath.CacheIndexFile(lname)) - index, err := repo.LoadIndexFile(idxFile) - if err != nil { - return indices, err - } - - // TODO: use constructor - cr := &repo.ChartRepository{ - Config: re, - IndexFile: index, - } - indices[lname] = cr - } - return indices, nil -} - -// writeLock writes a lockfile to disk -func writeLock(chartpath string, lock *chart.Lock, legacyLockfile bool) error { - data, err := yaml.Marshal(lock) - if err != nil { - return err - } - lockfileName := "Chart.lock" - if legacyLockfile { - lockfileName = "requirements.lock" - } - dest := filepath.Join(chartpath, lockfileName) - return os.WriteFile(dest, data, 0644) -} - -// archive a dep chart from local directory and save it into destPath -func tarFromLocalDir(chartpath, name, repo, version, destPath string) (string, error) { - if !strings.HasPrefix(repo, "file://") { - return "", errors.Errorf("wrong format: chart %s repository %s", name, repo) - } - - origPath, err := resolver.GetLocalPath(repo, chartpath) - if err != nil { - return "", err - } - - ch, err := loader.LoadDir(origPath) - if err != nil { - return "", err - } - - constraint, err := semver.NewConstraint(version) - if err != nil { - return "", errors.Wrapf(err, "dependency %s has an invalid version/constraint format", name) - } - - v, err := semver.NewVersion(ch.Metadata.Version) - if err != nil { - return "", err - } - - if constraint.Check(v) { - _, err = chartutil.Save(ch, destPath) - return ch.Metadata.Version, err - } - - return "", errors.Errorf("can't get a valid version for dependency %s", name) -} - -// The prefix to use for cache keys created by the manager for repo names -const managerKeyPrefix = "helm-manager-" - -// key is used to turn a name, such as a repository url, into a filesystem -// safe name that is unique for querying. To accomplish this a unique hash of -// the string is used. -func key(name string) (string, error) { - in := strings.NewReader(name) - hash := crypto.SHA256.New() - if _, err := io.Copy(hash, in); err != nil { - return "", nil - } - return hex.EncodeToString(hash.Sum(nil)), nil -} diff --git a/kubelink/pkg/downloader/manager_test.go b/kubelink/pkg/downloader/manager_test.go deleted file mode 100644 index db2487d16..000000000 --- a/kubelink/pkg/downloader/manager_test.go +++ /dev/null @@ -1,600 +0,0 @@ -/* -Copyright The Helm Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package downloader - -import ( - "bytes" - "os" - "path/filepath" - "reflect" - "testing" - - "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/chart/loader" - "helm.sh/helm/v3/pkg/chartutil" - "helm.sh/helm/v3/pkg/getter" - "helm.sh/helm/v3/pkg/repo/repotest" -) - -func TestVersionEquals(t *testing.T) { - tests := []struct { - name, v1, v2 string - expect bool - }{ - {name: "semver match", v1: "1.2.3-beta.11", v2: "1.2.3-beta.11", expect: true}, - {name: "semver match, build info", v1: "1.2.3-beta.11+a", v2: "1.2.3-beta.11+b", expect: true}, - {name: "string match", v1: "abcdef123", v2: "abcdef123", expect: true}, - {name: "semver mismatch", v1: "1.2.3-beta.11", v2: "1.2.3-beta.22", expect: false}, - {name: "semver mismatch, invalid semver", v1: "1.2.3-beta.11", v2: "stinkycheese", expect: false}, - } - - for _, tt := range tests { - if versionEquals(tt.v1, tt.v2) != tt.expect { - t.Errorf("%s: failed comparison of %q and %q (expect equal: %t)", tt.name, tt.v1, tt.v2, tt.expect) - } - } -} - -func TestNormalizeURL(t *testing.T) { - tests := []struct { - name, base, path, expect string - }{ - {name: "basic URL", base: "https://example.com", path: "http://helm.sh/foo", expect: "http://helm.sh/foo"}, - {name: "relative path", base: "https://helm.sh/charts", path: "foo", expect: "https://helm.sh/charts/foo"}, - {name: "Encoded path", base: "https://helm.sh/a%2Fb/charts", path: "foo", expect: "https://helm.sh/a%2Fb/charts/foo"}, - } - - for _, tt := range tests { - got, err := normalizeURL(tt.base, tt.path) - if err != nil { - t.Errorf("%s: error %s", tt.name, err) - continue - } else if got != tt.expect { - t.Errorf("%s: expected %q, got %q", tt.name, tt.expect, got) - } - } -} - -func TestFindChartURL(t *testing.T) { - var b bytes.Buffer - m := &Manager{ - Out: &b, - RepositoryConfig: repoConfig, - RepositoryCache: repoCache, - } - repos, err := m.loadChartRepositories() - if err != nil { - t.Fatal(err) - } - - name := "alpine" - version := "0.1.0" - repoURL := "http://example.com/charts" - - churl, username, password, insecureSkipTLSVerify, passcredentialsall, _, _, _, err := m.findChartURL(name, version, repoURL, repos) - if err != nil { - t.Fatal(err) - } - - if churl != "https://charts.helm.sh/stable/alpine-0.1.0.tgz" { - t.Errorf("Unexpected URL %q", churl) - } - if username != "" { - t.Errorf("Unexpected username %q", username) - } - if password != "" { - t.Errorf("Unexpected password %q", password) - } - if passcredentialsall != false { - t.Errorf("Unexpected passcredentialsall %t", passcredentialsall) - } - if insecureSkipTLSVerify { - t.Errorf("Unexpected insecureSkipTLSVerify %t", insecureSkipTLSVerify) - } - - name = "tlsfoo" - version = "1.2.3" - repoURL = "https://example-https-insecureskiptlsverify.com" - - churl, username, password, insecureSkipTLSVerify, passcredentialsall, _, _, _, err = m.findChartURL(name, version, repoURL, repos) - if err != nil { - t.Fatal(err) - } - - if !insecureSkipTLSVerify { - t.Errorf("Unexpected insecureSkipTLSVerify %t", insecureSkipTLSVerify) - } - if churl != "https://example.com/tlsfoo-1.2.3.tgz" { - t.Errorf("Unexpected URL %q", churl) - } - if username != "" { - t.Errorf("Unexpected username %q", username) - } - if password != "" { - t.Errorf("Unexpected password %q", password) - } - if passcredentialsall != false { - t.Errorf("Unexpected passcredentialsall %t", passcredentialsall) - } -} - -func TestGetRepoNames(t *testing.T) { - b := bytes.NewBuffer(nil) - m := &Manager{ - Out: b, - RepositoryConfig: repoConfig, - RepositoryCache: repoCache, - } - tests := []struct { - name string - req []*chart.Dependency - expect map[string]string - err bool - }{ - { - name: "no repo definition, but references a url", - req: []*chart.Dependency{ - {Name: "oedipus-rex", Repository: "http://example.com/test"}, - }, - expect: map[string]string{"http://example.com/test": "http://example.com/test"}, - }, - { - name: "no repo definition failure -- stable repo", - req: []*chart.Dependency{ - {Name: "oedipus-rex", Repository: "stable"}, - }, - err: true, - }, - { - name: "no repo definition failure", - req: []*chart.Dependency{ - {Name: "oedipus-rex", Repository: "http://example.com"}, - }, - expect: map[string]string{"oedipus-rex": "testing"}, - }, - { - name: "repo from local path", - req: []*chart.Dependency{ - {Name: "local-dep", Repository: "file://./testdata/signtest"}, - }, - expect: map[string]string{"local-dep": "file://./testdata/signtest"}, - }, - { - name: "repo alias (alias:)", - req: []*chart.Dependency{ - {Name: "oedipus-rex", Repository: "alias:testing"}, - }, - expect: map[string]string{"oedipus-rex": "testing"}, - }, - { - name: "repo alias (@)", - req: []*chart.Dependency{ - {Name: "oedipus-rex", Repository: "@testing"}, - }, - expect: map[string]string{"oedipus-rex": "testing"}, - }, - { - name: "repo from local chart under charts path", - req: []*chart.Dependency{ - {Name: "local-subchart", Repository: ""}, - }, - expect: map[string]string{}, - }, - } - - for _, tt := range tests { - l, err := m.resolveRepoNames(tt.req) - if err != nil { - if tt.err { - continue - } - t.Fatal(err) - } - - if tt.err { - t.Fatalf("Expected error in test %q", tt.name) - } - - // m1 and m2 are the maps we want to compare - eq := reflect.DeepEqual(l, tt.expect) - if !eq { - t.Errorf("%s: expected map %v, got %v", tt.name, l, tt.name) - } - } -} - -func TestDownloadAll(t *testing.T) { - chartPath := t.TempDir() - m := &Manager{ - Out: new(bytes.Buffer), - RepositoryConfig: repoConfig, - RepositoryCache: repoCache, - ChartPath: chartPath, - } - signtest, err := loader.LoadDir(filepath.Join("testdata", "signtest")) - if err != nil { - t.Fatal(err) - } - if err := chartutil.SaveDir(signtest, filepath.Join(chartPath, "testdata")); err != nil { - t.Fatal(err) - } - - local, err := loader.LoadDir(filepath.Join("testdata", "local-subchart")) - if err != nil { - t.Fatal(err) - } - if err := chartutil.SaveDir(local, filepath.Join(chartPath, "charts")); err != nil { - t.Fatal(err) - } - - signDep := &chart.Dependency{ - Name: signtest.Name(), - Repository: "file://./testdata/signtest", - Version: signtest.Metadata.Version, - } - localDep := &chart.Dependency{ - Name: local.Name(), - Repository: "", - Version: local.Metadata.Version, - } - - // create a 'tmpcharts' directory to test #5567 - if err := os.MkdirAll(filepath.Join(chartPath, "tmpcharts"), 0755); err != nil { - t.Fatal(err) - } - if err := m.downloadAll([]*chart.Dependency{signDep, localDep}); err != nil { - t.Error(err) - } - - if _, err := os.Stat(filepath.Join(chartPath, "charts", "signtest-0.1.0.tgz")); os.IsNotExist(err) { - t.Error(err) - } - - // A chart with a bad name like this cannot be loaded and saved. Handling in - // the loading and saving will return an error about the invalid name. In - // this case, the chart needs to be created directly. - badchartyaml := `apiVersion: v2 -description: A Helm chart for Kubernetes -name: ../bad-local-subchart -version: 0.1.0` - if err := os.MkdirAll(filepath.Join(chartPath, "testdata", "bad-local-subchart"), 0755); err != nil { - t.Fatal(err) - } - err = os.WriteFile(filepath.Join(chartPath, "testdata", "bad-local-subchart", "Chart.yaml"), []byte(badchartyaml), 0644) - if err != nil { - t.Fatal(err) - } - - badLocalDep := &chart.Dependency{ - Name: "../bad-local-subchart", - Repository: "file://./testdata/bad-local-subchart", - Version: "0.1.0", - } - - err = m.downloadAll([]*chart.Dependency{badLocalDep}) - if err == nil { - t.Fatal("Expected error for bad dependency name") - } -} - -func TestUpdateBeforeBuild(t *testing.T) { - // Set up a fake repo - srv, err := repotest.NewTempServerWithCleanup(t, "testdata/*.tgz*") - if err != nil { - t.Fatal(err) - } - defer srv.Stop() - if err := srv.LinkIndices(); err != nil { - t.Fatal(err) - } - dir := func(p ...string) string { - return filepath.Join(append([]string{srv.Root()}, p...)...) - } - - // Save dep - d := &chart.Chart{ - Metadata: &chart.Metadata{ - Name: "dep-chart", - Version: "0.1.0", - APIVersion: "v1", - }, - } - if err := chartutil.SaveDir(d, dir()); err != nil { - t.Fatal(err) - } - // Save a chart - c := &chart.Chart{ - Metadata: &chart.Metadata{ - Name: "with-dependency", - Version: "0.1.0", - APIVersion: "v2", - Dependencies: []*chart.Dependency{{ - Name: d.Metadata.Name, - Version: ">=0.1.0", - Repository: "file://../dep-chart", - }}, - }, - } - if err := chartutil.SaveDir(c, dir()); err != nil { - t.Fatal(err) - } - - // Set-up a manager - b := bytes.NewBuffer(nil) - g := getter.Providers{getter.Provider{ - Schemes: []string{"http", "https"}, - New: getter.NewHTTPGetter, - }} - m := &Manager{ - ChartPath: dir(c.Metadata.Name), - Out: b, - Getters: g, - RepositoryConfig: dir("repositories.yaml"), - RepositoryCache: dir(), - } - - // Update before Build. see issue: https://github.com/helm/helm/issues/7101 - err = m.Update() - if err != nil { - t.Fatal(err) - } - - err = m.Build() - if err != nil { - t.Fatal(err) - } -} - -// TestUpdateWithNoRepo is for the case of a dependency that has no repo listed. -// This happens when the dependency is in the charts directory and does not need -// to be fetched. -func TestUpdateWithNoRepo(t *testing.T) { - // Set up a fake repo - srv, err := repotest.NewTempServerWithCleanup(t, "testdata/*.tgz*") - if err != nil { - t.Fatal(err) - } - defer srv.Stop() - if err := srv.LinkIndices(); err != nil { - t.Fatal(err) - } - dir := func(p ...string) string { - return filepath.Join(append([]string{srv.Root()}, p...)...) - } - - // Setup the dependent chart - d := &chart.Chart{ - Metadata: &chart.Metadata{ - Name: "dep-chart", - Version: "0.1.0", - APIVersion: "v1", - }, - } - - // Save a chart with the dependency - c := &chart.Chart{ - Metadata: &chart.Metadata{ - Name: "with-dependency", - Version: "0.1.0", - APIVersion: "v2", - Dependencies: []*chart.Dependency{{ - Name: d.Metadata.Name, - Version: "0.1.0", - }}, - }, - } - if err := chartutil.SaveDir(c, dir()); err != nil { - t.Fatal(err) - } - - // Save dependent chart into the parents charts directory. If the chart is - // not in the charts directory Helm will return an error that it is not - // found. - if err := chartutil.SaveDir(d, dir(c.Metadata.Name, "charts")); err != nil { - t.Fatal(err) - } - - // Set-up a manager - b := bytes.NewBuffer(nil) - g := getter.Providers{getter.Provider{ - Schemes: []string{"http", "https"}, - New: getter.NewHTTPGetter, - }} - m := &Manager{ - ChartPath: dir(c.Metadata.Name), - Out: b, - Getters: g, - RepositoryConfig: dir("repositories.yaml"), - RepositoryCache: dir(), - } - - // Test the update - err = m.Update() - if err != nil { - t.Fatal(err) - } -} - -// This function is the skeleton test code of failing tests for #6416 and #6871 and bugs due to #5874. -// -// This function is used by below tests that ensures success of build operation -// with optional fields, alias, condition, tags, and even with ranged version. -// Parent chart includes local-subchart 0.1.0 subchart from a fake repository, by default. -// If each of these main fields (name, version, repository) is not supplied by dep param, default value will be used. -func checkBuildWithOptionalFields(t *testing.T, chartName string, dep chart.Dependency) { - // Set up a fake repo - srv, err := repotest.NewTempServerWithCleanup(t, "testdata/*.tgz*") - if err != nil { - t.Fatal(err) - } - defer srv.Stop() - if err := srv.LinkIndices(); err != nil { - t.Fatal(err) - } - dir := func(p ...string) string { - return filepath.Join(append([]string{srv.Root()}, p...)...) - } - - // Set main fields if not exist - if dep.Name == "" { - dep.Name = "local-subchart" - } - if dep.Version == "" { - dep.Version = "0.1.0" - } - if dep.Repository == "" { - dep.Repository = srv.URL() - } - - // Save a chart - c := &chart.Chart{ - Metadata: &chart.Metadata{ - Name: chartName, - Version: "0.1.0", - APIVersion: "v2", - Dependencies: []*chart.Dependency{&dep}, - }, - } - if err := chartutil.SaveDir(c, dir()); err != nil { - t.Fatal(err) - } - - // Set-up a manager - b := bytes.NewBuffer(nil) - g := getter.Providers{getter.Provider{ - Schemes: []string{"http", "https"}, - New: getter.NewHTTPGetter, - }} - m := &Manager{ - ChartPath: dir(chartName), - Out: b, - Getters: g, - RepositoryConfig: dir("repositories.yaml"), - RepositoryCache: dir(), - } - - // First build will update dependencies and create Chart.lock file. - err = m.Build() - if err != nil { - t.Fatal(err) - } - - // Second build should be passed. See PR #6655. - err = m.Build() - if err != nil { - t.Fatal(err) - } -} - -func TestBuild_WithoutOptionalFields(t *testing.T) { - // Dependency has main fields only (name/version/repository) - checkBuildWithOptionalFields(t, "without-optional-fields", chart.Dependency{}) -} - -func TestBuild_WithSemVerRange(t *testing.T) { - // Dependency version is the form of SemVer range - checkBuildWithOptionalFields(t, "with-semver-range", chart.Dependency{ - Version: ">=0.1.0", - }) -} - -func TestBuild_WithAlias(t *testing.T) { - // Dependency has an alias - checkBuildWithOptionalFields(t, "with-alias", chart.Dependency{ - Alias: "local-subchart-alias", - }) -} - -func TestBuild_WithCondition(t *testing.T) { - // Dependency has a condition - checkBuildWithOptionalFields(t, "with-condition", chart.Dependency{ - Condition: "some.condition", - }) -} - -func TestBuild_WithTags(t *testing.T) { - // Dependency has several tags - checkBuildWithOptionalFields(t, "with-tags", chart.Dependency{ - Tags: []string{"tag1", "tag2"}, - }) -} - -// Failing test for #6871 -func TestBuild_WithRepositoryAlias(t *testing.T) { - // Dependency repository is aliased in Chart.yaml - checkBuildWithOptionalFields(t, "with-repository-alias", chart.Dependency{ - Repository: "@test", - }) -} - -func TestErrRepoNotFound_Error(t *testing.T) { - type fields struct { - Repos []string - } - tests := []struct { - name string - fields fields - want string - }{ - { - name: "OK", - fields: fields{ - Repos: []string{"https://charts1.example.com", "https://charts2.example.com"}, - }, - want: "no repository definition for https://charts1.example.com, https://charts2.example.com", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - e := ErrRepoNotFound{ - Repos: tt.fields.Repos, - } - if got := e.Error(); got != tt.want { - t.Errorf("Error() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestKey(t *testing.T) { - tests := []struct { - name string - expect string - }{ - { - name: "file:////tmp", - expect: "afeed3459e92a874f6373aca264ce1459bfa91f9c1d6612f10ae3dc2ee955df3", - }, - { - name: "https://example.com/charts", - expect: "7065c57c94b2411ad774638d76823c7ccb56415441f5ab2f5ece2f3845728e5d", - }, - { - name: "foo/bar/baz", - expect: "15c46a4f8a189ae22f36f201048881d6c090c93583bedcf71f5443fdef224c82", - }, - } - - for _, tt := range tests { - o, err := key(tt.name) - if err != nil { - t.Fatalf("unable to generate key for %q with error: %s", tt.name, err) - } - if o != tt.expect { - t.Errorf("wrong key name generated for %q, expected %q but got %q", tt.name, tt.expect, o) - } - } -} diff --git a/kubelink/pkg/downloader/testdata/helm-test-key.pub b/kubelink/pkg/downloader/testdata/helm-test-key.pub deleted file mode 100644 index 38714f25adaf701b08e11fd559a587074bbde0e4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1243 zcmV<11SI>J0SyFKmTjH^2mr{k15wFPQdpTAAEclY4oV^-7nTy5x$&xF;PC4il}o-Pk4@#$knlp#(|J0GE?qli_lr;7-o zyY8vBsN_GJe;#w<`JdR7riNL&RJlcS)FG+W=91;dYS6NZ2tY?kZ8Sw9{r=e4|L3E{ zRod|EPC!PWgW&pe&4qiqKQAijj;G~fyjcC^m%0p54Tn{h%YFKd0VC4n=#~SRc@BVd znj66*)Om%*SEQfX{1*Tb0RRECT}WkYZ6H)-b98BLXCNq4XlZjGYh`&Lb7*gMY-AvB zZftoVVr3w8b7f>8W^ZyJbY*jNX>MmOAVg0fPES-IR8mz_R4yqXJZNQXZ7p6uVakV zT7GGV$jaKjyjfI_a~N1!Hk?5C$0wa&4)R=i$v7t&ZMycW#RkavpF%A?>MTT2anNDzOQUm<++zEOykJ9-@&c2QXq3owqf7fek`=L@+7iF zv;IW2Q>Q&r+V@cWDF&hAUUsCKlDinerKgvJUJCl$5gjb7NhM{mBP%!M^mX-iS8xFf zuLB{@MDqvtZzF#Bxd9CXSC(y_0SExW>8~h=U8|!do4*OJj2u#!KDe3v+1T+aVzU5di=Ji2)x37y$|Z2?YXImTjH_8w>yn2@r%kznCAv zhhp0`2mpxVj5j%o&5i)?`r7iES|8dA@p2kk@+XS(tjBGN)6>tm^=gayCn`gTEC*K74Y~{I_PREk) z)PstIMx1RxB@cK8%Mey%;nVnKriAKUk2Ky?dBMG3uXItKL$3N(#3P^pQa*K$l)wUy F^>pMLK0g2e diff --git a/kubelink/pkg/downloader/testdata/helm-test-key.secret b/kubelink/pkg/downloader/testdata/helm-test-key.secret deleted file mode 100644 index a966aef93ed97d01d764f29940738df6df2d9d24..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2545 zcmVclY4oV^-7nTy5x$&xF;PC4il}o-Pk4@#$knlp#(|J0GE?qli_lr;7-o zyY8vBsN_GJe;#w<`JdR7riNL&RJlcS)FG+W=91;dYS6NZ2tY?kZ8Sw9{r=e4|L3E{ zRod|EPC!PWgW&pe&4qiqKQAijj;G~fyjcC^m%0p54Tn{h%YFKd0VC4n=#~SRc@BVd znj66*)Om%*SEQfX{1*Tb0RRC22mUT~!#(ymA#eaSp1lpODzX${Vf^l{qDyu}xC-Z; zRnH<54GSVm<$?Ua1k#(+mu~3_*CIx=sPuoZB#9t`5)>)SncaZ0<~%)I$~BM-5aP3W z%`ewoaI;P40uHnDeE!9-_o2Lr{wDfL45jGGU-JZ36T9ToJqMX(TnRN-EvGi{o6aI#oT_2HU(J8=theYZsj5h?ml@F2 zqCpxqkdZi=~i+&Z}q^cR< zq>lNT5cnJ5X@K!3vOww0B>@Bg*7x*i59vbegj}$ELl?K2l`+`uY;jn;@-#}^!(c8$ z&Y`@LLxZ_Y>^#gGbxsy-2s=w7cVmR@z_%b#0_e^qDmIrpKw6U7N;6^TN}@&nxKj6i zje++&m}XQA&G8O8FX86?Frxrjmu5ktfDRyHBb|j&n&H#v>T!Mdmk8Y#1OV>P(*gow;}0v-BdsmdUSV3M9tIkRO0OTBw16eYCzxs>OEG!?i}$^8yY+hFlb3GJ~F z@#2Vimrfeb0(o3X?>!tSIROL!mGC1>cHXGVp;VD$oE{N!h=IF(C(PNLd6^nZO^!ix zHnE%@Y*d~bJl_M}WW0D1EM+&xdQI5#y67>-8{P4^*?j9-RL!3>e89fC4fbJFTGXY* zQA`Z&jZ*qV!0N>p>(<2RFPDhHj^h*B*O(i139Dwv{>MY%puY021Or@)I~ufINM&qo zAXH^@bZKs9AShI5X>%ZJWqBZTXm53FWFT*DY^`Z*m}XWpi|CZf7na zL{A`2PgEdOQdLt_E-4^9Xk~0|Ep%mbbZKs9Kxk!bZ7y?YK8XQ01QP)Y03iheSC(y_ z0viJb3ke7Z0|gZd2?z@X76JnS00JHX0vCV)3JDN|JHMD8!G~grMF;@2`RLQ-(ihS@ zk(ZDi8>PUMNBttVp^;f=(#~5ORUCP*V~o^Verbou%G$oXSyYd67+6|1oIv=;C!Jsp z@?3ezI42oxy7sHZ2FUrJLM=V)2rULvozb@g^vZ~+Ui10l{t^9T2DBYydv1DFI?mTjH^2mrz9 zuPBIJtD_~GzX`6498#D*yg_W@HI~u}LQvFZ zjHz2I7O5nm<}d0gU&SbRw}dGu2{gYWzK!Qb2tL4r=Ttf(&pz_gadeY}n}E@spby2h zn?Jq8s}cOpE&?`36cTnn-abrV^*hkY1rlNa6>W(?OAePZXMfE&?IzWku7z=T;E66)b)o_po?XSOFY;U!8IY8l|z)F~<`!sdiAt3+}0RRC2 z2mUKrERA52qzq^qU4-%uqeMA@h`$YTvMnKwO3MFdg819*{h|i5{tcC;Av-jm`%7`? zISDa>*_u$~x5)kpVt_aYB_e`#K)Xd5tcJ05BQ>ps?qeo`#OS{-ilRZ+9`nljqxsy1 zp;Lu#*--$l?6qncfhI%m^w(3lOt}ywL5?%+_Ov|T=-O)O#|1&>=}51a%Sb~KTR2_K z!};{n;NgPO;;v%0;n-j>b-Y|l)x=^&d84lKmr8o*+q*$Sul50u>9%n+e!b~90-}xc znpRXgsh*hBzGXpmnXaxdFnD1FEnbiC?537`DY#mL7&iHNEY4|+!A|s9dFssoYIy_z z+imR1K+cnVPeX&M1X~ed#U~gsS6HR0zgm1NR~u@{BN;*5Gvl)42%Kq{=4gSyFIAOo zw)ZGKn^3RZn+iXfb*zL1mnJJGsvTLnDB5DF8)!;+KX&@(mJ7k5LnTlXYxI(#)c`4{ zo6I4Djv|uTRI{JJ9glvUHq0WkzV7H91OVbY6u%#c1Z-!^cIjhIC)Ek7Hx7cRvtc6M zYLV(#kP^D1#2+7pDzLBFanZFqRw>On{`4qC48A)&{zk{n0CZEKY$SfN1Rk^!_V}?Oa05R<~;U7Vou+rQZcj7^Zr@2q2}K8g2gzsQ|Y$Hp^5`riTL^4T#Q?}_!b9ge@36zaVNe`|(D|D@%b z?q#ETmMPVDW6=SC^ zp>(H_BkTP!*5u$7;(xt$0Z3AJ%*#wE`2MxJYbiqwMV z55Sy@$Oto*a)E^@IG(B#1R_APzVCxJvjDnj!`b?U+KFs4f-?w1(lA6SB!RPKJ5Wx? zlJL}niiAd-Z9pXtcm~T5R%GGR_+_Sq>RpdC-c)(PyDc zVQyr3R8em|NM&qo0PNJuio!4y2H>vq6nTN^{F#IDc zVQyr3R8em|NM&qo0PI;!Z{s!-&Y8brUgz=_Sktr}$Ea^Xvp|8iX|P!oD2ie|mc|mX z6v>j56T{7aFGN{RptN<1*ba)-bCFFAIYWuhe96m92l8R?O^z<`H5TgZ&=5k1>0}bG zLWuTN3`du{-*J36nocjyKpfnXKSAjOx-;==UG2^NM}SuTM9xd2XRsQwlzif(4e|dK zd`qf;q&gX}G!DKi7vwYr@=RkvGiXi^TQzG4KIDSE^{zVnQ|$P^LRFGKiUZike<8*# z{*Onaj{hgY=CLE|my8|%0~JgTD{>4VF*=~sVa28w)d|0wGg8BYv-qqfgS&OPO6ZZHjWOhV=w{qp0jiKm`e}7wAQ%b!RMqDWXdd{ zz>wrpXYas~!XQ@!7DN7Q9CgahK~siRbpijkj+XL)Qn;5PhyQ)W;YY33V04^WnFN*` zD5;4vetq}pE*M9QXEJoI;9%JCzjnq)X#?!#|M*4xA6<+){+|MWSN~s=Rb~wc3-mI9 zt9U}-d#TF@uqI`>sRDZ*g7ve(po$;d=kdC257cLhc~iQC{EYQ?!kG+tx!{Q@qI^B6 zYa*N;ZT^3Fe|7!CdtRgm)Ul8M6ESSZ|H-uD|49%7Iz3=v6~R4v$VijJKq-{IivA&| zCNYP39{YigFwmCVbI#buoM8S`Kh7bQj*?*9x~T;`Agsu(!ON(~nzX7OqF<=vKeEJ> z)h)9Giw+A4i^A#e;`HZiQiyBkB|M$hS&LG{ht9ST#$-&KR`}ShFIx8n|ViWC6ihh>Q4(d&FZbS zwzqfY?IgA%@H_lgnotSGashYlS&90`8}00960pc^bi03-ka*J$OZ diff --git a/kubelink/pkg/downloader/testdata/signtest-0.1.0.tgz.prov b/kubelink/pkg/downloader/testdata/signtest-0.1.0.tgz.prov deleted file mode 100644 index d325bb266..000000000 --- a/kubelink/pkg/downloader/testdata/signtest-0.1.0.tgz.prov +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA512 - -apiVersion: v1 -description: A Helm chart for Kubernetes -name: signtest -version: 0.1.0 - -... -files: - signtest-0.1.0.tgz: sha256:e5ef611620fb97704d8751c16bab17fedb68883bfb0edc76f78a70e9173f9b55 ------BEGIN PGP SIGNATURE----- - -wsBcBAEBCgAQBQJcoosfCRCEO7+YH8GHYgAA220IALAs8T8NPgkcLvHu+5109cAN -BOCNPSZDNsqLZW/2Dc9cKoBG7Jen4Qad+i5l9351kqn3D9Gm6eRfAWcjfggRobV/ -9daZ19h0nl4O1muQNAkjvdgZt8MOP3+PB3I3/Tu2QCYjI579SLUmuXlcZR5BCFPR -PJy+e3QpV2PcdeU2KZLG4tjtlrq+3QC9ZHHEJLs+BVN9d46Dwo6CxJdHJrrrAkTw -M8MhA92vbiTTPRSCZI9x5qDAwJYhoq0oxLflpuL2tIlo3qVoCsaTSURwMESEHO32 -XwYG7BaVDMELWhAorBAGBGBwWFbJ1677qQ2gd9CN0COiVhekWlFRcnn60800r84= -=k9Y9 ------END PGP SIGNATURE----- \ No newline at end of file diff --git a/kubelink/pkg/downloader/testdata/signtest/.helmignore b/kubelink/pkg/downloader/testdata/signtest/.helmignore deleted file mode 100644 index 435b756d8..000000000 --- a/kubelink/pkg/downloader/testdata/signtest/.helmignore +++ /dev/null @@ -1,5 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -.git diff --git a/kubelink/pkg/downloader/testdata/signtest/Chart.yaml b/kubelink/pkg/downloader/testdata/signtest/Chart.yaml deleted file mode 100644 index f1f73723a..000000000 --- a/kubelink/pkg/downloader/testdata/signtest/Chart.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -description: A Helm chart for Kubernetes -name: signtest -version: 0.1.0 diff --git a/kubelink/pkg/downloader/testdata/signtest/alpine/Chart.yaml b/kubelink/pkg/downloader/testdata/signtest/alpine/Chart.yaml deleted file mode 100644 index eec261220..000000000 --- a/kubelink/pkg/downloader/testdata/signtest/alpine/Chart.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -description: Deploy a basic Alpine Linux pod -home: https://helm.sh/helm -name: alpine -sources: -- https://github.com/helm/helm -version: 0.1.0 diff --git a/kubelink/pkg/downloader/testdata/signtest/alpine/README.md b/kubelink/pkg/downloader/testdata/signtest/alpine/README.md deleted file mode 100644 index 28bebae07..000000000 --- a/kubelink/pkg/downloader/testdata/signtest/alpine/README.md +++ /dev/null @@ -1,9 +0,0 @@ -This example was generated using the command `helm create alpine`. - -The `templates/` directory contains a very simple pod resource with a -couple of parameters. - -The `values.yaml` file contains the default values for the -`alpine-pod.yaml` template. - -You can install this example using `helm install ./alpine`. diff --git a/kubelink/pkg/downloader/testdata/signtest/alpine/templates/alpine-pod.yaml b/kubelink/pkg/downloader/testdata/signtest/alpine/templates/alpine-pod.yaml deleted file mode 100644 index 5bbae10af..000000000 --- a/kubelink/pkg/downloader/testdata/signtest/alpine/templates/alpine-pod.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: {{.Release.Name}}-{{.Chart.Name}} - labels: - app.kubernetes.io/managed-by: {{.Release.Service}} - chartName: {{.Chart.Name}} - chartVersion: {{.Chart.Version | quote}} -spec: - restartPolicy: {{default "Never" .restart_policy}} - containers: - - name: waiter - image: "alpine:3.3" - command: ["/bin/sleep","9000"] diff --git a/kubelink/pkg/downloader/testdata/signtest/alpine/values.yaml b/kubelink/pkg/downloader/testdata/signtest/alpine/values.yaml deleted file mode 100644 index bb6c06ae4..000000000 --- a/kubelink/pkg/downloader/testdata/signtest/alpine/values.yaml +++ /dev/null @@ -1,2 +0,0 @@ -# The pod name -name: my-alpine diff --git a/kubelink/pkg/downloader/testdata/signtest/templates/pod.yaml b/kubelink/pkg/downloader/testdata/signtest/templates/pod.yaml deleted file mode 100644 index 9b00ccaf7..000000000 --- a/kubelink/pkg/downloader/testdata/signtest/templates/pod.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: signtest -spec: - restartPolicy: Never - containers: - - name: waiter - image: "alpine:3.3" - command: ["/bin/sleep","9000"] diff --git a/kubelink/pkg/downloader/testdata/signtest/values.yaml b/kubelink/pkg/downloader/testdata/signtest/values.yaml deleted file mode 100644 index e69de29bb..000000000 diff --git a/kubelink/pkg/service/helmApplicationService/helmAppService.go b/kubelink/pkg/service/helmApplicationService/helmAppService.go index 37dac9f9f..ad817c5a3 100644 --- a/kubelink/pkg/service/helmApplicationService/helmAppService.go +++ b/kubelink/pkg/service/helmApplicationService/helmAppService.go @@ -28,11 +28,11 @@ import ( "github.com/devtron-labs/kubelink/converter" error2 "github.com/devtron-labs/kubelink/error" repository "github.com/devtron-labs/kubelink/pkg/cluster" - "github.com/devtron-labs/kubelink/pkg/downloader" "github.com/devtron-labs/kubelink/pkg/service/commonHelmService" chart2 "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/cli" + downloader2 "helm.sh/helm/v3/pkg/downloader" "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/registry" "helm.sh/helm/v3/pkg/storage/driver" @@ -563,17 +563,7 @@ func (impl *HelmAppServiceImpl) UpgradeRelease(ctx context.Context, request *cli if err != nil { return nil, err } - - opts := []registry.ClientOption{ - registry.ClientOptDebug(false), - registry.ClientOptEnableCache(true), - registry.ClientOptWriter(os.Stderr), - } - //if plainHTTP { - // opts = append(opts, registry.ClientOptPlainHTTP()) - //} - - registryClient, err := registry.NewClient(opts...) + registryClient, err := registry.NewClient() if err != nil { impl.logger.Errorw(HELM_CLIENT_ERROR, "err", err) return nil, err @@ -588,116 +578,9 @@ func (impl *HelmAppServiceImpl) UpgradeRelease(ctx context.Context, request *cli } return nil, err } - - dirPath := "/tmp/dir" - outputChartPathDir := fmt.Sprintf("%s", dirPath) - err = os.MkdirAll(outputChartPathDir, os.ModePerm) - if err != nil { - return nil, err - } - - defer func() { - err := os.RemoveAll(outputChartPathDir) - if err != nil { - fmt.Println("error in deleting dir", " dir: ", outputChartPathDir, " err: ", err) - } - }() - abpath, err := chartutil.Save(helmRelease.Chart, outputChartPathDir) + impl.logger.Debug("Updating the dependencies if required") + err = impl.updateChartDependencies(helmRelease, registryClient) if err != nil { - fmt.Println("error in saving chartdata in the destination dir ", " dir : ", outputChartPathDir, " err : ", err) - return nil, err - } - - // Unpack the .tgz file to a directory - h, err := os.Open(abpath) - if err != nil { - return nil, err - } - if err := chartutil.Expand(dirPath, h); err != nil { - fmt.Println("error unpacking chart", "dir:", abpath, "err:", err) - return nil, err - } - - outputChartPathDir = filepath.Join(dirPath, helmRelease.Chart.Metadata.Name) - - outputBuffer := bytes.NewBuffer(nil) - - settings := cli.New() - err = helmClient.SetEnvSettings(&helmClient.Options{ - RepositoryConfig: helmClient.DefaultRepositoryConfigPath, - RepositoryCache: helmClient.DefaultCachePath, - Linting: true, - }, settings) - - manager := &downloader.Manager{ - ChartPath: outputChartPathDir, - Out: outputBuffer, - Getters: getter.All(settings), - RepositoryConfig: helmClient.DefaultRepositoryConfigPath, - RepositoryCache: helmClient.DefaultCachePath, - RegistryClient: registryClient, - } - // Update dependencies before building the chart - err = manager.Update() - err = manager.Build() - if err != nil { - impl.logger.Errorw("Error updating chart dependencies", "err", err) - return nil, err - } - - // Step 1: Locate the .tgz file in the charts directory - chartsDir := filepath.Join(outputChartPathDir, "charts") - files, err := os.ReadDir(chartsDir) - if err != nil { - impl.logger.Errorw("Error reading charts directory", "dir", chartsDir, "err", err) - return nil, err - } - - for _, file := range files { - if strings.HasSuffix(file.Name(), ".tgz") { - tgzPath := filepath.Join(chartsDir, file.Name()) - - // Step 2: Expand the .tgz file - //expandedChartDir := filepath.Join(chartsDir, strings.TrimSuffix(file.Name(), ".tgz")) - tgzFile, err := os.Open(tgzPath) - if err != nil { - impl.logger.Errorw("Error opening tgz file", "file", tgzPath, "err", err) - return nil, err - } - - err = chartutil.Expand(chartsDir, tgzFile) - //tgzFile.Close() // Close the file after expanding - if err != nil { - impl.logger.Errorw("Error expanding tgz file", "file", tgzPath, "err", err) - return nil, err - } - - chartsDir = filepath.Join(chartsDir, helmRelease.Chart.Metadata.Dependencies[0].Name) - - // Step 3: Load the expanded chart - expandedChart, err := loader.LoadDir(chartsDir) - if err != nil { - impl.logger.Errorw("Error loading expanded chart", "dir", chartsDir, "err", err) - return nil, err - } - - // Step 4: Set the expanded chart as a dependency - helmRelease.Chart.SetDependencies(expandedChart) - } - } - - // Step 5: Update the Chart.Lock file - lockFilePath := filepath.Join(outputChartPathDir, "Chart.lock") - lockFileData, err := os.ReadFile(lockFilePath) - if err != nil { - impl.logger.Errorw("Error reading Chart.lock file", "file", lockFilePath, "err", err) - return nil, err - } - - helmRelease.Chart.Lock = &chart2.Lock{} - err = yaml.Unmarshal(lockFileData, helmRelease.Chart.Lock) - if err != nil { - impl.logger.Errorw("Error unmarshalling Chart.lock data", "file", lockFilePath, "err", err) return nil, err } @@ -742,7 +625,144 @@ func (impl *HelmAppServiceImpl) UpgradeRelease(ctx context.Context, request *cli } return upgradeReleaseResponse, nil } +func (impl *HelmAppServiceImpl) updateChartDependencies(helmRelease *release.Release, registry *registry.Client) error { + + // Step 1: Check if Chart.yaml has dependencies + if helmRelease.Chart.Metadata.Dependencies == nil || len(helmRelease.Chart.Metadata.Dependencies) == 0 { + impl.logger.Infow("No dependencies listed in Chart.yaml, skipping update.") + return nil + } + + // Step 1: Update chart dependencies + dirPath := "/tmp/dir" + outputChartPathDir := fmt.Sprintf("%s", dirPath) + err := os.MkdirAll(outputChartPathDir, os.ModePerm) + if err != nil { + return err + } + defer func() { + err := os.RemoveAll(outputChartPathDir) + if err != nil { + fmt.Println("error in deleting dir", " dir: ", outputChartPathDir, " err: ", err) + } + }() + abpath, err := chartutil.Save(helmRelease.Chart, outputChartPathDir) + if err != nil { + fmt.Println("error in saving chartdata in the destination dir ", " dir : ", outputChartPathDir, " err : ", err) + return err + } + + // Unpack the .tgz file to a directory + h, err := os.Open(abpath) + if err != nil { + return err + } + if err := chartutil.Expand(dirPath, h); err != nil { + fmt.Println("error unpacking chart", "dir:", abpath, "err:", err) + return err + } + + outputChartPathDir = filepath.Join(dirPath, helmRelease.Chart.Metadata.Name) + outputBuffer := bytes.NewBuffer(nil) + settings := cli.New() + err = helmClient.SetEnvSettings(&helmClient.Options{ + RepositoryConfig: helmClient.DefaultRepositoryConfigPath, + RepositoryCache: helmClient.DefaultCachePath, + Linting: true, + }, settings) + manager := &downloader2.Manager{ + ChartPath: outputChartPathDir, + Out: outputBuffer, + Getters: getter.All(settings), + RepositoryConfig: helmClient.DefaultRepositoryConfigPath, + RepositoryCache: helmClient.DefaultCachePath, + RegistryClient: registry, + } + err = manager.Update() + if err != nil { + impl.logger.Errorw("Error updating chart dependencies", "err", err) + return err + } + + // Step 2: Check and process .tgz files in charts directory + chartsDir := filepath.Join(outputChartPathDir, "charts") + if err := impl.processTGZFiles(chartsDir, helmRelease); err != nil { + return err + } + + // Step 3: Update the Chart.lock file if it exists + lockFilePath := filepath.Join(outputChartPathDir, "Chart.lock") + if _, err := os.Stat(lockFilePath); os.IsNotExist(err) { + impl.logger.Infow("No Chart.lock file found, skipping lock file update", "file", lockFilePath) + return nil + } + + if err := impl.updateChartLock(lockFilePath, helmRelease); err != nil { + return err + } + return nil +} + +// processTGZFiles locates and processes .tgz files in the charts directory. +func (impl *HelmAppServiceImpl) processTGZFiles(chartsDir string, helmRelease *release.Release) error { + files, err := os.ReadDir(chartsDir) + if err != nil { + impl.logger.Errorw("Error reading charts directory", "dir", chartsDir, "err", err) + return err + } + + for _, file := range files { + if strings.HasSuffix(file.Name(), ".tgz") { + tgzPath := filepath.Join(chartsDir, file.Name()) + + if err := impl.expandAndSetDependency(tgzPath, chartsDir, helmRelease, file.Name()); err != nil { + return err + } + } + } + return nil +} + +// expandAndSetDependency expands a .tgz file and sets it as a dependency. +func (impl *HelmAppServiceImpl) expandAndSetDependency(tgzPath, chartsDir string, helmRelease *release.Release, fileName string) error { + tgzFile, err := os.Open(tgzPath) + if err != nil { + impl.logger.Errorw("Error opening tgz file", "file", tgzPath, "err", err) + return err + } + defer tgzFile.Close() + + if err := chartutil.Expand(chartsDir, tgzFile); err != nil { + impl.logger.Errorw("Error expanding tgz file", "file", tgzPath, "err", err) + return err + } + + expandedChartDir := filepath.Join(chartsDir, fileName) + expandedChart, err := loader.LoadDir(expandedChartDir) + if err != nil { + impl.logger.Errorw("Error loading expanded chart", "dir", expandedChartDir, "err", err) + return err + } + helmRelease.Chart.SetDependencies(expandedChart) + return nil +} +// updateChartLock updates the Chart.lock file based on its contents. +func (impl *HelmAppServiceImpl) updateChartLock(lockFilePath string, helmRelease *release.Release) error { + lockFileData, err := os.ReadFile(lockFilePath) + if err != nil { + impl.logger.Errorw("Error reading Chart.lock file", "file", lockFilePath, "err", err) + return err + } + + helmRelease.Chart.Lock = &chart2.Lock{} + if err := yaml.Unmarshal(lockFileData, helmRelease.Chart.Lock); err != nil { + impl.logger.Errorw("Error unmarshalling Chart.lock data", "file", lockFilePath, "err", err) + return err + } + + return nil +} func (impl *HelmAppServiceImpl) GetDeploymentDetail(request *client.DeploymentDetailRequest) (*client.DeploymentDetailResponse, error) { releaseIdentifier := request.ReleaseIdentifier helmReleases, err := impl.getHelmReleaseHistory(releaseIdentifier.ClusterConfig, releaseIdentifier.ReleaseNamespace, releaseIdentifier.ReleaseName, impl.helmReleaseConfig.MaxCountForHelmRelease) From c151de9bc0e6dc680337a1aa6f5ce81d67e6d05b Mon Sep 17 00:00:00 2001 From: Rajeev Date: Sun, 17 Nov 2024 17:08:05 +0530 Subject: [PATCH 4/9] handled the dynamic file of charts directory tgz expanded dirs --- .../helmApplicationService/helmAppService.go | 69 +++++++++++++++---- 1 file changed, 54 insertions(+), 15 deletions(-) diff --git a/kubelink/pkg/service/helmApplicationService/helmAppService.go b/kubelink/pkg/service/helmApplicationService/helmAppService.go index ad817c5a3..8af5788b4 100644 --- a/kubelink/pkg/service/helmApplicationService/helmAppService.go +++ b/kubelink/pkg/service/helmApplicationService/helmAppService.go @@ -579,9 +579,14 @@ func (impl *HelmAppServiceImpl) UpgradeRelease(ctx context.Context, request *cli return nil, err } impl.logger.Debug("Updating the dependencies if required") - err = impl.updateChartDependencies(helmRelease, registryClient) - if err != nil { - return nil, err + + // perform Dependency Update in case we detect any dependency listed in chart.Metadata + if helmRelease.Chart.Metadata.Dependencies != nil || len(helmRelease.Chart.Metadata.Dependencies) > 0 { + impl.logger.Infow("Dependencies listed in Chart.yaml, performing dependency update before upgrading") + err = impl.updateChartDependencies(helmRelease, registryClient) + if err != nil { + return nil, err + } } updateChartSpec := &helmClient.ChartSpec{ @@ -626,13 +631,6 @@ func (impl *HelmAppServiceImpl) UpgradeRelease(ctx context.Context, request *cli return upgradeReleaseResponse, nil } func (impl *HelmAppServiceImpl) updateChartDependencies(helmRelease *release.Release, registry *registry.Client) error { - - // Step 1: Check if Chart.yaml has dependencies - if helmRelease.Chart.Metadata.Dependencies == nil || len(helmRelease.Chart.Metadata.Dependencies) == 0 { - impl.logger.Infow("No dependencies listed in Chart.yaml, skipping update.") - return nil - } - // Step 1: Update chart dependencies dirPath := "/tmp/dir" outputChartPathDir := fmt.Sprintf("%s", dirPath) @@ -715,7 +713,7 @@ func (impl *HelmAppServiceImpl) processTGZFiles(chartsDir string, helmRelease *r if strings.HasSuffix(file.Name(), ".tgz") { tgzPath := filepath.Join(chartsDir, file.Name()) - if err := impl.expandAndSetDependency(tgzPath, chartsDir, helmRelease, file.Name()); err != nil { + if err := impl.expandAndSetDependency(tgzPath, helmRelease); err != nil { return err } } @@ -724,20 +722,61 @@ func (impl *HelmAppServiceImpl) processTGZFiles(chartsDir string, helmRelease *r } // expandAndSetDependency expands a .tgz file and sets it as a dependency. -func (impl *HelmAppServiceImpl) expandAndSetDependency(tgzPath, chartsDir string, helmRelease *release.Release, fileName string) error { +func (impl *HelmAppServiceImpl) expandAndSetDependency(tgzPath string, helmRelease *release.Release) error { + // Create a temporary directory for expanding the chart + tempDir, err := os.MkdirTemp("", "helmDependencyChart*") + if err != nil { + impl.logger.Errorw("Error creating temporary directory", "err", err) + return err + } + defer func() { + // Clean up the temporary directory + if removeErr := os.RemoveAll(tempDir); removeErr != nil { + impl.logger.Warnw("Error removing temporary directory", "dir", tempDir, "err", removeErr) + } + }() + + // Open the .tgz file tgzFile, err := os.Open(tgzPath) if err != nil { impl.logger.Errorw("Error opening tgz file", "file", tgzPath, "err", err) return err } - defer tgzFile.Close() + defer func(tgzFile *os.File) { + err := tgzFile.Close() + if err != nil { + impl.logger.Errorw("Error closing tgz file", "file", tgzPath, "err", err) + } + }(tgzFile) - if err := chartutil.Expand(chartsDir, tgzFile); err != nil { + // Expand the .tgz file into the temporary directory + if err := chartutil.Expand(tempDir, tgzFile); err != nil { impl.logger.Errorw("Error expanding tgz file", "file", tgzPath, "err", err) return err } - expandedChartDir := filepath.Join(chartsDir, fileName) + // Dynamically find the expanded chart directory + entries, err := os.ReadDir(tempDir) + if err != nil { + impl.logger.Errorw("Error reading contents of temporary directory", "dir", tempDir, "err", err) + return err + } + + var expandedChartDir string + for _, entry := range entries { + if entry.IsDir() { + expandedChartDir = filepath.Join(tempDir, entry.Name()) + break + } + } + + if expandedChartDir == "" { + err := fmt.Errorf("no expanded chart directory found in %s", tempDir) + impl.logger.Errorw("Error locating expanded chart directory", "dir", tempDir, "err", err) + return err + } + + // Load the expanded chart expandedChart, err := loader.LoadDir(expandedChartDir) if err != nil { impl.logger.Errorw("Error loading expanded chart", "dir", expandedChartDir, "err", err) From f1c3b6d0082f964c2a55f17d59c86c34e663dc22 Mon Sep 17 00:00:00 2001 From: Rajeev Date: Thu, 21 Nov 2024 14:36:31 +0530 Subject: [PATCH 5/9] done basic structuring of utils --- kubelink/pkg/helmClient/client.go | 5 + kubelink/pkg/helmClient/interface.go | 2 + .../helmApplicationService/helmAppService.go | 179 +----------------- .../service/helmApplicationService/utils.go | 167 ++++++++++++++++ 4 files changed, 175 insertions(+), 178 deletions(-) diff --git a/kubelink/pkg/helmClient/client.go b/kubelink/pkg/helmClient/client.go index 100a3007a..b9ad81701 100644 --- a/kubelink/pkg/helmClient/client.go +++ b/kubelink/pkg/helmClient/client.go @@ -48,6 +48,7 @@ const ( CHART_WORKING_DIR_PATH = "/tmp/charts/" DefaultCachePath = "/home/devtron/devtroncd/.helmcache" DefaultRepositoryConfigPath = "/home/devtron/devtroncd/.helmrepo" + DefaultTempDirectory = "/tmp/dir/" ) // NewClientFromRestConf returns a new Helm client constructed with the provided REST config options @@ -830,3 +831,7 @@ func GetChartSavedDir(helmChart *chart.Chart) (string, error) { return absFilePath, nil } + +func (c *HelmClient) GetProviders() getter.Providers { + return c.Providers +} diff --git a/kubelink/pkg/helmClient/interface.go b/kubelink/pkg/helmClient/interface.go index 957697d47..afafc4d5d 100644 --- a/kubelink/pkg/helmClient/interface.go +++ b/kubelink/pkg/helmClient/interface.go @@ -19,6 +19,7 @@ package helmClient import ( "context" "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/release" "helm.sh/helm/v3/pkg/repo" ) @@ -40,4 +41,5 @@ type Client interface { RollbackRelease(spec *ChartSpec, version int) error TemplateChart(spec *ChartSpec, options *HelmTemplateOptions, chartData []byte, returnChartBytes bool) ([]byte, []byte, error) GetNotes(spec *ChartSpec, options *HelmTemplateOptions) ([]byte, error) + GetProviders() getter.Providers } diff --git a/kubelink/pkg/service/helmApplicationService/helmAppService.go b/kubelink/pkg/service/helmApplicationService/helmAppService.go index ceb123d3a..b0ec47d5b 100644 --- a/kubelink/pkg/service/helmApplicationService/helmAppService.go +++ b/kubelink/pkg/service/helmApplicationService/helmAppService.go @@ -29,16 +29,11 @@ import ( error2 "github.com/devtron-labs/kubelink/error" repository "github.com/devtron-labs/kubelink/pkg/cluster" "github.com/devtron-labs/kubelink/pkg/service/commonHelmService" - chart2 "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" - "helm.sh/helm/v3/pkg/cli" - downloader2 "helm.sh/helm/v3/pkg/downloader" - "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/registry" "helm.sh/helm/v3/pkg/storage/driver" "net/url" "path" - "sigs.k8s.io/yaml" "strings" "time" @@ -583,7 +578,7 @@ func (impl *HelmAppServiceImpl) UpgradeRelease(ctx context.Context, request *cli // perform Dependency Update in case we detect any dependency listed in chart.Metadata if helmRelease.Chart.Metadata.Dependencies != nil || len(helmRelease.Chart.Metadata.Dependencies) > 0 { impl.logger.Infow("Dependencies listed in Chart.yaml, performing dependency update before upgrading") - err = impl.updateChartDependencies(helmRelease, registryClient) + err = UpdateChartDependencies(helmClientObj.GetProviders(), helmRelease, registryClient) if err != nil { return nil, err } @@ -630,178 +625,6 @@ func (impl *HelmAppServiceImpl) UpgradeRelease(ctx context.Context, request *cli } return upgradeReleaseResponse, nil } -func (impl *HelmAppServiceImpl) updateChartDependencies(helmRelease *release.Release, registry *registry.Client) error { - // Step 1: Update chart dependencies - dirPath := "/tmp/dir" - outputChartPathDir := fmt.Sprintf("%s", dirPath) - err := os.MkdirAll(outputChartPathDir, os.ModePerm) - if err != nil { - return err - } - defer func() { - err := os.RemoveAll(outputChartPathDir) - if err != nil { - fmt.Println("error in deleting dir", " dir: ", outputChartPathDir, " err: ", err) - } - }() - abpath, err := chartutil.Save(helmRelease.Chart, outputChartPathDir) - if err != nil { - fmt.Println("error in saving chartdata in the destination dir ", " dir : ", outputChartPathDir, " err : ", err) - return err - } - - // Unpack the .tgz file to a directory - h, err := os.Open(abpath) - if err != nil { - return err - } - if err := chartutil.Expand(dirPath, h); err != nil { - fmt.Println("error unpacking chart", "dir:", abpath, "err:", err) - return err - } - - outputChartPathDir = filepath.Join(dirPath, helmRelease.Chart.Metadata.Name) - outputBuffer := bytes.NewBuffer(nil) - settings := cli.New() - err = helmClient.SetEnvSettings(&helmClient.Options{ - RepositoryConfig: helmClient.DefaultRepositoryConfigPath, - RepositoryCache: helmClient.DefaultCachePath, - Linting: true, - }, settings) - manager := &downloader2.Manager{ - ChartPath: outputChartPathDir, - Out: outputBuffer, - Getters: getter.All(settings), - RepositoryConfig: helmClient.DefaultRepositoryConfigPath, - RepositoryCache: helmClient.DefaultCachePath, - RegistryClient: registry, - } - err = manager.Update() - if err != nil { - impl.logger.Errorw("Error updating chart dependencies", "err", err) - return err - } - - // Step 2: Check and process .tgz files in charts directory - chartsDir := filepath.Join(outputChartPathDir, "charts") - if err := impl.processTGZFiles(chartsDir, helmRelease); err != nil { - return err - } - - // Step 3: Update the Chart.lock file if it exists - lockFilePath := filepath.Join(outputChartPathDir, "Chart.lock") - if _, err := os.Stat(lockFilePath); os.IsNotExist(err) { - impl.logger.Infow("No Chart.lock file found, skipping lock file update", "file", lockFilePath) - return nil - } - - if err := impl.updateChartLock(lockFilePath, helmRelease); err != nil { - return err - } - return nil -} - -// processTGZFiles locates and processes .tgz files in the charts directory. -func (impl *HelmAppServiceImpl) processTGZFiles(chartsDir string, helmRelease *release.Release) error { - files, err := os.ReadDir(chartsDir) - if err != nil { - impl.logger.Errorw("Error reading charts directory", "dir", chartsDir, "err", err) - return err - } - - for _, file := range files { - if strings.HasSuffix(file.Name(), ".tgz") { - tgzPath := filepath.Join(chartsDir, file.Name()) - - if err := impl.expandAndSetDependency(tgzPath, helmRelease); err != nil { - return err - } - } - } - return nil -} - -// expandAndSetDependency expands a .tgz file and sets it as a dependency. -func (impl *HelmAppServiceImpl) expandAndSetDependency(tgzPath string, helmRelease *release.Release) error { - // Create a temporary directory for expanding the chart - tempDir, err := os.MkdirTemp("", "helmDependencyChart*") - if err != nil { - impl.logger.Errorw("Error creating temporary directory", "err", err) - return err - } - defer func() { - // Clean up the temporary directory - if removeErr := os.RemoveAll(tempDir); removeErr != nil { - impl.logger.Warnw("Error removing temporary directory", "dir", tempDir, "err", removeErr) - } - }() - - // Open the .tgz file - tgzFile, err := os.Open(tgzPath) - if err != nil { - impl.logger.Errorw("Error opening tgz file", "file", tgzPath, "err", err) - return err - } - defer func(tgzFile *os.File) { - err := tgzFile.Close() - if err != nil { - impl.logger.Errorw("Error closing tgz file", "file", tgzPath, "err", err) - } - }(tgzFile) - - // Expand the .tgz file into the temporary directory - if err := chartutil.Expand(tempDir, tgzFile); err != nil { - impl.logger.Errorw("Error expanding tgz file", "file", tgzPath, "err", err) - return err - } - - // Dynamically find the expanded chart directory - entries, err := os.ReadDir(tempDir) - if err != nil { - impl.logger.Errorw("Error reading contents of temporary directory", "dir", tempDir, "err", err) - return err - } - - var expandedChartDir string - for _, entry := range entries { - if entry.IsDir() { - expandedChartDir = filepath.Join(tempDir, entry.Name()) - break - } - } - - if expandedChartDir == "" { - err := fmt.Errorf("no expanded chart directory found in %s", tempDir) - impl.logger.Errorw("Error locating expanded chart directory", "dir", tempDir, "err", err) - return err - } - - // Load the expanded chart - expandedChart, err := loader.LoadDir(expandedChartDir) - if err != nil { - impl.logger.Errorw("Error loading expanded chart", "dir", expandedChartDir, "err", err) - return err - } - helmRelease.Chart.SetDependencies(expandedChart) - return nil -} - -// updateChartLock updates the Chart.lock file based on its contents. -func (impl *HelmAppServiceImpl) updateChartLock(lockFilePath string, helmRelease *release.Release) error { - lockFileData, err := os.ReadFile(lockFilePath) - if err != nil { - impl.logger.Errorw("Error reading Chart.lock file", "file", lockFilePath, "err", err) - return err - } - - helmRelease.Chart.Lock = &chart2.Lock{} - if err := yaml.Unmarshal(lockFileData, helmRelease.Chart.Lock); err != nil { - impl.logger.Errorw("Error unmarshalling Chart.lock data", "file", lockFilePath, "err", err) - return err - } - - return nil -} func (impl *HelmAppServiceImpl) GetDeploymentDetail(request *client.DeploymentDetailRequest) (*client.DeploymentDetailResponse, error) { releaseIdentifier := request.ReleaseIdentifier helmReleases, err := impl.getHelmReleaseHistory(releaseIdentifier.ClusterConfig, releaseIdentifier.ReleaseNamespace, releaseIdentifier.ReleaseName, impl.helmReleaseConfig.MaxCountForHelmRelease) diff --git a/kubelink/pkg/service/helmApplicationService/utils.go b/kubelink/pkg/service/helmApplicationService/utils.go index b2162c1d6..2ad6bbc7b 100644 --- a/kubelink/pkg/service/helmApplicationService/utils.go +++ b/kubelink/pkg/service/helmApplicationService/utils.go @@ -16,10 +16,24 @@ package helmApplicationService import ( + "bytes" "errors" + "fmt" client "github.com/devtron-labs/kubelink/grpc" + "github.com/devtron-labs/kubelink/pkg/helmClient" "github.com/devtron-labs/kubelink/pkg/k8sInformer" + chart2 "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" + downloader2 "helm.sh/helm/v3/pkg/downloader" + "helm.sh/helm/v3/pkg/getter" + "helm.sh/helm/v3/pkg/registry" + "helm.sh/helm/v3/pkg/release" + "os" + "path/filepath" + "sigs.k8s.io/yaml" "strconv" + "strings" ) func getUniqueReleaseIdentifierName(releaseIdentifier *client.ReleaseIdentifier) string { @@ -33,3 +47,156 @@ const ( func IsReleaseNotFoundInCacheError(err error) bool { return errors.Is(err, k8sInformer.ErrorCacheMissReleaseNotFound) } + +// UpdateChartLock updates the Chart.lock file based on its contents. +func UpdateChartLock(lockFilePath string, helmRelease *release.Release) error { + lockFileData, err := os.ReadFile(lockFilePath) + if err != nil { + return fmt.Errorf("error reading Chart.lock file at %s: %w", lockFilePath, err) + } + helmRelease.Chart.Lock = &chart2.Lock{} + if err := yaml.Unmarshal(lockFileData, helmRelease.Chart.Lock); err != nil { + return fmt.Errorf("error unmarshalling Chart.lock file at %s: %w", lockFilePath, err) + } + return nil +} + +// ProcessTGZFiles locates and processes .tgz files in the charts directory. +func ProcessTGZFiles(chartsDir string, helmRelease *release.Release) error { + files, err := os.ReadDir(chartsDir) + if err != nil { + return fmt.Errorf("error reading charts directory in dir %s :%w", chartsDir, err) + + } + for _, file := range files { + if strings.HasSuffix(file.Name(), ".tgz") { + tgzPath := filepath.Join(chartsDir, file.Name()) + + if err := expandAndSetDependency(tgzPath, helmRelease); err != nil { + return err + } + } + } + return nil +} + +// expandAndSetDependency expands a .tgz file and sets it as a dependency. +func expandAndSetDependency(tgzPath string, helmRelease *release.Release) error { + // Create a temporary directory for expanding the chart + tempDir, err := os.MkdirTemp("", "helmDependencyChart*") + if err != nil { + return fmt.Errorf("error creating temporary directory: %w", err) + } + defer func() { + // Clean up the temporary directory + if removeErr := os.RemoveAll(tempDir); removeErr != nil { + fmt.Sprintf("error in removing temporary directory %s: %s", tempDir, removeErr.Error()) + } + }() + + // Open the .tgz file + tgzFile, err := os.Open(tgzPath) + if err != nil { + return fmt.Errorf("error opening tgz file of tgzPath %s: %w", tgzPath, err) + } + defer func(tgzFile *os.File) { + err := tgzFile.Close() + if err != nil { + fmt.Errorf("error in closing tgz file of tgzPath %s: %w", tgzPath, err) + } + }(tgzFile) + + // Expand the .tgz file into the temporary directory + if err := chartutil.Expand(tempDir, tgzFile); err != nil { + return fmt.Errorf("error expanding tgz file having filePath %s : %w", tgzPath, err) + } + + // Dynamically find the expanded chart directory + entries, err := os.ReadDir(tempDir) + if err != nil { + return fmt.Errorf("error reading contents of temporary directory %s: %w", tempDir, err) + } + + var expandedChartDir string + for _, entry := range entries { + if entry.IsDir() { + expandedChartDir = filepath.Join(tempDir, entry.Name()) + break + } + } + + if expandedChartDir == "" { + err := fmt.Errorf("no expanded chart directory found in %s", tempDir) + return fmt.Errorf("error locating expanded chart directory found in %s: %w", tempDir, err) + } + + // Load the expanded chart + expandedChart, err := loader.LoadDir(expandedChartDir) + if err != nil { + return fmt.Errorf("error loading expanded chart : %w", err) + } + helmRelease.Chart.SetDependencies(expandedChart) + return nil +} + +func UpdateChartDependencies(providers getter.Providers, helmRelease *release.Release, registry *registry.Client) error { + // Step 1: Update chart dependencies + outputChartPathDir := fmt.Sprintf("%s", helmClient.DefaultTempDirectory) + err := os.MkdirAll(outputChartPathDir, os.ModePerm) + if err != nil { + return err + } + defer func() { + err := os.RemoveAll(outputChartPathDir) + if err != nil { + fmt.Println("error in deleting dir", " dir: ", outputChartPathDir, " err: ", err) + } + }() + abpath, err := chartutil.Save(helmRelease.Chart, outputChartPathDir) + if err != nil { + fmt.Println("error in saving chartdata in the destination dir ", " dir : ", outputChartPathDir, " err : ", err) + return err + } + + // Unpack the .tgz file to a directory + h, err := os.Open(abpath) + if err != nil { + return err + } + if err := chartutil.Expand(helmClient.DefaultTempDirectory, h); err != nil { + fmt.Println("error unpacking chart", "dir:", abpath, "err:", err) + return err + } + outputChartPathDir = filepath.Join(helmClient.DefaultTempDirectory, helmRelease.Chart.Metadata.Name) + outputBuffer := bytes.NewBuffer(nil) + manager := &downloader2.Manager{ + ChartPath: outputChartPathDir, + Out: outputBuffer, + Getters: providers, + RepositoryConfig: helmClient.DefaultRepositoryConfigPath, + RepositoryCache: helmClient.DefaultCachePath, + RegistryClient: registry, + } + err = manager.Update() + if err != nil { + return fmt.Errorf("error updating chart dependencies: %w", err) + } + + // Step 2: Check and process .tgz files in charts directory + chartsDir := filepath.Join(outputChartPathDir, "charts") + if err := ProcessTGZFiles(chartsDir, helmRelease); err != nil { + return err + } + + // Step 3: Update the Chart.lock file if it exists + lockFilePath := filepath.Join(outputChartPathDir, "Chart.lock") + if _, err := os.Stat(lockFilePath); os.IsNotExist(err) { + fmt.Sprintf("No Chart.lock file found, skipping lock file update, %s", lockFilePath) + return nil + } + + if err := UpdateChartLock(lockFilePath, helmRelease); err != nil { + return err + } + return nil +} From b5ff093918f0090d08ae04ec47e61472f7aa876b Mon Sep 17 00:00:00 2001 From: Rajeev Date: Mon, 25 Nov 2024 18:18:39 +0530 Subject: [PATCH 6/9] resolved the comments and refactored the code --- kubelink/pkg/helmClient/client.go | 19 +- .../helmApplicationService/helmAppService.go | 176 +++++++++++++++++- .../service/helmApplicationService/utils.go | 105 +---------- 3 files changed, 184 insertions(+), 116 deletions(-) diff --git a/kubelink/pkg/helmClient/client.go b/kubelink/pkg/helmClient/client.go index b9ad81701..426484146 100644 --- a/kubelink/pkg/helmClient/client.go +++ b/kubelink/pkg/helmClient/client.go @@ -49,6 +49,7 @@ const ( DefaultCachePath = "/home/devtron/devtroncd/.helmcache" DefaultRepositoryConfigPath = "/home/devtron/devtroncd/.helmrepo" DefaultTempDirectory = "/tmp/dir/" + TemHelmChartDirectory = "helmDependencyChart" ) // NewClientFromRestConf returns a new Helm client constructed with the provided REST config options @@ -57,7 +58,7 @@ func NewClientFromRestConf(options *RestConfClientOptions) (Client, error) { clientGetter := NewRESTClientGetter(options.Namespace, nil, options.RestConfig) - err := SetEnvSettings(options.Options, settings) + err := setEnvSettings(options.Options, settings) if err != nil { return nil, err } @@ -67,7 +68,7 @@ func NewClientFromRestConf(options *RestConfClientOptions) (Client, error) { // newClient returns a new Helm client via the provided options and REST config func newClient(options *Options, clientGetter genericclioptions.RESTClientGetter, settings *cli.EnvSettings) (Client, error) { - err := SetEnvSettings(options, settings) + err := setEnvSettings(options, settings) if err != nil { return nil, err } @@ -100,8 +101,8 @@ func newClient(options *Options, clientGetter genericclioptions.RESTClientGetter }, nil } -// SetEnvSettings sets the client's environment settings based on the provided client configuration -func SetEnvSettings(options *Options, settings *cli.EnvSettings) error { +// setEnvSettings sets the client's environment settings based on the provided client configuration +func setEnvSettings(options *Options, settings *cli.EnvSettings) error { if options == nil { options = &Options{ RepositoryConfig: DefaultRepositoryConfigPath, @@ -800,11 +801,13 @@ func GetChartBytes(helmChart *chart.Chart) ([]byte, error) { absFilePath, err := GetChartSavedDir(helmChart) if err != nil { - fmt.Println("error in getting saved chart data directory path", "err", err) + fmt.Errorf("error in getting saved chart data directory path %w", err) + return nil, err } chartBytes, err := os.ReadFile(absFilePath) if err != nil { - fmt.Println("error in reading chartdata from the file ", " filePath : ", absFilePath, " err : ", err) + fmt.Errorf("error in reading chartdata from the file filePath : %s err : %w", absFilePath, err) + return nil, err } return chartBytes, nil @@ -820,12 +823,12 @@ func GetChartSavedDir(helmChart *chart.Chart) (string, error) { defer func() { err := os.RemoveAll(outputChartPathDir) if err != nil { - fmt.Println("error in deleting dir", " dir: ", outputChartPathDir, " err: ", err) + fmt.Errorf("error in deleting dir, %s, err: %w", outputChartPathDir, err) } }() absFilePath, err := chartutil.Save(helmChart, outputChartPathDir) if err != nil { - fmt.Println("error in saving chartdata in the destination dir ", " dir : ", outputChartPathDir, " err : ", err) + fmt.Errorf("error in saving chartdata in the destination dir %s, err: %w", outputChartPathDir, err) return "", err } diff --git a/kubelink/pkg/service/helmApplicationService/helmAppService.go b/kubelink/pkg/service/helmApplicationService/helmAppService.go index b0ec47d5b..bf9e5b368 100644 --- a/kubelink/pkg/service/helmApplicationService/helmAppService.go +++ b/kubelink/pkg/service/helmApplicationService/helmAppService.go @@ -29,11 +29,15 @@ import ( error2 "github.com/devtron-labs/kubelink/error" repository "github.com/devtron-labs/kubelink/pkg/cluster" "github.com/devtron-labs/kubelink/pkg/service/commonHelmService" + "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" + downloader2 "helm.sh/helm/v3/pkg/downloader" + "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/registry" "helm.sh/helm/v3/pkg/storage/driver" "net/url" "path" + "sigs.k8s.io/yaml" "strings" "time" @@ -573,12 +577,10 @@ func (impl *HelmAppServiceImpl) UpgradeRelease(ctx context.Context, request *cli } return nil, err } - impl.logger.Debug("Updating the dependencies if required") - // perform Dependency Update in case we detect any dependency listed in chart.Metadata - if helmRelease.Chart.Metadata.Dependencies != nil || len(helmRelease.Chart.Metadata.Dependencies) > 0 { - impl.logger.Infow("Dependencies listed in Chart.yaml, performing dependency update before upgrading") - err = UpdateChartDependencies(helmClientObj.GetProviders(), helmRelease, registryClient) + if len(helmRelease.Chart.Metadata.Dependencies) != 0 { + impl.logger.Infow("Dependencies listed in Chart.yaml, performing dependency update before upgrading", "dependencies", helmRelease.Chart.Metadata.Dependencies) + err = impl.updateChartDependencies(helmClientObj.GetProviders(), helmRelease, registryClient) if err != nil { return nil, err } @@ -1930,3 +1932,167 @@ func podMetadataAdapter(podmetadatas []*commonBean.PodMetadata) []*client.PodMet } return podMetadatas } + +func (impl *HelmAppServiceImpl) updateChartDependencies(providers getter.Providers, helmRelease *release.Release, registry *registry.Client) error { + // Step 1: Update chart dependencies + outputChartPathDir := fmt.Sprintf("%s", helmClient.DefaultTempDirectory) + err := os.MkdirAll(outputChartPathDir, os.ModePerm) + if err != nil { + return err + } + defer func() { + err := os.RemoveAll(outputChartPathDir) + if err != nil { + impl.logger.Errorw("error in deleting dir", " dir: ", outputChartPathDir, " err: ", err) + } + }() + abpath, err := chartutil.Save(helmRelease.Chart, outputChartPathDir) + if err != nil { + impl.logger.Errorw("error in saving chartData in the destination dir", "outputChartPathDir", outputChartPathDir, " err: ", err) + return err + } + + // Unpack the .tgz file to a directory + h, err := os.Open(abpath) + if err != nil { + return err + } + if err := chartutil.Expand(helmClient.DefaultTempDirectory, h); err != nil { + impl.logger.Errorw("error in unpacking the chart", "dir", abpath, "err", err) + return err + } + outputChartPathDir = filepath.Join(helmClient.DefaultTempDirectory, helmRelease.Chart.Metadata.Name) + outputBuffer := bytes.NewBuffer(nil) + manager := &downloader2.Manager{ + ChartPath: outputChartPathDir, + Out: outputBuffer, + Getters: providers, + RepositoryConfig: helmClient.DefaultRepositoryConfigPath, + RepositoryCache: helmClient.DefaultCachePath, + RegistryClient: registry, + } + err = manager.Update() + if err != nil { + impl.logger.Errorw("error updating chart dependencies", "err", err) + return err + } + + // Step 2: Check and process .tgz files in charts directory + chartsDir := filepath.Join(outputChartPathDir, "charts") + if err := impl.processTGZFiles(chartsDir, helmRelease); err != nil { + impl.logger.Errorw("error in processing TGZ files", "err", err) + return err + } + + // Step 3: Update the Chart.lock file if it exists + lockFilePath := filepath.Join(outputChartPathDir, "Chart.lock") + if _, err := os.Stat(lockFilePath); os.IsNotExist(err) { + impl.logger.Infow("No Chart.lock file found, skipping lock file update", "lockFilePath", lockFilePath) + return nil + } + + if err := impl.updateChartLock(lockFilePath, helmRelease); err != nil { + impl.logger.Errorw("error in updating the Chart Lock", "lockFilePath", lockFilePath, "err", err) + return err + } + return nil +} + +// UpdateChartLock updates the Chart.lock file based on its contents. +func (impl *HelmAppServiceImpl) updateChartLock(lockFilePath string, helmRelease *release.Release) error { + lockFileData, err := os.ReadFile(lockFilePath) + if err != nil { + impl.logger.Errorw("error reading Chart.lock file at", "lockFilePath", lockFilePath, "err", err) + return err + } + + helmRelease.Chart.Lock = &chart.Lock{} + err = yaml.Unmarshal(lockFileData, helmRelease.Chart.Lock) + if err != nil { + impl.logger.Errorw("error unmarshalling Chart.lock file at ", "lockFilePath", lockFilePath, "err", err) + return err + } + return nil +} + +// expandAndSetDependency expands a .tgz file and sets it as a dependency. +func (impl *HelmAppServiceImpl) expandAndSetDependency(tgzPath string, helmRelease *release.Release) error { + // Create a temporary directory for expanding the chart + tempDir, err := os.MkdirTemp("", helmClient.TemHelmChartDirectory) + if err != nil { + impl.logger.Errorw("error in creating temporary directory", "err", err) + return err + } + defer func() { + // Clean up the temporary directory + if removeErr := os.RemoveAll(tempDir); removeErr != nil { + impl.logger.Errorw("error in removing temp Directory", "tempDir", tempDir, "err", err) + } + }() + + // Open the .tgz file + tgzFile, err := os.Open(tgzPath) + if err != nil { + impl.logger.Errorw("error opening tgz file of tgzPath", "tgzPath", tgzPath, "err", err) + return err + } + defer func(tgzFile *os.File) { + err := tgzFile.Close() + if err != nil { + impl.logger.Errorw("error in closing tgz file of tgzPath ", "tgzPath", tgzPath, "err", err) + } + }(tgzFile) + + // Expand the .tgz file into the temporary directory + if err := chartutil.Expand(tempDir, tgzFile); err != nil { + impl.logger.Errorw("error expanding tgz file having filePath", "tgzPath", tgzPath, "err", err) + return err + } + + // Dynamically find the expanded chart directory + entries, err := os.ReadDir(tempDir) + if err != nil { + impl.logger.Errorw("error reading contents of temporary directory", "tempDir", tempDir, "err", err) + return err + } + + var expandedChartDir string + for _, entry := range entries { + if entry.IsDir() { + expandedChartDir = filepath.Join(tempDir, entry.Name()) + break + } + } + + if expandedChartDir == "" { + impl.logger.Errorw("error locating expanded chart directory found in", "tempDir", tempDir, "err", err) + return err + } + + // Load the expanded chart + expandedChart, err := loader.LoadDir(expandedChartDir) + if err != nil { + impl.logger.Errorw("error loading expanded chart", "err", err) + return err + } + helmRelease.Chart.SetDependencies(expandedChart) + return nil +} + +// ProcessTGZFiles locates and processes .tgz files in the charts directory. +func (impl *HelmAppServiceImpl) processTGZFiles(chartsDir string, helmRelease *release.Release) error { + files, err := os.ReadDir(chartsDir) + if err != nil { + impl.logger.Errorw("error reading charts directory in dir", "chartsDir", chartsDir, "err", err) + return err + } + for _, file := range files { + if strings.HasSuffix(file.Name(), ".tgz") { + tgzPath := filepath.Join(chartsDir, file.Name()) + if err := impl.expandAndSetDependency(tgzPath, helmRelease); err != nil { + return err + } + } + } + return nil +} diff --git a/kubelink/pkg/service/helmApplicationService/utils.go b/kubelink/pkg/service/helmApplicationService/utils.go index 2ad6bbc7b..cc262d2f9 100644 --- a/kubelink/pkg/service/helmApplicationService/utils.go +++ b/kubelink/pkg/service/helmApplicationService/utils.go @@ -16,24 +16,16 @@ package helmApplicationService import ( - "bytes" "errors" "fmt" client "github.com/devtron-labs/kubelink/grpc" - "github.com/devtron-labs/kubelink/pkg/helmClient" "github.com/devtron-labs/kubelink/pkg/k8sInformer" - chart2 "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/chartutil" - downloader2 "helm.sh/helm/v3/pkg/downloader" - "helm.sh/helm/v3/pkg/getter" - "helm.sh/helm/v3/pkg/registry" "helm.sh/helm/v3/pkg/release" "os" "path/filepath" - "sigs.k8s.io/yaml" "strconv" - "strings" ) func getUniqueReleaseIdentifierName(releaseIdentifier *client.ReleaseIdentifier) string { @@ -48,44 +40,13 @@ func IsReleaseNotFoundInCacheError(err error) bool { return errors.Is(err, k8sInformer.ErrorCacheMissReleaseNotFound) } -// UpdateChartLock updates the Chart.lock file based on its contents. -func UpdateChartLock(lockFilePath string, helmRelease *release.Release) error { - lockFileData, err := os.ReadFile(lockFilePath) - if err != nil { - return fmt.Errorf("error reading Chart.lock file at %s: %w", lockFilePath, err) - } - helmRelease.Chart.Lock = &chart2.Lock{} - if err := yaml.Unmarshal(lockFileData, helmRelease.Chart.Lock); err != nil { - return fmt.Errorf("error unmarshalling Chart.lock file at %s: %w", lockFilePath, err) - } - return nil -} - -// ProcessTGZFiles locates and processes .tgz files in the charts directory. -func ProcessTGZFiles(chartsDir string, helmRelease *release.Release) error { - files, err := os.ReadDir(chartsDir) - if err != nil { - return fmt.Errorf("error reading charts directory in dir %s :%w", chartsDir, err) - - } - for _, file := range files { - if strings.HasSuffix(file.Name(), ".tgz") { - tgzPath := filepath.Join(chartsDir, file.Name()) - - if err := expandAndSetDependency(tgzPath, helmRelease); err != nil { - return err - } - } - } - return nil -} - // expandAndSetDependency expands a .tgz file and sets it as a dependency. func expandAndSetDependency(tgzPath string, helmRelease *release.Release) error { // Create a temporary directory for expanding the chart tempDir, err := os.MkdirTemp("", "helmDependencyChart*") if err != nil { - return fmt.Errorf("error creating temporary directory: %w", err) + fmt.Errorf("error creating temporary directory: %w", err) + return err } defer func() { // Clean up the temporary directory @@ -138,65 +99,3 @@ func expandAndSetDependency(tgzPath string, helmRelease *release.Release) error helmRelease.Chart.SetDependencies(expandedChart) return nil } - -func UpdateChartDependencies(providers getter.Providers, helmRelease *release.Release, registry *registry.Client) error { - // Step 1: Update chart dependencies - outputChartPathDir := fmt.Sprintf("%s", helmClient.DefaultTempDirectory) - err := os.MkdirAll(outputChartPathDir, os.ModePerm) - if err != nil { - return err - } - defer func() { - err := os.RemoveAll(outputChartPathDir) - if err != nil { - fmt.Println("error in deleting dir", " dir: ", outputChartPathDir, " err: ", err) - } - }() - abpath, err := chartutil.Save(helmRelease.Chart, outputChartPathDir) - if err != nil { - fmt.Println("error in saving chartdata in the destination dir ", " dir : ", outputChartPathDir, " err : ", err) - return err - } - - // Unpack the .tgz file to a directory - h, err := os.Open(abpath) - if err != nil { - return err - } - if err := chartutil.Expand(helmClient.DefaultTempDirectory, h); err != nil { - fmt.Println("error unpacking chart", "dir:", abpath, "err:", err) - return err - } - outputChartPathDir = filepath.Join(helmClient.DefaultTempDirectory, helmRelease.Chart.Metadata.Name) - outputBuffer := bytes.NewBuffer(nil) - manager := &downloader2.Manager{ - ChartPath: outputChartPathDir, - Out: outputBuffer, - Getters: providers, - RepositoryConfig: helmClient.DefaultRepositoryConfigPath, - RepositoryCache: helmClient.DefaultCachePath, - RegistryClient: registry, - } - err = manager.Update() - if err != nil { - return fmt.Errorf("error updating chart dependencies: %w", err) - } - - // Step 2: Check and process .tgz files in charts directory - chartsDir := filepath.Join(outputChartPathDir, "charts") - if err := ProcessTGZFiles(chartsDir, helmRelease); err != nil { - return err - } - - // Step 3: Update the Chart.lock file if it exists - lockFilePath := filepath.Join(outputChartPathDir, "Chart.lock") - if _, err := os.Stat(lockFilePath); os.IsNotExist(err) { - fmt.Sprintf("No Chart.lock file found, skipping lock file update, %s", lockFilePath) - return nil - } - - if err := UpdateChartLock(lockFilePath, helmRelease); err != nil { - return err - } - return nil -} From 134125721275c5f03a9117b6a4898f208c9b29e8 Mon Sep 17 00:00:00 2001 From: Rajeev Date: Mon, 25 Nov 2024 19:30:07 +0530 Subject: [PATCH 7/9] added log --- kubelink/pkg/service/helmApplicationService/helmAppService.go | 1 + 1 file changed, 1 insertion(+) diff --git a/kubelink/pkg/service/helmApplicationService/helmAppService.go b/kubelink/pkg/service/helmApplicationService/helmAppService.go index bf9e5b368..d18b369c5 100644 --- a/kubelink/pkg/service/helmApplicationService/helmAppService.go +++ b/kubelink/pkg/service/helmApplicationService/helmAppService.go @@ -582,6 +582,7 @@ func (impl *HelmAppServiceImpl) UpgradeRelease(ctx context.Context, request *cli impl.logger.Infow("Dependencies listed in Chart.yaml, performing dependency update before upgrading", "dependencies", helmRelease.Chart.Metadata.Dependencies) err = impl.updateChartDependencies(helmClientObj.GetProviders(), helmRelease, registryClient) if err != nil { + impl.logger.Errorw("error in updating chart Dependencies", "err", err) return nil, err } } From f468dfb90e64efe5950383fda345022d54416727 Mon Sep 17 00:00:00 2001 From: Rajeev Date: Tue, 26 Nov 2024 18:08:03 +0530 Subject: [PATCH 8/9] removed the default settings cache paths --- kubelink/pkg/helmClient/client.go | 14 +++++++++++--- kubelink/pkg/helmClient/interface.go | 2 ++ .../helmApplicationService/helmAppService.go | 11 +++++------ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/kubelink/pkg/helmClient/client.go b/kubelink/pkg/helmClient/client.go index 426484146..60d0d3b65 100644 --- a/kubelink/pkg/helmClient/client.go +++ b/kubelink/pkg/helmClient/client.go @@ -105,9 +105,9 @@ func newClient(options *Options, clientGetter genericclioptions.RESTClientGetter func setEnvSettings(options *Options, settings *cli.EnvSettings) error { if options == nil { options = &Options{ - RepositoryConfig: DefaultRepositoryConfigPath, - RepositoryCache: DefaultCachePath, - Linting: true, + //RepositoryConfig: DefaultRepositoryConfigPath, + //RepositoryCache: DefaultCachePath, + Linting: true, } } @@ -838,3 +838,11 @@ func GetChartSavedDir(helmChart *chart.Chart) (string, error) { func (c *HelmClient) GetProviders() getter.Providers { return c.Providers } + +func (c *HelmClient) GetRepositoryConfig() string { + return c.Settings.RepositoryConfig +} + +func (c *HelmClient) GetRepositoryCache() string { + return c.Settings.RepositoryCache +} diff --git a/kubelink/pkg/helmClient/interface.go b/kubelink/pkg/helmClient/interface.go index afafc4d5d..eaf1c93af 100644 --- a/kubelink/pkg/helmClient/interface.go +++ b/kubelink/pkg/helmClient/interface.go @@ -42,4 +42,6 @@ type Client interface { TemplateChart(spec *ChartSpec, options *HelmTemplateOptions, chartData []byte, returnChartBytes bool) ([]byte, []byte, error) GetNotes(spec *ChartSpec, options *HelmTemplateOptions) ([]byte, error) GetProviders() getter.Providers + GetRepositoryConfig() string + GetRepositoryCache() string } diff --git a/kubelink/pkg/service/helmApplicationService/helmAppService.go b/kubelink/pkg/service/helmApplicationService/helmAppService.go index d18b369c5..e43a07982 100644 --- a/kubelink/pkg/service/helmApplicationService/helmAppService.go +++ b/kubelink/pkg/service/helmApplicationService/helmAppService.go @@ -32,7 +32,6 @@ import ( "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" downloader2 "helm.sh/helm/v3/pkg/downloader" - "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/registry" "helm.sh/helm/v3/pkg/storage/driver" "net/url" @@ -580,7 +579,7 @@ func (impl *HelmAppServiceImpl) UpgradeRelease(ctx context.Context, request *cli // perform Dependency Update in case we detect any dependency listed in chart.Metadata if len(helmRelease.Chart.Metadata.Dependencies) != 0 { impl.logger.Infow("Dependencies listed in Chart.yaml, performing dependency update before upgrading", "dependencies", helmRelease.Chart.Metadata.Dependencies) - err = impl.updateChartDependencies(helmClientObj.GetProviders(), helmRelease, registryClient) + err = impl.updateChartDependencies(helmClientObj, helmRelease, registryClient) if err != nil { impl.logger.Errorw("error in updating chart Dependencies", "err", err) return nil, err @@ -1934,7 +1933,7 @@ func podMetadataAdapter(podmetadatas []*commonBean.PodMetadata) []*client.PodMet return podMetadatas } -func (impl *HelmAppServiceImpl) updateChartDependencies(providers getter.Providers, helmRelease *release.Release, registry *registry.Client) error { +func (impl *HelmAppServiceImpl) updateChartDependencies(client helmClient.Client, helmRelease *release.Release, registry *registry.Client) error { // Step 1: Update chart dependencies outputChartPathDir := fmt.Sprintf("%s", helmClient.DefaultTempDirectory) err := os.MkdirAll(outputChartPathDir, os.ModePerm) @@ -1967,9 +1966,9 @@ func (impl *HelmAppServiceImpl) updateChartDependencies(providers getter.Provide manager := &downloader2.Manager{ ChartPath: outputChartPathDir, Out: outputBuffer, - Getters: providers, - RepositoryConfig: helmClient.DefaultRepositoryConfigPath, - RepositoryCache: helmClient.DefaultCachePath, + Getters: client.GetProviders(), + RepositoryConfig: client.GetRepositoryConfig(), + RepositoryCache: client.GetRepositoryCache(), RegistryClient: registry, } err = manager.Update() From 56435bdfc09746e3cbdedde5ae7c148ab31c7c23 Mon Sep 17 00:00:00 2001 From: Rajeev Date: Tue, 26 Nov 2024 20:18:23 +0530 Subject: [PATCH 9/9] added additional check for before update --- .../helmApplicationService/helmAppService.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/kubelink/pkg/service/helmApplicationService/helmAppService.go b/kubelink/pkg/service/helmApplicationService/helmAppService.go index e43a07982..b420201e8 100644 --- a/kubelink/pkg/service/helmApplicationService/helmAppService.go +++ b/kubelink/pkg/service/helmApplicationService/helmAppService.go @@ -29,6 +29,7 @@ import ( error2 "github.com/devtron-labs/kubelink/error" repository "github.com/devtron-labs/kubelink/pkg/cluster" "github.com/devtron-labs/kubelink/pkg/service/commonHelmService" + "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" downloader2 "helm.sh/helm/v3/pkg/downloader" @@ -577,12 +578,17 @@ func (impl *HelmAppServiceImpl) UpgradeRelease(ctx context.Context, request *cli return nil, err } // perform Dependency Update in case we detect any dependency listed in chart.Metadata - if len(helmRelease.Chart.Metadata.Dependencies) != 0 { - impl.logger.Infow("Dependencies listed in Chart.yaml, performing dependency update before upgrading", "dependencies", helmRelease.Chart.Metadata.Dependencies) - err = impl.updateChartDependencies(helmClientObj, helmRelease, registryClient) - if err != nil { - impl.logger.Errorw("error in updating chart Dependencies", "err", err) - return nil, err + if req := helmRelease.Chart.Metadata.Dependencies; req != nil { + if err := action.CheckDependencies(helmRelease.Chart, req); err != nil { + impl.logger.Errorw("An error occurred while checking for chart dependencies. You may need to run `helm dependency build` to fetch missing dependencies", "err", err) + if len(helmRelease.Chart.Metadata.Dependencies) != 0 { + impl.logger.Infow("Dependencies listed in Chart.yaml, performing dependency update before upgrading", "dependencies", helmRelease.Chart.Metadata.Dependencies) + err = impl.updateChartDependencies(helmClientObj, helmRelease, registryClient) + if err != nil { + impl.logger.Errorw("error in updating chart Dependencies", "err", err) + return nil, err + } + } } }