Add initial registry module related code

Author: pysysops

cmd/install.go

@@ -69,7 +69,7 @@ func init() {
 func getModule(moduleName string, moduleMeta module, wg *sync.WaitGroup) {
 	defer wg.Done()
 
-	moduleSource, modulePath := splitAddrSubdir(moduleMeta.Source)
+	moduleSource, modulePath := getter.SourceDirSubdir(moduleMeta.Source)
 
 	moduleVersion := ""
 	if len(moduleMeta.Version) > 0 {
@@ -85,7 +85,7 @@ func getModule(moduleName string, moduleMeta module, wg *sync.WaitGroup) {
 	switch {
 	case xt.IsLocalSourceAddr(moduleSource):
 		xt.CopyFile(moduleName, moduleSource, directory)
-	case isRegistrySourceAddr(moduleSource):
+	case xt.IsRegistrySourceAddr(moduleSource):
 		source, version := getRegistrySource(moduleName, moduleSource, moduleVersion)
 		getWithGoGetter(moduleName, source, version, directory)
 	default:
@@ -112,19 +112,12 @@ func getModule(moduleName string, moduleMeta module, wg *sync.WaitGroup) {
 var registryBaseURL = "https://registry.terraform.io/v1/modules"
 var githubDownloadURLRe = regexp.MustCompile(`https://[^/]+/repos/([^/]+)/([^/]+)/tarball/([^/]+)/.*`)
 
-func isRegistrySourceAddr(addr string) bool {
-	nameRegex := "[0-9A-Za-z](?:[0-9A-Za-z-_]{0,62}[0-9A-Za-z])?"
-	providerRegex := "[0-9a-z]{1,64}"
-	registryRegex := regexp.MustCompile(
-		fmt.Sprintf("^(%s)\\/(%s)\\/(%s)(?:\\/\\/(.*))?$", nameRegex, nameRegex, providerRegex))
-	return registryRegex.MatchString(addr)
-}
-
 func getRegistrySource(name string, source string, version string) (string, string) {
 	logVersion := "latest"
 	if len(version) > 0 {
 		logVersion = version
 	}
+
 	jww.INFO.Printf("[%s] Looking up %s version %s in Terraform registry", name, source, logVersion)
 
 	src := strings.Split(source, "/")
@@ -150,6 +143,7 @@ func getRegistrySource(name string, source string, version string) (string, stri
 	if len(resp.Header["X-Terraform-Get"]) > 0 {
 		githubDownloadURL = resp.Header["X-Terraform-Get"][0]
 	}
+	jww.INFO.Printf("[%s] %v", name, registryDownloadURL)
 
 	if githubDownloadURLRe.MatchString(githubDownloadURL) {
 		matches := githubDownloadURLRe.FindStringSubmatch(githubDownloadURL)
@@ -159,7 +153,7 @@ func getRegistrySource(name string, source string, version string) (string, stri
 	}
 	err = errors.New("Unable to find module or version download url")
 	xt.CheckIfError(name, err)
-	return "", "" // Never reacbhes here
+	return "", "" // Never reaches here
 }
 
 // Handle modules from other sources to reflect:
@@ -248,9 +242,3 @@ func getWithGoGetter(name string, source string, version string, directory strin
 	err = client.Get()
 	xt.CheckIfError(name, err)
 }
-
-// The subDir portion will be returned as empty if no subdir separator
-// ("//") is present in the address.
-func splitAddrSubdir(addr string) (packageAddr, subDir string) {
-	return getter.SourceDirSubdir(addr)
-}

pkg/xterrafile/registry.go

@@ -0,0 +1,95 @@
+// Copyright © 2019 Tim Birkett <tim.birkett@devopsmakers.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package xterrafile
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+
+	"github.com/blang/semver"
+	"github.com/hashicorp/terraform/registry"
+	"github.com/hashicorp/terraform/registry/regsrc"
+	"github.com/hashicorp/terraform/svchost/disco"
+
+	jww "github.com/spf13/jwalterweatherman"
+)
+
+// IsRegistrySourceAddr check an address is a valid registry address
+func IsRegistrySourceAddr(addr string) bool {
+	jww.DEBUG.Printf("Testing if %s is a registry source", addr)
+	_, err := regsrc.ParseModuleSource(addr)
+	return err == nil
+}
+
+// GetRegistrySource retrieves a modules download source from a Terraform registry
+func GetRegistrySource(name string, source string, version string) bool {
+	modSrc, err := getModSrc(source)
+	CheckIfError(name, err)
+
+	version, err = getRegistryVersion(modSrc, version, nil)
+	CheckIfError(name, err)
+
+	jww.ERROR.Printf("[%s] Found module version %s at %s", name, version, modSrc.Host())
+
+	return true
+}
+
+// Helper function to return a valid version
+func getRegistryVersion(modSrc *regsrc.Module, version string, services *disco.Disco) (string, error) {
+	// Don't log from Terraform's HTTP client
+	log.SetFlags(0)
+	log.SetOutput(ioutil.Discard)
+
+	regClient := registry.NewClient(services, nil)
+	regClientResp, err := regClient.ModuleVersions(modSrc)
+	if err != nil {
+		return "", err
+	}
+
+	validModuleVersionRange, err := semver.ParseRange(version)
+	if err != nil {
+		return "", err
+	}
+
+	regModule := regClientResp.Modules[0]
+	for _, moduleVersion := range regModule.Versions {
+		v, _ := semver.ParseTolerant(moduleVersion.Version)
+
+		if validModuleVersionRange(v) {
+			return v.String(), nil
+		}
+	}
+	err = fmt.Errorf(
+		"Unable to find a valid version at %s newest version is %s",
+		modSrc.Host(),
+		regModule.Versions[0].Version)
+	return "", err
+}
+
+// Helper function to parse and return a module source
+func getModSrc(source string) (*regsrc.Module, error) {
+	modSrc, err := regsrc.ParseModuleSource(source)
+	if err != nil {
+		return nil, err
+	}
+	return modSrc, nil
+}

pkg/xterrafile/registry_test.go

@@ -0,0 +1,71 @@
+package xterrafile
+
+import (
+	"testing"
+
+	"github.com/hashicorp/terraform/registry/test"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIsRegistrySourceAddr(t *testing.T) {
+	assert.False(t, IsRegistrySourceAddr("./some/path"), "non-registry path should be false")
+	assert.True(t, IsRegistrySourceAddr("terraform-digitalocean-modules/droplet/digitalocean"),
+		"registry path should be true")
+	assert.True(t, IsRegistrySourceAddr(
+		"terraform-digitalocean-modules/droplet/digitalocean//examples/test"),
+		"registry path with sub-path should be true")
+	assert.True(t, IsRegistrySourceAddr("app.terraform.io/terraform-digitalocean-modules/droplet/digitalocean"),
+		"private registry path should be true")
+	assert.True(t, IsRegistrySourceAddr(
+		"app.terraform.io/terraform-digitalocean-modules/droplet/digitalocean//examples/test"),
+		"private registry path with sub-path should be true")
+}
+
+func TestGetRegistrySource(t *testing.T) {
+	assert.True(t, GetRegistrySource(
+		"droplet",
+		"terraform-digitalocean-modules/droplet/digitalocean",
+		">=0.1.2"),
+		"private registry path with sub-path should be true")
+	// assert.True(t, GetRegistrySource(
+	// 	"droplet2",
+	// 	"app.terraform.io/terraform-digitalocean-modules/droplet/digitalocean",
+	// 	">= 1.0.0"),
+	// 	"private registry path with sub-path should be true")
+}
+
+func TestGetModSrc(t *testing.T) {
+	module1String := "terraform-digitalocean-modules/droplet/digitalocean"
+	module1Src, err := getModSrc(module1String)
+	assert.Equal(t, nil, err, "error should be nil")
+	assert.Equal(t, "registry.terraform.io", module1Src.Host().Normalized(), "host should be public repo")
+
+	module2String := "app.terraform.io/terraform-digitalocean-modules/droplet/digitalocean"
+	module2Src, err := getModSrc(module2String)
+	assert.Equal(t, nil, err, "error should be nil")
+	assert.Equal(t, "app.terraform.io", module2Src.Host().Normalized(), "host should be private repo")
+
+	module3String := "---.io/terraform-digitalocean-modules/droplet/digitalocean"
+	module3Src, err := getModSrc(module3String)
+	assert.NotEqual(t, nil, err, "error should be present")
+	assert.Panics(t, func() { module3Src.Host().Normalized() }, "accessing host should panic")
+}
+
+func TestGetRegistryVersion(t *testing.T) {
+	server := test.Registry()
+	defer server.Close()
+
+	modSrc, _ := getModSrc("example.com/test-versions/name/provider")
+	version, _ := getRegistryVersion(modSrc, ">= 2.0.0 < 2.2.0", test.Disco(server))
+	assert.Equal(t, "2.1.1", version, "version should be >= 2.0.0 < 2.2.0")
+
+	_, err := getRegistryVersion(modSrc, ">= 3.0.0", test.Disco(server))
+	assert.Error(t, err, "should have returned an error")
+
+	_, err = getRegistryVersion(modSrc, "not.a.version", test.Disco(server))
+	assert.Error(t, err, "should return an error")
+
+	modSrc, _ = getModSrc("invalid.com/test-versions/name/provider")
+	_, err = getRegistryVersion(modSrc, ">= 3.0.0", test.Disco(server))
+	assert.Error(t, err, "should return an error")
+}

pkg/xterrafile/util_test.go

@@ -3,6 +3,7 @@ package xterrafile
 import (
 	"bytes"
 	"errors"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -15,6 +16,7 @@ func TestCheckIfError(t *testing.T) {
 	// Capture logging
 	var outputBuf bytes.Buffer
 	jww.SetStdoutOutput(&outputBuf)
+	defer jww.SetStdoutOutput(os.Stdout)
 
 	// override osExit to test for usage
 	oldOsExit := osExit