Config changes, improved version computation and build area synthesis

- typescript packages: library config field is replaced with "packaging"
- go packages: library config field is replaced with "packaging"
Thanks @AlexTugarev for the suggestion

- packages no longer depend on the whole build file but just their own config
- safer naming convention for linked dependencies in the build area

Author: Christian Weichel

cmd/build.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"io/ioutil"
 	"os"
+	"path/filepath"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -51,7 +52,7 @@ var buildCmd = &cobra.Command{
 		} else {
 			localCacheLoc = os.Getenv(leeway.EnvvarCacheDir)
 			if localCacheLoc == "" {
-				localCacheLoc = os.TempDir()
+				localCacheLoc = filepath.Join(os.TempDir(), "cache")
 			}
 		}
 		log.WithField("location", localCacheLoc).Debug("set up local cache")

go.sum

@@ -63,7 +63,6 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/typefox/gitpod v0.4.11 h1:x2OXJfgezSy6FEi3zd590kSM6W8JQoczpTeRO822d/c=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=

pkg/leeway/build.go

@@ -305,13 +305,7 @@ func (p *Package) build(buildctx *buildContext) (err error) {
 		log.WithField("package", p.FullName()).WithField("version", version).Info("build succeded")
 	}()
 
-	pkgdir := p.FullName()
-	pkgdir = strings.Replace(pkgdir, "/", "-", -1)
-	pkgdir = strings.Replace(pkgdir, ":", "--", -1)
-	pkgdir = pkgdir + "." + version
-	// components in the workspace root would otherwise start with -- which breaks a lot of shell commands
-	pkgdir = strings.TrimPrefix(pkgdir, "--")
-
+	pkgdir := p.FilesystemSafeName() + "." + version
 	builddir := filepath.Join(buildctx.BuildDir(), pkgdir)
 	if _, err := os.Stat(builddir); !os.IsNotExist(err) {
 		err := os.RemoveAll(builddir)
@@ -324,14 +318,16 @@ func (p *Package) build(buildctx *buildContext) (err error) {
 		return err
 	}
 
-	cpargs := []string{"--parents"}
-	for _, src := range p.Sources {
-		cpargs = append(cpargs, strings.TrimPrefix(src, p.C.Origin+"/"))
-	}
-	cpargs = append(cpargs, builddir)
-	err = run(getRunPrefix(p), nil, p.C.Origin, "cp", cpargs...)
-	if err != nil {
-		return err
+	if len(p.Sources) > 0 {
+		cpargs := []string{"--parents"}
+		for _, src := range p.Sources {
+			cpargs = append(cpargs, strings.TrimPrefix(src, p.C.Origin+"/"))
+		}
+		cpargs = append(cpargs, builddir)
+		err = run(getRunPrefix(p), nil, p.C.Origin, "cp", cpargs...)
+		if err != nil {
+			return err
+		}
 	}
 
 	result, _ := buildctx.LocalCache.Location(p)
@@ -386,31 +382,44 @@ func (p *Package) buildTypescript(buildctx *buildContext, wd, result string) (er
 		commands = append(commands, []string{"cp", filepath.Join(p.C.Origin, cfg.TSConfig), "."})
 	}
 
+	if cfg.Packaging == TypescriptOfflineMirror {
+		commands = append(commands, [][]string{
+			{"mkdir", "_mirror"},
+			{"sh", "-c", "echo yarn-offline-mirror \"./_mirror\" > .yarnrc"},
+		}...)
+	}
+
 	pkgYarnLock := "pkg-yarn.lock"
 	for _, deppkg := range p.GetTransitiveDependencies() {
 		builtpkg, ok := buildctx.LocalCache.Location(deppkg)
 		if !ok {
 			return PkgNotBuiltErr{deppkg}
 		}
 
+		tgt := deppkg.FilesystemSafeName()
+		if cfg.Packaging == TypescriptOfflineMirror {
+			fn := fmt.Sprintf("%s.tar.gz", tgt)
+			commands = append(commands, []string{"cp", builtpkg, filepath.Join("_mirror", fn)})
+			builtpkg = filepath.Join(wd, "_mirror", fn)
+		}
+
 		if deppkg.Type == TypescriptPackage {
 			cfg, ok := deppkg.Config.(TypescriptPkgConfig)
-			if !ok || !cfg.Library {
-				return xerrors.Errorf("cannot depend on typescript application packages: %s", deppkg.FullName())
+			if !ok || cfg.Packaging != TypescriptLibrary {
+				return xerrors.Errorf("can only depend on typescript libraries: %s", deppkg.FullName())
 			}
 
 			// make previously built package availabe through yarn lock
 			commands = append(commands, []string{"sh", "-c", fmt.Sprintf("tar Ozfx %s package/%s | sed '/resolved /c\\  resolved \"file://%s\"' >> yarn.lock", builtpkg, pkgYarnLock, builtpkg)})
 		} else {
-			tgt := strings.Replace(deppkg.FullName(), "/", "-", -1)
 			commands = append(commands, [][]string{
 				{"mkdir", tgt},
 				{"tar", "xfz", builtpkg, "-C", tgt},
 			}...)
 		}
 	}
 
-	if cfg.Library {
+	if cfg.Packaging == TypescriptLibrary {
 		// We can't modify the `yarn pack` generated tar file without runnign the risk of yarn blocking when attempting to unpack it again. Thus, we must include the pkgYarnLock in the npm
 		// package we're building. To this end, we modify the package.json of the source package.
 		pkgJSONFilename := filepath.Join(wd, "package.json")
@@ -452,14 +461,27 @@ func (p *Package) buildTypescript(buildctx *buildContext, wd, result string) (er
 		yarnMutex = "network"
 	}
 	yarnCache := filepath.Join(buildctx.BuildDir(), "yarn-cache")
-	commands = append(commands, []string{"yarn", "install", "--mutex", yarnMutex, "--cache-folder", yarnCache})
-	if len(cfg.BuildCmd) == 0 {
+	if len(cfg.Commands.Install) == 0 {
+		commands = append(commands, []string{"yarn", "install", "--mutex", yarnMutex, "--cache-folder", yarnCache})
+	} else {
+		commands = append(commands, cfg.Commands.Install)
+	}
+	if len(cfg.Commands.Build) == 0 {
 		commands = append(commands, []string{"npx", "tsc"})
 	} else {
-		commands = append(commands, cfg.BuildCmd)
+		commands = append(commands, cfg.Commands.Build)
 	}
 
-	if cfg.Library {
+	if cfg.Packaging == TypescriptOfflineMirror {
+		dst := filepath.Join("_mirror", fmt.Sprintf("%s.tar.gz", p.FilesystemSafeName()))
+		commands = append(commands, [][]string{
+			{"sh", "-c", fmt.Sprintf("yarn generate-lock-entry --resolved file://./%s > _mirror/content_yarn.lock", dst)},
+			{"sh", "-c", "cat yarn.lock >> _mirror/content_yarn.lock"},
+			{"yarn", "pack", "--filename", dst},
+			{"sh", "-c", "#!/bin/bash\n" + `echo cd \$\(dirname \"\${BASH_SOURCE[0]}\"\) \&\& sed \'s?resolved \"file://.\*\/?resolved \"file:\/\/\'\$\(pwd\)\'\/?g\' content_yarn.lock > _mirror/get_yarn_lock.sh`},
+			{"tar", "cfz", result, "-C", "_mirror", "."},
+		}...)
+	} else if cfg.Packaging == TypescriptLibrary {
 		commands = append(commands, [][]string{
 			{"sh", "-c", fmt.Sprintf("yarn generate-lock-entry --resolved file://%s > %s", result, pkgYarnLock)},
 			{"yarn", "pack", "--filename", result},
@@ -491,7 +513,7 @@ func (p *Package) buildGo(buildctx *buildContext, wd, result string) (err error)
 				return PkgNotBuiltErr{dep}
 			}
 
-			tgt := filepath.Join("_deps", strings.Replace(dep.FullName(), "/", "-", -1))
+			tgt := filepath.Join("_deps", dep.FilesystemSafeName())
 			commands = append(commands, [][]string{
 				{"mkdir", tgt},
 				{"tar", "xfz", builtpkg, "-C", tgt},
@@ -518,7 +540,7 @@ func (p *Package) buildGo(buildctx *buildContext, wd, result string) (err error)
 	if !cfg.DontCheckGoFmt {
 		commands = append(commands, []string{"sh", "-c", `if [ ! $(go fmt ./... | wc -l) -eq 0 ]; then echo; echo; echo please gofmt your code; echo; echo; exit 1; fi`})
 	}
-	if !cfg.Library {
+	if cfg.Packaging == GoApp {
 		cmd := []string{"go", "build"}
 		cmd = append(cmd, cfg.BuildFlags...)
 		cmd = append(cmd, ".")
@@ -554,7 +576,7 @@ func (p *Package) buildDocker(buildctx *buildContext, wd, result string) (err er
 			return PkgNotBuiltErr{dep}
 		}
 
-		tgt := strings.Replace(dep.FullName(), "/", "-", -1)
+		tgt := dep.FilesystemSafeName()
 		commands = append(commands, [][]string{
 			{"mkdir", tgt},
 			{"tar", "xfz", fn, "-C", tgt},
@@ -619,7 +641,7 @@ func (p *Package) buildGeneric(buildctx *buildContext, wd, result string) (err e
 			return PkgNotBuiltErr{dep}
 		}
 
-		tgt := strings.Replace(dep.FullName(), "/", "-", -1)
+		tgt := dep.FilesystemSafeName()
 		commands = append(commands, [][]string{
 			{"mkdir", tgt},
 			{"tar", "xfz", fn, "-C", tgt},

pkg/leeway/package.go

@@ -213,7 +213,6 @@ func loadComponent(workspace *Workspace, path string, args Arguments) (Component
 
 		// add component BUILD file and additional sources to package sources
 		completeSources := make(map[string]struct{})
-		completeSources[path] = struct{}{}
 		for _, src := range pkg.Sources {
 			completeSources[src] = struct{}{}
 		}
@@ -315,7 +314,8 @@ type Package struct {
 	versionCache string
 
 	packageInternal
-	Config PackageConfig `yaml:"config"`
+	Config  PackageConfig `yaml:"config"`
+	rawYAML []byte
 
 	dependencies []*Package
 }
@@ -376,6 +376,16 @@ func (p *Package) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	}
 	*p = Package{packageInternal: tpe}
 
+	var buf yaml.MapSlice
+	err = unmarshal(&buf)
+	if err != nil {
+		return err
+	}
+	p.rawYAML, err = yaml.Marshal(buf)
+	if err != nil {
+		return err
+	}
+
 	cfg, err := unmarshalTypeDependentConfig(tpe.Type, unmarshal)
 	if err != nil {
 		return err
@@ -394,6 +404,12 @@ func unmarshalTypeDependentConfig(tpe PackageType, unmarshal func(interface{}) e
 		if err := unmarshal(&cfg); err != nil {
 			return nil, err
 		}
+		if cfg.Config.Packaging == "" {
+			cfg.Config.Packaging = TypescriptApp
+		}
+		if err := cfg.Config.Validate(); err != nil {
+			return nil, err
+		}
 		return cfg.Config, nil
 	case GoPackage:
 		var cfg struct {
@@ -402,6 +418,12 @@ func unmarshalTypeDependentConfig(tpe PackageType, unmarshal func(interface{}) e
 		if err := unmarshal(&cfg); err != nil {
 			return nil, err
 		}
+		if cfg.Config.Packaging == "" {
+			cfg.Config.Packaging = GoApp
+		}
+		if err := cfg.Config.Validate(); err != nil {
+			return nil, err
+		}
 		return cfg.Config, nil
 	case DockerPackage:
 		var cfg struct {
@@ -435,12 +457,40 @@ type PackageConfig interface {
 
 // TypescriptPkgConfig configures a typescript package
 type TypescriptPkgConfig struct {
-	YarnLock string   `yaml:"yarnLock,omitempty"`
-	TSConfig string   `yaml:"tsconfig"`
-	Library  bool     `yaml:"library,omitempty"`
-	BuildCmd []string `yaml:"buildCommand,omitempty"`
+	YarnLock  string              `yaml:"yarnLock,omitempty"`
+	TSConfig  string              `yaml:"tsconfig"`
+	Packaging TypescriptPackaging `yaml:"packaging,omitempty"`
+	Commands  struct {
+		Install []string `yaml:"install,omitempty"`
+		Build   []string `yaml:"build,omitempty"`
+	} `yaml:"commands,omitempty"`
+}
+
+// Validate ensures this config can be acted upon/is valid
+func (cfg TypescriptPkgConfig) Validate() error {
+	switch cfg.Packaging {
+	case TypescriptLibrary:
+	case TypescriptOfflineMirror:
+	case TypescriptApp:
+	default:
+		return xerrors.Errorf("unknown packaging: %s", cfg.Packaging)
+	}
+
+	return nil
 }
 
+// TypescriptPackaging configures the packaging method of a typescript package
+type TypescriptPackaging string
+
+const (
+	// TypescriptLibrary means the package will be created using `yarn pack`
+	TypescriptLibrary TypescriptPackaging = "library"
+	// TypescriptOfflineMirror means that the package will become a yarn offline mirror
+	TypescriptOfflineMirror TypescriptPackaging = "offline-mirror"
+	// TypescriptApp simply tars the build directory
+	TypescriptApp TypescriptPackaging = "app"
+)
+
 // AdditionalSources returns a list of unresolved sources coming in through this configuration
 func (cfg TypescriptPkgConfig) AdditionalSources() []string {
 	var res []string
@@ -455,13 +505,35 @@ func (cfg TypescriptPkgConfig) AdditionalSources() []string {
 
 // GoPkgConfig configures a Go package
 type GoPkgConfig struct {
-	Library        bool     `yaml:"library,omitempty"`
-	Generate       bool     `yaml:"generate,omitempty"`
-	DontTest       bool     `yaml:"dontTest,omitempty"`
-	DontCheckGoFmt bool     `yaml:"dontCheckGoFmt,omitempty"`
-	BuildFlags     []string `yaml:"buildFlags,omitempty"`
+	Packaging      GoPackaging `yaml:"packaging,omitempty"`
+	Generate       bool        `yaml:"generate,omitempty"`
+	DontTest       bool        `yaml:"dontTest,omitempty"`
+	DontCheckGoFmt bool        `yaml:"dontCheckGoFmt,omitempty"`
+	BuildFlags     []string    `yaml:"buildFlags,omitempty"`
+}
+
+// Validate ensures this config can be acted upon/is valid
+func (cfg GoPkgConfig) Validate() error {
+	switch cfg.Packaging {
+	case GoLibrary:
+	case GoApp:
+	default:
+		return xerrors.Errorf("unknown packaging: %s", cfg.Packaging)
+	}
+
+	return nil
 }
 
+// GoPackaging configures the packaging method of a Go package
+type GoPackaging string
+
+const (
+	// GoLibrary means the package can be imported in another package
+	GoLibrary GoPackaging = "library"
+	// GoApp runs go build and tars the build directory
+	GoApp GoPackaging = "app"
+)
+
 // AdditionalSources returns a list of unresolved sources coming in through this configuration
 func (cfg GoPkgConfig) AdditionalSources() []string {
 	return []string{}
@@ -510,6 +582,16 @@ func (p *Package) FullName() string {
 	return fmt.Sprintf("%s:%s", p.C.Name, p.Name)
 }
 
+// FilesystemSafeName returns a string that is safe to use in a Unix filesystem as directory or filename
+func (p *Package) FilesystemSafeName() string {
+	pkgdir := p.FullName()
+	pkgdir = strings.Replace(pkgdir, "/", "-", -1)
+	pkgdir = strings.Replace(pkgdir, ":", "--", -1)
+	// components in the workspace root would otherwise start with -- which breaks a lot of shell commands
+	pkgdir = strings.TrimPrefix(pkgdir, "--")
+	return pkgdir
+}
+
 // resolveSources resolves any glob expression in the source list
 func (p *Package) resolveSources() error {
 	var res []string
@@ -633,6 +715,14 @@ func (p *Package) Version() (string, error) {
 	}
 
 	h := sha1.New()
+	_, err = h.Write(p.rawYAML)
+	if err != nil {
+		return "", err
+	}
+	_, err = io.WriteString(h, strings.Join(manifest, "\n"))
+	if err != nil {
+		return "", err
+	}
 	_, err = io.WriteString(h, strings.Join(manifest, "\n"))
 	if err != nil {
 		return "", err