Run example tests from provider module (#2786)

Fixes #2788

Author example tests within the provider module so the provider and unit
tests are executing in the same process.

1. Start the provider host from the test setup.
2. Grab the host port and pass to the ProgramTest PULUMI_DEBUG_PROVIDERS
env.
3. Use a deferred, cancellable context to shut down the provider at the
end of the test.

Benefits:
- Diagnose provider behaviour by easily starting program tests in debug
mode within an IDE and breakpoint straight through to provider calls.
- Generates test-coverage data on which parts of the provider are or
aren't tested via program tests.
- Shorten testing feedback loop by removing requirement to build SDKs
and provider binary before re-running tests.

Specifics:
- Implicitly label non-e2e tests via build tags as `unit`.
- Local runs via `make test_provider` exclude integration tests by
default for speed.
   - CI runs all tests by default
   - All tests are runnable within a development environment

Aside: remove unused schema string in provider setup.

## Long Term (inspirational) Plan

1. Move all program tests into provider module as YAML.
2. Build simple matrix testing to automatically convert YAML to each
language and confirm correctness.
3. Use [HTTP session capture and
replay](https://pkg.go.dev/github.com/google/go-replayers/httpreplay) to
enable very fast testing of whole provider end-to-end without hitting
the real backend for local development.
4. Run capture only on nightly job and run all PRs in replay mode by
default.
5. Remove SDKs and leave only the single provider module to get rid of
mono-repo layout (also fixing go.work oddities).

Author: danielrbradley

.github/workflows/build-test.yml

@@ -216,7 +216,7 @@ jobs:
         run: make --touch codegen schema provider
 
       - name: Test Provider Library
-        run: make test_provider
+        run: PROVIDER_TEST_TAGS=all make test_provider
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v3

Makefile

@@ -122,9 +122,11 @@ arm2pulumi_coverage_report: .make/provider_mod_download provider/cmd/$(PROVIDER)
 	(cd provider/pkg/arm2pulumi/internal/testdata && if [ ! -d azure-quickstart-templates ]; then git clone https://github.com/Azure/azure-quickstart-templates && cd azure-quickstart-templates && git checkout 3b2757465c2de537e333f5e2d1c3776c349b8483; fi)
 	(cd provider && go test -v -tags=coverage -run TestQuickstartTemplateCoverage github.com/pulumi/pulumi-azure-native/v2/provider/pkg/arm2pulumi/internal/test)
 
+# Use PROVIDER_TEST_TAGS=all to run all tests including examples integrate tests
+PROVIDER_TEST_TAGS ?= unit # Default to unit tests only for quick local feedback
 .PHONY: test_provider
 test_provider: .make/provider_mod_download .make/provider_prebuild provider/cmd/$(PROVIDER)/*.go $(PROVIDER_PKG)
-	cd provider && go test -v -coverprofile="coverage.txt" -coverpkg=./... $(PROVIDER_PKGS)
+	cd provider && go test -v --tags=$(PROVIDER_TEST_TAGS) -coverprofile="coverage.txt" -coverpkg=./... $(PROVIDER_PKGS)
 
 .PHONY: lint_provider
 lint_provider: .make/provider_mod_download provider/cmd/$(PROVIDER)/*.go $(PROVIDER_PKG)

examples/api/Pulumi.yaml

@@ -12,16 +12,6 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestAccApiTs(t *testing.T) {
-	skipIfShort(t)
-	test := getJSBaseOptions(t).
-		With(integration.ProgramTestOptions{
-			Dir: filepath.Join(getCwd(t), "api"),
-		})
-
-	integration.ProgramTest(t, &test)
-}
-
 func TestAccAppServiceTs(t *testing.T) {
 	skipIfShort(t)
 	test := getJSBaseOptions(t).

examples/api/index.ts

@@ -29,5 +29,5 @@ func unsafeStringToBytes(data string) []byte {
 }
 
 func main() {
-	provider.Serve(providerName, version.Version, unsafeStringToBytes(pulumiSchema), pulumiSchema, unsafeStringToBytes(azureApiResources))
+	provider.Serve(providerName, version.Version, unsafeStringToBytes(pulumiSchema), unsafeStringToBytes(azureApiResources))
 }

examples/api/package.json

@@ -73,15 +73,14 @@ type azureNativeProvider struct {
 	config          map[string]string
 	schemaBytes     []byte
 	metadataBytes   []byte
-	schemaString    string // optimization, this and schemaBytes should have same underlying repr
 	fullPkgSpec     *schema.PackageSpec
 	fullResourceMap *resources.AzureAPIMetadata
 	converter       *resources.SdkShapeConverter
 	customResources map[string]*resources.CustomResource
 	rgLocationMap   map[string]string
 }
 
-func makeProvider(host *provider.HostClient, name, version string, schemaBytes []byte, schemaString string,
+func makeProvider(host *provider.HostClient, name, version string, schemaBytes []byte,
 	azureAPIResourcesBytes []byte) (rpc.ResourceProviderServer, error) {
 	autorest.Count429AsRetry = false
 	// Creating a REST client, defaulting to Pulumi Partner ID until the Configure method is invoked.

examples/api/tsconfig.json

@@ -0,0 +1,130 @@
+// Copyright 2023, Pulumi Corporation.  All rights reserved.
+
+// Disable running if we've specifically selected unit tests to run as this is an integration test.
+// This is easier than having to remember to explicitly tag every unit test with `//go:build unit || all`.
+//go:build !unit
+
+package provider
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"google.golang.org/grpc"
+
+	rp "github.com/pulumi/pulumi/pkg/v3/resource/provider"
+	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/util/rpcutil"
+	pulumirpc "github.com/pulumi/pulumi/sdk/v3/proto/go"
+)
+
+func TestStorageBlob(t *testing.T) {
+	runTestProgram(t, "storage-blob")
+}
+
+func TestApi(t *testing.T) {
+	runTestProgram(t, "api")
+}
+
+// runTestProgram runs an example from ./examples/<initialDir>
+// Any editDirs are applied in order, and the program is run after each edit. e.g. ./examples/<editDir>
+func runTestProgram(t *testing.T, initialDir string, editDirs ...string) {
+	if t.Skipped() {
+		return
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	port, err := startProvider(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	opts, err := getTestOptions(initialDir, editDirs, port)
+	if err != nil {
+		t.Fatal(err)
+		return
+	}
+
+	integration.ProgramTest(t, opts)
+}
+
+func getTestOptions(initialDir string, editDirs []string, port int) (*integration.ProgramTestOptions, error) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		return nil, err
+	}
+	azureLocation := getLocation()
+	test := integration.ProgramTestOptions{
+		Dir:                  filepath.Join(cwd, "test-programs", initialDir),
+		ExpectRefreshChanges: true,
+		Config: map[string]string{
+			"azure-native:location": azureLocation,
+		},
+		Env: []string{
+			fmt.Sprintf("PULUMI_DEBUG_PROVIDERS=azure-native:%d", port),
+		},
+	}
+	for _, editDir := range editDirs {
+		test.EditDirs = append(test.EditDirs, integration.EditDir{
+			Dir: filepath.Join(cwd, "test-programs", editDir),
+		})
+	}
+	return &test, nil
+}
+
+// startProvider starts the provider in a goProc and returns the port it's listening on.
+// To shut down the provider, cancel the context.
+func startProvider(ctx context.Context) (int, error) {
+	providerName := "azure-native"
+	cancelChannel := make(chan bool)
+	go func() {
+		<-ctx.Done()
+		close(cancelChannel)
+	}()
+	var host *rp.HostClient // We don't initialize this - to mimic attach mode
+	version := ""
+	schemaBytes, err := os.ReadFile(filepath.Join("..", "..", "..", "bin", "schema-full.json"))
+	if err != nil {
+		return 0, err
+	}
+	azureAPIResourcesBytes, err := os.ReadFile(filepath.Join("..", "..", "..", "bin", "metadata-compact.json"))
+	if err != nil {
+		return 0, err
+	}
+	handle, err := rpcutil.ServeWithOptions(rpcutil.ServeOptions{
+		Cancel: cancelChannel,
+		Init: func(srv *grpc.Server) error {
+			prov, proverr := makeProvider(host, providerName, version, schemaBytes, azureAPIResourcesBytes)
+			if proverr != nil {
+				return fmt.Errorf("failed to create resource provider: %v", proverr)
+			}
+			pulumirpc.RegisterResourceProviderServer(srv, prov)
+			return nil
+		},
+		Options: rpcutil.OpenTracingServerInterceptorOptions(nil),
+	})
+	if err != nil {
+		return 0, fmt.Errorf("fatal: %v", err)
+	}
+
+	return handle.Port, nil
+}
+
+func getLocation() string {
+	azureLocation := os.Getenv("ARM_LOCATION")
+	if azureLocation == "" {
+		azureLocation = "westus2"
+		fmt.Println("Defaulting location to 'westus2'. You can override using the ARM_LOCATION variable.")
+	}
+
+	return azureLocation
+}
+
+func skipIfShort(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping long-running test in short mode")
+	}
+}