From fd8330c02725a0d7654612e369ad743b96778e93 Mon Sep 17 00:00:00 2001 From: Marc Nuri Date: Thu, 27 Nov 2025 07:08:38 +0100 Subject: [PATCH] test(mcp): use envtest built-in features to apply CRDs Trying to speed up the test cases that require additional kubernetes APIs. Each time a test suite or test case requires an additional API (like OpenShift Routes or Kiali CRDs), provided by k8s.io/apiextensions, a fixed 2 seconds delay is introduced. There's no way to reduce that delay by using envtest features or APIServer CLI flags. The delays are introduced by kube-aggregator, apiextensions, and others: https://github.com/kubernetes/kube-aggregator/blob/a00232cf7e758c2771b3542033bec19e69e6451a/pkg/controllers/openapiv3/controller.go#L34 https://github.com/kubernetes/apiextensions-apiserver/blob/8db5ab628dd026827c1c9677944432db70c065c3/pkg/apiserver/customresource_handler.go#L381-L394 To mitigate this, we're going to declare all CRDs used in the test in the common_test.go TestMain function. This will make sure we only have a single delay at the beginning of the entire test suite instead of per test suite or test case. The CRDs are NOT served by default. For each test suite we'll be able to enable them or disable them by using the EnvTestEnableCRD and EnvTestDisableCRD functions. These will also ensure that the API server lists the new APIs or not. Signed-off-by: Marc Nuri --- go.mod | 2 +- pkg/mcp/common_crd_test.go | 137 +++++++++++++++++++++++++++++++++++++ pkg/mcp/common_test.go | 120 +++++--------------------------- pkg/mcp/resources_test.go | 2 +- 4 files changed, 155 insertions(+), 106 deletions(-) create mode 100644 pkg/mcp/common_crd_test.go diff --git a/go.mod b/go.mod index f9f1d232..b84f28a0 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( github.com/google/jsonschema-go v0.3.0 github.com/mark3labs/mcp-go v0.43.1 github.com/modelcontextprotocol/go-sdk v1.1.0 - github.com/pkg/errors v0.9.1 github.com/spf13/afero v1.15.0 github.com/spf13/cobra v1.10.1 github.com/spf13/pflag v1.0.10 @@ -104,6 +103,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.22.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect diff --git a/pkg/mcp/common_crd_test.go b/pkg/mcp/common_crd_test.go new file mode 100644 index 00000000..6f658337 --- /dev/null +++ b/pkg/mcp/common_crd_test.go @@ -0,0 +1,137 @@ +package mcp + +import ( + "context" + "fmt" + "time" + + "golang.org/x/sync/errgroup" + apiextensionsv1spec "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/discovery" + "k8s.io/utils/ptr" +) + +func CRD(group, version, resource, kind, singular string, namespaced bool) *apiextensionsv1spec.CustomResourceDefinition { + scope := "Cluster" + if namespaced { + scope = "Namespaced" + } + crd := &apiextensionsv1spec.CustomResourceDefinition{ + TypeMeta: metav1.TypeMeta{ + APIVersion: apiextensionsv1spec.SchemeGroupVersion.String(), + Kind: "CustomResourceDefinition", + }, + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s.%s", resource, group)}, + Spec: apiextensionsv1spec.CustomResourceDefinitionSpec{ + Group: group, + Versions: []apiextensionsv1spec.CustomResourceDefinitionVersion{ + { + Name: version, + Served: false, + Storage: true, + Schema: &apiextensionsv1spec.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1spec.JSONSchemaProps{ + Type: "object", + XPreserveUnknownFields: ptr.To(true), + }, + }, + }, + }, + Scope: apiextensionsv1spec.ResourceScope(scope), + Names: apiextensionsv1spec.CustomResourceDefinitionNames{ + Plural: resource, + Singular: singular, + Kind: kind, + ShortNames: []string{singular}, + }, + }, + } + return crd +} + +func EnvTestEnableCRD(ctx context.Context, group, version, resource string) error { + apiExtensionsV1Client := apiextensionsv1.NewForConfigOrDie(envTestRestConfig) + _, err := apiExtensionsV1Client.CustomResourceDefinitions().Patch( + ctx, + fmt.Sprintf("%s.%s", resource, group), + types.JSONPatchType, + []byte(`[{"op": "replace", "path": "/spec/versions/0/served", "value": true}]`), + metav1.PatchOptions{}) + if err != nil { + return err + } + return EnvTestWaitForAPIResourceCondition(ctx, group, version, resource, true) +} + +func EnvTestDisableCRD(ctx context.Context, group, version, resource string) error { + apiExtensionsV1Client := apiextensionsv1.NewForConfigOrDie(envTestRestConfig) + _, err := apiExtensionsV1Client.CustomResourceDefinitions().Patch( + ctx, + fmt.Sprintf("%s.%s", resource, group), + types.JSONPatchType, + []byte(`[{"op": "replace", "path": "/spec/versions/0/served", "value": false}]`), + metav1.PatchOptions{}) + if err != nil { + return err + } + return EnvTestWaitForAPIResourceCondition(ctx, group, version, resource, false) +} + +func EnvTestWaitForAPIResourceCondition(ctx context.Context, group, version, resource string, shouldBeAvailable bool) error { + discoveryClient, err := discovery.NewDiscoveryClientForConfig(envTestRestConfig) + if err != nil { + return fmt.Errorf("failed to create discovery client: %w", err) + } + + groupVersion := fmt.Sprintf("%s/%s", group, version) + if group == "" { + groupVersion = version + } + + return wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, 10*time.Second, true, func(ctx context.Context) (bool, error) { + resourceList, err := discoveryClient.ServerResourcesForGroupVersion(groupVersion) + if err != nil { + // If we're waiting for the resource to be unavailable and we get an error, it might be gone + if !shouldBeAvailable { + return true, nil + } + // Otherwise, keep polling + return false, nil + } + + // Check if the resource exists in the list + found := false + for _, apiResource := range resourceList.APIResources { + if apiResource.Name == resource { + found = true + break + } + } + + // Return true if the condition is met + if shouldBeAvailable { + return found, nil + } + return !found, nil + }) +} + +// EnvTestInOpenShift sets up the kubernetes environment to seem to be running OpenShift +func EnvTestInOpenShift(ctx context.Context) error { + tasks, _ := errgroup.WithContext(ctx) + tasks.Go(func() error { return EnvTestEnableCRD(ctx, "project.openshift.io", "v1", "projects") }) + tasks.Go(func() error { return EnvTestEnableCRD(ctx, "route.openshift.io", "v1", "routes") }) + return tasks.Wait() +} + +// EnvTestInOpenShiftClear clears the kubernetes environment so it no longer seems to be running OpenShift +func EnvTestInOpenShiftClear(ctx context.Context) error { + tasks, _ := errgroup.WithContext(ctx) + tasks.Go(func() error { return EnvTestDisableCRD(ctx, "project.openshift.io", "v1", "projects") }) + tasks.Go(func() error { return EnvTestDisableCRD(ctx, "route.openshift.io", "v1", "routes") }) + return tasks.Wait() +} diff --git a/pkg/mcp/common_test.go b/pkg/mcp/common_test.go index 06dfc06b..fe173e83 100644 --- a/pkg/mcp/common_test.go +++ b/pkg/mcp/common_test.go @@ -2,29 +2,20 @@ package mcp import ( "context" - "encoding/json" - "fmt" "os" "path/filepath" "runtime" "testing" - "time" "github.com/mark3labs/mcp-go/client/transport" - "github.com/pkg/errors" "github.com/spf13/afero" "github.com/stretchr/testify/suite" - "golang.org/x/sync/errgroup" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apiextensionsv1spec "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - toolswatch "k8s.io/client-go/tools/watch" - "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/tools/setup-envtest/env" "sigs.k8s.io/controller-runtime/tools/setup-envtest/remote" @@ -49,6 +40,8 @@ func TestMain(m *testing.M) { // Set high rate limits to avoid client-side throttling in tests _ = os.Setenv("KUBE_CLIENT_QPS", "1000") _ = os.Setenv("KUBE_CLIENT_BURST", "2000") + //// Enable control plane output to see API server logs + //_ = os.Setenv("KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT", "true") envTestDir, err := store.DefaultStoreDir() if err != nil { panic(err) @@ -73,7 +66,21 @@ func TestMain(m *testing.M) { versionDir := envTestEnv.Platform.BaseName(*envTestEnv.Version.AsConcrete()) envTest = &envtest.Environment{ BinaryAssetsDirectory: filepath.Join(envTestDir, "k8s", versionDir), + CRDs: []*apiextensionsv1spec.CustomResourceDefinition{ + CRD("project.openshift.io", "v1", "projects", "Project", "project", false), + CRD("route.openshift.io", "v1", "routes", "Route", "route", true), + }, } + // Configure API server for faster CRD establishment and test performance + envTest.ControlPlane.GetAPIServer().Configure(). + // Increase concurrent request limits for faster parallel operations + Set("max-requests-inflight", "1000"). + Set("max-mutating-requests-inflight", "500"). + // Speed up namespace cleanup with more workers + Set("delete-collection-workers", "10") //. + // Enable verbose logging for debugging + //Set("v", "9") + adminSystemMasterBaseConfig, _ := envTest.Start() au := test.Must(envTest.AddUser(envTestUser, adminSystemMasterBaseConfig)) envTestRestConfig = au.Config() @@ -195,98 +202,3 @@ func (s *BaseMcpSuite) InitMcpClient(options ...transport.StreamableHTTPCOption) s.Require().NoError(err, "Expected no error creating MCP server") s.McpClient = test.NewMcpClient(s.T(), s.mcpServer.ServeHTTP(), options...) } - -// EnvTestInOpenShift sets up the kubernetes environment to seem to be running OpenShift -func EnvTestInOpenShift(ctx context.Context) error { - crdTemplate := ` - { - "apiVersion": "apiextensions.k8s.io/v1", - "kind": "CustomResourceDefinition", - "metadata": {"name": "%s"}, - "spec": { - "group": "%s", - "versions": [{ - "name": "v1","served": true,"storage": true, - "schema": {"openAPIV3Schema": {"type": "object","x-kubernetes-preserve-unknown-fields": true}} - }], - "scope": "%s", - "names": {"plural": "%s","singular": "%s","kind": "%s"} - } - }` - tasks, _ := errgroup.WithContext(ctx) - tasks.Go(func() error { - return EnvTestCrdApply(ctx, fmt.Sprintf(crdTemplate, "projects.project.openshift.io", "project.openshift.io", - "Cluster", "projects", "project", "Project")) - }) - tasks.Go(func() error { - return EnvTestCrdApply(ctx, fmt.Sprintf(crdTemplate, "routes.route.openshift.io", "route.openshift.io", - "Namespaced", "routes", "route", "Route")) - }) - return tasks.Wait() -} - -// EnvTestInOpenShiftClear clears the kubernetes environment so it no longer seems to be running OpenShift -func EnvTestInOpenShiftClear(ctx context.Context) error { - tasks, _ := errgroup.WithContext(ctx) - tasks.Go(func() error { return EnvTestCrdDelete(ctx, "projects.project.openshift.io") }) - tasks.Go(func() error { return EnvTestCrdDelete(ctx, "routes.route.openshift.io") }) - return tasks.Wait() -} - -// EnvTestCrdWaitUntilReady waits for a CRD to be established -func EnvTestCrdWaitUntilReady(ctx context.Context, name string) error { - apiExtensionClient := apiextensionsv1.NewForConfigOrDie(envTestRestConfig) - watcher, err := apiExtensionClient.CustomResourceDefinitions().Watch(ctx, metav1.ListOptions{ - FieldSelector: "metadata.name=" + name, - }) - if err != nil { - return fmt.Errorf("unable to watch CRDs: %w", err) - } - _, err = toolswatch.UntilWithoutRetry(ctx, watcher, func(event watch.Event) (bool, error) { - for _, c := range event.Object.(*apiextensionsv1spec.CustomResourceDefinition).Status.Conditions { - if c.Type == apiextensionsv1spec.Established && c.Status == apiextensionsv1spec.ConditionTrue { - return true, nil - } - } - return false, nil - }) - if err != nil { - return fmt.Errorf("failed to wait for CRD: %w", err) - } - return nil -} - -// EnvTestCrdApply creates a CRD from the provided resource string and waits for it to be established -func EnvTestCrdApply(ctx context.Context, resource string) error { - apiExtensionsV1Client := apiextensionsv1.NewForConfigOrDie(envTestRestConfig) - var crd = &apiextensionsv1spec.CustomResourceDefinition{} - err := json.Unmarshal([]byte(resource), crd) - if err != nil { - return fmt.Errorf("failed to create CRD %v", err) - } - _, err = apiExtensionsV1Client.CustomResourceDefinitions().Create(ctx, crd, metav1.CreateOptions{}) - if err != nil { - return fmt.Errorf("failed to create CRD %v", err) - } - return EnvTestCrdWaitUntilReady(ctx, crd.Name) -} - -// crdDelete deletes a CRD by name and waits for it to be removed -func EnvTestCrdDelete(ctx context.Context, name string) error { - apiExtensionsV1Client := apiextensionsv1.NewForConfigOrDie(envTestRestConfig) - err := apiExtensionsV1Client.CustomResourceDefinitions().Delete(ctx, name, metav1.DeleteOptions{ - GracePeriodSeconds: ptr.To(int64(0)), - }) - iteration := 0 - for iteration < 100 { - if _, derr := apiExtensionsV1Client.CustomResourceDefinitions().Get(ctx, name, metav1.GetOptions{}); derr != nil { - break - } - time.Sleep(5 * time.Millisecond) - iteration++ - } - if err != nil { - return errors.Wrap(err, "failed to delete CRD") - } - return nil -} diff --git a/pkg/mcp/resources_test.go b/pkg/mcp/resources_test.go index a1f176ff..005a74b4 100644 --- a/pkg/mcp/resources_test.go +++ b/pkg/mcp/resources_test.go @@ -401,7 +401,7 @@ func (s *ResourcesSuite) TestResourcesCreateOrUpdate() { _, err = apiExtensionsV1Client.CustomResourceDefinitions().Get(s.T().Context(), "customs.example.com", metav1.GetOptions{}) s.Nilf(err, "custom resource definition not found") }) - s.Require().NoError(EnvTestCrdWaitUntilReady(s.T().Context(), "customs.example.com")) + s.Require().NoError(EnvTestWaitForAPIResourceCondition(s.T().Context(), "example.com", "v1", "customs", true)) }) s.Run("resources_create_or_update creates custom resource", func() {