Dev (#304)

* update comments

* fix msw again

* should fix scrolling in table tests

* properly test table scrolling with mocked scroll

* fix test fetching more

* update postgres with index extensions

* clean mrt

* clean mrt and gen

* data testids

* default rtl timeout

* filters

* update tests

* fixmes

* refactor

* tygo e2e typescript model gen setup

* attempt at golangcilint generated file fix

* e2e users example

* e2e teams

Author: danicc097

.env.e2e

@@ -1,10 +1,10 @@
 APP_ENV=e2e
 # using container name resolution when dockerized
-DB_PORT=5432
+DB_PORT=5656
 EXPOSED_POSTGRES_PORT=5656
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres
-POSTGRES_SERVER=postgres_db_openapi-go-gin
+POSTGRES_SERVER=localhost
 POSTGRES_PORT=5432
 POSTGRES_DB=postgres_e2e
 PROJECT_PREFIX=openapi-go-gin
@@ -24,6 +24,7 @@ FRONTEND_PORT=80
 OIDC_CLIENT_ID=web
 OIDC_CLIENT_SECRET=secret
 AUTH_SERVER_UI_PROFILE="https://authserver.local.localhost/oidc/profile"
+MOCK_OIDC_SERVER_PATH_PREFIX="/oidc"
 OIDC_DOMAIN="authserver.local.localhost"
 OIDC_ISSUER="https://authserver.local.localhost/oidc"
 OIDC_SCOPES="openid profile email auth"

.golangci.yml

@@ -1,5 +1,9 @@
 # https://github.com/MarioCarrion/todo-api-microservice-example/blob/main/.golangci.yml
-
+run:
+  skip-files: # ignore yaml error...
+    - ".*\\.xo\\.go$"
+    - ".*\\.gen\\.go$"
+    - ".*\\.pb\\.go$"
 linters:
   enable-all: true
   disable:

.vscode/settings.json

@@ -5,7 +5,7 @@
     "--config=${workspaceFolder}/.golangci.yml",
     "--issues-exit-code=0",
     // golangci possibly broke generated code skipping in recent releases
-    // "--fix",
+    "--fix",
     "--fast" // breaks some linters like errcheck, if its too slow then enable back
   ],
   "go.lintOnSave": "package",

cmd/initial-data/e2e/models.go

@@ -0,0 +1,22 @@
+package e2e
+
+import "github.com/danicc097/openapi-go-gin-postgres-sqlc/internal/models"
+
+/*
+ NOTE: tygo just looks the ast from the current package.
+ All spec models available in tstype as `models` namespace
+*/
+
+type User struct {
+	Username  string        `json:"username"`
+	Email     string        `json:"email"`
+	FirstName *string       `json:"firstName"`
+	LastName  *string       `json:"lastName"`
+	Scopes    models.Scopes `json:"scopes"    tstype:"models.Scopes"`
+	Role      models.Role   `json:"role"      tstype:"models.Role"`
+}
+
+type Team struct {
+	Name        string             `json:"name"`
+	ProjectName models.ProjectName `json:"projectName" tstype:"models.Project"`
+}

cmd/initial-data/main.go

@@ -2,24 +2,27 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"log"
 	"math/rand"
 	"os"
+	"path"
 	"strconv"
 	"sync"
 	"time"
 
+	"github.com/danicc097/openapi-go-gin-postgres-sqlc/cmd/initial-data/e2e"
 	"github.com/danicc097/openapi-go-gin-postgres-sqlc/internal"
 	"github.com/danicc097/openapi-go-gin-postgres-sqlc/internal/envvar"
 	"github.com/danicc097/openapi-go-gin-postgres-sqlc/internal/models"
 	"github.com/danicc097/openapi-go-gin-postgres-sqlc/internal/repos"
 	"github.com/danicc097/openapi-go-gin-postgres-sqlc/internal/repos/postgresql"
 	"github.com/danicc097/openapi-go-gin-postgres-sqlc/internal/repos/postgresql/gen/db"
 	"github.com/danicc097/openapi-go-gin-postgres-sqlc/internal/services"
-	"github.com/danicc097/openapi-go-gin-postgres-sqlc/internal/testutil"
 	"github.com/danicc097/openapi-go-gin-postgres-sqlc/internal/utils/pointers"
+	"github.com/gzuidhof/tygo/tygo"
 	"github.com/jackc/pgx/v5/pgxpool"
 	"go.uber.org/zap"
 )
@@ -35,6 +38,15 @@ const (
 	month = 30 * day
 )
 
+const e2eTestDataDir = "e2e/__tests__/data"
+
+/**
+ * TODO: for env = E2E marshal objects at the end to e2e/testdata json file
+ * which includes useful e2e static info that doesn't change between this cli's runs (ie no ids or random data)
+ * We can generate typescript types from these go structs as well if its somehow needed (since they won't be the same as openapi types).
+ * We have library mode with https://github.com/gzuidhof/tygo so this is trivial and more than enough for E2E
+ *.
+ */
 func main() {
 	var err error
 	var env, scopePolicyPath, rolePolicyPath string
@@ -56,7 +68,6 @@ func main() {
 
 	repositories := services.CreateRepos()
 
-	// TODO: services.Create(logger, repositories, pool)
 	svc := services.New(logger, repositories, pool)
 
 	ctx := context.Background()
@@ -108,8 +119,8 @@ func main() {
 	for i := 0; i < 50; i++ {
 		u, err := svc.User.Register(ctx, pool, services.UserRegisterParams{
 			Username:   "user_" + strconv.Itoa(i),
-			FirstName:  pointers.New(testutil.RandomFirstName()),
-			LastName:   pointers.New(testutil.RandomLastName()),
+			FirstName:  pointers.New("First name " + fmt.Sprintf("%v", i)),
+			LastName:   pointers.New("Last name " + fmt.Sprintf("%v", i)),
 			Email:      "user_" + strconv.Itoa(i) + "@mail.com",
 			ExternalID: "external_id_user_" + strconv.Itoa(i),
 			// Scopes: []models.Scope{models.}, // TODO:
@@ -141,25 +152,31 @@ func main() {
 	 *
 	 **/
 	logger.Info("Creating teams...")
+	var teams []*db.Team
 
 	teamDemo, err := svc.Team.Create(ctx, pool, &db.TeamCreateParams{
 		ProjectID:   internal.ProjectIDByName[models.ProjectDemo],
 		Name:        "Team 1",
 		Description: "Team 1 description",
 	})
 	handleError(err, teamDemo)
+	teams = append(teams, teamDemo)
+
 	teamDemo2, err := svc.Team.Create(ctx, pool, &db.TeamCreateParams{
 		ProjectID:   internal.ProjectIDByName[models.ProjectDemoTwo],
 		Name:        "Team 2-1",
 		Description: "Team 2-1 description",
 	})
 	handleError(err, teamDemo2)
+	teams = append(teams, teamDemo2)
+
 	team2Demo2, err := svc.Team.Create(ctx, pool, &db.TeamCreateParams{
 		ProjectID:   internal.ProjectIDByName[models.ProjectDemoTwo],
 		Name:        "Team 2-2",
 		Description: "Team 2-2 description",
 	})
 	handleError(err, team2Demo2)
+	teams = append(teams, team2Demo2)
 
 	for i, u := range users {
 		users[i], err = svc.User.AssignTeam(ctx, pool, u.UserID, teamDemo.TeamID)
@@ -429,6 +446,54 @@ func main() {
 	// handleError(err)
 	// fmt.Printf("wis len: %v - First workitem found:\n", len(wis))
 	// format.PrintJSONByTag(wis[0], "db")
+
+	if cfg.AppEnv == internal.AppEnvE2E {
+		println("Generating E2E fixtures")
+		config := &tygo.Config{
+			Packages: []*tygo.PackageConfig{
+				{
+					Path: "github.com/danicc097/openapi-go-gin-postgres-sqlc/cmd/initial-data/e2e",
+					TypeMappings: map[string]string{
+						"time.Time":     "string /* RFC3339Nano */",
+						"uuid.UUID":     "string /* uuid */",
+						"uuid.NullUUID": "null | string /* uuid */",
+					},
+					// to import actual values from models package, do it explicitly
+					Frontmatter: `import type * as models from "client/gen/model";`,
+					OutputPath:  path.Join(e2eTestDataDir, "initial-data.ts"),
+				},
+			},
+		}
+		gen := tygo.New(config)
+		handleError(gen.Generate())
+	}
+	uu := make(map[string]e2e.User)
+	for _, u := range users {
+		role, _ := svc.Authorization.RoleByRank(u.RoleRank)
+		uu[u.Email] = e2e.User{
+			Username:  u.Username,
+			Email:     u.Email,
+			FirstName: u.FirstName,
+			LastName:  u.LastName,
+			Role:      role.Name,
+			Scopes:    u.Scopes,
+		}
+	}
+
+	uuj, err := json.MarshalIndent(uu, "", "  ")
+	handleError(err)
+	handleError(os.WriteFile(path.Join(e2eTestDataDir, "users.json"), uuj, 0o644))
+
+	tt := make([]e2e.Team, len(teams))
+	for i, t := range teams {
+		tt[i] = e2e.Team{
+			Name:        t.Name,
+			ProjectName: t.ProjectJoin.Name,
+		}
+	}
+	ttj, err := json.MarshalIndent(tt, "", "  ")
+	handleError(err)
+	handleError(os.WriteFile(path.Join(e2eTestDataDir, "teams.json"), ttj, 0o644))
 }
 
 func errAndExit(out []byte, err error) {

db/migrations/0000002_init.up.sql

@@ -5,6 +5,10 @@ create schema if not exists "cache";
 
 create extension if not exists pg_stat_statements schema extensions;
 
+create extension if not exists hypopg schema extensions;
+
+create extension if not exists index_advisor schema extensions;
+
 create extension if not exists pg_trgm schema extensions;
 
 create extension if not exists btree_gin schema extensions;
@@ -316,21 +320,13 @@ alter table user_api_keys
 
 comment on column user_api_keys.user_api_key_id is '"properties":private';
 
--- -- pg13 alt for CONSTRAINT uq_external_id UNIQUE NULLS NOT DISTINCT (external_id)
--- create unique index on users (user_id , external_id)
--- where
---   external_id is not null;
--- create unique index on users (user_id)
--- where
---   external_id is null;
--- composite on id, deleted_at, email, deleted_at, etc. will not improve speed
 -- create unique index on users (user_id) where deleted_at is null; -- helps if you have much more deleted rows only
 -- create index on users (deleted_at);  - not worth the extra overhead.
--- for finding all deleted users exclusively
+-- does get used when filtering deleted users exclusively and there's few of them
 create index on users (deleted_at)
 where (deleted_at is not null);
 
-create index on users (updated_at);
+create index on users using gin (role_rank , age , username gin_trgm_ops , email gin_trgm_ops , full_name gin_trgm_ops);
 
 -- notification_types are append-only
 create type notification_type as ENUM (

docker/postgres/Dockerfile

@@ -1,10 +1,15 @@
 FROM postgres:16.2
 
-RUN apt-get update && apt-get install -y curl postgresql-$PG_MAJOR-plpgsql-check postgresql-$PG_MAJOR-rum
+RUN apt-get update && apt-get install -y curl postgresql-$PG_MAJOR-plpgsql-check postgresql-$PG_MAJOR-rum postgresql-$PG_MAJOR-hypopg
 # 0.3.1
 ENV SUPA_AUDIT_COMMIT="0b87d13840e26fc68d0e01e9917c67c894121f6f"
 
 RUN curl -SL "https://github.com/danicc097/supa_audit/archive/${SUPA_AUDIT_COMMIT}.tar.gz" -o /supa_audit.tar.gz \
   && tar -xzvf /supa_audit.tar.gz -C /usr/share/postgresql/$PG_MAJOR/extension/ --strip-components=1 \
   && rm /supa_audit.tar.gz \
   && chown -R postgres:postgres /usr/share/postgresql/$PG_MAJOR/extension/supa_audit*
+
+RUN curl -SL "https://github.com/supabase/index_advisor/archive/refs/tags/v0.2.0.tar.gz" -o /index_advisor.tar.gz \
+  && tar -xzvf /index_advisor.tar.gz -C /usr/share/postgresql/$PG_MAJOR/extension/ --strip-components=1 \
+  && rm /index_advisor.tar.gz \
+  && chown -R postgres:postgres /usr/share/postgresql/$PG_MAJOR/extension/index_advisor*

e2e/__tests__/data/initial-data.ts

@@ -0,0 +1,17 @@
+// Code generated by tygo. DO NOT EDIT.
+import type * as models from "client/gen/model";
+//////////
+// source: models.go
+
+export interface User {
+  username: string;
+  email: string;
+  firstName?: string;
+  lastName?: string;
+  scopes: models.Scopes;
+  role: models.Role;
+}
+export interface Team {
+  name: string;
+  projectName: models.Project;
+}

e2e/__tests__/data/teams.json

@@ -0,0 +1,14 @@
+[
+  {
+    "name": "Team 1",
+    "projectName": "demo"
+  },
+  {
+    "name": "Team 2-1",
+    "projectName": "demo_two"
+  },
+  {
+    "name": "Team 2-2",
+    "projectName": "demo_two"
+  }
+]