Dev (#291)

* TODO - oapi runtime support for discriminators

* showcase dynamic filters and update runtime now that it supports discriminators

* update bad notes on indexes

* refactor cache index creation in post migrations

* update runtime

* notes

* standard filter modes labels - fix filter inputs

* log before oapi 400 middleware

* update pagination params and filters

* notes to fix runtime package for arrays

* update towards support for discriminator in query params

* go mod update

* attempt at discriminator json parsing

* working example of query params with array oneof

* generate fixed query params with arrays in anyof oneof allof and frontend filters update

* no right section now that we have filter mode labels

* fix filters and heleprs

* dont export oidc mock user models

* css

Author: danicc097

bin/.helpers.sh

@@ -495,20 +495,10 @@ docker.postgres.drop_and_recreate_db() {
 
   docker.postgres.wait_until_ready
 
-  docker.postgres psql --no-psqlrc \
-    -U "$POSTGRES_USER" \
-    -d "postgres" \
-    -c "CREATE DATABASE db_template OWNER $POSTGRES_USER;" 2>/dev/null || true
-
   echo "${RED}${BOLD}Dropping database $db.${OFF}"
-  docker.postgres \
-    dropdb --if-exists -f "$db"
+  docker.postgres dropdb --force --if-exists -f "$db"
 
-  echo "${BLUE}${BOLD}Creating database $db.${OFF}"
-  docker.postgres psql --no-psqlrc \
-    -U "$POSTGRES_USER" \
-    -d db_template \
-    -c "CREATE DATABASE $db OWNER $POSTGRES_USER;"
+  docker.postgres.create_db $db
 }
 
 # Create database `db`.
@@ -518,12 +508,7 @@ docker.postgres.create_db() {
   docker.postgres.wait_until_ready
 
   echo "${BLUE}${BOLD}Creating database $db.${OFF}"
-  {
-    docker.postgres psql --no-psqlrc -U "$POSTGRES_USER" \
-      -tc "SELECT 1 FROM pg_database WHERE datname = '$db'" |
-      grep -q 1
-  } ||
-    docker.postgres psql --no-psqlrc -U "$POSTGRES_USER" -c "CREATE DATABASE $db" ||
+  docker.postgres createdb $db -U "$POSTGRES_USER" -O "$POSTGRES_USER" ||
     echo "Skipping $db database creation"
 }
 

bin/project

@@ -1393,7 +1393,6 @@ pre_build_cmd="project setup-swagger-ui"
 # Run backend.
 x.run.backend() {
   $pre_build_cmd
-  run_shared_services up -d --build --remove-orphans --force-recreate --wait
   go run ./cmd/rest-server/main.go -env=.env.$X_ENV
 }
 
@@ -1402,7 +1401,6 @@ x.run.backend-hr() {
   # TODO replace healthcheck with adhoc calls and bring services up in btaches
   # to prevent either bombarding with req or having to wait too long at startup.
   # see https://github.com/moby/moby/issues/33410
-  run_shared_services up -d --build --remove-orphans --force-recreate --wait
 
   # https://github.com/cosmtrek/air/blob/master/air_example.toml
   # NOTE: building binary unreliable, leads to bin not found.

cmd/initial-data/main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"encoding/json"
 	"flag"
 	"fmt"
 	"log"
@@ -105,16 +104,6 @@ func main() {
 		os.Exit(0)
 	}
 
-	// TODO: use users which will exist in auth server. that way we can test out these users as well.
-	// no need to do it for local.json. as for e2e, we dont want any initial data apart from the superadmin at all
-	// so that it mimics real usage from an empty project.
-	authServerUsersPath := "cmd/oidc-server/data/users/base.json"
-	usersBlob, err := os.ReadFile(authServerUsersPath)
-	handleError(err)
-	var uu map[string]*models.AuthServerUser
-	err = json.Unmarshal(usersBlob, &uu)
-	handleError(err)
-
 	logger.Info("Creating users...")
 	for i := 0; i < 50; i++ {
 		u, err := svc.User.Register(ctx, pool, services.UserRegisterParams{
@@ -254,7 +243,7 @@ func main() {
 	demoWorkItems := []*db.WorkItem{}
 	var wg sync.WaitGroup
 	semaphore := make(chan struct{}, 2000)
-	for i := 1; i <= 200; i++ {
+	for i := 1; i <= 1000; i++ {
 		semaphore <- struct{}{} // acquire
 		wg.Add(1)
 

cmd/oapi-codegen/oapi-templates/inline.tmpl

@@ -4,6 +4,8 @@ var swaggerSpec = []string{
     "{{.}}",{{end}}
 }
 
+var spec, _ = GetSwagger()
+
 // GetSwagger returns the content of the embedded swagger specification file
 // or error if failed to decode
 func decodeSpec() ([]byte, error) {
@@ -40,7 +42,7 @@ func PathToRawSpec(pathToFile string) map[string]func() ([]byte, error) {
     if len(pathToFile) > 0 {
         res[pathToFile] = rawSpec
     }
-    {{ if .ImportMapping }}
+    {{/* {{ if .ImportMapping }}
     pathPrefix := path.Dir(pathToFile)
     {{ end }}
     {{ range $key, $value := .ImportMapping }}
@@ -50,7 +52,7 @@ func PathToRawSpec(pathToFile string) map[string]func() ([]byte, error) {
         }
         res[rawPath] = rawFunc
     }
-    {{- end }}
+    {{- end }} */}}
     return res
 }
 

cmd/oapi-codegen/oapi-templates/union.tmpl

@@ -8,7 +8,33 @@
         // As{{$name}} returns the union data inside the {{$typeName}} as a {{$name}}
         func (t {{$typeName}}) As{{$name}}() ({{$name}}, error) {
             var body {{$name}}
-            err := json.Unmarshal(t.union, &body)
+            if !t.FromQueryParams {
+              err := json.Unmarshal(t.union, &body)
+              return body, err
+            }
+            // arrays are passed as maps, so reconstruct json properly
+            var unionm map[string]interface{}
+            err := json.Unmarshal(t.union, &unionm)
+            if err != nil {
+              return {{$name}}{}, err
+            }
+
+            s := "{{$typeName}}"
+            discriminatorSchema:= "{{$name}}"
+            fp, ok := spec.Components.Schemas[s]
+            if !ok {
+              return {{$name}}{}, fmt.Errorf("schema %s not found", s)
+            }
+            union, err := openapi.ReconstructQueryParamsValues(fp.Value, unionm, discriminatorSchema)
+            if err != nil {
+              return {{$name}}{}, fmt.Errorf("could not reconstruct query params for schema %s: %v", discriminatorSchema, err)
+            }
+            jsonUnion, err := json.Marshal(union)
+            if err != nil {
+              return {{$name}}{}, fmt.Errorf("reconstructed params are not valid json: %v", err)
+            }
+            err = json.Unmarshal(jsonUnion, &body)
+
             return body, err
         }
 

cmd/oidc-server/main.go

@@ -6,8 +6,8 @@ import (
 
 	oidc_server "github.com/danicc097/oidc-server/v3"
 	"github.com/danicc097/oidc-server/v3/storage"
-	"github.com/danicc097/openapi-go-gin-postgres-sqlc/internal/models"
 	"github.com/zitadel/oidc/v2/pkg/oidc"
+	"golang.org/x/text/language"
 )
 
 const (
@@ -39,7 +39,7 @@ func getPrivateClaimsFromScopesFunc(ctx context.Context, userID, clientID string
 	return claims, nil
 }
 
-func setUserInfoFunc(user *models.AuthServerUser, userInfo *oidc.UserInfo, scope, clientID string) {
+func setUserInfoFunc(user *AuthServerUser, userInfo *oidc.UserInfo, scope, clientID string) {
 	switch scope {
 	case oidc.ScopeOpenID:
 		userInfo.Subject = user.ID()
@@ -75,7 +75,7 @@ func main() {
 
 	flag.Parse()
 
-	config := oidc_server.Config[models.AuthServerUser]{
+	config := oidc_server.Config[AuthServerUser]{
 		SetUserInfoFunc:                setUserInfoFunc,
 		GetPrivateClaimsFromScopesFunc: getPrivateClaimsFromScopesFunc,
 		PathPrefix:                     pathPrefix,
@@ -93,3 +93,41 @@ func main() {
 
 	oidc_server.Run(config)
 }
+
+// AuthServerUser implements oidc-server storage.User.
+// It is used for development and testing purposes only.
+// nolint: revive
+// Still cannot access common interface fields:
+//
+//	https://go101.org/generics/888-the-status-quo-of-go-custom-generics.html
+type AuthServerUser struct {
+	ID_               string       `json:"id"` // need exported for unmarshalling
+	Username_         string       `json:"username"`
+	Password_         string       `json:"password"`
+	FirstName         string       `json:"firstName"`
+	LastName          string       `json:"lastName"`
+	Email             string       `json:"email"`
+	EmailVerified     bool         `json:"emailVerified"`
+	Phone             string       `json:"phone"`
+	PhoneVerified     bool         `json:"phoneVerified"`
+	PreferredLanguage language.Tag `json:"preferredLanguage"`
+	IsAdmin_          bool         `json:"isAdmin"`
+}
+
+func (u AuthServerUser) ID() string {
+	return u.ID_
+}
+
+func (u AuthServerUser) Username() string {
+	return u.Username_
+}
+
+func (u AuthServerUser) IsAdmin() bool {
+	return u.IsAdmin_
+}
+
+func (u AuthServerUser) Password() string {
+	return u.Password_
+}
+
+var _ storage.User = (*AuthServerUser)(nil)

db/migrations/0000002_init.up.sql

@@ -270,6 +270,7 @@ create table users (
   user_id uuid default gen_random_uuid () primary key
   , username text not null unique
   , email text not null unique
+  , age int
   , first_name text
   , last_name text
   , full_name text generated always as ( case when first_name is null then

db/post-migrations/003_cache_tables_triggers.up.sql

@@ -208,6 +208,23 @@ end;
 $$
 language plpgsql;
 
+-- NOTE: concurrently not supported in migration transactions
+create or replace function create_or_update_index (idx_name text , project_name text , idx_def text)
+  returns VOID
+  as $$
+begin
+  if idx_def <> '' and not same_index_definition (idx_name , idx_def) then
+    raise notice 'recreating differing index: %' , idx_name;
+    execute FORMAT('create index newidx on cache__%I %s;' , project_name , idx_def);
+    execute FORMAT('drop index if exists %I;' , idx_name);
+    execute FORMAT('alter index newidx rename to %I;' , idx_name);
+  else
+    raise notice 'skipping identical create index statement: %' , idx_name;
+  end if;
+end;
+$$
+language plpgsql;
+
 --
 --
 -- Sync project work item tables cache
@@ -224,9 +241,14 @@ begin
     work_items_table_name
   from
     projects loop
+      --
+      -- sync cache table - idempotent
+      --
       perform
         create_or_update_work_item_cache_table (project_name);
-
+      --
+      -- at least one gin index mandatory for all projects
+      --
       idx_name := FORMAT('public.cache__%I_gin_index' , project_name);
 
 
@@ -237,18 +259,23 @@ begin
        - gin is apparently not too helpful for very short string searches.
        - btree not usable for text except for equals and startsWith.
 
-       create index on cache__demo_work_items using gin (
+
+       make sure the index used in explain is the one being tested...
+
+       drop index if exists lastmsg;
+       create index lastmsg on cache__demo_work_items using btree (
+       last_message_at desc);
+
+       drop index if exists abc;
+       create index abc on cache__demo_work_items using gin (
        description gin_trgm_ops
        , last_message_at
        , reopened);
 
        set enable_seqscan = "off";
-       explain analyze select * from cache__demo_work_items where description  ilike '%54%' order by last_message_at  desc;
-
-       1000 rows dataset: (to properly test index, rows returned must be >0)
-       Index Scan Backward using cache__demo_work_items_last_message_at_idx on cache__demo_work_items  (cost=0.28..58.22 rows=20 width=145) (actual time=0.059..0.725 rows=20 loops=1)
-       Filter: (description ~~* '%54%'::text)
-       Rows Removed by Filter: 980
+       -- with 1000 rows
+       explain analyze select * from cache__demo_work_items where description  ilike '%4%' order by last_message_at  desc limit 20;
+       ->  Index Scan using lastmsg on cache__demo_work_items  (cost=0.28..58.23 rows=263 width=145) (actual time=0.018..0.160 rows=20 loops=1)
        */
       case project_name
       when 'demo_work_items' then
@@ -257,7 +284,6 @@ begin
         , line gin_trgm_ops
         , description gin_trgm_ops
         , ref gin_trgm_ops
-        , last_message_at
         , reopened)';
       when 'demo_two_work_items' then
         idx_def := 'using gin (
@@ -268,22 +294,27 @@ begin
         idx_def := ''; raise exception 'No index definition found for cache__%' , project_name;
       end case;
 
-      if idx_def <> '' and not same_index_definition (idx_name , idx_def) then
-        raise notice 'recreating differing index: %' , idx_name;
+      perform
+        create_or_update_index (idx_name , project_name , idx_def);
         --
-        execute FORMAT('create index newidx on cache__%I %s;' , project_name , idx_def);
+        -- adhoc indexes. TODO: abstract away idx create/replace
         --
-        execute FORMAT('drop index if exists %I;' , idx_name);
+        case project_name
+        when 'demo_work_items' then
+          idx_name := FORMAT('public.cache__%I_last_message_at_index' , project_name);
+          --
+          idx_def := 'using btree (last_message_at)';
+          perform
+            create_or_update_index (idx_name , project_name , idx_def);
+          else
+        end case;
         --
-        execute FORMAT('alter index newidx rename to %I;' , idx_name);
-      else
-        raise notice 'skipping identical create index statement: %' , idx_name;
-        end if;
-
-      execute FORMAT('create or replace trigger work_items_sync_trigger_%1$I
+        -- triggers
+        --
+        execute FORMAT('create or replace trigger work_items_sync_trigger_%1$I
         after insert or update on %1$I for each row
         execute function sync_work_items (%1$s);' , project_name);
-    end loop;
-end;
+        end loop;
+      end;
 $BODY$
 language plpgsql;

e2e/client/gen/model/dbUser.ts

@@ -9,6 +9,7 @@ import type { Scopes } from './scopes'
 import type { DbUserID } from './dbUserID'
 
 export interface DbUser {
+  age?: number | null
   createdAt: Date
   deletedAt?: Date | null
   email: string

e2e/client/gen/model/getPaginatedUsersParams.ts

@@ -6,13 +6,15 @@
  * OpenAPI spec version: 2.0.0
  */
 import type { Direction } from './direction'
+import type { GetPaginatedUsersQueryParameters } from './getPaginatedUsersQueryParameters'
 import type { GetPaginatedUsersFilterObjectsItem } from './getPaginatedUsersFilterObjectsItem'
 import type { GetPaginatedUsersNestedObj } from './getPaginatedUsersNestedObj'
 
 export type GetPaginatedUsersParams = {
   limit: number
   direction: Direction
   cursor: string
+  searchQuery?: GetPaginatedUsersQueryParameters
   filter?: {
     bools?: boolean[]
     ints?: number[]