Dev (#290)

* notes for squashed test migrations

* filters update - gin btree index notes - proper index normalization for checks

* recreate test db before gen

Author: danicc097

bin/project

@@ -659,12 +659,15 @@ clean_yq_array() {
 }
 
 go_test() {
-  local cache_opt="-count=1"
+  local cache_opt="-count=1" exit_code=1
   cache_all "$GEN_CACHE/go-test.md5" .env.$X_ENV db/ >/dev/null && cache_opt=""
 
   set -x
   APP_ENV="$X_ENV" go test ${go_test_flags[@]} $cache_opt $@
+  exit_code=$?
   set +x
+
+  return $exit_code
 }
 
 # Run post-generation scripts in the internal package.
@@ -1227,9 +1230,9 @@ x.test.backend.setup() {
     # NOTE: tests run independently in Go so we can't have a function be called and run
     # only once before any test starts
     run_shared_services up -d --build --remove-orphans --force-recreate --wait
-    x.gen
     # no need to migrate, done on every test run internally
     docker.postgres.drop_and_recreate_db $POSTGRES_TEST_DB
+    x.gen
   } 2>&4 | xlog >&3; } 4>&1 | xerr >&3; } 3>&1
   xsetup.backup.cleanup
 }
@@ -1350,8 +1353,7 @@ test_backend_watch() {
       --format='%T %f' \
       --timefmt='%s' \
       . && clear &&
-      go_test -tags $tags "$@" &&
-      echo "${GREEN}✓ All tests passing${OFF}"
+      { go_test -tags $tags "$@" && echo "${GREEN}✓ All tests passing${OFF}"; } || echo "${RED}X Tests failed${OFF}"
   done
 }
 

cmd/initial-data/main.go

@@ -254,7 +254,7 @@ func main() {
 	demoWorkItems := []*db.WorkItem{}
 	var wg sync.WaitGroup
 	semaphore := make(chan struct{}, 2000)
-	for i := 1; i <= 100; i++ {
+	for i := 1; i <= 200; i++ {
 		semaphore <- struct{}{} // acquire
 		wg.Add(1)
 

db/post-migrations/003_cache_tables_triggers.up.sql

@@ -1,4 +1,5 @@
-create or replace function remove_comments (input_string text)
+-- NOTE: do not use `extensions.` prefix for indexes. Already in search_path.
+create or replace function normalize_index_def (input_string text)
   returns text
   as $$
 declare
@@ -8,12 +9,15 @@ begin
   output_string := REGEXP_REPLACE(input_string , '/\*([^*]|\*+[^*/])*\*+/' , '' , 'g');
   -- Remove -- style comments
   output_string := REGEXP_REPLACE(output_string , '--.*?\n' , '' , 'g');
-  -- Replace newlines with spaces
+  -- Replace newlines
   output_string := REGEXP_REPLACE(output_string , E'[\n\r]+' , ' ' , 'g');
-  -- Replace consecutive spaces with a single space
+  -- No ; in stored def
+  output_string := REGEXP_REPLACE(output_string , ';$' , '' , 'g');
   output_string := REGEXP_REPLACE(output_string , ' +' , ' ' , 'g');
-  -- Trim leading and trailing whitespace
-  output_string := TRIM(output_string);
+  output_string := REGEXP_REPLACE(output_string , '\s*\(\s*' , '(' , 'g');
+  output_string := REGEXP_REPLACE(output_string , '\s*\)\s*' , ')' , 'g');
+  output_string := REGEXP_REPLACE(output_string , '\s*,\s*' , ',' , 'g');
+  output_string := TRIM(LOWER(output_string));
 
   return output_string;
 end;
@@ -36,15 +40,13 @@ begin
     return false;
   end if;
 
-  execute FORMAT('SELECT remove_comments(pg_get_indexdef(''%I''::regclass));' , index_name) into existing_def;
-  new_index_def := remove_comments (new_index_def);
-  if existing_def is null then
-    return true;
-  ELSIF existing_def <> new_index_def then
-    return true;
-  else
+  execute FORMAT('SELECT normalize_index_def(pg_get_indexdef(''%I''::regclass));' , index_name) into existing_def;
+  new_index_def := normalize_index_def (new_index_def);
+  if existing_def is null or existing_def <> new_index_def then
     return false;
   end if;
+
+  return true;
 end;
 $$
 language plpgsql;
@@ -225,31 +227,57 @@ begin
       perform
         create_or_update_work_item_cache_table (project_name);
 
-      idx_name := FORMAT('cache__%I_gin_index' , project_name);
+      idx_name := FORMAT('public.cache__%I_gin_index' , project_name);
+
+
+      /*
+       - gin cannot use sort index, just btrees (or using btree_gin since pg12). the only alternative would be RUM
+       but its not guaranteed to be always faster. also doesnt support like op unless changing code:
+       see https://github.com/postgrespro/rum/issues/34 for supporting LIKE.
+       - gin is apparently not too helpful for very short string searches.
+       - btree not usable for text except for equals and startsWith.
+
+       create index on cache__demo_work_items using gin (
+       description gin_trgm_ops
+       , last_message_at
+       , reopened);
 
+       set enable_seqscan = "off";
+       explain analyze select * from cache__demo_work_items where description  ilike '%54%' order by last_message_at  desc;
+
+       1000 rows dataset: (to properly test index, rows returned must be >0)
+       Index Scan Backward using cache__demo_work_items_last_message_at_idx on cache__demo_work_items  (cost=0.28..58.22 rows=20 width=145) (actual time=0.059..0.725 rows=20 loops=1)
+       Filter: (description ~~* '%54%'::text)
+       Rows Removed by Filter: 980
+       */
       case project_name
       when 'demo_work_items' then
         idx_def := 'using gin (
-        title extensions.gin_trgm_ops
-        , line extensions.gin_trgm_ops
-        , ref extensions.gin_trgm_ops
+        title gin_trgm_ops
+        , line gin_trgm_ops
+        , description gin_trgm_ops
+        , ref gin_trgm_ops
+        , last_message_at
         , reopened)';
       when 'demo_two_work_items' then
         idx_def := 'using gin (
-        title extensions.gin_trgm_ops
+        title gin_trgm_ops
+        , description gin_trgm_ops
         )';
       else
         idx_def := ''; raise exception 'No index definition found for cache__%' , project_name;
       end case;
 
       if idx_def <> '' and not same_index_definition (idx_name , idx_def) then
-          execute FORMAT('create index newidx on cache__%I %s;' , project_name , idx_def);
-          --
-          execute FORMAT('drop index if exists %s;' , idx_name);
-          --
-          execute FORMAT('alter index newidx rename to %s;' , idx_name);
-        else
-          raise notice 'skipping identical create index statement: %' , idx_name;
+        raise notice 'recreating differing index: %' , idx_name;
+        --
+        execute FORMAT('create index newidx on cache__%I %s;' , project_name , idx_def);
+        --
+        execute FORMAT('drop index if exists %I;' , idx_name);
+        --
+        execute FORMAT('alter index newidx rename to %I;' , idx_name);
+      else
+        raise notice 'skipping identical create index statement: %' , idx_name;
         end if;
 
       execute FORMAT('create or replace trigger work_items_sync_trigger_%1$I

e2e/client/gen/model/paginationFilterPrimitive.ts

@@ -8,6 +8,7 @@
 import type { PaginationFilterPrimitiveFilterMode } from './paginationFilterPrimitiveFilterMode'
 
 export interface PaginationFilterPrimitive {
+  caseSensitive?: boolean | null
   filterMode: PaginationFilterPrimitiveFilterMode
   value: string | null
 }

frontend/src/client-validator/gen/dereferenced-schema.json

@@ -6410,6 +6410,12 @@
             "string",
             "null"
           ]
+        },
+        "caseSensitive": {
+          "type": [
+            "boolean",
+            "null"
+          ]
         }
       },
       "required": [
@@ -6483,6 +6489,12 @@
                 "string",
                 "null"
               ]
+            },
+            "caseSensitive": {
+              "type": [
+                "boolean",
+                "null"
+              ]
             }
           },
           "required": [
@@ -6562,6 +6574,12 @@
                     "string",
                     "null"
                   ]
+                },
+                "caseSensitive": {
+                  "type": [
+                    "boolean",
+                    "null"
+                  ]
                 }
               },
               "required": [
@@ -6654,6 +6672,12 @@
                       "string",
                       "null"
                     ]
+                  },
+                  "caseSensitive": {
+                    "type": [
+                      "boolean",
+                      "null"
+                    ]
                   }
                 },
                 "required": [
@@ -13334,6 +13358,12 @@
             "string",
             "null"
           ]
+        },
+        "caseSensitive": {
+          "type": [
+            "boolean",
+            "null"
+          ]
         }
       },
       "required": [
@@ -13407,6 +13437,12 @@
                 "string",
                 "null"
               ]
+            },
+            "caseSensitive": {
+              "type": [
+                "boolean",
+                "null"
+              ]
             }
           },
           "required": [
@@ -13486,6 +13522,12 @@
                     "string",
                     "null"
                   ]
+                },
+                "caseSensitive": {
+                  "type": [
+                    "boolean",
+                    "null"
+                  ]
                 }
               },
               "required": [
@@ -13578,6 +13620,12 @@
                       "string",
                       "null"
                     ]
+                  },
+                  "caseSensitive": {
+                    "type": [
+                      "boolean",
+                      "null"
+                    ]
                   }
                 },
                 "required": [

frontend/src/client-validator/gen/models.ts

@@ -614,6 +614,7 @@ export interface WorkItemBase {
 export interface PaginationFilterPrimitive {
   filterMode: PaginationFilterModes
   value: string | null
+  caseSensitive?: boolean | null
 }
 export interface PaginationFilterArray {
   filterMode: PaginationFilterModes

frontend/src/client-validator/gen/schema.json

@@ -2448,6 +2448,12 @@
             "string",
             "null"
           ]
+        },
+        "caseSensitive": {
+          "type": [
+            "boolean",
+            "null"
+          ]
         }
       },
       "required": [

frontend/src/gen/model/paginationFilterPrimitive.ts

@@ -9,6 +9,7 @@ import type * as EntityIDs from 'src/gen/entity-ids'
 import type { PaginationFilterPrimitiveFilterMode } from './paginationFilterPrimitiveFilterMode';
 
 export interface PaginationFilterPrimitive {
+  caseSensitive?: boolean | null;
   filterMode: PaginationFilterPrimitiveFilterMode;
   value: string | null;
 }

frontend/src/types/schema.d.ts

@@ -808,6 +808,7 @@ export interface components {
     PaginationFilterPrimitive: {
       filterMode: components["schemas"]["PaginationFilterModes"];
       value: string | null;
+      caseSensitive?: boolean | null;
     };
     PaginationFilterArray: {
       filterMode: components["schemas"]["PaginationFilterModes"];