Skip to content

Commit ee06924

Browse files
zinanrensivethe
zinanren
authored and
sivethe
committed
Merged PR 1555452: [OSS] Enable distributed tests in documentdb
### Does this PR have any customer impact? No ### Type (Feature, Refactoring, Bugfix, DevOps, Testing, Perf, etc) Refactoring ### Does it involve schema level changes? (Table, Column, Index, UDF, etc level changes) No ### Are you introducing any new config? If yes, do you have tests with and without them being set? No ### Release Note (Refer [Template](../docs/release_notes/pgmongo_template.md)) ### Description Enable distributed tests in documentdb, and clean up helio references from documentdb_distributed Add substitution to make pg_helio_distributed tests work Add substitution to make pgmongo/helio_compat tests work Move udfs bson_aggregation_redact to documentdb ---- #### AI description (iteration 1) #### PR Classification New feature #### PR Summary This pull request enables distributed tests in DocumentDB by adding a new set of regression tests for BSON aggregation comparison operators. - Added `bson_aggregation_comparison_operators_tests.out` with extensive test cases for BSON comparison operators (`$cmp`, `$eq`, `$ne`, `$gt`, `$gte`, `$lt`, `$lte`).
1 parent 489b3fe commit ee06924

File tree

431 files changed

+29820
-29762
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

431 files changed

+29820
-29762
lines changed

‎internal/pg_documentdb_distributed/documentdb_distributed.control‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@ default_version = '0.100-0'
33
module_pathname = '$libdir/pg_documentdb_distributed'
44
relocatable = false
55
superuser = true
6-
requires = 'documentdb_core, documentdb, citus'
6+
requires = 'citus, documentdb_core, documentdb'

‎internal/pg_documentdb_distributed/src/distribution/cluster_operations.c‎

Lines changed: 40 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,13 @@ extern char *ApiExtensionName;
3434
extern char *ApiGucPrefix;
3535
extern char *ClusterAdminRole;
3636

37-
char *ApiDistributedSchemaName = "helio_api_distributed";
38-
char *DistributedExtensionName = "pg_helio_distributed";
37+
char *ApiDistributedSchemaName = "documentdb_api_distributed";
38+
char *DistributedExtensionName = "documentdb_distributed";
3939
bool CreateDistributedFunctions = false;
4040
bool CreateIndexBuildQueueTable = false;
4141

42+
extern char * GetIndexQueueName(void);
43+
4244
static char * GetClusterInitializedVersion(void);
4345
static void DistributeCrudFunctions(void);
4446
static void ScheduleIndexBuildTasks(char *extensionPrefix);
@@ -62,7 +64,7 @@ static bool VersionEquals(ExtensionVersion versionA, ExtensionVersion versionB);
6264
static void GetInstalledVersion(ExtensionVersion *installedVersion);
6365
static void ParseVersionString(ExtensionVersion *extensionVersion, char *versionString);
6466
static bool SetupCluster(bool isInitialize);
65-
static void SetPermissionsForHelioReadOnlyRole(void);
67+
static void SetPermissionsForReadOnlyRole(void);
6668
static ArrayType * GetCollectionIds(void);
6769

6870
PG_FUNCTION_INFO_V1(command_initialize_cluster);
@@ -225,7 +227,7 @@ SetupCluster(bool isInitialize)
225227
char *oldExtensionPrefix = ExtensionObjectPrefix;
226228
UnscheduleIndexBuildTasks(oldExtensionPrefix);
227229

228-
char *extensionPrefix = "helio";
230+
char *extensionPrefix = ExtensionObjectPrefixV2;
229231
ScheduleIndexBuildTasks(extensionPrefix);
230232
}
231233

@@ -238,14 +240,14 @@ SetupCluster(bool isInitialize)
238240
if (ShouldRunSetupForVersion(lastUpgradeVersion, installedVersion, DocDB_V0, 15, 0))
239241
{
240242
/* reduce the Index background cron job schedule to 2 seconds by default. */
241-
char *extensionPrefix = "helio";
243+
char *extensionPrefix = ExtensionObjectPrefixV2;
242244
UnscheduleIndexBuildTasks(extensionPrefix);
243245
ScheduleIndexBuildTasks(extensionPrefix);
244246
}
245247

246248
if (ShouldRunSetupForVersion(lastUpgradeVersion, installedVersion, DocDB_V0, 17, 1))
247249
{
248-
SetPermissionsForHelioReadOnlyRole();
250+
SetPermissionsForReadOnlyRole();
249251
}
250252

251253
if (ShouldRunSetupForVersion(lastUpgradeVersion, installedVersion, DocDB_V0, 21, 0))
@@ -255,7 +257,9 @@ SetupCluster(bool isInitialize)
255257
StringInfo cmdStr = makeStringInfo();
256258
bool isNull = false;
257259
appendStringInfo(cmdStr,
258-
"GRANT helio_admin_role, helio_readonly_role TO %s WITH ADMIN OPTION;",
260+
"GRANT %s, %s TO %s WITH ADMIN OPTION;",
261+
ApiAdminRoleV2,
262+
ApiReadOnlyRole,
259263
quote_identifier(ClusterAdminRole));
260264
ExtensionExecuteQueryViaSPI(cmdStr->data, false, SPI_OK_UTILITY,
261265
&isNull);
@@ -443,13 +447,13 @@ CreateIndexBuildQueueCore()
443447

444448
StringInfo dropStr = makeStringInfo();
445449
appendStringInfo(dropStr,
446-
"DROP TABLE IF EXISTS helio_api_catalog.helio_index_queue;");
450+
"DROP TABLE IF EXISTS %s;", GetIndexQueueName());
447451
ExtensionExecuteQueryViaSPI(dropStr->data, readOnly, SPI_OK_UTILITY,
448452
&isNull);
449453

450454
StringInfo createStr = makeStringInfo();
451455
appendStringInfo(createStr,
452-
"CREATE TABLE IF NOT EXISTS helio_api_catalog.helio_index_queue ("
456+
"CREATE TABLE IF NOT EXISTS %s ("
453457
"index_cmd text not null,"
454458

455459
/* 'C' for CREATE INDEX and 'R' for REINDEX */
@@ -471,32 +475,36 @@ CreateIndexBuildQueueCore()
471475
/* update_time shows the time when request was updated in the table */
472476
"update_time timestamp with time zone DEFAULT now()"
473477

474-
")", CoreSchemaName);
478+
")", GetIndexQueueName(), CoreSchemaName);
475479

476480
ExtensionExecuteQueryViaSPI(createStr->data, readOnly, SPI_OK_UTILITY,
477481
&isNull);
478482

479483
resetStringInfo(createStr);
480484
appendStringInfo(createStr,
481-
"CREATE INDEX IF NOT EXISTS helio_index_queue_indexid_cmdtype on helio_api_catalog.helio_index_queue (index_id, cmd_type)");
485+
"CREATE INDEX IF NOT EXISTS %s_index_queue_indexid_cmdtype on %s (index_id, cmd_type)",
486+
ExtensionObjectPrefixV2, GetIndexQueueName());
482487
ExtensionExecuteQueryViaSPI(createStr->data, readOnly, SPI_OK_UTILITY,
483488
&isNull);
484489

485490
resetStringInfo(createStr);
486491
appendStringInfo(createStr,
487-
"CREATE INDEX IF NOT EXISTS helio_index_queue_cmdtype_collectionid_cmdstatus on helio_api_catalog.helio_index_queue (cmd_type, collection_id, index_cmd_status)");
492+
"CREATE INDEX IF NOT EXISTS %s_index_queue_cmdtype_collectionid_cmdstatus on %s (cmd_type, collection_id, index_cmd_status)",
493+
ExtensionObjectPrefixV2, GetIndexQueueName());
488494
ExtensionExecuteQueryViaSPI(createStr->data, readOnly, SPI_OK_UTILITY,
489495
&isNull);
490496

491497
resetStringInfo(createStr);
492498
appendStringInfo(createStr,
493-
"GRANT SELECT ON TABLE helio_api_catalog.helio_index_queue TO public");
499+
"GRANT SELECT ON TABLE %s TO public", GetIndexQueueName());
494500
ExtensionExecuteQueryViaSPI(createStr->data, readOnly, SPI_OK_UTILITY,
495501
&isNull);
496502

497503
resetStringInfo(createStr);
498504
appendStringInfo(createStr,
499-
"GRANT ALL ON TABLE helio_api_catalog.helio_index_queue TO helio_admin_role, %s",
505+
"GRANT ALL ON TABLE %s TO %s, %s",
506+
GetIndexQueueName(),
507+
ApiAdminRoleV2,
500508
ApiAdminRole);
501509
ExtensionExecuteQueryViaSPI(createStr->data, readOnly, SPI_OK_UTILITY,
502510
&isNull);
@@ -519,7 +527,8 @@ CreateIndexBuildsTable()
519527
bool isNull = false;
520528
StringInfo createStr = makeStringInfo();
521529
appendStringInfo(createStr,
522-
"SELECT citus_add_local_table_to_metadata('helio_api_catalog.helio_index_queue')");
530+
"SELECT citus_add_local_table_to_metadata('%s')",
531+
GetIndexQueueName());
523532
ExtensionExecuteQueryViaSPI(createStr->data, readOnly, SPI_OK_SELECT,
524533
&isNull);
525534
}
@@ -802,20 +811,23 @@ AddUserColumnsToIndexQueue()
802811

803812
StringInfo cmdStr = makeStringInfo();
804813
appendStringInfo(cmdStr,
805-
"ALTER TABLE helio_api_catalog.helio_index_queue ADD COLUMN IF NOT EXISTS user_oid Oid;");
814+
"ALTER TABLE %s ADD COLUMN IF NOT EXISTS user_oid Oid;",
815+
GetIndexQueueName());
806816
ExtensionExecuteQueryViaSPI(cmdStr->data, readOnly, SPI_OK_UTILITY,
807817
&isNull);
808818

809819
/* We first drop the check constraint if it already exists. Some upgrade paths can create it before this function is executed. */
810820
resetStringInfo(cmdStr);
811821
appendStringInfo(cmdStr,
812-
"ALTER TABLE helio_api_catalog.helio_index_queue DROP CONSTRAINT IF EXISTS helio_index_queue_user_oid_check;");
822+
"ALTER TABLE %s DROP CONSTRAINT IF EXISTS %s_index_queue_user_oid_check;",
823+
GetIndexQueueName(), ExtensionObjectPrefixV2);
813824
ExtensionExecuteQueryViaSPI(cmdStr->data, readOnly, SPI_OK_UTILITY,
814825
&isNull);
815826

816827
resetStringInfo(cmdStr);
817828
appendStringInfo(cmdStr,
818-
"ALTER TABLE helio_api_catalog.helio_index_queue ADD CONSTRAINT helio_index_queue_user_oid_check CHECK (user_oid IS NULL OR user_oid != '0'::oid);");
829+
"ALTER TABLE %s ADD CONSTRAINT %s_index_queue_user_oid_check CHECK (user_oid IS NULL OR user_oid != '0'::oid);",
830+
GetIndexQueueName(), ExtensionObjectPrefixV2);
819831
ExtensionExecuteQueryViaSPI(cmdStr->data, readOnly, SPI_OK_UTILITY,
820832
&isNull);
821833
}
@@ -914,18 +926,18 @@ GetInstalledVersion(ExtensionVersion *installedVersion)
914926

915927

916928
/*
917-
* SetPermissionsForHelioReadOnlyRole - Set the right permissions for helio_readonly_role
929+
* SetPermissionsForReadOnlyRole - Set the right permissions for ApiReadOnlyRole
918930
*/
919931
static void
920-
SetPermissionsForHelioReadOnlyRole()
932+
SetPermissionsForReadOnlyRole()
921933
{
922934
bool readOnly = false;
923935
bool isNull = false;
924936
StringInfo cmdStr = makeStringInfo();
925937

926938
appendStringInfo(cmdStr,
927-
"GRANT SELECT ON TABLE %s.%s_cluster_data TO helio_readonly_role;",
928-
ApiDistributedSchemaName, ExtensionObjectPrefix);
939+
"GRANT SELECT ON TABLE %s.%s_cluster_data TO %s;",
940+
ApiDistributedSchemaName, ExtensionObjectPrefix, ApiReadOnlyRole);
929941
ExtensionExecuteQueryViaSPI(cmdStr->data, readOnly, SPI_OK_UTILITY,
930942
&isNull);
931943

@@ -946,8 +958,8 @@ SetPermissionsForHelioReadOnlyRole()
946958
int collection_id = DatumGetInt32(elements[i]);
947959
resetStringInfo(cmdStr);
948960
appendStringInfo(cmdStr,
949-
"GRANT SELECT ON %s.documents_%d TO helio_readonly_role;",
950-
ApiDataSchemaName, collection_id);
961+
"GRANT SELECT ON %s.documents_%d TO %s;",
962+
ApiDataSchemaName, collection_id, ApiReadOnlyRole);
951963
ExtensionExecuteQueryViaSPI(cmdStr->data, readOnly, SPI_OK_UTILITY,
952964
&isNull);
953965
}
@@ -992,8 +1004,10 @@ UpdateClusterMetadata(bool isInitialize)
9921004

9931005
Datum clusterVersionDatum = ExtensionExecuteQueryViaSPI(
9941006
FormatSqlQuery(
995-
"SELECT helio_core.bson_get_value_text(metadata, 'last_deploy_version') FROM %s.%s_cluster_data",
996-
ApiDistributedSchemaName, ExtensionObjectPrefix), true, SPI_OK_SELECT,
1007+
"SELECT %s.bson_get_value_text(metadata, 'last_deploy_version') FROM %s.%s_cluster_data",
1008+
CoreSchemaNameV2, ApiDistributedSchemaName, ExtensionObjectPrefix),
1009+
true,
1010+
SPI_OK_SELECT,
9971011
&isNull);
9981012
Assert(!isNull);
9991013

‎internal/pg_documentdb_distributed/src/test/regress/Makefile‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,9 @@ check-dist-minimal:
3333
$(call common_test,--schedule=./minimal_schedule, $(EXTRA_TESTS))
3434

3535
check-basic:
36-
# TODO(OSSRename)
37-
# $(call common_test,--schedule=./basic_schedule --schedule=./basic_schedule_core)
38-
$(call common_test,--schedule=./basic_schedule)
36+
$(call common_test,--schedule=./basic_schedule --schedule=./basic_schedule_core)
3937

4038
check-test-output:
4139
./validate_test_output.sh
4240

43-
all: check-basic # TODO(OSSRename) check-test-output
41+
all: check-basic check-test-output

‎internal/pg_documentdb_distributed/src/test/regress/basic_schedule‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
# test: documentdb_distributed_setup
2-
# test: public_api_schema
3-
# test: documentdb_distributed_test_helpers
1+
test: documentdb_distributed_setup
2+
test: public_api_schema
3+
test: documentdb_distributed_test_helpers

‎internal/pg_documentdb_distributed/src/test/regress/expected/and.out‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
-- Based on and.js
22
CREATE SCHEMA and1;
3-
SET search_path TO helio_core,helio_api,helio_api_catalog,helio_api_internal,public,and1;
3+
SET search_path TO documentdb_core,documentdb_api,documentdb_api_catalog,documentdb_api_internal,public,and1;
44
SET citus.next_shard_id TO 549000;
5-
SET helio_api.next_collection_id TO 5490;
6-
SET helio_api.next_collection_index_id TO 5490;
5+
SET documentdb.next_collection_id TO 5490;
6+
SET documentdb.next_collection_index_id TO 5490;
77
SELECT 1 FROM drop_collection('db','and1');
88
?column?
99
---------------------------------------------------------------------
@@ -23,7 +23,7 @@ SELECT 1 FROM insert_one('db','and1', '{"a":"foo"}');
2323
1
2424
(1 row)
2525

26-
SELECT helio_distributed_test_helpers.drop_primary_key('db', 'and1');
26+
SELECT documentdb_distributed_test_helpers.drop_primary_key('db', 'and1');
2727
drop_primary_key
2828
---------------------------------------------------------------------
2929

@@ -122,7 +122,7 @@ SELECT check1();
122122

123123
(1 row)
124124

125-
SELECT helio_api_internal.create_indexes_non_concurrently('db', helio_distributed_test_helpers.generate_create_index_arg('and1', 'index_1', '{"a.$**": 1}'), true);
125+
SELECT documentdb_api_internal.create_indexes_non_concurrently('db', documentdb_distributed_test_helpers.generate_create_index_arg('and1', 'index_1', '{"a.$**": 1}'), true);
126126
create_indexes_non_concurrently
127127
---------------------------------------------------------------------
128128
{ "raw" : { "defaultShard" : { "numIndexesBefore" : { "$numberInt" : "0" }, "numIndexesAfter" : { "$numberInt" : "1" }, "createdCollectionAutomatically" : false, "ok" : { "$numberInt" : "1" } } }, "ok" : { "$numberInt" : "1" } }

‎internal/pg_documentdb_distributed/src/test/regress/expected/and3.out‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
-- Based on and3.js
22
CREATE SCHEMA and3;
3-
SET search_path TO helio_core,helio_api,helio_api_catalog,helio_api_internal,public,and3;
3+
SET search_path TO documentdb_core,documentdb_api,documentdb_api_catalog,documentdb_api_internal,public,and3;
44
SET citus.next_shard_id TO 547000;
5-
SET helio_api.next_collection_id TO 5470;
6-
SET helio_api.next_collection_index_id TO 5470;
5+
SET documentdb.next_collection_id TO 5470;
6+
SET documentdb.next_collection_index_id TO 5470;
77
SELECT drop_collection('db','and3');
88
drop_collection
99
---------------------------------------------------------------------
@@ -23,13 +23,13 @@ SELECT 1 FROM insert_one('db','and3', '{"a":"foo"}');
2323
1
2424
(1 row)
2525

26-
SELECT helio_distributed_test_helpers.drop_primary_key('db', 'and3');
26+
SELECT documentdb_distributed_test_helpers.drop_primary_key('db', 'and3');
2727
drop_primary_key
2828
---------------------------------------------------------------------
2929

3030
(1 row)
3131

32-
SELECT helio_api_internal.create_indexes_non_concurrently('db', helio_distributed_test_helpers.generate_create_index_arg('and3', 'index_1', '{"a.$**": 1}'), true);
32+
SELECT documentdb_api_internal.create_indexes_non_concurrently('db', documentdb_distributed_test_helpers.generate_create_index_arg('and3', 'index_1', '{"a.$**": 1}'), true);
3333
create_indexes_non_concurrently
3434
---------------------------------------------------------------------
3535
{ "raw" : { "defaultShard" : { "numIndexesBefore" : { "$numberInt" : "0" }, "numIndexesAfter" : { "$numberInt" : "1" }, "createdCollectionAutomatically" : false, "ok" : { "$numberInt" : "1" } } }, "ok" : { "$numberInt" : "1" } }

‎internal/pg_documentdb_distributed/src/test/regress/expected/andor.out‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
-- Based on andor.js
22
CREATE SCHEMA andor;
3-
SET search_path TO helio_core,helio_api,helio_api_catalog,helio_api_internal,public,andor;
3+
SET search_path TO documentdb_core,documentdb_api,documentdb_api_catalog,documentdb_api_internal,public,andor;
44
SET citus.next_shard_id TO 548000;
5-
SET helio_api.next_collection_id TO 5480;
6-
SET helio_api.next_collection_index_id TO 5480;
7-
CREATE OR REPLACE FUNCTION andor.ok(query helio_core.bson)
5+
SET documentdb.next_collection_id TO 5480;
6+
SET documentdb.next_collection_index_id TO 5480;
7+
CREATE OR REPLACE FUNCTION andor.ok(query documentdb_core.bson)
88
RETURNS void
99
LANGUAGE plpgsql
1010
AS $$
@@ -27,7 +27,7 @@ NOTICE: creating collection
2727
1
2828
(1 row)
2929

30-
SELECT helio_distributed_test_helpers.drop_primary_key('db', 'andor');
30+
SELECT documentdb_distributed_test_helpers.drop_primary_key('db', 'andor');
3131
drop_primary_key
3232
---------------------------------------------------------------------
3333

@@ -79,7 +79,7 @@ SELECT test1();
7979

8080
(1 row)
8181

82-
SELECT helio_api_internal.create_indexes_non_concurrently('db', helio_distributed_test_helpers.generate_create_index_arg('andor', 'index_1', '{"a.$**": 1}'), true);
82+
SELECT documentdb_api_internal.create_indexes_non_concurrently('db', documentdb_distributed_test_helpers.generate_create_index_arg('andor', 'index_1', '{"a.$**": 1}'), true);
8383
create_indexes_non_concurrently
8484
---------------------------------------------------------------------
8585
{ "raw" : { "defaultShard" : { "numIndexesBefore" : { "$numberInt" : "0" }, "numIndexesAfter" : { "$numberInt" : "1" }, "createdCollectionAutomatically" : false, "ok" : { "$numberInt" : "1" } } }, "ok" : { "$numberInt" : "1" } }
@@ -155,7 +155,7 @@ SELECT test2();
155155

156156
(1 row)
157157

158-
SELECT helio_api_internal.create_indexes_non_concurrently('db', helio_distributed_test_helpers.generate_create_index_arg('andor', 'index_2', '{"a.$**": 1}'), true);
158+
SELECT documentdb_api_internal.create_indexes_non_concurrently('db', documentdb_distributed_test_helpers.generate_create_index_arg('andor', 'index_2', '{"a.$**": 1}'), true);
159159
create_indexes_non_concurrently
160160
---------------------------------------------------------------------
161161
{ "raw" : { "defaultShard" : { "numIndexesBefore" : { "$numberInt" : "1" }, "numIndexesAfter" : { "$numberInt" : "2" }, "createdCollectionAutomatically" : false, "ok" : { "$numberInt" : "1" } } }, "ok" : { "$numberInt" : "1" } }

0 commit comments

Comments
 (0)