Skip to content

Commit 94e2c3a

Browse files
ndeodharndeodhar
committed
[YSQL] Support alter table for colocated tables: yugabyte#4293
Summary: Add alter table support for colocated tables. Changes include: 1. AlterSchema RPC takes table ID as input. If there is no table ID, we use tablet's primary table ID. 2. SchemaPB now contains information about colocated table ID. Future diff: Tablet report sends a schema version for each tablet. Since a colocated tablet can have multiple tables, we need to send multiple schema versions (1 per table) and process that on the master. Test Plan: Added tests to yb_feature_colocation Reviewers: bogdan, hector Reviewed By: hector Subscribers: jason, yql Differential Revision: https://phabricator.dev.yugabyte.com/D8603
1 parent f92bfdf commit 94e2c3a

20 files changed

+338
-85
lines changed

‎src/postgres/src/test/regress/expected/yb_feature_colocation.out‎

Lines changed: 53 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,7 @@ SELECT * FROM tab_range;
310310
2
311311
(1 row)
312312

313+
TRUNCATE TABLE tab_range;
313314
-- truncate non-colocated table without index
314315
TRUNCATE TABLE tab_nonkey_noco;
315316
SELECT * FROM tab_nonkey_noco;
@@ -374,6 +375,54 @@ SELECT * FROM tab_range_nonkey_noco2;
374375
public | tab_range_range_pkey | index | yugabyte | tab_range_range
375376
(17 rows)
376377

378+
-- ALTER TABLE
379+
INSERT INTO tab_range (a) VALUES (0), (1), (2);
380+
INSERT INTO tab_range_nonkey2 (a, b) VALUES (0, 0), (1, 1);
381+
SELECT * FROM tab_range;
382+
a
383+
---
384+
0
385+
1
386+
2
387+
(3 rows)
388+
389+
SELECT * FROM tab_range_nonkey2;
390+
a | b
391+
---+---
392+
0 | 0
393+
1 | 1
394+
(2 rows)
395+
396+
-- Alter colocated tables
397+
ALTER TABLE tab_range ADD COLUMN x INT;
398+
ALTER TABLE tab_range_nonkey2 DROP COLUMN b;
399+
SELECT * FROM tab_range;
400+
a | x
401+
---+---
402+
0 |
403+
1 |
404+
2 |
405+
(3 rows)
406+
407+
SELECT * FROM tab_range_nonkey2;
408+
a
409+
---
410+
0
411+
1
412+
(2 rows)
413+
414+
ALTER TABLE tab_range_nonkey2 RENAME TO tab_range_nonkey2_renamed;
415+
SELECT * FROM tab_range_nonkey2_renamed;
416+
a
417+
---
418+
0
419+
1
420+
(2 rows)
421+
422+
SELECT * FROM tab_range_nonkey2;
423+
ERROR: relation "tab_range_nonkey2" does not exist
424+
LINE 1: SELECT * FROM tab_range_nonkey2;
425+
^
377426
-- DROP TABLE
378427
-- drop colocated table with default index
379428
DROP TABLE tab_range;
@@ -388,10 +437,10 @@ ERROR: relation "tab_nonkey_noco" does not exist
388437
LINE 1: SELECT * FROM tab_nonkey_noco;
389438
^
390439
--- drop colocated table with explicit index
391-
DROP TABLE tab_range_nonkey2;
392-
SELECT * FROM tab_range_nonkey2;
393-
ERROR: relation "tab_range_nonkey2" does not exist
394-
LINE 1: SELECT * FROM tab_range_nonkey2;
440+
DROP TABLE tab_range_nonkey2_renamed;
441+
SELECT * FROM tab_range_nonkey2_renamed;
442+
ERROR: relation "tab_range_nonkey2_renamed" does not exist
443+
LINE 1: SELECT * FROM tab_range_nonkey2_renamed;
395444
^
396445
-- drop non-colocated table with explicit index
397446
DROP TABLE tab_range_nonkey_noco2;

‎src/postgres/src/test/regress/sql/yb_feature_colocation.sql‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ SELECT * FROM tab_range;
114114
INSERT INTO tab_range VALUES (2);
115115
SELECT * FROM tab_range;
116116

117+
TRUNCATE TABLE tab_range;
118+
117119
-- truncate non-colocated table without index
118120
TRUNCATE TABLE tab_nonkey_noco;
119121
SELECT * FROM tab_nonkey_noco;
@@ -129,6 +131,24 @@ SELECT * FROM tab_range_nonkey_noco2;
129131
\dt
130132
\di
131133

134+
-- ALTER TABLE
135+
INSERT INTO tab_range (a) VALUES (0), (1), (2);
136+
INSERT INTO tab_range_nonkey2 (a, b) VALUES (0, 0), (1, 1);
137+
138+
SELECT * FROM tab_range;
139+
SELECT * FROM tab_range_nonkey2;
140+
141+
-- Alter colocated tables
142+
ALTER TABLE tab_range ADD COLUMN x INT;
143+
ALTER TABLE tab_range_nonkey2 DROP COLUMN b;
144+
145+
SELECT * FROM tab_range;
146+
SELECT * FROM tab_range_nonkey2;
147+
148+
ALTER TABLE tab_range_nonkey2 RENAME TO tab_range_nonkey2_renamed;
149+
SELECT * FROM tab_range_nonkey2_renamed;
150+
SELECT * FROM tab_range_nonkey2;
151+
132152
-- DROP TABLE
133153

134154
-- drop colocated table with default index
@@ -140,8 +160,8 @@ DROP TABLE tab_nonkey_noco;
140160
SELECT * FROM tab_nonkey_noco;
141161

142162
--- drop colocated table with explicit index
143-
DROP TABLE tab_range_nonkey2;
144-
SELECT * FROM tab_range_nonkey2;
163+
DROP TABLE tab_range_nonkey2_renamed;
164+
SELECT * FROM tab_range_nonkey2_renamed;
145165

146166
-- drop non-colocated table with explicit index
147167
DROP TABLE tab_range_nonkey_noco2;

‎src/yb/common/common.proto‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,16 @@ message ColumnSchemaPB {
226226
repeated QLJsonOperationPB OBSOLETE_json_operations = 13;
227227
}
228228

229+
message ColocatedTableIdentifierPB {
230+
// Note: an absent value means NULL.
231+
oneof value {
232+
// Colocated YSQL user tables use 4-byte PG table ID.
233+
uint32 pgtable_id = 1;
234+
// Colocated YSQL system tables use 16-byte UUID.
235+
bytes cotable_id = 2;
236+
}
237+
}
238+
229239
message TablePropertiesPB {
230240
optional uint64 default_time_to_live = 1;
231241
optional bool contain_counters = 2;
@@ -257,6 +267,7 @@ message TablePropertiesPB {
257267
message SchemaPB {
258268
repeated ColumnSchemaPB columns = 1;
259269
optional TablePropertiesPB table_properties = 2;
270+
optional ColocatedTableIdentifierPB colocated_table_id = 3;
260271
}
261272

262273
// This message contains the metadata of a secondary index of a table.

‎src/yb/common/schema.cc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -573,6 +573,8 @@ void SchemaBuilder::Reset() {
573573
num_key_columns_ = 0;
574574
next_id_ = kFirstColumnId;
575575
table_properties_.Reset();
576+
pgtable_id_ = 0;
577+
cotable_id_ = Uuid(boost::uuids::nil_uuid());
576578
}
577579

578580
void SchemaBuilder::Reset(const Schema& schema) {
@@ -594,6 +596,8 @@ void SchemaBuilder::Reset(const Schema& schema) {
594596
next_id_ = *std::max_element(col_ids_.begin(), col_ids_.end()) + 1;
595597
}
596598
table_properties_ = schema.table_properties_;
599+
pgtable_id_ = schema.pgtable_id_;
600+
cotable_id_ = schema.cotable_id_;
597601
}
598602

599603
Status SchemaBuilder::AddKeyColumn(const string& name, const shared_ptr<QLType>& type) {

‎src/yb/common/schema.h‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1169,8 +1169,28 @@ class SchemaBuilder {
11691169
return next_id_;
11701170
}
11711171

1172-
Schema Build() const { return Schema(cols_, col_ids_, num_key_columns_, table_properties_); }
1173-
Schema BuildWithoutIds() const { return Schema(cols_, num_key_columns_, table_properties_); }
1172+
void set_pgtable_id(PgTableOid pgtable_id) {
1173+
pgtable_id_ = pgtable_id;
1174+
}
1175+
1176+
PgTableOid pgtable_id() const {
1177+
return pgtable_id_;
1178+
}
1179+
1180+
void set_cotable_id(Uuid cotable_id) {
1181+
cotable_id_ = cotable_id;
1182+
}
1183+
1184+
Uuid cotable_id() const {
1185+
return cotable_id_;
1186+
}
1187+
1188+
Schema Build() const {
1189+
return Schema(cols_, col_ids_, num_key_columns_, table_properties_, cotable_id_, pgtable_id_);
1190+
}
1191+
Schema BuildWithoutIds() const {
1192+
return Schema(cols_, num_key_columns_, table_properties_, cotable_id_, pgtable_id_);
1193+
}
11741194

11751195
// assumes type is allowed in primary key -- this should be checked before getting here
11761196
// using DataType (not QLType) since primary key columns only support elementary types
@@ -1238,6 +1258,8 @@ class SchemaBuilder {
12381258
unordered_set<string> col_names_;
12391259
size_t num_key_columns_;
12401260
TableProperties table_properties_;
1261+
PgTableOid pgtable_id_ = 0;
1262+
Uuid cotable_id_ = Uuid(boost::uuids::nil_uuid());
12411263

12421264
DISALLOW_COPY_AND_ASSIGN(SchemaBuilder);
12431265
};

‎src/yb/common/wire_protocol.cc‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
#include <string>
3636
#include <vector>
3737

38+
#include "yb/common/entity_ids.h"
3839
#include "yb/common/row.h"
3940
#include "yb/gutil/port.h"
4041
#include "yb/gutil/stl_util.h"
@@ -311,8 +312,19 @@ Status AddHostPortPBs(const std::vector<Endpoint>& addrs,
311312
return Status::OK();
312313
}
313314

315+
void SchemaToColocatedTableIdentifierPB(
316+
const Schema& schema, ColocatedTableIdentifierPB* colocated_pb) {
317+
if (schema.has_pgtable_id()) {
318+
colocated_pb->set_pgtable_id(schema.pgtable_id());
319+
} else if (schema.has_cotable_id()) {
320+
colocated_pb->set_cotable_id(schema.cotable_id().ToString());
321+
}
322+
323+
}
324+
314325
void SchemaToPB(const Schema& schema, SchemaPB *pb, int flags) {
315326
pb->Clear();
327+
SchemaToColocatedTableIdentifierPB(schema, pb->mutable_colocated_table_id());
316328
SchemaToColumnPBs(schema, pb->mutable_columns(), flags);
317329
schema.table_properties().ToTablePropertiesPB(pb->mutable_table_properties());
318330
}
@@ -331,7 +343,24 @@ Status SchemaFromPB(const SchemaPB& pb, Schema *schema) {
331343

332344
// Convert the table properties.
333345
TableProperties table_properties = TableProperties::FromTablePropertiesPB(pb.table_properties());
334-
return schema->Reset(columns, column_ids, num_key_columns, table_properties);
346+
RETURN_NOT_OK(schema->Reset(columns, column_ids, num_key_columns, table_properties));
347+
348+
if (pb.has_colocated_table_id()) {
349+
switch (pb.colocated_table_id().value_case()) {
350+
case ColocatedTableIdentifierPB::kCotableId: {
351+
Uuid cotable_id;
352+
RETURN_NOT_OK(cotable_id.FromString(pb.colocated_table_id().cotable_id()));
353+
schema->set_cotable_id(cotable_id);
354+
break;
355+
}
356+
case ColocatedTableIdentifierPB::kPgtableId:
357+
schema->set_pgtable_id(pb.colocated_table_id().pgtable_id());
358+
break;
359+
case ColocatedTableIdentifierPB::VALUE_NOT_SET:
360+
break;
361+
}
362+
}
363+
return Status::OK();
335364
}
336365

337366
void ColumnSchemaToPB(const ColumnSchema& col_schema, ColumnSchemaPB *pb, int flags) {

‎src/yb/common/wire_protocol.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,10 @@ void SchemaToColumnPBs(
129129
google::protobuf::RepeatedPtrField<ColumnSchemaPB>* cols,
130130
int flags = 0);
131131

132+
// Extract the colocated table information of the given schema into a protobuf object.
133+
void SchemaToColocatedTableIdentifierPB(
134+
const Schema& schema, ColocatedTableIdentifierPB* colocated_pb);
135+
132136
YB_DEFINE_ENUM(UsePrivateIpMode, (cloud)(region)(zone)(never));
133137

134138
// Returns mode for selecting between private and public IP.

‎src/yb/master/async_rpc_tasks.cc‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -446,6 +446,14 @@ AsyncTabletLeaderTask::AsyncTabletLeaderTask(
446446
tablet_(tablet) {
447447
}
448448

449+
AsyncTabletLeaderTask::AsyncTabletLeaderTask(
450+
Master* master, ThreadPool* callback_pool, const scoped_refptr<TabletInfo>& tablet,
451+
const scoped_refptr<TableInfo>& table)
452+
: RetryingTSRpcTask(
453+
master, callback_pool, gscoped_ptr<TSPicker>(new PickLeaderReplica(tablet)), table),
454+
tablet_(tablet) {
455+
}
456+
449457
std::string AsyncTabletLeaderTask::description() const {
450458
return type_name() + " RPC for " + tablet_->ToString();
451459
}
@@ -626,7 +634,7 @@ void AsyncAlterTable::HandleResponse(int attempt) {
626634
if (state() == MonitoredTaskState::kComplete) {
627635
// TODO: proper error handling here. Not critical, since TSHeartbeat will retry on failure.
628636
WARN_NOT_OK(master_->catalog_manager()->HandleTabletSchemaVersionReport(
629-
tablet_.get(), schema_version_),
637+
tablet_.get(), schema_version_, table()),
630638
yb::Format(
631639
"$0 for $1 failed while running AsyncAlterTable::HandleResponse. response $2",
632640
description(), tablet_->ToString(), resp_.DebugString()));
@@ -647,6 +655,7 @@ bool AsyncAlterTable::SendRequest(int attempt) {
647655
req.set_schema_version(l->data().pb.version());
648656
req.set_dest_uuid(permanent_uuid());
649657
req.set_tablet_id(tablet_->tablet_id());
658+
req.set_alter_table_id(table_->id());
650659

651660
if (l->data().pb.has_wal_retention_secs()) {
652661
req.set_wal_retention_secs(l->data().pb.wal_retention_secs());

‎src/yb/master/async_rpc_tasks.h‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include "yb/gutil/gscoped_ptr.h"
2828
#include "yb/gutil/strings/substitute.h"
2929

30+
#include "yb/master/catalog_entity_info.h"
3031
#include "yb/rpc/rpc_controller.h"
3132

3233
#include "yb/server/monitored_task.h"
@@ -271,6 +272,10 @@ class AsyncTabletLeaderTask : public RetryingTSRpcTask {
271272
AsyncTabletLeaderTask(
272273
Master* master, ThreadPool* callback_pool, const scoped_refptr<TabletInfo>& tablet);
273274

275+
AsyncTabletLeaderTask(
276+
Master* master, ThreadPool* callback_pool, const scoped_refptr<TabletInfo>& tablet,
277+
const scoped_refptr<TableInfo>& table);
278+
274279
std::string description() const override;
275280

276281
protected:
@@ -358,10 +363,14 @@ class AsyncDeleteReplica : public RetrySpecificTSRpcTask {
358363
class AsyncAlterTable : public AsyncTabletLeaderTask {
359364
public:
360365
AsyncAlterTable(
361-
Master* master, ThreadPool* callback_pool, const scoped_refptr<TabletInfo>& tablet,
362-
bool has_wal_retention_secs = false)
366+
Master* master, ThreadPool* callback_pool, const scoped_refptr<TabletInfo>& tablet)
363367
: AsyncTabletLeaderTask(master, callback_pool, tablet) {}
364368

369+
AsyncAlterTable(
370+
Master* master, ThreadPool* callback_pool, const scoped_refptr<TabletInfo>& tablet,
371+
const scoped_refptr<TableInfo>& table)
372+
: AsyncTabletLeaderTask(master, callback_pool, tablet, table) {}
373+
365374
Type type() const override { return ASYNC_ALTER_TABLE; }
366375

367376
std::string type_name() const override { return "Alter Table"; }

‎src/yb/master/catalog_entity_info.cc‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ TabletInfo::TabletInfo(const scoped_refptr<TableInfo>& table, TabletId tablet_id
108108
: tablet_id_(std::move(tablet_id)),
109109
table_(table),
110110
last_update_time_(MonoTime::Now()),
111-
reported_schema_version_(0) {}
111+
reported_schema_version_({}) {}
112112

113113
TabletInfo::~TabletInfo() {
114114
}
@@ -178,18 +178,22 @@ MonoTime TabletInfo::last_update_time() const {
178178
return last_update_time_;
179179
}
180180

181-
bool TabletInfo::set_reported_schema_version(uint32_t version) {
181+
bool TabletInfo::set_reported_schema_version(const TableId& table_id, uint32_t version) {
182182
std::lock_guard<simple_spinlock> l(lock_);
183-
if (version > reported_schema_version_) {
184-
reported_schema_version_ = version;
183+
if (reported_schema_version_.count(table_id) == 0 ||
184+
version > reported_schema_version_[table_id]) {
185+
reported_schema_version_[table_id] = version;
185186
return true;
186187
}
187188
return false;
188189
}
189190

190-
uint32_t TabletInfo::reported_schema_version() const {
191+
uint32_t TabletInfo::reported_schema_version(const TableId& table_id) {
191192
std::lock_guard<simple_spinlock> l(lock_);
192-
return reported_schema_version_;
193+
if (reported_schema_version_.count(table_id) == 0) {
194+
return 0;
195+
}
196+
return reported_schema_version_[table_id];
193197
}
194198

195199
bool TabletInfo::colocated() const {
@@ -367,10 +371,10 @@ void TableInfo::GetTabletsInRange(const GetTableLocationsRequestPB* req, TabletI
367371
bool TableInfo::IsAlterInProgress(uint32_t version) const {
368372
shared_lock<decltype(lock_)> l(lock_);
369373
for (const TableInfo::TabletInfoMap::value_type& e : tablet_map_) {
370-
if (e.second->reported_schema_version() < version) {
374+
if (e.second->reported_schema_version(table_id_) < version) {
371375
VLOG(3) << "Table " << table_id_ << " ALTER in progress due to tablet "
372376
<< e.second->ToString() << " because reported schema "
373-
<< e.second->reported_schema_version() << " < expected " << version;
377+
<< e.second->reported_schema_version(table_id_) << " < expected " << version;
374378
return true;
375379
}
376380
}

0 commit comments

Comments
 (0)