Support JSON parameter for max_dynamic_paths and max_dynamic_types (#164)

Author: Junrin-Lee

parser/ast.go

@@ -3716,8 +3716,10 @@ func (j *JSONPath) String() string {
 }
 
 type JSONOption struct {
-	SkipPath  *JSONPath
-	SkipRegex *StringLiteral
+	SkipPath        *JSONPath
+	SkipRegex       *StringLiteral
+	MaxDynamicPaths *NumberLiteral
+	MaxDynamicTypes *NumberLiteral
 }
 
 func (j *JSONOption) String() string {
@@ -3730,6 +3732,16 @@ func (j *JSONOption) String() string {
 		builder.WriteString(" SKIP REGEXP ")
 		builder.WriteString(j.SkipRegex.String())
 	}
+	if j.MaxDynamicPaths != nil {
+		builder.WriteString("max_dynamic_paths")
+		builder.WriteByte('=')
+		builder.WriteString(j.MaxDynamicPaths.String())
+	}
+	if j.MaxDynamicTypes != nil {
+		builder.WriteString("max_dynamic_types")
+		builder.WriteByte('=')
+		builder.WriteString(j.MaxDynamicTypes.String())
+	}
 	return builder.String()
 }
 

parser/parser_column.go

@@ -964,6 +964,34 @@ func (p *Parser) parseJSONPath() (*JSONPath, error) {
 	}, nil
 }
 
+func (p *Parser) parseJSONMaxDynamicOptions(pos Pos) (*JSONOption, error) {
+	ident, err := p.parseIdent()
+	if err != nil {
+		return nil, err
+	}
+
+	if err := p.expectTokenKind(TokenKindSingleEQ); err != nil {
+		return nil, err
+	}
+
+	switch ident.Name {
+	case "max_dynamic_types":
+		number, err := p.parseNumber(pos)
+		if err != nil {
+			return nil, err
+		}
+		return &JSONOption{MaxDynamicTypes: number}, nil
+	case "max_dynamic_paths":
+		number, err := p.parseNumber(pos)
+		if err != nil {
+			return nil, err
+		}
+		return &JSONOption{MaxDynamicPaths: number}, nil
+	default:
+		return nil, fmt.Errorf("unexpected token kind: %s", p.lastTokenKind())
+	}
+}
+
 func (p *Parser) parseJSONOption() (*JSONOption, error) {
 	switch {
 	case p.tryConsumeKeywords(KeywordSkip):
@@ -983,6 +1011,8 @@ func (p *Parser) parseJSONOption() (*JSONOption, error) {
 		return &JSONOption{
 			SkipPath: jsonPath,
 		}, nil
+	case p.matchTokenKind(TokenKindIdent):
+		return p.parseJSONMaxDynamicOptions(p.Pos())
 	default:
 		return nil, fmt.Errorf("unexpected token kind: %s", p.lastTokenKind())
 	}

parser/testdata/ddl/create_table_basic.sql

@@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS test.events_local (
     f8 Datetime DEFAULT now(),
     f9 String MATERIALIZED toString(f7['f70']),
     f10 String ALIAS f11,
-    f12 JSON(SKIP a, SKIP a.b.c, SKIP REGEXP 'hello'),
+    f12 JSON(max_dynamic_types=10, max_dynamic_paths=3, SKIP a, SKIP a.b.c, SKIP REGEXP 'hello'),
 ) ENGINE = MergeTree
 PRIMARY KEY (f0, f1, f2)
 PARTITION BY toYYYYMMDD(f3)

parser/testdata/ddl/format/create_table_basic.sql

@@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS test.events_local (
     f8 Datetime DEFAULT now(),
     f9 String MATERIALIZED toString(f7['f70']),
     f10 String ALIAS f11,
-    f12 JSON(SKIP a, SKIP a.b.c, SKIP REGEXP 'hello'),
+    f12 JSON(max_dynamic_types=10, max_dynamic_paths=3, SKIP a, SKIP a.b.c, SKIP REGEXP 'hello'),
 ) ENGINE = MergeTree
 PRIMARY KEY (f0, f1, f2)
 PARTITION BY toYYYYMMDD(f3)
@@ -32,4 +32,4 @@ ORDER BY (f1,f2,f3)
 COMMENT 'Comment for table';
 
 -- Format SQL:
-CREATE TABLE IF NOT EXISTS test.events_local (f0 String, f1 String CODEC(ZSTD(1)), f2 VARCHAR(255), f3 Datetime, f4 Datetime, f5 Map(String, String), f6 String, f7 Nested(f70 UInt32, f71 UInt32, f72 DateTime, f73 Int64, f74 Int64, f75 String), f8 Datetime DEFAULT now(), f9 String MATERIALIZED toString(f7['f70']), f10 String ALIAS f11, f12 JSON(SKIP a, SKIP a.b.c,  SKIP REGEXP 'hello')) ENGINE = MergeTree PRIMARY KEY (f0, f1, f2) PARTITION BY toYYYYMMDD(f3) TTL f3 + INTERVAL 6 MONTH ORDER BY (f1, f2, f3) COMMENT 'Comment for table';
+CREATE TABLE IF NOT EXISTS test.events_local (f0 String, f1 String CODEC(ZSTD(1)), f2 VARCHAR(255), f3 Datetime, f4 Datetime, f5 Map(String, String), f6 String, f7 Nested(f70 UInt32, f71 UInt32, f72 DateTime, f73 Int64, f74 Int64, f75 String), f8 Datetime DEFAULT now(), f9 String MATERIALIZED toString(f7['f70']), f10 String ALIAS f11, f12 JSON(max_dynamic_types=10, max_dynamic_paths=3, SKIP a, SKIP a.b.c,  SKIP REGEXP 'hello')) ENGINE = MergeTree PRIMARY KEY (f0, f1, f2) PARTITION BY toYYYYMMDD(f3) TTL f3 + INTERVAL 6 MONTH ORDER BY (f1, f2, f3) COMMENT 'Comment for table';