Fix(#219): split_high_level function to ignore delimiters inside single quotes (#220)

* Fix: split_high_level function to ignore delimiters inside single quotes

Previously, the split function would incorrectly split strings containing delimiters inside single quotes.
this causes issue when splitting columns with default value that has ',' within it.
Now, delimiters within single quotes are ignored, ensuring proper splitting behavior.

* Add unit tests for split_high_level function

- Test ignoring delimiters inside single quotes
- Test ignoring delimiters inside parentheses
- Test realistic SQL scenarios including ENUM, SET, DEFAULT values
- Cover edge cases like empty strings and nested structures
- All 14 tests passing

---------

Co-authored-by: Jayakarthik K <iamjayakarthik@gmail.com>

Author: jayakarthik-jk

mysql_ch_replicator/converter.py

@@ -278,23 +278,67 @@ def strip_sql_name(name):
     return name
 
 
-def split_high_level(data, token):
-    results = []
-    level = 0
-    curr_data = ''
-    for c in data:
-        if c == token and level == 0:
-            results.append(curr_data.strip())
-            curr_data = ''
+def split_high_level(data, delimiter):
+    """
+    Split a string by a delimiter, ignoring delimiters inside parentheses or quotes.
+    
+    This function performs a context-aware split, respecting nested structures:
+    - Delimiters inside parentheses () are ignored
+    - Delimiters inside single quotes '' are ignored
+    - Handles nested parentheses at any depth
+    
+    Args:
+        data (str): The string to split
+        delimiter (str): The character to split on (typically ',' or ';')
+    
+    Returns:
+        list[str]: List of split segments with whitespace stripped
+    
+    Examples:
+        >>> split_high_level("a,b(c,d),e", ",")
+        ['a', 'b(c,d)', 'e']
+        
+        >>> split_high_level("name varchar(100) DEFAULT 'a,b',id int", ",")
+        ["name varchar(100) DEFAULT 'a,b'", 'id int']
+    """
+    if not data:
+        return []
+
+    segments = []
+    current_segment = []
+    paren_depth = 0
+    in_quotes = False
+
+    for i, char in enumerate(data):
+        # Handle quote toggling (ignore escaped quotes)
+        if char == "'" and (i == 0 or data[i - 1] != '\\'):
+            in_quotes = not in_quotes
+            current_segment.append(char)
+            continue
+
+        # Track parentheses depth only outside quotes
+        if not in_quotes:
+            if char == '(':
+                paren_depth += 1
+            elif char == ')':
+                paren_depth -= 1
+
+        # Split only at top level (outside parentheses and quotes)
+        if char == delimiter and paren_depth == 0 and not in_quotes:
+            segment_text = ''.join(current_segment).strip()
+            if segment_text:  # Only add non-empty segments
+                segments.append(segment_text)
+            current_segment = []
             continue
-        if c == '(':
-            level += 1
-        if c == ')':
-            level -= 1
-        curr_data += c
-    if curr_data:
-        results.append(curr_data.strip())
-    return results
+
+        current_segment.append(char)
+
+    # Add final segment if it exists
+    final_segment = ''.join(current_segment).strip()
+    if final_segment:
+        segments.append(final_segment)
+
+    return segments
 
 
 def strip_sql_comments(sql_statement):

tests/test_split_high_level.py

@@ -0,0 +1,114 @@
+import pytest
+from mysql_ch_replicator.converter import split_high_level
+
+
+@pytest.mark.parametrize("data,delimiter,expected", [
+    # Basic column definitions without quotes or parentheses
+    (
+        "id int NOT NULL, name varchar(255), age int",
+        ",",
+        ['id int NOT NULL', 'name varchar(255)', 'age int']
+    ),
+
+    # Column with DEFAULT value containing comma inside single quotes
+    (
+        "status varchar(50) DEFAULT 'active,pending', id int",
+        ",",
+        ["status varchar(50) DEFAULT 'active,pending'", 'id int']
+    ),
+
+    # Multiple columns with quoted DEFAULT values containing commas
+    (
+        "col1 varchar(50) DEFAULT 'value,with,commas', col2 int, col3 varchar(100) DEFAULT 'another,comma'",
+        ",",
+        ["col1 varchar(50) DEFAULT 'value,with,commas'", 'col2 int', "col3 varchar(100) DEFAULT 'another,comma'"]
+    ),
+
+    # ENUM definition with multiple values (commas inside parentheses)
+    (
+        "status enum('active','inactive','pending'), id int",
+        ",",
+        ["status enum('active','inactive','pending')", 'id int']
+    ),
+
+    # SET type with multiple values
+    (
+        "permissions set('read','write','execute'), user_id int",
+        ",",
+        ["permissions set('read','write','execute')", 'user_id int']
+    ),
+
+    # Column with DEFAULT containing single quote with comma
+    (
+        "description text DEFAULT 'User, Admin', created_at datetime",
+        ",",
+        ["description text DEFAULT 'User, Admin'", 'created_at datetime']
+    ),
+
+    # DECIMAL with precision and scale (comma inside parentheses)
+    (
+        "price decimal(10,2), quantity int",
+        ",",
+        ['price decimal(10,2)', 'quantity int']
+    ),
+
+    # Complex: ENUM + DEFAULT with commas in both
+    (
+        "type enum('type1','type2') DEFAULT 'type1', description varchar(255) DEFAULT 'desc,with,comma'",
+        ",",
+        ["type enum('type1','type2') DEFAULT 'type1'", "description varchar(255) DEFAULT 'desc,with,comma'"]
+    ),
+
+    # VARCHAR with length and DEFAULT containing comma
+    (
+        "name varchar(100) DEFAULT 'Last, First', id int NOT NULL",
+        ",",
+        ["name varchar(100) DEFAULT 'Last, First'", 'id int NOT NULL']
+    ),
+
+    # Empty string should return empty list
+    (
+        "",
+        ",",
+        []
+    ),
+
+    # Single column definition
+    (
+        "id int PRIMARY KEY",
+        ",",
+        ['id int PRIMARY KEY']
+    ),
+
+    # Multiple nested parentheses
+    (
+        "data1 varchar(100), func(arg1, arg2), data2 int",
+        ",",
+        ['data1 varchar(100)', 'func(arg1, arg2)', 'data2 int']
+    ),
+
+    # ALTER TABLE multi-statement with commas in DEFAULT values
+    (
+        "ADD COLUMN status varchar(50) DEFAULT 'new,value', DROP COLUMN old_col",
+        ",",
+        ["ADD COLUMN status varchar(50) DEFAULT 'new,value'", 'DROP COLUMN old_col']
+    ),
+
+    # Real-world example from MySQL CREATE TABLE
+    (
+        "`id` int NOT NULL AUTO_INCREMENT, `email` varchar(255) DEFAULT 'user@example.com', `status` enum('active','inactive') DEFAULT 'active'",
+        ",",
+        ["`id` int NOT NULL AUTO_INCREMENT", "`email` varchar(255) DEFAULT 'user@example.com'", "`status` enum('active','inactive') DEFAULT 'active'"]
+    ),
+])
+def test_split_high_level(data, delimiter, expected):
+    """
+    Test the split_high_level function with SQL column definitions.
+
+    This test verifies that the function correctly splits SQL statements by the delimiter
+    while ignoring delimiters that appear inside:
+    - Parentheses (e.g., enum values, function arguments, type precision)
+    - Single quotes (e.g., DEFAULT values, string literals)
+    """
+    result = split_high_level(data, delimiter)
+    assert result == expected, f"Failed for input: {data} with delimiter: {delimiter}"