From 5fb8e4db3c4a633ad8a98f1d15c9493ff1278e05 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Thu, 13 Nov 2025 16:39:24 +0530 Subject: [PATCH 1/2] fixed null value issue in CSV file type for import --- .../db/dataloader/core/util/ColumnUtils.java | 3 ++ .../dataloader/core/util/ColumnUtilsTest.java | 50 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java index 397d468a61..bd0635beef 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java @@ -83,6 +83,9 @@ public static Column createColumnFromValue( DataType dataType, ColumnInfo columnInfo, @Nullable String value) throws ColumnParsingException { String columnName = columnInfo.getColumnName(); + if (value != null && !dataType.equals(DataType.TEXT) && value.equals("null")) { + value = null; + } try { switch (dataType) { case BOOLEAN: diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java index 178809eef5..b27fe1db15 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java @@ -228,4 +228,54 @@ void getColumnsFromResult_withResultNull_withValidData_shouldReturnColumns() null, sourceRecord, false, mockMetadata, "namespace", "table"); assertEquals(8, columns.size()); } + + /** + * Tests that the string "null" (lowercase) is correctly treated as null for numeric, boolean, and + * date/time types. + */ + @Test + void createColumnFromValue_valueIsLowercaseNull_shouldReturnNullColumn() + throws ColumnParsingException { + String columnName = "testColumn"; + ColumnInfo columnInfo = ColumnInfo.builder().columnName(columnName).build(); + + // Integer type + Column intColumn = ColumnUtils.createColumnFromValue(DataType.INT, columnInfo, "null"); + assertEquals(IntColumn.ofNull(columnName), intColumn); + + // Double type + Column doubleColumn = ColumnUtils.createColumnFromValue(DataType.DOUBLE, columnInfo, "null"); + assertEquals(DoubleColumn.ofNull(columnName), doubleColumn); + + // Boolean type + Column boolColumn = ColumnUtils.createColumnFromValue(DataType.BOOLEAN, columnInfo, "null"); + assertEquals(BooleanColumn.ofNull(columnName), boolColumn); + + // Date type + Column dateColumn = ColumnUtils.createColumnFromValue(DataType.DATE, columnInfo, "null"); + assertEquals(DateColumn.ofNull(columnName), dateColumn); + + // Time type + Column timeColumn = ColumnUtils.createColumnFromValue(DataType.TIME, columnInfo, "null"); + assertEquals(TimeColumn.ofNull(columnName), timeColumn); + + // Timestamp type + Column timestampColumn = + ColumnUtils.createColumnFromValue(DataType.TIMESTAMP, columnInfo, "null"); + assertEquals(TimestampColumn.ofNull(columnName), timestampColumn); + } + + /** + * Tests that when the string value "null" is provided for TEXT columns, it is treated as a + * literal string and not converted to null. + */ + @Test + void createColumnFromValue_valueIsNullString_shouldRemainLiteralForTextType() + throws ColumnParsingException { + String columnName = "textColumn"; + ColumnInfo columnInfo = ColumnInfo.builder().columnName(columnName).build(); + + Column textCol = ColumnUtils.createColumnFromValue(DataType.TEXT, columnInfo, "null"); + assertEquals(TextColumn.of(columnName, "null"), textCol); + } } From a0fd37100a7f6bb23c9fb8224dde35cf7fbc01c4 Mon Sep 17 00:00:00 2001 From: Jishnu J Date: Wed, 19 Nov 2025 15:49:09 +0530 Subject: [PATCH 2/2] ignore case added --- .../db/dataloader/core/util/ColumnUtils.java | 2 +- .../dataloader/core/util/ColumnUtilsTest.java | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java index bd0635beef..40bd92d00e 100644 --- a/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java +++ b/data-loader/core/src/main/java/com/scalar/db/dataloader/core/util/ColumnUtils.java @@ -83,7 +83,7 @@ public static Column createColumnFromValue( DataType dataType, ColumnInfo columnInfo, @Nullable String value) throws ColumnParsingException { String columnName = columnInfo.getColumnName(); - if (value != null && !dataType.equals(DataType.TEXT) && value.equals("null")) { + if (value != null && !dataType.equals(DataType.TEXT) && value.equalsIgnoreCase("null")) { value = null; } try { diff --git a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java index b27fe1db15..1f00c04cd5 100644 --- a/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java +++ b/data-loader/core/src/test/java/com/scalar/db/dataloader/core/util/ColumnUtilsTest.java @@ -242,27 +242,49 @@ void createColumnFromValue_valueIsLowercaseNull_shouldReturnNullColumn() // Integer type Column intColumn = ColumnUtils.createColumnFromValue(DataType.INT, columnInfo, "null"); assertEquals(IntColumn.ofNull(columnName), intColumn); + intColumn = ColumnUtils.createColumnFromValue(DataType.INT, columnInfo, "Null"); + assertEquals(IntColumn.ofNull(columnName), intColumn); // Double type Column doubleColumn = ColumnUtils.createColumnFromValue(DataType.DOUBLE, columnInfo, "null"); assertEquals(DoubleColumn.ofNull(columnName), doubleColumn); + doubleColumn = ColumnUtils.createColumnFromValue(DataType.DOUBLE, columnInfo, "NULL"); + assertEquals(DoubleColumn.ofNull(columnName), doubleColumn); // Boolean type Column boolColumn = ColumnUtils.createColumnFromValue(DataType.BOOLEAN, columnInfo, "null"); assertEquals(BooleanColumn.ofNull(columnName), boolColumn); + boolColumn = ColumnUtils.createColumnFromValue(DataType.BOOLEAN, columnInfo, "nuLL"); + assertEquals(BooleanColumn.ofNull(columnName), boolColumn); + // Date type Column dateColumn = ColumnUtils.createColumnFromValue(DataType.DATE, columnInfo, "null"); assertEquals(DateColumn.ofNull(columnName), dateColumn); + dateColumn = ColumnUtils.createColumnFromValue(DataType.DATE, columnInfo, "NULL"); + assertEquals(DateColumn.ofNull(columnName), dateColumn); // Time type Column timeColumn = ColumnUtils.createColumnFromValue(DataType.TIME, columnInfo, "null"); assertEquals(TimeColumn.ofNull(columnName), timeColumn); + timeColumn = ColumnUtils.createColumnFromValue(DataType.TIME, columnInfo, "nuLL"); + assertEquals(TimeColumn.ofNull(columnName), timeColumn); + // Timestamp type Column timestampColumn = ColumnUtils.createColumnFromValue(DataType.TIMESTAMP, columnInfo, "null"); assertEquals(TimestampColumn.ofNull(columnName), timestampColumn); + timestampColumn = ColumnUtils.createColumnFromValue(DataType.TIMESTAMP, columnInfo, "NULL"); + assertEquals(TimestampColumn.ofNull(columnName), timestampColumn); + + // Timestamp type + Column timestamprtzColumn = + ColumnUtils.createColumnFromValue(DataType.TIMESTAMPTZ, columnInfo, "null"); + assertEquals(TimestampTZColumn.ofNull(columnName), timestamprtzColumn); + timestamprtzColumn = + ColumnUtils.createColumnFromValue(DataType.TIMESTAMPTZ, columnInfo, "Null"); + assertEquals(TimestampTZColumn.ofNull(columnName), timestamprtzColumn); } /** @@ -277,5 +299,14 @@ void createColumnFromValue_valueIsNullString_shouldRemainLiteralForTextType() Column textCol = ColumnUtils.createColumnFromValue(DataType.TEXT, columnInfo, "null"); assertEquals(TextColumn.of(columnName, "null"), textCol); + + textCol = ColumnUtils.createColumnFromValue(DataType.TEXT, columnInfo, "NULL"); + assertEquals(TextColumn.of(columnName, "NULL"), textCol); + + textCol = ColumnUtils.createColumnFromValue(DataType.TEXT, columnInfo, "Null"); + assertEquals(TextColumn.of(columnName, "Null"), textCol); + + textCol = ColumnUtils.createColumnFromValue(DataType.TEXT, columnInfo, "nuLL"); + assertEquals(TextColumn.of(columnName, "nuLL"), textCol); } }