Replace uniVocity CSV library with jackson-csv #702 review comments update -fixing csvConfig

Author: rkampani

core/src/main/java/org/jsmart/zerocode/core/db/DbCsvLoader.java

@@ -15,7 +15,6 @@
 import org.slf4j.LoggerFactory;
 
 
-
 /**
  * Data loading in the database from a CSV external source
  */

core/src/main/java/org/jsmart/zerocode/core/db/DbSqlExecutor.java

@@ -1,11 +1,9 @@
 package org.jsmart.zerocode.core.db;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.csv.CsvParser;
 import com.google.inject.Inject;
 import com.google.inject.name.Named;
 
-
 import org.apache.commons.dbutils.DbUtils;
 import org.jsmart.zerocode.core.di.provider.CsvParserProvider;
 import org.slf4j.Logger;

core/src/main/java/org/jsmart/zerocode/core/di/provider/CsvParserConfig.java

@@ -0,0 +1,72 @@
+package org.jsmart.zerocode.core.di.provider;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * /
+ *  Configuration class for parsing CSV files with Jackson's CSV module,
+ *  using a custom deserializer to maintain compatibility with non-standard formats previously handled by uniVocity.
+ *
+ */
+public class CsvParserConfig {
+
+    public static class CustomStringDeserializer extends StdDeserializer<String> {
+        // Immutable map of replacement rules: pattern → replacement
+        private static final Map<String, String> REPLACEMENTS = createReplacements();
+
+        /**
+         * Creates an immutable map of replacement rules for escape patterns.
+         * @return A map containing patterns and their replacements.
+         */
+        private static Map<String, String> createReplacements() {
+            Map<String, String> map = new HashMap<>();
+            map.put("\\'", "'"); // Backslash-escaped single quote
+            map.put("''", "'");  // Double single quote (single-quoted CSV)
+            map.put("\\\\", "\\"); // Double backslash to preserve literal backslash
+            return Collections.unmodifiableMap(map);
+        }
+
+        public CustomStringDeserializer() {
+            super(String.class);
+        }
+
+        /**
+         * Deserializes a String value from the CSV parser, applying custom escape pattern replacements.
+         * <p>
+         * The method processes the input string to handle non-standard escape patterns required
+         * for the expected output (e.g., <code>["a'c", "d\"f", "x\y"]</code>). It uses a stream-based
+         * approach to apply replacements only when the pattern is present, ensuring efficiency.
+         * <p>
+         * Without this deserializer, Jackson's default CSV parser may:
+         * <ul>
+         *     <li>Strip literal backslashes (e.g., <code>x\y</code> becomes <code>xy</code>).</li>
+         *     <li>Misinterpret single-quote escaping (e.g., <code>\'</code> or <code>''</code>).</li>
+         * </ul>
+         * <p>
+         * This implementation ensures compatibility with the previous CSV parsing library's behavior
+         * and handles inputs like <code>"a'c","d""f","x\y"</code> or <code>"a\'c","d\"f","x\y"</code>.
+         *
+         * @return The processed string with escape patterns replaced, or null if the input is null.
+         * @throws IOException If an I/O error occurs during parsing.
+         */
+        @Override
+        public String deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
+            final String value = p.getText();
+            if (Objects.isNull(value)) {
+                return null;
+            }
+            return REPLACEMENTS.entrySet().stream()
+                    .filter(entry -> value.contains(entry.getKey()))
+                    .reduce(value, (current, entry) -> current.replace(entry.getKey(), entry.getValue()), (v1, v2) -> v1);
+        }
+    }
+
+}

core/src/main/java/org/jsmart/zerocode/core/di/provider/CsvParserProvider.java

@@ -2,6 +2,7 @@
 
 
 import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.fasterxml.jackson.dataformat.csv.CsvMapper;
 import com.fasterxml.jackson.dataformat.csv.CsvParser;
 import com.fasterxml.jackson.dataformat.csv.CsvSchema;
@@ -20,7 +21,12 @@ public class CsvParserProvider implements Provider<JacksonCsvParserAdapter> {
 
     static {
         final CsvSchema schema = createCsvSchema();
-        final ObjectReader mapper =  new CsvMapper()
+        final CsvMapper csvMapper = new CsvMapper();
+        csvMapper.enable(CsvParser.Feature.TRIM_SPACES);
+        csvMapper.enable(CsvParser.Feature.ALLOW_TRAILING_COMMA);
+        csvMapper.registerModule(new SimpleModule()
+                .addDeserializer(String.class, new CsvParserConfig.CustomStringDeserializer()));
+        final ObjectReader mapper =  csvMapper
                 .enable(CsvParser.Feature.TRIM_SPACES)
                 .readerFor(String[].class)
                 .with(schema);
@@ -45,14 +51,13 @@ private String sanitizeLine(final String line) {
         if (StringUtils.isNotBlank(line) && !line.contains(CARRIAGE_RETURN)) {
             return line;
         }
-        return line.replace(CARRIAGE_RETURN, StringUtils.EMPTY);
+        return line.replace(CARRIAGE_RETURN, StringUtils.SPACE);
     }
 
     private static CsvSchema createCsvSchema() {
         return CsvSchema.builder()
                 .setColumnSeparator(',')
                 .setQuoteChar('\'')
-                .setEscapeChar('\'')
                 .setNullValue("")
                 .setLineSeparator(LINE_SEPARATOR)
                 .build();

core/src/main/java/org/jsmart/zerocode/core/di/provider/JacksonCsvParserAdapter.java

@@ -1,6 +1,7 @@
 package org.jsmart.zerocode.core.di.provider;
 
 import com.fasterxml.jackson.databind.ObjectReader;
+import org.apache.commons.lang3.StringUtils;
 
 import java.io.IOException;
 import java.io.StringReader;
@@ -13,6 +14,10 @@ public JacksonCsvParserAdapter(final ObjectReader mapper) {
     }
     @Override
     public String[] parseLine(final String line) throws IOException {
+        if (StringUtils.isEmpty(line)) return null ;
+        if(line.trim().isEmpty()) {
+            return new String[]{ null };
+        }
         return mapper.readValue(new StringReader(line));
     }
 }

core/src/test/java/org/jsmart/zerocode/core/db/DbCsvLoaderTest.java

@@ -16,7 +16,6 @@
 
 import com.google.inject.Inject;
 
-
 @RunWith(JukitoRunner.class)
 public class DbCsvLoaderTest extends DbTestBase{
 

core/src/test/java/org/jsmart/zerocode/core/di/CsvParserTest.java

@@ -0,0 +1,64 @@
+package org.jsmart.zerocode.core.di;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertNull;
+
+import org.jsmart.zerocode.core.di.main.ApplicationMainModule;
+import org.jsmart.zerocode.core.di.provider.CsvParserProvider;
+import org.jukito.JukitoRunner;
+import org.jukito.TestModule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import com.google.inject.Inject;
+
+
+@RunWith(JukitoRunner.class)
+public class CsvParserTest {
+
+    public static class JukitoModule extends TestModule {
+        @Override
+        protected void configureTest() {
+            ApplicationMainModule applicationMainModule = new ApplicationMainModule("config_hosts_test.properties");
+            install(applicationMainModule);
+        }
+    }
+
+    @Inject
+    CsvParserProvider parser;
+
+    @Test
+    public void testCsvparseSpaces() {
+        assertThat(parser.parseLine(" abc ,\t de f  , ghi "), is(new String[] { "abc", "de f", "ghi" }));
+    }
+
+    @Test
+    public void testCsvParseEmptyCell() {
+        assertThat(parser.parseLine(",,  , \t ,"), is(new String[] { null, null, null, null, null }));
+    }
+
+    @Test
+    public void testCsvparseEmptyLine() {
+        assertNull(parser.parseLine(""));
+        assertThat(parser.parseLine("  "), is(new String[] { null }));
+        assertThat(parser.parseLine(" \t "), is(new String[] { null }));
+    }
+
+    @Test
+    public void testCsvParseQuotesAndOtherChars() {
+        assertThat(parser.parseLine("a'c, d\"f, x\\y"), is(new String[] { "a'c", "d\"f", "x\\y" }));
+        assertThat(parser.parseLine("euro\u20AC, año, naïf"), is(new String[] { "euro\u20AC", "año", "naïf" }));
+    }
+
+    @Test
+    public void testCsvParseWindowsEndings() { // assume that lines where already splitted by \n
+        assertThat(parser.parseLine("abc, def\r"), is(new String[] { "abc", "def" }));
+        assertThat(parser.parseLine("abc, def \r"), is(new String[] { "abc", "def" }));
+        // empty lines
+        assertThat(parser.parseLine("\r"), is(new String[] { null })); // should be null?, see testCsvparseEmptyLine
+        assertThat(parser.parseLine("  \r"), is(new String[] { null }));
+        assertThat(parser.parseLine(" \t \r"), is(new String[] { null }));
+    }
+
+}