From d9b709695174f5b73c0eb41340d0e081146c437b Mon Sep 17 00:00:00 2001 From: alexeykap Date: Thu, 21 Mar 2019 15:37:28 +0400 Subject: [PATCH 1/3] issure #12 Add dont_change option for "table_transform" and "column_transform" This commit add option "dont_change" for parametres "table_transform" and "column_transform". With that option case of table names and column names does not changed via migration. --- README.md | 4 ++-- pom.xml | 8 ++++++++ src/main/java/net/twentyonesolutions/m2pg/Config.java | 3 +++ src/main/java/net/twentyonesolutions/m2pg/Util.java | 10 ++++++++++ 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index aef9890..9861793 100644 --- a/README.md +++ b/README.md @@ -134,9 +134,9 @@ Values that are wrapped in `%` symbols are treated as varaibles and evaluated at | +-- column_mapping struct - maps column names if needed, e.g. "group" -> "group_name" | -+-- table_transform string - ([""], "lower_case", "upper_case", "camel_to_snake_case") ++-- table_transform string - ([""], "lower_case", "upper_case", "camel_to_snake_case", "dont_change") | -+-- column_transform string - ([""], "lower_case", "upper_case", "camel_to_snake_case") ++-- column_transform string - ([""], "lower_case", "upper_case", "camel_to_snake_case", "dont_change") | +-- ddl | diff --git a/pom.xml b/pom.xml index 7e9639c..fab2d0d 100644 --- a/pom.xml +++ b/pom.xml @@ -85,6 +85,14 @@ + + maven-compiler-plugin + 3.1 + + true + C:\Program Files\Java\jdk1.8.0_202\bin\javac.exe + + diff --git a/src/main/java/net/twentyonesolutions/m2pg/Config.java b/src/main/java/net/twentyonesolutions/m2pg/Config.java index 7554287..9140b98 100644 --- a/src/main/java/net/twentyonesolutions/m2pg/Config.java +++ b/src/main/java/net/twentyonesolutions/m2pg/Config.java @@ -340,6 +340,9 @@ public static String transform(String input, String type){ if (type.equalsIgnoreCase("camel_to_snake_case")) return Util.convertCamelToSnakeCase(input); + + if (type.equalsIgnoreCase("dont_change")) + return Util.convertDontChangeCase(input); } return input; diff --git a/src/main/java/net/twentyonesolutions/m2pg/Util.java b/src/main/java/net/twentyonesolutions/m2pg/Util.java index 1085fb4..084dc97 100644 --- a/src/main/java/net/twentyonesolutions/m2pg/Util.java +++ b/src/main/java/net/twentyonesolutions/m2pg/Util.java @@ -71,6 +71,16 @@ public static String convertCamelToSnakeCase(String camelCaseString){ .replaceAll("([^_A-Z0-9])([A-Z0-9])", "$1_$2") .toLowerCase(); } + + /** + * If it is necessary to do not change case, add quote to input string; + * + * @param anyCaseString + * @return + */ + public static String convertDontChangeCase(String anyCaseString){ + return "\""+anyCaseString+"\""; + } public static Map flattenKeys(Map map){ From a858aa40bd161eebb8ab9da2a3018cb81d326d58 Mon Sep 17 00:00:00 2001 From: alexeykap Date: Fri, 22 Mar 2019 10:01:49 +0400 Subject: [PATCH 2/3] issue #14 Could not make dml migration from MSSQL type UNIQUEIDENTIFIER to uuid postrges type Add to dml config "postgres_type_casting" parameter. It is array of type's names which must be casted by postgress implictly. --- README.md | 2 ++ .../java/net/twentyonesolutions/m2pg/Config.java | 6 ++++-- .../java/net/twentyonesolutions/m2pg/Schema.java | 16 +++++++++++----- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9861793..3e32260 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,8 @@ Values that are wrapped in `%` symbols are treated as varaibles and evaluated at +-- after_all array of SQL commands to run after data copy | +-- recomended ([""], "all") - specifying "all" will execute recommendations + | + +-- implicitConversionTypes array of source column types that should be cast implicitly with postgres (for example "UNIQUEIDENTIFIER") | +-- threads (["cores", integer]) - number of concurrent connections | diff --git a/src/main/java/net/twentyonesolutions/m2pg/Config.java b/src/main/java/net/twentyonesolutions/m2pg/Config.java index 9140b98..87c6760 100644 --- a/src/main/java/net/twentyonesolutions/m2pg/Config.java +++ b/src/main/java/net/twentyonesolutions/m2pg/Config.java @@ -42,6 +42,7 @@ public class Config { Map dml, ddl; Map schemaMapping, tableMapping, columnMapping; Map columnDefaultReplace; + Map postgresTypeCasting; String name, source, target; @@ -87,6 +88,7 @@ private void parseDml(){ ,"source_column_quote_prefix" ,"source_column_quote_suffix" ,"threads" + ,"implicit_conversion_types" }; // populate result with config value or default of empty string @@ -95,7 +97,7 @@ private void parseDml(){ } // wrap single item String in List - for (String k : new String[]{ "execute.before_all", "execute.after_all" }){ + for (String k : new String[]{ "execute.before_all", "execute.after_all", "implicit_conversion_types"}){ Object v = result.get(k); if (v instanceof String){ result.put(k, new ArrayList(){{ add((String)v); }}); @@ -104,7 +106,7 @@ private void parseDml(){ mapSrc = (Map)config.get(prefix + "jdbc_type_mapping"); result.put("jdbc_type_mapping", getCaseInsensitiveMap(mapSrc, uppercaseValue)); - + this.dml = result; } diff --git a/src/main/java/net/twentyonesolutions/m2pg/Schema.java b/src/main/java/net/twentyonesolutions/m2pg/Schema.java index 9d13845..8767ad1 100644 --- a/src/main/java/net/twentyonesolutions/m2pg/Schema.java +++ b/src/main/java/net/twentyonesolutions/m2pg/Schema.java @@ -147,8 +147,7 @@ public String copyTable(String tableName, IProgress progress) throws IOException statSrc = conSrc.createStatement(); - PreparedStatement statInsert = conTgt.prepareStatement(qInsert); - + PreparedStatement statInsert = conTgt.prepareStatement(qInsert); statSrc.setFetchSize(1000); rs = statSrc.executeQuery(qSelect); @@ -156,7 +155,8 @@ public String copyTable(String tableName, IProgress progress) throws IOException ResultSetMetaData rsMetaData = rs.getMetaData(); Map jdbcTypeMapping = (Map) config.dml.get("jdbc_type_mapping"); - + List implicitConversionTypes = (List) config.dml.getOrDefault("implicit_conversion_types", Collections.EMPTY_LIST); + int columnCount = rsMetaData.getColumnCount(); int[] columnTypes = new int[columnCount]; @@ -172,6 +172,7 @@ public String copyTable(String tableName, IProgress progress) throws IOException // tgtType = Integer.parseInt(jdbcTypeMapping.getOrDefault(String.valueOf(srcType), String.valueOf(tgtType))); String srcTypeName = JDBCType.valueOf(srcType).getName(); // WARN: rsMetaData.getColumnTypeName(i) returns the vendor's name instead of JDBC name, e.g. ntext instead of longnvarchar for MSSQL + if (jdbcTypeMapping.containsKey(srcTypeName)) { String tgtTypeName = jdbcTypeMapping.get(srcTypeName); tgtType = JDBCType.valueOf(tgtTypeName).getVendorTypeNumber(); @@ -190,8 +191,13 @@ public String copyTable(String tableName, IProgress progress) throws IOException for (int i = 1; i <= columnCount; i++) { Object value = rs.getObject(i); values[i - 1] = value; - statInsert.setObject(i, value, columnTypes[i - 1]); - // statInsert.setObject(i, value, sqlTypes[i - 1]); // throws java.sql.SQLFeatureNotSupportedException: Method org.postgresql.jdbc.PgPreparedStatement.setObject is not yet implemented. + String sourceColumnType=new String(); + sourceColumnType=table.columns.get(i-1).type.toString(); + if(implicitConversionTypes.contains(sourceColumnType)){ + statInsert.setObject(i, value,java.sql.Types.OTHER); + }else{ + statInsert.setObject(i, value, columnTypes[i - 1]); // throws java.sql.SQLFeatureNotSupportedException: Method org.postgresql.jdbc.PgPreparedStatement.setObject is not yet implemented. + } } try { From d97d0296fbbf53c7e241a4aa421f3e9d60982c66 Mon Sep 17 00:00:00 2001 From: lehkap Date: Fri, 22 Mar 2019 17:17:21 +0400 Subject: [PATCH 3/3] issue #14 refactoring Refactor solution --- .../net/twentyonesolutions/m2pg/Schema.java | 47 ++++++++++--------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/src/main/java/net/twentyonesolutions/m2pg/Schema.java b/src/main/java/net/twentyonesolutions/m2pg/Schema.java index 8767ad1..f7bd146 100644 --- a/src/main/java/net/twentyonesolutions/m2pg/Schema.java +++ b/src/main/java/net/twentyonesolutions/m2pg/Schema.java @@ -163,22 +163,29 @@ public String copyTable(String tableName, IProgress progress) throws IOException // SQLType[] sqlTypes = new SQLType[columnCount]; // TODO: use this instead of columnTypes when pgjdbc will support setObject with SQLType for (int i = 1; i <= columnCount; i++) { - - int srcType = rsMetaData.getColumnType(i); - int tgtType = srcType; - - // translate unsupported types, e.g. nvarchar to varchar, dml jdbcTypeMapping is based on JDBC types, while ddl jdbcTypeMapping is based on SQL types - // if (jdbcTypeMapping.containsKey(String.valueOf(srcType))) - // tgtType = Integer.parseInt(jdbcTypeMapping.getOrDefault(String.valueOf(srcType), String.valueOf(tgtType))); - - String srcTypeName = JDBCType.valueOf(srcType).getName(); // WARN: rsMetaData.getColumnTypeName(i) returns the vendor's name instead of JDBC name, e.g. ntext instead of longnvarchar for MSSQL - - if (jdbcTypeMapping.containsKey(srcTypeName)) { - String tgtTypeName = jdbcTypeMapping.get(srcTypeName); - tgtType = JDBCType.valueOf(tgtTypeName).getVendorTypeNumber(); - } - - columnTypes[i - 1] = tgtType; + String sourceColumnType=new String(); + sourceColumnType=table.columns.get(i-1).type.toString(); + if(implicitConversionTypes.contains(sourceColumnType)){ + columnTypes[i - 1]=java.sql.Types.OTHER; + }else{ + int srcType = rsMetaData.getColumnType(i); + int tgtType = srcType; + + // translate unsupported types, e.g. nvarchar to varchar, dml jdbcTypeMapping is based on JDBC types, while ddl jdbcTypeMapping is based on SQL types + // if (jdbcTypeMapping.containsKey(String.valueOf(srcType))) + // tgtType = Integer.parseInt(jdbcTypeMapping.getOrDefault(String.valueOf(srcType), String.valueOf(tgtType))); + + + String srcTypeName = JDBCType.valueOf(srcType).getName();// WARN: rsMetaData.getColumnTypeName(i) returns the vendor's name instead of JDBC name, e.g. ntext instead of longnvarchar for MSSQL + + + if (jdbcTypeMapping.containsKey(srcTypeName)) { + String tgtTypeName = jdbcTypeMapping.get(srcTypeName); + + tgtType = JDBCType.valueOf(tgtTypeName).getVendorTypeNumber(); + } + columnTypes[i - 1] = tgtType; + } } boolean hasErrors = false; @@ -191,13 +198,7 @@ public String copyTable(String tableName, IProgress progress) throws IOException for (int i = 1; i <= columnCount; i++) { Object value = rs.getObject(i); values[i - 1] = value; - String sourceColumnType=new String(); - sourceColumnType=table.columns.get(i-1).type.toString(); - if(implicitConversionTypes.contains(sourceColumnType)){ - statInsert.setObject(i, value,java.sql.Types.OTHER); - }else{ - statInsert.setObject(i, value, columnTypes[i - 1]); // throws java.sql.SQLFeatureNotSupportedException: Method org.postgresql.jdbc.PgPreparedStatement.setObject is not yet implemented. - } + statInsert.setObject(i, value, columnTypes[i - 1]); // throws java.sql.SQLFeatureNotSupportedException: Method org.postgresql.jdbc.PgPreparedStatement.setObject is not yet implemented. } try {