Merge pull request #6 from vbhavsingh/master

To dump large data sets into files

Author: BernhardSchlegel

bin/sql-dumper.jar

@@ -1,13 +1,61 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
 
-    <groupId>de.bernhard-schlegel</groupId>
-    <artifactId>sql_dumper</artifactId>
-    <version>0.0.3</version>
+	<groupId>de.bernhard-schlegel</groupId>
+	<artifactId>sql_dumper</artifactId>
+	<version>0.0.3</version>
 
-    <dependencies>
-    </dependencies>
+	<dependencies>
+		<dependency>
+			<groupId>com.github.noraui</groupId>
+			<artifactId>ojdbc7</artifactId>
+			<version>12.1.0.2</version>
+		</dependency>
+	</dependencies>
+
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-jar-plugin</artifactId>
+				<configuration>
+					<archive>
+						<manifest>
+							<mainClass>sqldump.Main</mainClass>
+						</manifest>
+						<manifestEntries>
+						</manifestEntries>
+					</archive>
+				</configuration>
+			</plugin>
+
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-shade-plugin</artifactId>
+				<version>2.4.3</version>
+				<executions>
+					<execution>
+						<phase>package</phase>
+						<goals>
+							<goal>shade</goal>
+						</goals>
+					</execution>
+				</executions>
+				<configuration>
+					<finalName>sql-dumper</finalName>
+					<filters>
+						<filter>
+							<artifact>*:*</artifact>
+							<excludes>
+								<exclude>META-INF/*.SF</exclude>
+								<exclude>META-INF/*.DSA</exclude>
+								<exclude>META-INF/*.RSA</exclude>
+							</excludes>
+						</filter>
+					</filters>
+				</configuration>
+			</plugin>
+		</plugins>
+	</build>
 </project>

pom.xml

@@ -66,7 +66,6 @@ public void run(String[] args) {
 
         File folder = new File(sqlPath);
         File[] listOfSQLFiles = folder.listFiles(new FilenameFilter() {
-            @Override
             public boolean accept(File dir, String name) {
                 return name.endsWith(".sql");
             }

src/main/java/sqldump/SQLDump.java

@@ -82,33 +82,49 @@ public static void toFile(ResultSet rs, String filename, Boolean append, Log log
            fileWriter.flush();
            fileWriter.close();
 
-           PingPong ping2 = log.info("converting data to CSV (one dot corresponds with 50k lines)");
+           PingPong ping2 = log.info("converting data to CSV (one dot corresponds with 10k lines)");
            List<String> records = new ArrayList<String>();
+           int counter =0;
            while (rs.next()) {
-
+        	   
+        	   boolean StringType = (meta.getColumnType(1) == java.sql.Types.VARCHAR);
+        	   
                StringBuffer sbRow = new StringBuffer();
-               sbRow.append("\"" + rs.getString(1) + "\""  );
+               if(StringType) {
+            	   sbRow.append("\"" + rs.getString(1) + "\""  );
+               }else {
+            	   sbRow.append(rs.getString(1));
+               }
 
                for (int j = 2 ; j < numberOfColumns + 1 ; j ++ ) {
-                   sbRow.append(",\"" + rs.getString(j) + "\"");
+            	   StringType = (meta.getColumnType(j) == java.sql.Types.VARCHAR);
+            	   if(StringType) {
+            		   sbRow.append(",\"" + rs.getString(j) + "\"");
+            	   }else {
+            		   sbRow.append("," + rs.getString(j));
+            	   }
                }
                sbRow.append("\r\n");
                records.add(sbRow.toString());
 
-               if (records.size()%50000 == 0) {
+               if (records.size()%10000 == 0) {
+            	   counter+=records.size();
                    System.out.print(".");
+                   writeBuffered(filename, records, BUFFER_SIZE, true); // always apppend
+                   records = new ArrayList<String>();
+                   sbRow.setLength(0);
                }
 
            }
            log.finished(ping2);
-           System.out.print(" (" + records.size() + " entries)");
+           System.out.print(" (" + counter + " entries)");
            if (rs != null) {
                rs.close();
            }
 
-           PingPong ping3 = log.info("dumping CSV to file ...");
-           writeBuffered(filename, records, BUFFER_SIZE, true); // always apppend
-           log.finished(ping3);
+           //PingPong ping3 = log.info("dumping CSV to file ...");
+           //writeBuffered(filename, records, BUFFER_SIZE, true); // always apppend
+           //log.finished(ping3);
 
            log.finished(ping);