Merge pull request #22 from rasilon/dev

New Features

docker/multistage/TEST_config_live.xml

@@ -4,6 +4,11 @@
   <nRowsToLog>10000</nRowsToLog>
   <blockSize>1000</blockSize>
   <pollTimeout>500</pollTimeout>
+  <drivers>
+    <driver>org.postgresql.Driver</driver>
+    <driver>org.relique.jdbc.csv.CsvDriver</driver>
+  </drivers>
+
   <source>
     <dsn>jdbc:postgresql://testdb:5432/test</dsn>
     <username>test</username>

docker/multistage/small.csv

@@ -0,0 +1,4 @@
+id,dat
+1,banana
+2,potato
+3,nugget

docker/test_db/setup.sql

@@ -44,10 +44,14 @@ CREATE TABLE denormalised_personalia(
     lname text
 );
 
+CREATE TABLE test_csvjdbc(
+    id integer not null primary key,
+    dat text
+);
+
 GRANT SELECT ON ALL TABLES IN SCHEMA public TO test;
 GRANT SELECT,INSERT,UPDATE ON denormalised_personalia TO test;
 
-
 \c postgres
 CREATE TABLE public.container_ready AS SELECT 1 FROM(VALUES(1)) AS a(a);
 GRANT SELECT ON public.container_ready TO TEST;

install_extra/run_copying_job

@@ -30,9 +30,9 @@ fi
 /usr/bin/java \
     -Xms1g \
     -Xmx2g   \
+    -Dlog4j.configuration=file:"$LOG_PROPS" \
     -cp /usr/share/ujetl/lib/CopyingApp.jar \
     com.rasilon.ujetl.CopyingApp \
-    --log4j "$LOG_PROPS" \
     --config "/etc/ujetl/${JOBNAME}_config_live.xml"
 
 #rm -f $LOCKFILE

pom.xml

@@ -6,7 +6,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma
   <groupId>com.rasilon.ujetl</groupId>
   <artifactId>CopyingApp</artifactId>
   <packaging>jar</packaging>
-  <version>2.4.4</version>
+  <version>2.5.0</version>
   <name>uJETL</name>
   <url>https://github.com/rasilon/ujetl</url>
   <properties>
@@ -78,6 +78,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma
       <artifactId>postgresql</artifactId>
       <version>42.4.3</version>
     </dependency>
+    <dependency>
+      <groupId>net.sourceforge.csvjdbc</groupId>
+      <artifactId>csvjdbc</artifactId>
+      <version>1.0.40</version>
+    </dependency>
   </dependencies>
   <build>
     <plugins>

src/main/java/com/rasilon/ujetl/CopyingApp.java

@@ -34,10 +34,6 @@ public class CopyingApp {
     public static void main(String[] args) {
         CopyingAppCommandParser cli = new CopyingAppCommandParser(args);
         LoggerContext context = (org.apache.logging.log4j.core.LoggerContext) LogManager.getContext(false);
-        String log4jConfigLocation = cli.getLog4jConfigFile();
-        File file = new File(log4jConfigLocation);
-        context.setConfigLocation(file.toURI());
-        System.out.println("Config set from "+file.toURI());
 
         CopyingApp app = new CopyingApp(cli);
         try {
@@ -79,6 +75,7 @@ public class CopyingApp {
 
             Configuration config = configs.xml(cli.getConfigFile());
 
+            loadDrivers(config);
             String hardLimitSeconds = config.getString("hardLimitSeconds");
             if(hardLimitSeconds != null) {
                 TimeLimiter hardLimit = new TimeLimiter(Integer.decode(hardLimitSeconds).intValue(),true);
@@ -108,14 +105,14 @@ public class CopyingApp {
                 log.info(String.format("%s - Setting Row count interval to default of 100 rows.",jobName));
             }
 
-		  	Integer pollTimeout = null;
+            Integer pollTimeout = null;
-	  		try {
+            try {
-  				pollTimeout = new Integer(config.getString("pollTimeout"));
+                pollTimeout = new Integer(config.getString("pollTimeout"));
-				log.info(String.format("%s - Setting Poll timeout to %s milliseconds", jobName, pollTimeout));
+                log.info(String.format("%s - Setting Poll timeout to %s milliseconds", jobName, pollTimeout));
-			} catch(Exception e) {
+            } catch(Exception e) {
-				pollTimeout = new Integer(1000); // If we don't have a new setting, use the old default
+                pollTimeout = new Integer(1000); // If we don't have a new setting, use the old default
-				log.info(String.format("%s - Setting poll timeout to default of 1 second.",jobName));
+                log.info(String.format("%s - Setting poll timeout to default of 1 second.",jobName));
-			}
+            }
 
 
 
@@ -151,7 +148,7 @@ public class CopyingApp {
                         pollTimeout,
                         identifySourceSQL,
                         identifyDestinationSQL
-                        );
+                    );
                     j.start();
                     j.join();
 
@@ -182,7 +179,7 @@ public class CopyingApp {
                     pollTimeout,
                     identifySourceSQL,
                     identifyDestinationSQL
-                    );
+                );
                 j.start();
                 j.join();
             } else {
@@ -244,4 +241,21 @@ public class CopyingApp {
 
         return c;
     }
+
+    // Even with JDBC 4, some drivers don't play nicely with whatever
+    // the classloaders are up to.  So this allows us to force it the
+    // old fashioned way, and works around the
+    // "But it works fine when it's the /only/ driver!"
+    // cross-database problem
+    private void loadDrivers(Configuration config) {
+        String[] drivers = config.get(String[].class, "drivers.driver");
+        for(String d:drivers) {
+            try {
+                Class.forName(d);
+                log.info("Preloaded driver  "+d);
+            } catch(ClassNotFoundException e) {
+                log.error("Could not preload driver "+d,e);
+            }
+        }
+    }
 }

src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java

@@ -23,8 +23,4 @@ public class CopyingAppCommandParser {
 		return configFile;
 	}
 
-	public String getLog4jConfigFile() {
-		return log4jConfigFile;
-	}
-	
 }

src/test/java/com/rasilon/ujetl/TestConfig.java

@@ -0,0 +1,37 @@
+package com.rasilon.ujetl;
+
+import org.apache.commons.configuration2.Configuration;
+import org.apache.commons.configuration2.builder.fluent.Configurations;
+import org.apache.commons.configuration2.ex.ConfigurationException;
+
+import org.apache.commons.beanutils.PropertyUtils; // Why does config need this?
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.MethodOrderer.Alphanumeric;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+
+
+/**
+ * @author derryh
+ *
+ */
+public class TestConfig {
+
+    @Test
+    public void test001VerifyArrayOfDrivers() {
+        try {
+            Configurations configs = new Configurations();
+            Configuration config = configs.xml("TEST_config_live.xml");
+            String[] drivers = config.get(String[].class, "drivers.driver");
+            int ndrivers =drivers.length;
+            if(ndrivers != 3){
+                fail("Expected 3 drivers, but found "+ndrivers);
+            }
+        } catch(Exception e) {
+            fail(e.toString());
+        }
+    }
+
+}

src/test/java/com/rasilon/ujetl/TestParser.java

@@ -13,15 +13,12 @@ public class TestParser {
     public void test001Parset() {
         try {
             String[] args = {
-                "--log4j",
-                "log4j_test_banana.xml",
                 "--config",
                 "config_test_banana.xml"
             };
             CopyingAppCommandParser p = new CopyingAppCommandParser(args);
 
             assertEquals(p.getConfigFile(),"config_test_banana.xml");
-            assertEquals(p.getLog4jConfigFile(),"log4j_test_banana.xml");
 
         } catch(Exception e) {
             fail(e.toString());

src/test/resources/TEST_config_live.xml

@@ -4,6 +4,11 @@
   <nRowsToLog>10000</nRowsToLog>
   <blockSize>1000</blockSize>
   <pollTimeout>500</pollTimeout>
+  <drivers>
+      <driver>org.postgresql.Driver</driver>
+      <driver>org.h2.Driver</driver>
+      <driver>org.relique.jdbc.csv.CsvDriver</driver>
+  </drivers>
   <source>
     <dsn>jdbc:postgresql://localhost:5432/test</dsn>
     <username>test</username>