diff --git a/docker/multistage/TEST_config_live.xml b/docker/multistage/TEST_config_live.xml
index fa38eec..d1b7345 100644
--- a/docker/multistage/TEST_config_live.xml
+++ b/docker/multistage/TEST_config_live.xml
@@ -4,6 +4,11 @@
10000
1000
500
+
+ org.postgresql.Driver
+ org.relique.jdbc.csv.CsvDriver
+
+
jdbc:postgresql://testdb:5432/test
test
diff --git a/docker/multistage/small.csv b/docker/multistage/small.csv
new file mode 100644
index 0000000..37ddeff
--- /dev/null
+++ b/docker/multistage/small.csv
@@ -0,0 +1,4 @@
+id,dat
+1,banana
+2,potato
+3,nugget
diff --git a/docker/test_db/setup.sql b/docker/test_db/setup.sql
index ff4a1b7..2469967 100644
--- a/docker/test_db/setup.sql
+++ b/docker/test_db/setup.sql
@@ -44,10 +44,14 @@ CREATE TABLE denormalised_personalia(
lname text
);
+CREATE TABLE test_csvjdbc(
+ id integer not null primary key,
+ dat text
+);
+
GRANT SELECT ON ALL TABLES IN SCHEMA public TO test;
GRANT SELECT,INSERT,UPDATE ON denormalised_personalia TO test;
-
\c postgres
CREATE TABLE public.container_ready AS SELECT 1 FROM(VALUES(1)) AS a(a);
GRANT SELECT ON public.container_ready TO TEST;
diff --git a/install_extra/run_copying_job b/install_extra/run_copying_job
index bf51414..5dfa2b0 100755
--- a/install_extra/run_copying_job
+++ b/install_extra/run_copying_job
@@ -30,9 +30,9 @@ fi
/usr/bin/java \
-Xms1g \
-Xmx2g \
+ -Dlog4j.configuration=file:"$LOG_PROPS" \
-cp /usr/share/ujetl/lib/CopyingApp.jar \
com.rasilon.ujetl.CopyingApp \
- --log4j "$LOG_PROPS" \
--config "/etc/ujetl/${JOBNAME}_config_live.xml"
#rm -f $LOCKFILE
diff --git a/pom.xml b/pom.xml
index 304defa..3b55a96 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma
com.rasilon.ujetl
CopyingApp
jar
- 2.4.4
+ 2.5.0
uJETL
https://github.com/rasilon/ujetl
@@ -78,6 +78,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma
postgresql
42.4.3
+
+ net.sourceforge.csvjdbc
+ csvjdbc
+ 1.0.40
+
diff --git a/src/main/java/com/rasilon/ujetl/CopyingApp.java b/src/main/java/com/rasilon/ujetl/CopyingApp.java
index e1e47a5..fa830c8 100644
--- a/src/main/java/com/rasilon/ujetl/CopyingApp.java
+++ b/src/main/java/com/rasilon/ujetl/CopyingApp.java
@@ -34,10 +34,6 @@ public class CopyingApp {
public static void main(String[] args) {
CopyingAppCommandParser cli = new CopyingAppCommandParser(args);
LoggerContext context = (org.apache.logging.log4j.core.LoggerContext) LogManager.getContext(false);
- String log4jConfigLocation = cli.getLog4jConfigFile();
- File file = new File(log4jConfigLocation);
- context.setConfigLocation(file.toURI());
- System.out.println("Config set from "+file.toURI());
CopyingApp app = new CopyingApp(cli);
try {
@@ -79,6 +75,7 @@ public class CopyingApp {
Configuration config = configs.xml(cli.getConfigFile());
+ loadDrivers(config);
String hardLimitSeconds = config.getString("hardLimitSeconds");
if(hardLimitSeconds != null) {
TimeLimiter hardLimit = new TimeLimiter(Integer.decode(hardLimitSeconds).intValue(),true);
@@ -108,14 +105,14 @@ public class CopyingApp {
log.info(String.format("%s - Setting Row count interval to default of 100 rows.",jobName));
}
- Integer pollTimeout = null;
- try {
- pollTimeout = new Integer(config.getString("pollTimeout"));
- log.info(String.format("%s - Setting Poll timeout to %s milliseconds", jobName, pollTimeout));
- } catch(Exception e) {
- pollTimeout = new Integer(1000); // If we don't have a new setting, use the old default
- log.info(String.format("%s - Setting poll timeout to default of 1 second.",jobName));
- }
+ Integer pollTimeout = null;
+ try {
+ pollTimeout = new Integer(config.getString("pollTimeout"));
+ log.info(String.format("%s - Setting Poll timeout to %s milliseconds", jobName, pollTimeout));
+ } catch(Exception e) {
+ pollTimeout = new Integer(1000); // If we don't have a new setting, use the old default
+ log.info(String.format("%s - Setting poll timeout to default of 1 second.",jobName));
+ }
@@ -151,7 +148,7 @@ public class CopyingApp {
pollTimeout,
identifySourceSQL,
identifyDestinationSQL
- );
+ );
j.start();
j.join();
@@ -182,7 +179,7 @@ public class CopyingApp {
pollTimeout,
identifySourceSQL,
identifyDestinationSQL
- );
+ );
j.start();
j.join();
} else {
@@ -244,4 +241,21 @@ public class CopyingApp {
return c;
}
+
+ // Even with JDBC 4, some drivers don't play nicely with whatever
+ // the classloaders are up to. So this allows us to force it the
+ // old fashioned way, and works around the
+ // "But it works fine when it's the /only/ driver!"
+ // cross-database problem
+ private void loadDrivers(Configuration config) {
+ String[] drivers = config.get(String[].class, "drivers.driver");
+ for(String d:drivers) {
+ try {
+ Class.forName(d);
+ log.info("Preloaded driver "+d);
+ } catch(ClassNotFoundException e) {
+ log.error("Could not preload driver "+d,e);
+ }
+ }
+ }
}
diff --git a/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java b/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java
index b700221..97d2dc8 100644
--- a/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java
+++ b/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java
@@ -23,8 +23,4 @@ public class CopyingAppCommandParser {
return configFile;
}
- public String getLog4jConfigFile() {
- return log4jConfigFile;
- }
-
}
diff --git a/src/test/java/com/rasilon/ujetl/TestConfig.java b/src/test/java/com/rasilon/ujetl/TestConfig.java
new file mode 100644
index 0000000..05f9d82
--- /dev/null
+++ b/src/test/java/com/rasilon/ujetl/TestConfig.java
@@ -0,0 +1,37 @@
+package com.rasilon.ujetl;
+
+import org.apache.commons.configuration2.Configuration;
+import org.apache.commons.configuration2.builder.fluent.Configurations;
+import org.apache.commons.configuration2.ex.ConfigurationException;
+
+import org.apache.commons.beanutils.PropertyUtils; // Why does config need this?
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.MethodOrderer.Alphanumeric;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+
+
+/**
+ * @author derryh
+ *
+ */
+public class TestConfig {
+
+ @Test
+ public void test001VerifyArrayOfDrivers() {
+ try {
+ Configurations configs = new Configurations();
+ Configuration config = configs.xml("TEST_config_live.xml");
+ String[] drivers = config.get(String[].class, "drivers.driver");
+ int ndrivers =drivers.length;
+ if(ndrivers != 3){
+ fail("Expected 3 drivers, but found "+ndrivers);
+ }
+ } catch(Exception e) {
+ fail(e.toString());
+ }
+ }
+
+}
diff --git a/src/test/java/com/rasilon/ujetl/TestParser.java b/src/test/java/com/rasilon/ujetl/TestParser.java
index 13366fd..398f046 100644
--- a/src/test/java/com/rasilon/ujetl/TestParser.java
+++ b/src/test/java/com/rasilon/ujetl/TestParser.java
@@ -13,15 +13,12 @@ public class TestParser {
public void test001Parset() {
try {
String[] args = {
- "--log4j",
- "log4j_test_banana.xml",
"--config",
"config_test_banana.xml"
};
CopyingAppCommandParser p = new CopyingAppCommandParser(args);
assertEquals(p.getConfigFile(),"config_test_banana.xml");
- assertEquals(p.getLog4jConfigFile(),"log4j_test_banana.xml");
} catch(Exception e) {
fail(e.toString());
diff --git a/src/test/resources/TEST_config_live.xml b/src/test/resources/TEST_config_live.xml
index 338faff..210d8f1 100644
--- a/src/test/resources/TEST_config_live.xml
+++ b/src/test/resources/TEST_config_live.xml
@@ -4,6 +4,11 @@
10000
1000
500
+
+ org.postgresql.Driver
+ org.h2.Driver
+ org.relique.jdbc.csv.CsvDriver
+
jdbc:postgresql://localhost:5432/test
test