diff --git a/docker/multistage/TEST_config_live.xml b/docker/multistage/TEST_config_live.xml index fa38eec..d1b7345 100644 --- a/docker/multistage/TEST_config_live.xml +++ b/docker/multistage/TEST_config_live.xml @@ -4,6 +4,11 @@ 10000 1000 500 + + org.postgresql.Driver + org.relique.jdbc.csv.CsvDriver + + jdbc:postgresql://testdb:5432/test test diff --git a/docker/multistage/small.csv b/docker/multistage/small.csv new file mode 100644 index 0000000..37ddeff --- /dev/null +++ b/docker/multistage/small.csv @@ -0,0 +1,4 @@ +id,dat +1,banana +2,potato +3,nugget diff --git a/docker/test_db/setup.sql b/docker/test_db/setup.sql index ff4a1b7..2469967 100644 --- a/docker/test_db/setup.sql +++ b/docker/test_db/setup.sql @@ -44,10 +44,14 @@ CREATE TABLE denormalised_personalia( lname text ); +CREATE TABLE test_csvjdbc( + id integer not null primary key, + dat text +); + GRANT SELECT ON ALL TABLES IN SCHEMA public TO test; GRANT SELECT,INSERT,UPDATE ON denormalised_personalia TO test; - \c postgres CREATE TABLE public.container_ready AS SELECT 1 FROM(VALUES(1)) AS a(a); GRANT SELECT ON public.container_ready TO TEST; diff --git a/install_extra/run_copying_job b/install_extra/run_copying_job index bf51414..5dfa2b0 100755 --- a/install_extra/run_copying_job +++ b/install_extra/run_copying_job @@ -30,9 +30,9 @@ fi /usr/bin/java \ -Xms1g \ -Xmx2g \ + -Dlog4j.configuration=file:"$LOG_PROPS" \ -cp /usr/share/ujetl/lib/CopyingApp.jar \ com.rasilon.ujetl.CopyingApp \ - --log4j "$LOG_PROPS" \ --config "/etc/ujetl/${JOBNAME}_config_live.xml" #rm -f $LOCKFILE diff --git a/pom.xml b/pom.xml index 304defa..3b55a96 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma com.rasilon.ujetl CopyingApp jar - 2.4.4 + 2.5.0 uJETL https://github.com/rasilon/ujetl @@ -78,6 +78,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma postgresql 42.4.3 + + net.sourceforge.csvjdbc + csvjdbc + 1.0.40 + diff --git a/src/main/java/com/rasilon/ujetl/CopyingApp.java b/src/main/java/com/rasilon/ujetl/CopyingApp.java index e1e47a5..fa830c8 100644 --- a/src/main/java/com/rasilon/ujetl/CopyingApp.java +++ b/src/main/java/com/rasilon/ujetl/CopyingApp.java @@ -34,10 +34,6 @@ public class CopyingApp { public static void main(String[] args) { CopyingAppCommandParser cli = new CopyingAppCommandParser(args); LoggerContext context = (org.apache.logging.log4j.core.LoggerContext) LogManager.getContext(false); - String log4jConfigLocation = cli.getLog4jConfigFile(); - File file = new File(log4jConfigLocation); - context.setConfigLocation(file.toURI()); - System.out.println("Config set from "+file.toURI()); CopyingApp app = new CopyingApp(cli); try { @@ -79,6 +75,7 @@ public class CopyingApp { Configuration config = configs.xml(cli.getConfigFile()); + loadDrivers(config); String hardLimitSeconds = config.getString("hardLimitSeconds"); if(hardLimitSeconds != null) { TimeLimiter hardLimit = new TimeLimiter(Integer.decode(hardLimitSeconds).intValue(),true); @@ -108,14 +105,14 @@ public class CopyingApp { log.info(String.format("%s - Setting Row count interval to default of 100 rows.",jobName)); } - Integer pollTimeout = null; - try { - pollTimeout = new Integer(config.getString("pollTimeout")); - log.info(String.format("%s - Setting Poll timeout to %s milliseconds", jobName, pollTimeout)); - } catch(Exception e) { - pollTimeout = new Integer(1000); // If we don't have a new setting, use the old default - log.info(String.format("%s - Setting poll timeout to default of 1 second.",jobName)); - } + Integer pollTimeout = null; + try { + pollTimeout = new Integer(config.getString("pollTimeout")); + log.info(String.format("%s - Setting Poll timeout to %s milliseconds", jobName, pollTimeout)); + } catch(Exception e) { + pollTimeout = new Integer(1000); // If we don't have a new setting, use the old default + log.info(String.format("%s - Setting poll timeout to default of 1 second.",jobName)); + } @@ -151,7 +148,7 @@ public class CopyingApp { pollTimeout, identifySourceSQL, identifyDestinationSQL - ); + ); j.start(); j.join(); @@ -182,7 +179,7 @@ public class CopyingApp { pollTimeout, identifySourceSQL, identifyDestinationSQL - ); + ); j.start(); j.join(); } else { @@ -244,4 +241,21 @@ public class CopyingApp { return c; } + + // Even with JDBC 4, some drivers don't play nicely with whatever + // the classloaders are up to. So this allows us to force it the + // old fashioned way, and works around the + // "But it works fine when it's the /only/ driver!" + // cross-database problem + private void loadDrivers(Configuration config) { + String[] drivers = config.get(String[].class, "drivers.driver"); + for(String d:drivers) { + try { + Class.forName(d); + log.info("Preloaded driver "+d); + } catch(ClassNotFoundException e) { + log.error("Could not preload driver "+d,e); + } + } + } } diff --git a/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java b/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java index b700221..97d2dc8 100644 --- a/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java +++ b/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java @@ -23,8 +23,4 @@ public class CopyingAppCommandParser { return configFile; } - public String getLog4jConfigFile() { - return log4jConfigFile; - } - } diff --git a/src/test/java/com/rasilon/ujetl/TestConfig.java b/src/test/java/com/rasilon/ujetl/TestConfig.java new file mode 100644 index 0000000..05f9d82 --- /dev/null +++ b/src/test/java/com/rasilon/ujetl/TestConfig.java @@ -0,0 +1,37 @@ +package com.rasilon.ujetl; + +import org.apache.commons.configuration2.Configuration; +import org.apache.commons.configuration2.builder.fluent.Configurations; +import org.apache.commons.configuration2.ex.ConfigurationException; + +import org.apache.commons.beanutils.PropertyUtils; // Why does config need this? + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.MethodOrderer.Alphanumeric; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + + +/** + * @author derryh + * + */ +public class TestConfig { + + @Test + public void test001VerifyArrayOfDrivers() { + try { + Configurations configs = new Configurations(); + Configuration config = configs.xml("TEST_config_live.xml"); + String[] drivers = config.get(String[].class, "drivers.driver"); + int ndrivers =drivers.length; + if(ndrivers != 3){ + fail("Expected 3 drivers, but found "+ndrivers); + } + } catch(Exception e) { + fail(e.toString()); + } + } + +} diff --git a/src/test/java/com/rasilon/ujetl/TestParser.java b/src/test/java/com/rasilon/ujetl/TestParser.java index 13366fd..398f046 100644 --- a/src/test/java/com/rasilon/ujetl/TestParser.java +++ b/src/test/java/com/rasilon/ujetl/TestParser.java @@ -13,15 +13,12 @@ public class TestParser { public void test001Parset() { try { String[] args = { - "--log4j", - "log4j_test_banana.xml", "--config", "config_test_banana.xml" }; CopyingAppCommandParser p = new CopyingAppCommandParser(args); assertEquals(p.getConfigFile(),"config_test_banana.xml"); - assertEquals(p.getLog4jConfigFile(),"log4j_test_banana.xml"); } catch(Exception e) { fail(e.toString()); diff --git a/src/test/resources/TEST_config_live.xml b/src/test/resources/TEST_config_live.xml index 338faff..210d8f1 100644 --- a/src/test/resources/TEST_config_live.xml +++ b/src/test/resources/TEST_config_live.xml @@ -4,6 +4,11 @@ 10000 1000 500 + + org.postgresql.Driver + org.h2.Driver + org.relique.jdbc.csv.CsvDriver + jdbc:postgresql://localhost:5432/test test