From 866d02fb525ad611f0dfdaddf1e561ba5a0c3b0e Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Tue, 11 Jul 2023 14:07:37 +0100 Subject: [PATCH 1/4] Change how log4j finds its config. log4j is reacting less well to reconfigurations, so this moves the config file location to earlier in the process, so that no class sees an unconfigured instance. --- install_extra/run_copying_job | 2 +- pom.xml | 2 +- src/main/java/com/rasilon/ujetl/CopyingApp.java | 4 ---- src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java | 4 ---- src/test/java/com/rasilon/ujetl/TestParser.java | 3 --- 5 files changed, 2 insertions(+), 13 deletions(-) diff --git a/install_extra/run_copying_job b/install_extra/run_copying_job index bf51414..5dfa2b0 100755 --- a/install_extra/run_copying_job +++ b/install_extra/run_copying_job @@ -30,9 +30,9 @@ fi /usr/bin/java \ -Xms1g \ -Xmx2g \ + -Dlog4j.configuration=file:"$LOG_PROPS" \ -cp /usr/share/ujetl/lib/CopyingApp.jar \ com.rasilon.ujetl.CopyingApp \ - --log4j "$LOG_PROPS" \ --config "/etc/ujetl/${JOBNAME}_config_live.xml" #rm -f $LOCKFILE diff --git a/pom.xml b/pom.xml index 304defa..043daad 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma com.rasilon.ujetl CopyingApp jar - 2.4.4 + 2.5.0 uJETL https://github.com/rasilon/ujetl diff --git a/src/main/java/com/rasilon/ujetl/CopyingApp.java b/src/main/java/com/rasilon/ujetl/CopyingApp.java index e1e47a5..1100d93 100644 --- a/src/main/java/com/rasilon/ujetl/CopyingApp.java +++ b/src/main/java/com/rasilon/ujetl/CopyingApp.java @@ -34,10 +34,6 @@ public class CopyingApp { public static void main(String[] args) { CopyingAppCommandParser cli = new CopyingAppCommandParser(args); LoggerContext context = (org.apache.logging.log4j.core.LoggerContext) LogManager.getContext(false); - String log4jConfigLocation = cli.getLog4jConfigFile(); - File file = new File(log4jConfigLocation); - context.setConfigLocation(file.toURI()); - System.out.println("Config set from "+file.toURI()); CopyingApp app = new CopyingApp(cli); try { diff --git a/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java b/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java index b700221..97d2dc8 100644 --- a/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java +++ b/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java @@ -23,8 +23,4 @@ public class CopyingAppCommandParser { return configFile; } - public String getLog4jConfigFile() { - return log4jConfigFile; - } - } diff --git a/src/test/java/com/rasilon/ujetl/TestParser.java b/src/test/java/com/rasilon/ujetl/TestParser.java index 13366fd..398f046 100644 --- a/src/test/java/com/rasilon/ujetl/TestParser.java +++ b/src/test/java/com/rasilon/ujetl/TestParser.java @@ -13,15 +13,12 @@ public class TestParser { public void test001Parset() { try { String[] args = { - "--log4j", - "log4j_test_banana.xml", "--config", "config_test_banana.xml" }; CopyingAppCommandParser p = new CopyingAppCommandParser(args); assertEquals(p.getConfigFile(),"config_test_banana.xml"); - assertEquals(p.getLog4jConfigFile(),"log4j_test_banana.xml"); } catch(Exception e) { fail(e.toString()); From 1b1ba551c8bd47b319d60b7d208c9fe71d6c76c8 Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Tue, 11 Jul 2023 15:04:33 +0100 Subject: [PATCH 2/4] Add forced driver loading --- docker/multistage/TEST_config_live.xml | 5 +++ docker/multistage/small.csv | 4 ++ docker/test_db/setup.sql | 6 ++- pom.xml | 5 +++ .../java/com/rasilon/ujetl/CopyingApp.java | 8 ++++ .../java/com/rasilon/ujetl/TestConfig.java | 37 +++++++++++++++++++ src/test/resources/TEST_config_live.xml | 5 +++ 7 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 docker/multistage/small.csv create mode 100644 src/test/java/com/rasilon/ujetl/TestConfig.java diff --git a/docker/multistage/TEST_config_live.xml b/docker/multistage/TEST_config_live.xml index fa38eec..d1b7345 100644 --- a/docker/multistage/TEST_config_live.xml +++ b/docker/multistage/TEST_config_live.xml @@ -4,6 +4,11 @@ 10000 1000 500 + + org.postgresql.Driver + org.relique.jdbc.csv.CsvDriver + + jdbc:postgresql://testdb:5432/test test diff --git a/docker/multistage/small.csv b/docker/multistage/small.csv new file mode 100644 index 0000000..37ddeff --- /dev/null +++ b/docker/multistage/small.csv @@ -0,0 +1,4 @@ +id,dat +1,banana +2,potato +3,nugget diff --git a/docker/test_db/setup.sql b/docker/test_db/setup.sql index ff4a1b7..2469967 100644 --- a/docker/test_db/setup.sql +++ b/docker/test_db/setup.sql @@ -44,10 +44,14 @@ CREATE TABLE denormalised_personalia( lname text ); +CREATE TABLE test_csvjdbc( + id integer not null primary key, + dat text +); + GRANT SELECT ON ALL TABLES IN SCHEMA public TO test; GRANT SELECT,INSERT,UPDATE ON denormalised_personalia TO test; - \c postgres CREATE TABLE public.container_ready AS SELECT 1 FROM(VALUES(1)) AS a(a); GRANT SELECT ON public.container_ready TO TEST; diff --git a/pom.xml b/pom.xml index 043daad..3b55a96 100644 --- a/pom.xml +++ b/pom.xml @@ -78,6 +78,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma postgresql 42.4.3 + + net.sourceforge.csvjdbc + csvjdbc + 1.0.40 + diff --git a/src/main/java/com/rasilon/ujetl/CopyingApp.java b/src/main/java/com/rasilon/ujetl/CopyingApp.java index 1100d93..60dd65d 100644 --- a/src/main/java/com/rasilon/ujetl/CopyingApp.java +++ b/src/main/java/com/rasilon/ujetl/CopyingApp.java @@ -75,6 +75,7 @@ public class CopyingApp { Configuration config = configs.xml(cli.getConfigFile()); + loadDrivers(config); String hardLimitSeconds = config.getString("hardLimitSeconds"); if(hardLimitSeconds != null) { TimeLimiter hardLimit = new TimeLimiter(Integer.decode(hardLimitSeconds).intValue(),true); @@ -240,4 +241,11 @@ public class CopyingApp { return c; } + + private void loadDrivers(Configuration config){ + String[] drivers = config.get(String[].class, "drivers.driver"); + for(String d:drivers){ + log.info("Would load "+d); + } + } } diff --git a/src/test/java/com/rasilon/ujetl/TestConfig.java b/src/test/java/com/rasilon/ujetl/TestConfig.java new file mode 100644 index 0000000..bbf855b --- /dev/null +++ b/src/test/java/com/rasilon/ujetl/TestConfig.java @@ -0,0 +1,37 @@ +package com.rasilon.ujetl; + +import org.apache.commons.configuration2.Configuration; +import org.apache.commons.configuration2.builder.fluent.Configurations; +import org.apache.commons.configuration2.ex.ConfigurationException; + +import org.apache.commons.beanutils.PropertyUtils; // Why does config need this? + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.MethodOrderer.Alphanumeric; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + + +/** + * @author derryh + * + */ +public class TestConfig { + + @Test + public void test002verifyH2Works() { + try { + Configurations configs = new Configurations(); + Configuration config = configs.xml("TEST_config_live.xml"); + String[] drivers = config.get(String[].class, "drivers.driver"); + int ndrivers =drivers.length; + if(ndrivers != 3){ + fail("Expected 3 drivers, but found "+ndrivers); + } + } catch(Exception e) { + fail(e.toString()); + } + } + +} diff --git a/src/test/resources/TEST_config_live.xml b/src/test/resources/TEST_config_live.xml index 338faff..210d8f1 100644 --- a/src/test/resources/TEST_config_live.xml +++ b/src/test/resources/TEST_config_live.xml @@ -4,6 +4,11 @@ 10000 1000 500 + + org.postgresql.Driver + org.h2.Driver + org.relique.jdbc.csv.CsvDriver + jdbc:postgresql://localhost:5432/test test From 9a8716f33e7a4a6f0638b9c6beee5747518ccfe9 Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Wed, 12 Jul 2023 08:45:48 +0100 Subject: [PATCH 3/4] Add the real implementation --- .../java/com/rasilon/ujetl/CopyingApp.java | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/rasilon/ujetl/CopyingApp.java b/src/main/java/com/rasilon/ujetl/CopyingApp.java index 60dd65d..fa830c8 100644 --- a/src/main/java/com/rasilon/ujetl/CopyingApp.java +++ b/src/main/java/com/rasilon/ujetl/CopyingApp.java @@ -105,14 +105,14 @@ public class CopyingApp { log.info(String.format("%s - Setting Row count interval to default of 100 rows.",jobName)); } - Integer pollTimeout = null; - try { - pollTimeout = new Integer(config.getString("pollTimeout")); - log.info(String.format("%s - Setting Poll timeout to %s milliseconds", jobName, pollTimeout)); - } catch(Exception e) { - pollTimeout = new Integer(1000); // If we don't have a new setting, use the old default - log.info(String.format("%s - Setting poll timeout to default of 1 second.",jobName)); - } + Integer pollTimeout = null; + try { + pollTimeout = new Integer(config.getString("pollTimeout")); + log.info(String.format("%s - Setting Poll timeout to %s milliseconds", jobName, pollTimeout)); + } catch(Exception e) { + pollTimeout = new Integer(1000); // If we don't have a new setting, use the old default + log.info(String.format("%s - Setting poll timeout to default of 1 second.",jobName)); + } @@ -148,7 +148,7 @@ public class CopyingApp { pollTimeout, identifySourceSQL, identifyDestinationSQL - ); + ); j.start(); j.join(); @@ -179,7 +179,7 @@ public class CopyingApp { pollTimeout, identifySourceSQL, identifyDestinationSQL - ); + ); j.start(); j.join(); } else { @@ -242,10 +242,20 @@ public class CopyingApp { return c; } - private void loadDrivers(Configuration config){ + // Even with JDBC 4, some drivers don't play nicely with whatever + // the classloaders are up to. So this allows us to force it the + // old fashioned way, and works around the + // "But it works fine when it's the /only/ driver!" + // cross-database problem + private void loadDrivers(Configuration config) { String[] drivers = config.get(String[].class, "drivers.driver"); - for(String d:drivers){ - log.info("Would load "+d); + for(String d:drivers) { + try { + Class.forName(d); + log.info("Preloaded driver "+d); + } catch(ClassNotFoundException e) { + log.error("Could not preload driver "+d,e); + } } } } From a88ac568485aa46d3bb2de41eeb814076bdee7df Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Wed, 12 Jul 2023 08:48:21 +0100 Subject: [PATCH 4/4] Correct test name; CnP error --- src/test/java/com/rasilon/ujetl/TestConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/rasilon/ujetl/TestConfig.java b/src/test/java/com/rasilon/ujetl/TestConfig.java index bbf855b..05f9d82 100644 --- a/src/test/java/com/rasilon/ujetl/TestConfig.java +++ b/src/test/java/com/rasilon/ujetl/TestConfig.java @@ -20,7 +20,7 @@ import static org.junit.jupiter.api.Assertions.fail; public class TestConfig { @Test - public void test002verifyH2Works() { + public void test001VerifyArrayOfDrivers() { try { Configurations configs = new Configurations(); Configuration config = configs.xml("TEST_config_live.xml");