diff --git a/build_util/build_rpm b/build_util/build_rpm deleted file mode 100755 index 769915f..0000000 --- a/build_util/build_rpm +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -set -e - -cd /root -cp -Rv build build2 -cd build2 - -SPEC=$(ls *.spec) -VER=$(grep Version $SPEC | awk '{print $2}') -tar cvf $HOME/rpmbuild/SOURCES/uJETL-${VER}.tar.gz --show-transformed --transform="s/^\./uJETL-${VER}/" . -rpmbuild -ba $SPEC -cp /root/rpmbuild/RPMS/x86_64/* /root/build/ diff --git a/build_util/build_rpms_in_docker b/build_util/build_rpms_in_docker deleted file mode 100755 index acc6fee..0000000 --- a/build_util/build_rpms_in_docker +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -e - -docker build --rm -t local/c7-buildhost docker/build - -docker run -it --rm -v `pwd`:/root/build local/c7-buildhost /root/build/build_util/build_rpm diff --git a/build_util/create_run_docker b/build_util/create_run_docker index 1acbb42..864fdf6 100755 --- a/build_util/create_run_docker +++ b/build_util/create_run_docker @@ -1,2 +1,3 @@ #!/bin/bash docker build --target deploy -t rasilon/ujetl docker/multistage +docker tag rasilon/ujetl:latest rasilon/ujetl:$(xpath -q -e '/project/version/text()' pom.xml) diff --git a/build_util/push_docker_images b/build_util/push_docker_images new file mode 100755 index 0000000..14afd64 --- /dev/null +++ b/build_util/push_docker_images @@ -0,0 +1,4 @@ +#!/bin/bash +docker push rasilon/ujetl:latest +docker push rasilon/ujetl:$(xpath -q -e '/project/version/text()' pom.xml) + diff --git a/docker/multistage/Dockerfile b/docker/multistage/Dockerfile index f529941..dfc23a9 100644 --- a/docker/multistage/Dockerfile +++ b/docker/multistage/Dockerfile @@ -1,17 +1,13 @@ -FROM centos:centos7 as builder -RUN yum -y install epel-release java-1.8.0-openjdk-devel -RUN yum -y groupinstall 'Development Tools' -RUN yum -y install git maven -RUN git clone https://github.com/rasilon/ujetl.git -RUN cd ujetl && mvn package +FROM ubuntu:22.04 as builder +RUN apt-get update && apt-get -y upgrade +RUN apt-get -y install openjdk-19-jdk-headless maven git +RUN git clone --single-branch --branch main https://github.com/rasilon/ujetl.git +RUN cd ujetl && mvn -e package - - - -FROM openjdk:8-alpine as runner +FROM openjdk:11 as runner LABEL maintainer="Derry Hamilton " -RUN apk update && apk upgrade && apk add bash +RUN apt update && apt upgrade -y && apt install -y bash RUN mkdir -p /usr/share/ujetl/lib/ /var/ujetl /etc/ujetl @@ -24,7 +20,7 @@ CMD ["/ujetl_entrypoint"] FROM runner as tester COPY TEST_config_live.xml /var/ujetl/ COPY wait_for_postgres / -RUN apk add postgresql-client +RUN apt-get install -y postgresql-client FROM runner as deploy diff --git a/docker/multistage/TEST_config_live.xml b/docker/multistage/TEST_config_live.xml index fa38eec..d1b7345 100644 --- a/docker/multistage/TEST_config_live.xml +++ b/docker/multistage/TEST_config_live.xml @@ -4,6 +4,11 @@ 10000 1000 500 + + org.postgresql.Driver + org.relique.jdbc.csv.CsvDriver + + jdbc:postgresql://testdb:5432/test test diff --git a/docker/multistage/small.csv b/docker/multistage/small.csv new file mode 100644 index 0000000..37ddeff --- /dev/null +++ b/docker/multistage/small.csv @@ -0,0 +1,4 @@ +id,dat +1,banana +2,potato +3,nugget diff --git a/docker/multistage/ujetl_entrypoint b/docker/multistage/ujetl_entrypoint index 9979de6..03870df 100755 --- a/docker/multistage/ujetl_entrypoint +++ b/docker/multistage/ujetl_entrypoint @@ -10,10 +10,11 @@ ls echo Starting run loop for file in *.xml do - /usr/bin/java \ + /usr/local/openjdk-11/bin/java \ -Xms1g \ -Xmx2g \ -cp /usr/share/ujetl/lib/CopyingApp.jar \ + -Dlog4j.configurationFile="$LOG_PROPS" \ com.rasilon.ujetl.CopyingApp \ --log4j "$LOG_PROPS" \ --config "$file" diff --git a/docker/test_db/setup.sql b/docker/test_db/setup.sql index ff4a1b7..2469967 100644 --- a/docker/test_db/setup.sql +++ b/docker/test_db/setup.sql @@ -44,10 +44,14 @@ CREATE TABLE denormalised_personalia( lname text ); +CREATE TABLE test_csvjdbc( + id integer not null primary key, + dat text +); + GRANT SELECT ON ALL TABLES IN SCHEMA public TO test; GRANT SELECT,INSERT,UPDATE ON denormalised_personalia TO test; - \c postgres CREATE TABLE public.container_ready AS SELECT 1 FROM(VALUES(1)) AS a(a); GRANT SELECT ON public.container_ready TO TEST; diff --git a/install_extra/run_copying_job b/install_extra/run_copying_job index bf51414..be923bd 100755 --- a/install_extra/run_copying_job +++ b/install_extra/run_copying_job @@ -30,9 +30,9 @@ fi /usr/bin/java \ -Xms1g \ -Xmx2g \ + -Dlog4j.configurationFile="$LOG_PROPS" \ -cp /usr/share/ujetl/lib/CopyingApp.jar \ com.rasilon.ujetl.CopyingApp \ - --log4j "$LOG_PROPS" \ --config "/etc/ujetl/${JOBNAME}_config_live.xml" #rm -f $LOCKFILE diff --git a/pom.xml b/pom.xml index 25d9bbf..6970fdb 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma com.rasilon.ujetl CopyingApp jar - 2.2.1 + 2.5.2 uJETL https://github.com/rasilon/ujetl @@ -35,13 +35,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma com.h2database h2 - 1.4.199 + 2.2.220 test org.apache.commons commons-lang3 - 3.9 + 3.18.0 commons-logging @@ -51,12 +51,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma org.apache.commons commons-configuration2 - 2.4 + 2.10.1 commons-beanutils commons-beanutils - 1.9.3 + 1.11.0 com.beust @@ -66,27 +66,31 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma org.apache.logging.log4j log4j-api - 2.11.2 + 2.17.1 org.apache.logging.log4j log4j-core - 2.11.2 + 2.25.3 org.postgresql postgresql - 42.2.5 + 42.7.2 + + + net.sourceforge.csvjdbc + csvjdbc + 1.0.40 maven-compiler-plugin - 2.3.2 + 3.8.0 - 1.8 - 1.8 + 11 diff --git a/src/main/java/com/rasilon/ujetl/CopyingApp.java b/src/main/java/com/rasilon/ujetl/CopyingApp.java index e1e47a5..fa830c8 100644 --- a/src/main/java/com/rasilon/ujetl/CopyingApp.java +++ b/src/main/java/com/rasilon/ujetl/CopyingApp.java @@ -34,10 +34,6 @@ public class CopyingApp { public static void main(String[] args) { CopyingAppCommandParser cli = new CopyingAppCommandParser(args); LoggerContext context = (org.apache.logging.log4j.core.LoggerContext) LogManager.getContext(false); - String log4jConfigLocation = cli.getLog4jConfigFile(); - File file = new File(log4jConfigLocation); - context.setConfigLocation(file.toURI()); - System.out.println("Config set from "+file.toURI()); CopyingApp app = new CopyingApp(cli); try { @@ -79,6 +75,7 @@ public class CopyingApp { Configuration config = configs.xml(cli.getConfigFile()); + loadDrivers(config); String hardLimitSeconds = config.getString("hardLimitSeconds"); if(hardLimitSeconds != null) { TimeLimiter hardLimit = new TimeLimiter(Integer.decode(hardLimitSeconds).intValue(),true); @@ -108,14 +105,14 @@ public class CopyingApp { log.info(String.format("%s - Setting Row count interval to default of 100 rows.",jobName)); } - Integer pollTimeout = null; - try { - pollTimeout = new Integer(config.getString("pollTimeout")); - log.info(String.format("%s - Setting Poll timeout to %s milliseconds", jobName, pollTimeout)); - } catch(Exception e) { - pollTimeout = new Integer(1000); // If we don't have a new setting, use the old default - log.info(String.format("%s - Setting poll timeout to default of 1 second.",jobName)); - } + Integer pollTimeout = null; + try { + pollTimeout = new Integer(config.getString("pollTimeout")); + log.info(String.format("%s - Setting Poll timeout to %s milliseconds", jobName, pollTimeout)); + } catch(Exception e) { + pollTimeout = new Integer(1000); // If we don't have a new setting, use the old default + log.info(String.format("%s - Setting poll timeout to default of 1 second.",jobName)); + } @@ -151,7 +148,7 @@ public class CopyingApp { pollTimeout, identifySourceSQL, identifyDestinationSQL - ); + ); j.start(); j.join(); @@ -182,7 +179,7 @@ public class CopyingApp { pollTimeout, identifySourceSQL, identifyDestinationSQL - ); + ); j.start(); j.join(); } else { @@ -244,4 +241,21 @@ public class CopyingApp { return c; } + + // Even with JDBC 4, some drivers don't play nicely with whatever + // the classloaders are up to. So this allows us to force it the + // old fashioned way, and works around the + // "But it works fine when it's the /only/ driver!" + // cross-database problem + private void loadDrivers(Configuration config) { + String[] drivers = config.get(String[].class, "drivers.driver"); + for(String d:drivers) { + try { + Class.forName(d); + log.info("Preloaded driver "+d); + } catch(ClassNotFoundException e) { + log.error("Could not preload driver "+d,e); + } + } + } } diff --git a/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java b/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java index d0d4d73..97d2dc8 100644 --- a/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java +++ b/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java @@ -12,7 +12,7 @@ public class CopyingAppCommandParser { private String configFile; @Parameter(names = {"-log4j","--log4j"}, description = "Log4J config file for this run") - private String log4jConfigFile = "/etc/ppl/default_log4j_config.properties"; + private String log4jConfigFile = "/etc/ujetl/default_log4j_config.properties"; public CopyingAppCommandParser(String[] args) { super(); @@ -23,8 +23,4 @@ public class CopyingAppCommandParser { return configFile; } - public String getLog4jConfigFile() { - return log4jConfigFile; - } - } diff --git a/src/test/java/com/rasilon/ujetl/TestConfig.java b/src/test/java/com/rasilon/ujetl/TestConfig.java new file mode 100644 index 0000000..05f9d82 --- /dev/null +++ b/src/test/java/com/rasilon/ujetl/TestConfig.java @@ -0,0 +1,37 @@ +package com.rasilon.ujetl; + +import org.apache.commons.configuration2.Configuration; +import org.apache.commons.configuration2.builder.fluent.Configurations; +import org.apache.commons.configuration2.ex.ConfigurationException; + +import org.apache.commons.beanutils.PropertyUtils; // Why does config need this? + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.MethodOrderer.Alphanumeric; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + + +/** + * @author derryh + * + */ +public class TestConfig { + + @Test + public void test001VerifyArrayOfDrivers() { + try { + Configurations configs = new Configurations(); + Configuration config = configs.xml("TEST_config_live.xml"); + String[] drivers = config.get(String[].class, "drivers.driver"); + int ndrivers =drivers.length; + if(ndrivers != 3){ + fail("Expected 3 drivers, but found "+ndrivers); + } + } catch(Exception e) { + fail(e.toString()); + } + } + +} diff --git a/src/test/java/com/rasilon/ujetl/TestJob.java b/src/test/java/com/rasilon/ujetl/TestJob.java index 853aeb4..63ab4c8 100644 --- a/src/test/java/com/rasilon/ujetl/TestJob.java +++ b/src/test/java/com/rasilon/ujetl/TestJob.java @@ -42,7 +42,7 @@ public class TestJob { dest, "jUnit Test Config", "jUnit Test Job", - "SELECT -1 AS key", + "SELECT -1 AS \"key\"", "SELECT id,dat FROM src WHERE id > ?", "INSERT INTO dest VALUES(?,?)", null, diff --git a/src/test/java/com/rasilon/ujetl/TestParser.java b/src/test/java/com/rasilon/ujetl/TestParser.java index 13366fd..398f046 100644 --- a/src/test/java/com/rasilon/ujetl/TestParser.java +++ b/src/test/java/com/rasilon/ujetl/TestParser.java @@ -13,15 +13,12 @@ public class TestParser { public void test001Parset() { try { String[] args = { - "--log4j", - "log4j_test_banana.xml", "--config", "config_test_banana.xml" }; CopyingAppCommandParser p = new CopyingAppCommandParser(args); assertEquals(p.getConfigFile(),"config_test_banana.xml"); - assertEquals(p.getLog4jConfigFile(),"log4j_test_banana.xml"); } catch(Exception e) { fail(e.toString()); diff --git a/src/test/java/com/rasilon/ujetl/TestPrePost.java b/src/test/java/com/rasilon/ujetl/TestPrePost.java index 77ff2a6..55f3ee1 100644 --- a/src/test/java/com/rasilon/ujetl/TestPrePost.java +++ b/src/test/java/com/rasilon/ujetl/TestPrePost.java @@ -42,7 +42,7 @@ public class TestPrePost { dest, "jUnit Test Config", "jUnit Test Job", - "SELECT -1 AS key", + "SELECT -1 AS \"key\"", "SELECT id,dat FROM src WHERE id > ?", "INSERT INTO tmp_dest VALUES(?,?)", "CREATE TEMP TABLE tmp_dest(id bigint not null primary key, dat varchar);", diff --git a/src/test/resources/TEST_config_live.xml b/src/test/resources/TEST_config_live.xml index 338faff..210d8f1 100644 --- a/src/test/resources/TEST_config_live.xml +++ b/src/test/resources/TEST_config_live.xml @@ -4,6 +4,11 @@ 10000 1000 500 + + org.postgresql.Driver + org.h2.Driver + org.relique.jdbc.csv.CsvDriver + jdbc:postgresql://localhost:5432/test test diff --git a/uJETL.spec b/uJETL.spec deleted file mode 100644 index 89b5fe5..0000000 --- a/uJETL.spec +++ /dev/null @@ -1,33 +0,0 @@ -Summary: Java app to facilitate moving data between databases. -Name: uJETL -Version: 2.2.1 -Release: 1 -Group: Applications/Database -License: All rights reserved. -Source: uJETL-%{version}.tar.gz -URL: https://github.com/rasilon/ujetl.git -Distribution: derryh -Vendor: derryh -Packager: Derry Hamilton -#BuildRoot: . - -%description -A very small ETL app - -%prep -%setup - -%build -#mvn -Dmaven.test.skip=true clean package -true - -%install -mkdir -p $RPM_BUILD_ROOT/usr/share/ujetl/lib $RPM_BUILD_ROOT/etc/ujetl $RPM_BUILD_ROOT/usr/bin -cp target/CopyingApp-*-jar-with-dependencies.jar $RPM_BUILD_ROOT/usr/share/ujetl/lib/CopyingApp.jar -cp install_extra/run_copying_job $RPM_BUILD_ROOT/usr/bin -cp install_extra/copying_defaults_log4j.xml $RPM_BUILD_ROOT/etc/ujetl - -%files -/usr/share/ujetl/lib/CopyingApp.jar -/usr/bin/run_copying_job -/etc/ujetl/copying_defaults_log4j.xml