diff --git a/build_util/build_rpm b/build_util/build_rpm new file mode 100755 index 0000000..769915f --- /dev/null +++ b/build_util/build_rpm @@ -0,0 +1,12 @@ +#!/bin/bash +set -e + +cd /root +cp -Rv build build2 +cd build2 + +SPEC=$(ls *.spec) +VER=$(grep Version $SPEC | awk '{print $2}') +tar cvf $HOME/rpmbuild/SOURCES/uJETL-${VER}.tar.gz --show-transformed --transform="s/^\./uJETL-${VER}/" . +rpmbuild -ba $SPEC +cp /root/rpmbuild/RPMS/x86_64/* /root/build/ diff --git a/build_util/build_rpms_in_docker b/build_util/build_rpms_in_docker new file mode 100755 index 0000000..acc6fee --- /dev/null +++ b/build_util/build_rpms_in_docker @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +docker build --rm -t local/c7-buildhost docker/build + +docker run -it --rm -v `pwd`:/root/build local/c7-buildhost /root/build/build_util/build_rpm diff --git a/build_util/create_run_docker b/build_util/create_run_docker index 864fdf6..1acbb42 100755 --- a/build_util/create_run_docker +++ b/build_util/create_run_docker @@ -1,3 +1,2 @@ #!/bin/bash docker build --target deploy -t rasilon/ujetl docker/multistage -docker tag rasilon/ujetl:latest rasilon/ujetl:$(xpath -q -e '/project/version/text()' pom.xml) diff --git a/build_util/push_docker_images b/build_util/push_docker_images deleted file mode 100755 index 14afd64..0000000 --- a/build_util/push_docker_images +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -docker push rasilon/ujetl:latest -docker push rasilon/ujetl:$(xpath -q -e '/project/version/text()' pom.xml) - diff --git a/docker/multistage/Dockerfile b/docker/multistage/Dockerfile index dfc23a9..6a7550d 100644 --- a/docker/multistage/Dockerfile +++ b/docker/multistage/Dockerfile @@ -1,13 +1,19 @@ -FROM ubuntu:22.04 as builder -RUN apt-get update && apt-get -y upgrade -RUN apt-get -y install openjdk-19-jdk-headless maven git -RUN git clone --single-branch --branch main https://github.com/rasilon/ujetl.git -RUN cd ujetl && mvn -e package +FROM centos:centos8 as builder +RUN yum -y install epel-release java-11-openjdk-devel +RUN yum -y groupinstall 'Development Tools' +RUN yum -y install git maven +RUN alternatives --set javac /usr/lib/jvm/java-11-openjdk-11.*/bin/javac +RUN alternatives --set java /usr/lib/jvm/java-11-openjdk-11.*/bin/java +RUN git clone --single-branch --branch master https://github.com/rasilon/ujetl.git +RUN cd ujetl && JAVA_HOME=/usr/lib/jvm/java-11 mvn -e package + + + FROM openjdk:11 as runner LABEL maintainer="Derry Hamilton " -RUN apt update && apt upgrade -y && apt install -y bash +RUN apt update && apt upgrade && apt install bash RUN mkdir -p /usr/share/ujetl/lib/ /var/ujetl /etc/ujetl diff --git a/docker/multistage/TEST_config_live.xml b/docker/multistage/TEST_config_live.xml index d1b7345..fa38eec 100644 --- a/docker/multistage/TEST_config_live.xml +++ b/docker/multistage/TEST_config_live.xml @@ -4,11 +4,6 @@ 10000 1000 500 - - org.postgresql.Driver - org.relique.jdbc.csv.CsvDriver - - jdbc:postgresql://testdb:5432/test test diff --git a/docker/multistage/small.csv b/docker/multistage/small.csv deleted file mode 100644 index 37ddeff..0000000 --- a/docker/multistage/small.csv +++ /dev/null @@ -1,4 +0,0 @@ -id,dat -1,banana -2,potato -3,nugget diff --git a/docker/test_db/setup.sql b/docker/test_db/setup.sql index 2469967..ff4a1b7 100644 --- a/docker/test_db/setup.sql +++ b/docker/test_db/setup.sql @@ -44,14 +44,10 @@ CREATE TABLE denormalised_personalia( lname text ); -CREATE TABLE test_csvjdbc( - id integer not null primary key, - dat text -); - GRANT SELECT ON ALL TABLES IN SCHEMA public TO test; GRANT SELECT,INSERT,UPDATE ON denormalised_personalia TO test; + \c postgres CREATE TABLE public.container_ready AS SELECT 1 FROM(VALUES(1)) AS a(a); GRANT SELECT ON public.container_ready TO TEST; diff --git a/install_extra/run_copying_job b/install_extra/run_copying_job index be923bd..bf51414 100755 --- a/install_extra/run_copying_job +++ b/install_extra/run_copying_job @@ -30,9 +30,9 @@ fi /usr/bin/java \ -Xms1g \ -Xmx2g \ - -Dlog4j.configurationFile="$LOG_PROPS" \ -cp /usr/share/ujetl/lib/CopyingApp.jar \ com.rasilon.ujetl.CopyingApp \ + --log4j "$LOG_PROPS" \ --config "/etc/ujetl/${JOBNAME}_config_live.xml" #rm -f $LOCKFILE diff --git a/pom.xml b/pom.xml index 6970fdb..53418c4 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma com.rasilon.ujetl CopyingApp jar - 2.5.2 + 2.3.1 uJETL https://github.com/rasilon/ujetl @@ -35,13 +35,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma com.h2database h2 - 2.2.220 + 1.4.199 test org.apache.commons commons-lang3 - 3.18.0 + 3.9 commons-logging @@ -51,12 +51,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma org.apache.commons commons-configuration2 - 2.10.1 + 2.7 commons-beanutils commons-beanutils - 1.11.0 + 1.9.4 com.beust @@ -66,22 +66,17 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma org.apache.logging.log4j log4j-api - 2.17.1 + 2.15.0 org.apache.logging.log4j log4j-core - 2.25.3 + 2.15.0 org.postgresql postgresql - 42.7.2 - - - net.sourceforge.csvjdbc - csvjdbc - 1.0.40 + 42.2.5 diff --git a/src/main/java/com/rasilon/ujetl/CopyingApp.java b/src/main/java/com/rasilon/ujetl/CopyingApp.java index fa830c8..e1e47a5 100644 --- a/src/main/java/com/rasilon/ujetl/CopyingApp.java +++ b/src/main/java/com/rasilon/ujetl/CopyingApp.java @@ -34,6 +34,10 @@ public class CopyingApp { public static void main(String[] args) { CopyingAppCommandParser cli = new CopyingAppCommandParser(args); LoggerContext context = (org.apache.logging.log4j.core.LoggerContext) LogManager.getContext(false); + String log4jConfigLocation = cli.getLog4jConfigFile(); + File file = new File(log4jConfigLocation); + context.setConfigLocation(file.toURI()); + System.out.println("Config set from "+file.toURI()); CopyingApp app = new CopyingApp(cli); try { @@ -75,7 +79,6 @@ public class CopyingApp { Configuration config = configs.xml(cli.getConfigFile()); - loadDrivers(config); String hardLimitSeconds = config.getString("hardLimitSeconds"); if(hardLimitSeconds != null) { TimeLimiter hardLimit = new TimeLimiter(Integer.decode(hardLimitSeconds).intValue(),true); @@ -105,14 +108,14 @@ public class CopyingApp { log.info(String.format("%s - Setting Row count interval to default of 100 rows.",jobName)); } - Integer pollTimeout = null; - try { - pollTimeout = new Integer(config.getString("pollTimeout")); - log.info(String.format("%s - Setting Poll timeout to %s milliseconds", jobName, pollTimeout)); - } catch(Exception e) { - pollTimeout = new Integer(1000); // If we don't have a new setting, use the old default - log.info(String.format("%s - Setting poll timeout to default of 1 second.",jobName)); - } + Integer pollTimeout = null; + try { + pollTimeout = new Integer(config.getString("pollTimeout")); + log.info(String.format("%s - Setting Poll timeout to %s milliseconds", jobName, pollTimeout)); + } catch(Exception e) { + pollTimeout = new Integer(1000); // If we don't have a new setting, use the old default + log.info(String.format("%s - Setting poll timeout to default of 1 second.",jobName)); + } @@ -148,7 +151,7 @@ public class CopyingApp { pollTimeout, identifySourceSQL, identifyDestinationSQL - ); + ); j.start(); j.join(); @@ -179,7 +182,7 @@ public class CopyingApp { pollTimeout, identifySourceSQL, identifyDestinationSQL - ); + ); j.start(); j.join(); } else { @@ -241,21 +244,4 @@ public class CopyingApp { return c; } - - // Even with JDBC 4, some drivers don't play nicely with whatever - // the classloaders are up to. So this allows us to force it the - // old fashioned way, and works around the - // "But it works fine when it's the /only/ driver!" - // cross-database problem - private void loadDrivers(Configuration config) { - String[] drivers = config.get(String[].class, "drivers.driver"); - for(String d:drivers) { - try { - Class.forName(d); - log.info("Preloaded driver "+d); - } catch(ClassNotFoundException e) { - log.error("Could not preload driver "+d,e); - } - } - } } diff --git a/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java b/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java index 97d2dc8..d0d4d73 100644 --- a/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java +++ b/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java @@ -12,7 +12,7 @@ public class CopyingAppCommandParser { private String configFile; @Parameter(names = {"-log4j","--log4j"}, description = "Log4J config file for this run") - private String log4jConfigFile = "/etc/ujetl/default_log4j_config.properties"; + private String log4jConfigFile = "/etc/ppl/default_log4j_config.properties"; public CopyingAppCommandParser(String[] args) { super(); @@ -23,4 +23,8 @@ public class CopyingAppCommandParser { return configFile; } + public String getLog4jConfigFile() { + return log4jConfigFile; + } + } diff --git a/src/test/java/com/rasilon/ujetl/TestConfig.java b/src/test/java/com/rasilon/ujetl/TestConfig.java deleted file mode 100644 index 05f9d82..0000000 --- a/src/test/java/com/rasilon/ujetl/TestConfig.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.rasilon.ujetl; - -import org.apache.commons.configuration2.Configuration; -import org.apache.commons.configuration2.builder.fluent.Configurations; -import org.apache.commons.configuration2.ex.ConfigurationException; - -import org.apache.commons.beanutils.PropertyUtils; // Why does config need this? - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.MethodOrderer.Alphanumeric; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; - - -/** - * @author derryh - * - */ -public class TestConfig { - - @Test - public void test001VerifyArrayOfDrivers() { - try { - Configurations configs = new Configurations(); - Configuration config = configs.xml("TEST_config_live.xml"); - String[] drivers = config.get(String[].class, "drivers.driver"); - int ndrivers =drivers.length; - if(ndrivers != 3){ - fail("Expected 3 drivers, but found "+ndrivers); - } - } catch(Exception e) { - fail(e.toString()); - } - } - -} diff --git a/src/test/java/com/rasilon/ujetl/TestJob.java b/src/test/java/com/rasilon/ujetl/TestJob.java index 63ab4c8..853aeb4 100644 --- a/src/test/java/com/rasilon/ujetl/TestJob.java +++ b/src/test/java/com/rasilon/ujetl/TestJob.java @@ -42,7 +42,7 @@ public class TestJob { dest, "jUnit Test Config", "jUnit Test Job", - "SELECT -1 AS \"key\"", + "SELECT -1 AS key", "SELECT id,dat FROM src WHERE id > ?", "INSERT INTO dest VALUES(?,?)", null, diff --git a/src/test/java/com/rasilon/ujetl/TestParser.java b/src/test/java/com/rasilon/ujetl/TestParser.java index 398f046..13366fd 100644 --- a/src/test/java/com/rasilon/ujetl/TestParser.java +++ b/src/test/java/com/rasilon/ujetl/TestParser.java @@ -13,12 +13,15 @@ public class TestParser { public void test001Parset() { try { String[] args = { + "--log4j", + "log4j_test_banana.xml", "--config", "config_test_banana.xml" }; CopyingAppCommandParser p = new CopyingAppCommandParser(args); assertEquals(p.getConfigFile(),"config_test_banana.xml"); + assertEquals(p.getLog4jConfigFile(),"log4j_test_banana.xml"); } catch(Exception e) { fail(e.toString()); diff --git a/src/test/java/com/rasilon/ujetl/TestPrePost.java b/src/test/java/com/rasilon/ujetl/TestPrePost.java index 55f3ee1..77ff2a6 100644 --- a/src/test/java/com/rasilon/ujetl/TestPrePost.java +++ b/src/test/java/com/rasilon/ujetl/TestPrePost.java @@ -42,7 +42,7 @@ public class TestPrePost { dest, "jUnit Test Config", "jUnit Test Job", - "SELECT -1 AS \"key\"", + "SELECT -1 AS key", "SELECT id,dat FROM src WHERE id > ?", "INSERT INTO tmp_dest VALUES(?,?)", "CREATE TEMP TABLE tmp_dest(id bigint not null primary key, dat varchar);", diff --git a/src/test/resources/TEST_config_live.xml b/src/test/resources/TEST_config_live.xml index 210d8f1..338faff 100644 --- a/src/test/resources/TEST_config_live.xml +++ b/src/test/resources/TEST_config_live.xml @@ -4,11 +4,6 @@ 10000 1000 500 - - org.postgresql.Driver - org.h2.Driver - org.relique.jdbc.csv.CsvDriver - jdbc:postgresql://localhost:5432/test test diff --git a/uJETL.spec b/uJETL.spec new file mode 100644 index 0000000..ce6f8ef --- /dev/null +++ b/uJETL.spec @@ -0,0 +1,33 @@ +Summary: Java app to facilitate moving data between databases. +Name: uJETL +Version: 2.3.1 +Release: 1 +Group: Applications/Database +License: All rights reserved. +Source: uJETL-%{version}.tar.gz +URL: https://github.com/rasilon/ujetl.git +Distribution: derryh +Vendor: derryh +Packager: Derry Hamilton +#BuildRoot: . + +%description +A very small ETL app + +%prep +%setup + +%build +#mvn -Dmaven.test.skip=true clean package +true + +%install +mkdir -p $RPM_BUILD_ROOT/usr/share/ujetl/lib $RPM_BUILD_ROOT/etc/ujetl $RPM_BUILD_ROOT/usr/bin +cp target/CopyingApp-*-jar-with-dependencies.jar $RPM_BUILD_ROOT/usr/share/ujetl/lib/CopyingApp.jar +cp install_extra/run_copying_job $RPM_BUILD_ROOT/usr/bin +cp install_extra/copying_defaults_log4j.xml $RPM_BUILD_ROOT/etc/ujetl + +%files +/usr/share/ujetl/lib/CopyingApp.jar +/usr/bin/run_copying_job +/etc/ujetl/copying_defaults_log4j.xml