build_util/build_rpm

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+
+cd /root
+cp -Rv build build2
+cd build2
+
+SPEC=$(ls *.spec)
+VER=$(grep Version $SPEC | awk '{print $2}')
+tar cvf $HOME/rpmbuild/SOURCES/uJETL-${VER}.tar.gz --show-transformed --transform="s/^\./uJETL-${VER}/"  .
+rpmbuild -ba $SPEC
+cp /root/rpmbuild/RPMS/x86_64/* /root/build/

build_util/build_rpms_in_docker

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+docker build --rm -t local/c7-buildhost docker/build
+
+docker run -it --rm -v `pwd`:/root/build local/c7-buildhost /root/build/build_util/build_rpm

build_util/create_run_docker

@@ -1,3 +1,2 @@
 #!/bin/bash
 docker build --target deploy -t rasilon/ujetl docker/multistage
-docker tag rasilon/ujetl:latest rasilon/ujetl:$(xpath -q -e '/project/version/text()' pom.xml)

build_util/push_docker_images

@@ -1,4 +0,0 @@
-#!/bin/bash
-docker push rasilon/ujetl:latest
-docker push rasilon/ujetl:$(xpath -q -e '/project/version/text()' pom.xml)
-

docker/multistage/Dockerfile

@@ -1,13 +1,19 @@
-FROM ubuntu:22.04 as builder
+FROM centos:centos8 as builder
-RUN apt-get update && apt-get -y upgrade
+RUN yum -y install epel-release java-11-openjdk-devel
-RUN apt-get -y install openjdk-19-jdk-headless maven git
+RUN yum -y groupinstall 'Development Tools'
-RUN git clone --single-branch --branch main https://github.com/rasilon/ujetl.git
+RUN yum -y install git maven
-RUN cd ujetl && mvn -e package
+RUN alternatives --set javac /usr/lib/jvm/java-11-openjdk-11.*/bin/javac
+RUN alternatives --set java /usr/lib/jvm/java-11-openjdk-11.*/bin/java
+RUN git clone --single-branch --branch master https://github.com/rasilon/ujetl.git
+RUN cd ujetl && JAVA_HOME=/usr/lib/jvm/java-11 mvn -e package
+
+
+
 
 FROM openjdk:11 as runner
 LABEL maintainer="Derry Hamilton <derryh@rasilon.net>"
 
-RUN apt update && apt upgrade -y && apt install -y bash
+RUN apt update && apt upgrade && apt install bash
 
 RUN mkdir -p /usr/share/ujetl/lib/ /var/ujetl /etc/ujetl
 

docker/multistage/TEST_config_live.xml

@@ -4,11 +4,6 @@
   <nRowsToLog>10000</nRowsToLog>
   <blockSize>1000</blockSize>
   <pollTimeout>500</pollTimeout>
-  <drivers>
-    <driver>org.postgresql.Driver</driver>
-    <driver>org.relique.jdbc.csv.CsvDriver</driver>
-  </drivers>
-
   <source>
     <dsn>jdbc:postgresql://testdb:5432/test</dsn>
     <username>test</username>

docker/multistage/small.csv

@@ -1,4 +0,0 @@
-id,dat
-1,banana
-2,potato
-3,nugget

docker/test_db/setup.sql

@@ -44,14 +44,10 @@ CREATE TABLE denormalised_personalia(
     lname text
 );
 
-CREATE TABLE test_csvjdbc(
-    id integer not null primary key,
-    dat text
-);
-
 GRANT SELECT ON ALL TABLES IN SCHEMA public TO test;
 GRANT SELECT,INSERT,UPDATE ON denormalised_personalia TO test;
 
+
 \c postgres
 CREATE TABLE public.container_ready AS SELECT 1 FROM(VALUES(1)) AS a(a);
 GRANT SELECT ON public.container_ready TO TEST;

install_extra/run_copying_job

@@ -30,9 +30,9 @@ fi
 /usr/bin/java \
     -Xms1g \
     -Xmx2g   \
-    -Dlog4j.configurationFile="$LOG_PROPS" \
     -cp /usr/share/ujetl/lib/CopyingApp.jar \
     com.rasilon.ujetl.CopyingApp \
+    --log4j "$LOG_PROPS" \
     --config "/etc/ujetl/${JOBNAME}_config_live.xml"
 
 #rm -f $LOCKFILE

pom.xml

@@ -6,7 +6,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma
   <groupId>com.rasilon.ujetl</groupId>
   <artifactId>CopyingApp</artifactId>
   <packaging>jar</packaging>
-  <version>2.5.2</version>
+  <version>2.3.1</version>
   <name>uJETL</name>
   <url>https://github.com/rasilon/ujetl</url>
   <properties>
@@ -35,13 +35,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma
     <dependency>
       <groupId>com.h2database</groupId>
       <artifactId>h2</artifactId>
-      <version>2.2.220</version>
+      <version>1.4.199</version>
       <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-lang3</artifactId>
-      <version>3.18.0</version>
+      <version>3.9</version>
     </dependency>
     <dependency>
       <groupId>commons-logging</groupId>
@@ -51,12 +51,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-configuration2</artifactId>
-      <version>2.10.1</version>
+      <version>2.7</version>
     </dependency>
     <dependency>
       <groupId>commons-beanutils</groupId>
       <artifactId>commons-beanutils</artifactId>
-      <version>1.11.0</version>
+      <version>1.9.4</version>
     </dependency>
     <dependency>
       <groupId>com.beust</groupId>
@@ -66,22 +66,17 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-api</artifactId>
-      <version>2.17.1</version>
+      <version>2.15.0</version>
     </dependency>
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-core</artifactId>
-      <version>2.25.3</version>
+      <version>2.15.0</version>
     </dependency>
     <dependency>
       <groupId>org.postgresql</groupId>
       <artifactId>postgresql</artifactId>
-      <version>42.7.2</version>
+      <version>42.2.5</version>
-    </dependency>
-    <dependency>
-      <groupId>net.sourceforge.csvjdbc</groupId>
-      <artifactId>csvjdbc</artifactId>
-      <version>1.0.40</version>
     </dependency>
   </dependencies>
   <build>

src/main/java/com/rasilon/ujetl/CopyingApp.java

@@ -34,6 +34,10 @@ public class CopyingApp {
     public static void main(String[] args) {
         CopyingAppCommandParser cli = new CopyingAppCommandParser(args);
         LoggerContext context = (org.apache.logging.log4j.core.LoggerContext) LogManager.getContext(false);
+        String log4jConfigLocation = cli.getLog4jConfigFile();
+        File file = new File(log4jConfigLocation);
+        context.setConfigLocation(file.toURI());
+        System.out.println("Config set from "+file.toURI());
 
         CopyingApp app = new CopyingApp(cli);
         try {
@@ -75,7 +79,6 @@ public class CopyingApp {
 
             Configuration config = configs.xml(cli.getConfigFile());
 
-            loadDrivers(config);
             String hardLimitSeconds = config.getString("hardLimitSeconds");
             if(hardLimitSeconds != null) {
                 TimeLimiter hardLimit = new TimeLimiter(Integer.decode(hardLimitSeconds).intValue(),true);
@@ -105,14 +108,14 @@ public class CopyingApp {
                 log.info(String.format("%s - Setting Row count interval to default of 100 rows.",jobName));
             }
 
-            Integer pollTimeout = null;
+		  	Integer pollTimeout = null;
-            try {
+	  		try {
-                pollTimeout = new Integer(config.getString("pollTimeout"));
+  				pollTimeout = new Integer(config.getString("pollTimeout"));
-                log.info(String.format("%s - Setting Poll timeout to %s milliseconds", jobName, pollTimeout));
+				log.info(String.format("%s - Setting Poll timeout to %s milliseconds", jobName, pollTimeout));
-            } catch(Exception e) {
+			} catch(Exception e) {
-                pollTimeout = new Integer(1000); // If we don't have a new setting, use the old default
+				pollTimeout = new Integer(1000); // If we don't have a new setting, use the old default
-                log.info(String.format("%s - Setting poll timeout to default of 1 second.",jobName));
+				log.info(String.format("%s - Setting poll timeout to default of 1 second.",jobName));
-            }
+			}
 
 
 
@@ -148,7 +151,7 @@ public class CopyingApp {
                         pollTimeout,
                         identifySourceSQL,
                         identifyDestinationSQL
-                    );
+                        );
                     j.start();
                     j.join();
 
@@ -179,7 +182,7 @@ public class CopyingApp {
                     pollTimeout,
                     identifySourceSQL,
                     identifyDestinationSQL
-                );
+                    );
                 j.start();
                 j.join();
             } else {
@@ -241,21 +244,4 @@ public class CopyingApp {
 
         return c;
     }
-
-    // Even with JDBC 4, some drivers don't play nicely with whatever
-    // the classloaders are up to.  So this allows us to force it the
-    // old fashioned way, and works around the
-    // "But it works fine when it's the /only/ driver!"
-    // cross-database problem
-    private void loadDrivers(Configuration config) {
-        String[] drivers = config.get(String[].class, "drivers.driver");
-        for(String d:drivers) {
-            try {
-                Class.forName(d);
-                log.info("Preloaded driver  "+d);
-            } catch(ClassNotFoundException e) {
-                log.error("Could not preload driver "+d,e);
-            }
-        }
-    }
 }

src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java

@@ -12,7 +12,7 @@ public class CopyingAppCommandParser {
 	private String configFile;
 	
 	@Parameter(names = {"-log4j","--log4j"}, description = "Log4J config file for this run")
-	private String log4jConfigFile = "/etc/ujetl/default_log4j_config.properties";
+	private String log4jConfigFile = "/etc/ppl/default_log4j_config.properties";
 
 	public CopyingAppCommandParser(String[] args) {
 		super();
@@ -23,4 +23,8 @@ public class CopyingAppCommandParser {
 		return configFile;
 	}
 
+	public String getLog4jConfigFile() {
+		return log4jConfigFile;
+	}
+	
 }

src/test/java/com/rasilon/ujetl/TestConfig.java

@@ -1,37 +0,0 @@
-package com.rasilon.ujetl;
-
-import org.apache.commons.configuration2.Configuration;
-import org.apache.commons.configuration2.builder.fluent.Configurations;
-import org.apache.commons.configuration2.ex.ConfigurationException;
-
-import org.apache.commons.beanutils.PropertyUtils; // Why does config need this?
-
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.MethodOrderer.Alphanumeric;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.fail;
-
-
-/**
- * @author derryh
- *
- */
-public class TestConfig {
-
-    @Test
-    public void test001VerifyArrayOfDrivers() {
-        try {
-            Configurations configs = new Configurations();
-            Configuration config = configs.xml("TEST_config_live.xml");
-            String[] drivers = config.get(String[].class, "drivers.driver");
-            int ndrivers =drivers.length;
-            if(ndrivers != 3){
-                fail("Expected 3 drivers, but found "+ndrivers);
-            }
-        } catch(Exception e) {
-            fail(e.toString());
-        }
-    }
-
-}

src/test/java/com/rasilon/ujetl/TestJob.java

@@ -42,7 +42,7 @@ public class TestJob {
                 dest,
                 "jUnit Test Config",
                 "jUnit Test Job",
-                "SELECT -1 AS \"key\"",
+                "SELECT -1 AS key",
                 "SELECT id,dat FROM src WHERE id > ?",
                 "INSERT INTO dest VALUES(?,?)",
                 null,

src/test/java/com/rasilon/ujetl/TestParser.java

@@ -13,12 +13,15 @@ public class TestParser {
     public void test001Parset() {
         try {
             String[] args = {
+                "--log4j",
+                "log4j_test_banana.xml",
                 "--config",
                 "config_test_banana.xml"
             };
             CopyingAppCommandParser p = new CopyingAppCommandParser(args);
 
             assertEquals(p.getConfigFile(),"config_test_banana.xml");
+            assertEquals(p.getLog4jConfigFile(),"log4j_test_banana.xml");
 
         } catch(Exception e) {
             fail(e.toString());

src/test/java/com/rasilon/ujetl/TestPrePost.java

@@ -42,7 +42,7 @@ public class TestPrePost {
                 dest,
                 "jUnit Test Config",
                 "jUnit Test Job",
-                "SELECT -1 AS \"key\"",
+                "SELECT -1 AS key",
                 "SELECT id,dat FROM src WHERE id > ?",
                 "INSERT INTO tmp_dest VALUES(?,?)",
                 "CREATE TEMP TABLE tmp_dest(id bigint not null primary key, dat varchar);",

src/test/resources/TEST_config_live.xml

@@ -4,11 +4,6 @@
   <nRowsToLog>10000</nRowsToLog>
   <blockSize>1000</blockSize>
   <pollTimeout>500</pollTimeout>
-  <drivers>
-      <driver>org.postgresql.Driver</driver>
-      <driver>org.h2.Driver</driver>
-      <driver>org.relique.jdbc.csv.CsvDriver</driver>
-  </drivers>
   <source>
     <dsn>jdbc:postgresql://localhost:5432/test</dsn>
     <username>test</username>

uJETL.spec

@@ -0,0 +1,33 @@
+Summary: Java app to facilitate moving data between databases.
+Name: uJETL
+Version: 2.3.1
+Release: 1
+Group: Applications/Database
+License: All rights reserved.
+Source: uJETL-%{version}.tar.gz
+URL:  https://github.com/rasilon/ujetl.git
+Distribution: derryh
+Vendor: derryh
+Packager: Derry Hamilton <derryh@rasilon.net>
+#BuildRoot: .
+
+%description
+A very small ETL app
+
+%prep
+%setup
+
+%build
+#mvn -Dmaven.test.skip=true clean package
+true
+
+%install
+mkdir -p $RPM_BUILD_ROOT/usr/share/ujetl/lib $RPM_BUILD_ROOT/etc/ujetl $RPM_BUILD_ROOT/usr/bin
+cp target/CopyingApp-*-jar-with-dependencies.jar $RPM_BUILD_ROOT/usr/share/ujetl/lib/CopyingApp.jar
+cp install_extra/run_copying_job $RPM_BUILD_ROOT/usr/bin
+cp install_extra/copying_defaults_log4j.xml $RPM_BUILD_ROOT/etc/ujetl
+
+%files
+/usr/share/ujetl/lib/CopyingApp.jar
+/usr/bin/run_copying_job
+/etc/ujetl/copying_defaults_log4j.xml