2026-04-11 18:39:30 +00:00
20 changed files with 124 additions and 220 deletions
--- a/build_util/build_rpm
+++ b/build_util/build_rpm
@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+
+cd /root
+cp -Rv build build2
+cd build2
+
+SPEC=$(ls *.spec)
+VER=$(grep Version $SPEC | awk '{print $2}')
+tar cvf $HOME/rpmbuild/SOURCES/uJETL-${VER}.tar.gz --show-transformed --transform="s/^\./uJETL-${VER}/"  .
+rpmbuild -ba $SPEC
+cp /root/rpmbuild/RPMS/x86_64/* /root/build/
--- a/build_util/build_rpms_in_docker
+++ b/build_util/build_rpms_in_docker
@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+docker build --rm -t local/c7-buildhost docker/build
+
+docker run -it --rm -v `pwd`:/root/build local/c7-buildhost /root/build/build_util/build_rpm
--- a/build_util/create_run_docker
+++ b/build_util/create_run_docker
@ -1,3 +1,2 @@
 #!/bin/bash
 docker build --target deploy -t rasilon/ujetl docker/multistage
-docker tag rasilon/ujetl:latest rasilon/ujetl:$(xpath -q -e '/project/version/text()' pom.xml)
--- a/build_util/push_docker_images
+++ b/build_util/push_docker_images
@ -1,4 +0,0 @@
-#!/bin/bash
-docker push rasilon/ujetl:latest
-docker push rasilon/ujetl:$(xpath -q -e '/project/version/text()' pom.xml)
-
--- a/docker/multistage/Dockerfile
+++ b/docker/multistage/Dockerfile
@ -1,13 +1,17 @@
-FROM ubuntu:22.04 as builder
-RUN apt-get update && apt-get -y upgrade
-RUN apt-get -y install openjdk-19-jdk-headless maven git
-RUN git clone --single-branch --branch main https://github.com/rasilon/ujetl.git
-RUN cd ujetl && mvn -e package
+FROM centos:centos7 as builder
+RUN yum -y install epel-release java-1.8.0-openjdk-devel
+RUN yum -y groupinstall 'Development Tools'
+RUN yum -y install git maven
+RUN git clone https://github.com/rasilon/ujetl.git
+RUN cd ujetl && mvn package

-FROM openjdk:11 as runner
+
+
+
+FROM openjdk:8-alpine as runner
 LABEL maintainer="Derry Hamilton <derryh@rasilon.net>"

-RUN apt update && apt upgrade -y && apt install -y bash
+RUN apk update && apk upgrade && apk add bash

 RUN mkdir -p /usr/share/ujetl/lib/ /var/ujetl /etc/ujetl

@ -20,7 +24,7 @@ CMD ["/ujetl_entrypoint"]
 FROM runner as tester
 COPY TEST_config_live.xml /var/ujetl/
 COPY wait_for_postgres /
-RUN apt-get  install -y postgresql-client
+RUN apk add postgresql-client


 FROM runner as deploy
--- a/docker/multistage/TEST_config_live.xml
+++ b/docker/multistage/TEST_config_live.xml
@ -4,11 +4,6 @@
  <nRowsToLog>10000</nRowsToLog>
  <blockSize>1000</blockSize>
  <pollTimeout>500</pollTimeout>
-  <drivers>
-    <driver>org.postgresql.Driver</driver>
-    <driver>org.relique.jdbc.csv.CsvDriver</driver>
-  </drivers>
-
  <source>
    <dsn>jdbc:postgresql://testdb:5432/test</dsn>
    <username>test</username>
@ -23,8 +18,6 @@
  <jobs>
    <job>
      <name>test</name>
-      <identifySourceSQL>select 'PID:'||pg_backend_pid()</identifySourceSQL>
-      <identifyDestinationSQL>select 'PID:'||pg_backend_pid()</identifyDestinationSQL>
      <key>select coalesce(-1,max(id),-1) as key from dest</key>
      <select>
          select
@ -58,8 +51,6 @@
    </job>
    <job>
      <name>test upsert</name>
-      <identifySourceSQL>select 'PID:'||pg_backend_pid()</identifySourceSQL>
-      <identifyDestinationSQL>select 'PID:'||pg_backend_pid()</identifyDestinationSQL>
      <key>select -1 as key</key>
      <select>
          select
@ -93,8 +84,6 @@
    </job>
    <job>
        <name>denormalise</name>
-      <identifySourceSQL>select 'PID:'||pg_backend_pid()</identifySourceSQL>
-      <identifyDestinationSQL>select 'PID:'||pg_backend_pid()</identifyDestinationSQL>
        <key>select -1 as key</key>
        <select>select person_id,fname,lname from normalised_personalia p join normalised_first_names f using(fid) join normalised_last_names l using(lid) where ?::integer is not null;</select>
        <insert>
--- a/docker/multistage/small.csv
+++ b/docker/multistage/small.csv
@ -1,4 +0,0 @@
-id,dat
-1,banana
-2,potato
-3,nugget
--- a/docker/multistage/ujetl_entrypoint
+++ b/docker/multistage/ujetl_entrypoint
@ -10,11 +10,10 @@ ls
 echo Starting run loop
 for file in *.xml
 do
-	/usr/local/openjdk-11/bin/java \
+	/usr/bin/java \
 		-Xms1g \
 		-Xmx2g   \
 		-cp /usr/share/ujetl/lib/CopyingApp.jar \
-		-Dlog4j.configurationFile="$LOG_PROPS" \
 		com.rasilon.ujetl.CopyingApp \
 		--log4j "$LOG_PROPS" \
 		--config "$file"
--- a/docker/test_db/setup.sql
+++ b/docker/test_db/setup.sql
@ -44,14 +44,10 @@ CREATE TABLE denormalised_personalia(
    lname text
 );

-CREATE TABLE test_csvjdbc(
-    id integer not null primary key,
-    dat text
-);
-
 GRANT SELECT ON ALL TABLES IN SCHEMA public TO test;
 GRANT SELECT,INSERT,UPDATE ON denormalised_personalia TO test;

+
 \c postgres
 CREATE TABLE public.container_ready AS SELECT 1 FROM(VALUES(1)) AS a(a);
 GRANT SELECT ON public.container_ready TO TEST;
--- a/install_extra/run_copying_job
+++ b/install_extra/run_copying_job
@ -30,9 +30,9 @@ fi
 /usr/bin/java \
    -Xms1g \
    -Xmx2g   \
-    -Dlog4j.configurationFile="$LOG_PROPS" \
    -cp /usr/share/ujetl/lib/CopyingApp.jar \
    com.rasilon.ujetl.CopyingApp \
+    --log4j "$LOG_PROPS" \
    --config "/etc/ujetl/${JOBNAME}_config_live.xml"

 #rm -f $LOCKFILE
--- a/pom.xml
+++ b/pom.xml
@ -6,7 +6,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma
  <groupId>com.rasilon.ujetl</groupId>
  <artifactId>CopyingApp</artifactId>
  <packaging>jar</packaging>
-  <version>2.5.2</version>
+  <version>2.1.7</version>
  <name>uJETL</name>
  <url>https://github.com/rasilon/ujetl</url>
  <properties>
@ -35,13 +35,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma
    <dependency>
      <groupId>com.h2database</groupId>
      <artifactId>h2</artifactId>
-      <version>2.2.220</version>
+      <version>1.4.199</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.commons</groupId>
      <artifactId>commons-lang3</artifactId>
-      <version>3.18.0</version>
+      <version>3.9</version>
    </dependency>
    <dependency>
      <groupId>commons-logging</groupId>
@ -51,12 +51,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma
    <dependency>
      <groupId>org.apache.commons</groupId>
      <artifactId>commons-configuration2</artifactId>
-      <version>2.10.1</version>
+      <version>2.4</version>
    </dependency>
    <dependency>
      <groupId>commons-beanutils</groupId>
      <artifactId>commons-beanutils</artifactId>
-      <version>1.11.0</version>
+      <version>1.9.3</version>
    </dependency>
    <dependency>
      <groupId>com.beust</groupId>
@ -66,31 +66,27 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma
    <dependency>
      <groupId>org.apache.logging.log4j</groupId>
      <artifactId>log4j-api</artifactId>
-      <version>2.17.1</version>
+      <version>2.11.2</version>
    </dependency>
    <dependency>
      <groupId>org.apache.logging.log4j</groupId>
      <artifactId>log4j-core</artifactId>
-      <version>2.25.3</version>
+      <version>2.11.2</version>
    </dependency>
    <dependency>
      <groupId>org.postgresql</groupId>
      <artifactId>postgresql</artifactId>
-      <version>42.7.2</version>
-    </dependency>
-    <dependency>
-      <groupId>net.sourceforge.csvjdbc</groupId>
-      <artifactId>csvjdbc</artifactId>
-      <version>1.0.40</version>
+      <version>42.2.5</version>
    </dependency>
  </dependencies>
  <build>
    <plugins>
      <plugin>
        <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.8.0</version>
+        <version>2.3.2</version>
        <configuration>
-          <release>11</release>
+          <source>1.8</source>
+          <target>1.8</target>
        </configuration>
      </plugin>
      <plugin>
--- a/src/main/java/com/rasilon/ujetl/CopyingApp.java
+++ b/src/main/java/com/rasilon/ujetl/CopyingApp.java
@ -34,6 +34,10 @@ public class CopyingApp {
    public static void main(String[] args) {
        CopyingAppCommandParser cli = new CopyingAppCommandParser(args);
        LoggerContext context = (org.apache.logging.log4j.core.LoggerContext) LogManager.getContext(false);
+        String log4jConfigLocation = cli.getLog4jConfigFile();
+        File file = new File(log4jConfigLocation);
+        context.setConfigLocation(file.toURI());
+        System.out.println("Config set from "+file.toURI());

        CopyingApp app = new CopyingApp(cli);
        try {
@ -75,7 +79,6 @@ public class CopyingApp {

            Configuration config = configs.xml(cli.getConfigFile());

-            loadDrivers(config);
            String hardLimitSeconds = config.getString("hardLimitSeconds");
            if(hardLimitSeconds != null) {
                TimeLimiter hardLimit = new TimeLimiter(Integer.decode(hardLimitSeconds).intValue(),true);
@ -130,25 +133,8 @@ public class CopyingApp {
                    String tabInsert = config.getString("jobs.job("+i+").insert");
                    String preTarget = config.getString("jobs.job("+i+").preTarget");
                    String postTarget = config.getString("jobs.job("+i+").postTarget");
-                    String identifySourceSQL = config.getString("jobs.job.identifySourceSQL");
-                    String identifyDestinationSQL = config.getString("jobs.job.identifyDestinationSQL");

-                    Job j = new Job(
-                        sConn,
-                        dConn,
-                        tabName,
-                        jobName,
-                        tabKey,
-                        tabSelect,
-                        tabInsert,
-                        preTarget,
-                        postTarget,
-                        nRowsToLog,
-                        blockSize,
-                        pollTimeout,
-                        identifySourceSQL,
-                        identifyDestinationSQL
-                    );
+                    Job j = new Job(sConn,dConn,tabName,jobName,tabKey,tabSelect,tabInsert,preTarget,postTarget,nRowsToLog,blockSize,pollTimeout);
                    j.start();
                    j.join();

@ -160,26 +146,7 @@ public class CopyingApp {
                String tabInsert = config.getString("jobs.job.insert");
                String preTarget = config.getString("jobs.job.preTarget");
                String postTarget = config.getString("jobs.job.postTarget");
-                String identifySourceSQL = config.getString("jobs.job.identifySourceSQL");
-                String identifyDestinationSQL = config.getString("jobs.job.identifyDestinationSQL");
-
-
-                Job j = new Job(
-                    sConn,
-                    dConn,
-                    tabName,
-                    jobName,
-                    tabKey,
-                    tabSelect,
-                    tabInsert,
-                    preTarget,
-                    postTarget,
-                    nRowsToLog,
-                    blockSize,
-                    pollTimeout,
-                    identifySourceSQL,
-                    identifyDestinationSQL
-                );
+                Job j = new Job(sConn,dConn,tabName,jobName,tabKey,tabSelect,tabInsert,preTarget,postTarget,nRowsToLog,blockSize,pollTimeout);
                j.start();
                j.join();
            } else {
@ -241,21 +208,4 @@ public class CopyingApp {

        return c;
    }
-
-    // Even with JDBC 4, some drivers don't play nicely with whatever
-    // the classloaders are up to.  So this allows us to force it the
-    // old fashioned way, and works around the
-    // "But it works fine when it's the /only/ driver!"
-    // cross-database problem
-    private void loadDrivers(Configuration config) {
-        String[] drivers = config.get(String[].class, "drivers.driver");
-        for(String d:drivers) {
-            try {
-                Class.forName(d);
-                log.info("Preloaded driver  "+d);
-            } catch(ClassNotFoundException e) {
-                log.error("Could not preload driver "+d,e);
-            }
-        }
-    }
 }
--- a/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java
+++ b/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java
@ -12,7 +12,7 @@ public class CopyingAppCommandParser {
 	private String configFile;
 	
 	@Parameter(names = {"-log4j","--log4j"}, description = "Log4J config file for this run")
-	private String log4jConfigFile = "/etc/ujetl/default_log4j_config.properties";
+	private String log4jConfigFile = "/etc/ppl/default_log4j_config.properties";

 	public CopyingAppCommandParser(String[] args) {
 		super();
@ -23,4 +23,8 @@ public class CopyingAppCommandParser {
 		return configFile;
 	}

+	public String getLog4jConfigFile() {
+		return log4jConfigFile;
+	}
+	
 }
--- a/src/main/java/com/rasilon/ujetl/Job.java
+++ b/src/main/java/com/rasilon/ujetl/Job.java
@ -22,29 +22,25 @@ import org.apache.logging.log4j.Logger;
 public class Job extends Thread {
    static Logger log = org.apache.logging.log4j.LogManager.getLogger(Job.class);

-    private Connection sConn;
-    private Connection dConn;
-    private String name;
-    private String jobName;
-    private String key;
-    private String select;
-    private String insert;
-    private String preTarget;
-    private String postTarget;
-    private Integer nRowsToLog;
-    private Integer blockSize;
-    private Integer pollTimeout;
-    private String identifySourceSQL;
-    private String identifyDestinationSQL;
+    Connection sConn;
+    Connection dConn;
+    String name;
+    String jobName;
+    String key;
+    String select;
+    String insert;
+    String preTarget;
+    String postTarget;
+    Integer nRowsToLog;
+    Integer blockSize;
+    Integer pollTimeout;

-    private BlockingQueue<List<String>> resultBuffer;
-    private AtomicBoolean producerLive;
-    private AtomicBoolean threadsExit = new AtomicBoolean(false);;
-    private String sourceID;
-    private String destID;
+    BlockingQueue<List<String>> resultBuffer;
+    AtomicBoolean producerLive;
+    AtomicBoolean threadsExit = new AtomicBoolean(false);;


-    public Job(Connection sConn,Connection dConn,String name,String jobName,String key,String select,String insert,String preTarget,String postTarget,Integer nRowsToLog,Integer blockSize,Integer pollTimeout,String identifySourceSQL, String identifyDestinationSQL) {
+    public Job(Connection sConn,Connection dConn,String name,String jobName,String key,String select,String insert,String preTarget,String postTarget,Integer nRowsToLog,Integer blockSize,Integer pollTimeout) {
        this.sConn = sConn;
        this.dConn = dConn;
        this.name = name;
@ -57,8 +53,6 @@ public class Job extends Thread {
        this.nRowsToLog = nRowsToLog;
        this.blockSize = blockSize;
        this.pollTimeout = pollTimeout;
-        this.identifySourceSQL = identifySourceSQL;
-        this.identifyDestinationSQL = identifyDestinationSQL;

        resultBuffer = new ArrayBlockingQueue<List<String>>( 3 * blockSize);
        producerLive = new AtomicBoolean(true);
@ -184,28 +178,15 @@ public class Job extends Thread {
        try {
            ResultSet rs;

-            if(identifySourceSQL != null) sourceID = getSingleString(identifySourceSQL,sConn);
-            if(identifyDestinationSQL != null) destID = getSingleString(identifyDestinationSQL,dConn);
-
-            if(sourceID != null || destID != null){
-                log.info(String.format(
-                            "%s - Processing table: %s with source: %s, dest: %s",
-                            jobName,
-                            name,
-                            sourceID==null?"":sourceID,
-                            destID==null?"":destID
-                            ));
-            }else{
            log.info(String.format("%s - Processing table: %s",jobName,name));
-            }
            if(preTarget != null){
-                log.info(String.format("%s - Trying to execute preTarget SQL",jobName));
+                log.info("Trying to execute preTarget SQL");
                PreparedStatement s = dConn.prepareStatement(preTarget);
                s.executeUpdate();
                s.close();
                dConn.commit();
            }else{
-                log.info(String.format("%s - No preTarget; skipping.",jobName));
+                log.info("No preTarget; skipping.");
            }

            log.debug("Trying to execute: "+key);
@ -245,13 +226,13 @@ public class Job extends Thread {
            c.join();

            if(postTarget != null){
-                log.info(String.format("%s - Trying to execute postTarget SQL",jobName));
+                log.info("Trying to execute postTarget SQL");
                PreparedStatement s = dConn.prepareStatement(postTarget);
                s.executeUpdate();
                s.close();
                dConn.commit();
            }else{
-                log.info(String.format("%s - No postTarget; skipping.",jobName));
+                log.info("No postTarget; skipping.");
            }


@ -261,16 +242,4 @@ public class Job extends Thread {
            throw new RuntimeException(e);
        }
    }
-
-    private String getSingleString(String sql, Connection conn){
-        try{
-            PreparedStatement s = conn.prepareStatement(sql);
-            ResultSet r = s.executeQuery();
-            r.next();
-            return r.getString(1);
-        } catch(SQLException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
 }
--- a/src/test/java/com/rasilon/ujetl/TestConfig.java
+++ b/src/test/java/com/rasilon/ujetl/TestConfig.java
@ -1,37 +0,0 @@
-package com.rasilon.ujetl;
-
-import org.apache.commons.configuration2.Configuration;
-import org.apache.commons.configuration2.builder.fluent.Configurations;
-import org.apache.commons.configuration2.ex.ConfigurationException;
-
-import org.apache.commons.beanutils.PropertyUtils; // Why does config need this?
-
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.MethodOrderer.Alphanumeric;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.fail;
-
-
-/**
- * @author derryh
- *
- */
-public class TestConfig {
-
-    @Test
-    public void test001VerifyArrayOfDrivers() {
-        try {
-            Configurations configs = new Configurations();
-            Configuration config = configs.xml("TEST_config_live.xml");
-            String[] drivers = config.get(String[].class, "drivers.driver");
-            int ndrivers =drivers.length;
-            if(ndrivers != 3){
-                fail("Expected 3 drivers, but found "+ndrivers);
-            }
-        } catch(Exception e) {
-            fail(e.toString());
-        }
-    }
-
-}
--- a/src/test/java/com/rasilon/ujetl/TestJob.java
+++ b/src/test/java/com/rasilon/ujetl/TestJob.java
@ -42,16 +42,14 @@ public class TestJob {
                dest,
                "jUnit Test Config",
                "jUnit Test Job",
-                "SELECT -1 AS \"key\"",
+                "SELECT -1 AS key",
                "SELECT id,dat FROM src WHERE id > ?",
                "INSERT INTO dest VALUES(?,?)",
                null,
                null,
                100,
                100,
-                100,
-                "select 'PID:'||session_id()",
-                "select 'PID:'||session_id()"
+                100
            );
            j.start();
            j.join();
--- a/src/test/java/com/rasilon/ujetl/TestParser.java
+++ b/src/test/java/com/rasilon/ujetl/TestParser.java
@ -13,12 +13,15 @@ public class TestParser {
    public void test001Parset() {
        try {
            String[] args = {
+                "--log4j",
+                "log4j_test_banana.xml",
                "--config",
                "config_test_banana.xml"
            };
            CopyingAppCommandParser p = new CopyingAppCommandParser(args);

            assertEquals(p.getConfigFile(),"config_test_banana.xml");
+            assertEquals(p.getLog4jConfigFile(),"log4j_test_banana.xml");

        } catch(Exception e) {
            fail(e.toString());
--- a/src/test/java/com/rasilon/ujetl/TestPrePost.java
+++ b/src/test/java/com/rasilon/ujetl/TestPrePost.java
@ -42,16 +42,14 @@ public class TestPrePost {
                dest,
                "jUnit Test Config",
                "jUnit Test Job",
-                "SELECT -1 AS \"key\"",
+                "SELECT -1 AS key",
                "SELECT id,dat FROM src WHERE id > ?",
                "INSERT INTO tmp_dest VALUES(?,?)",
                "CREATE TEMP TABLE tmp_dest(id bigint not null primary key, dat varchar);",
                "INSERT INTO dest SELECT * from tmp_dest;",
                100,
                100,
-                100,
-                "select 'PID:'||session_id()",
-                "select 'PID:'||session_id()"
+                100
            );
            j.start();
            j.join();
--- a/src/test/resources/TEST_config_live.xml
+++ b/src/test/resources/TEST_config_live.xml
@ -4,11 +4,6 @@
  <nRowsToLog>10000</nRowsToLog>
  <blockSize>1000</blockSize>
  <pollTimeout>500</pollTimeout>
-  <drivers>
-      <driver>org.postgresql.Driver</driver>
-      <driver>org.h2.Driver</driver>
-      <driver>org.relique.jdbc.csv.CsvDriver</driver>
-  </drivers>
  <source>
    <dsn>jdbc:postgresql://localhost:5432/test</dsn>
    <username>test</username>
@ -23,8 +18,6 @@
  <jobs>
    <job>
      <name>test</name>
-      <identifySourceSQL>select 'PID:'||pg_backend_pid()</identifySourceSQL>
-      <identifyDestinationSQL>select 'PID:'||pg_backend_pid()</identifyDestinationSQL>
      <key>select coalesce(-1,max(id),-1) as key from dest</key>
      <select>
          select
--- a/uJETL.spec
+++ b/uJETL.spec
@ -0,0 +1,33 @@
+Summary: Java app to facilitate moving data between databases.
+Name: uJETL
+Version: 2.1.7
+Release: 1
+Group: Applications/Database
+License: All rights reserved.
+Source: uJETL-%{version}.tar.gz
+URL:  https://github.com/rasilon/ujetl.git
+Distribution: derryh
+Vendor: derryh
+Packager: Derry Hamilton <derryh@rasilon.net>
+#BuildRoot: .
+
+%description
+A very small ETL app
+
+%prep
+%setup
+
+%build
+#mvn -Dmaven.test.skip=true clean package
+true
+
+%install
+mkdir -p $RPM_BUILD_ROOT/usr/share/ujetl/lib $RPM_BUILD_ROOT/etc/ujetl $RPM_BUILD_ROOT/usr/bin
+cp target/CopyingApp-*-jar-with-dependencies.jar $RPM_BUILD_ROOT/usr/share/ujetl/lib/CopyingApp.jar
+cp install_extra/run_copying_job $RPM_BUILD_ROOT/usr/bin
+cp install_extra/copying_defaults_log4j.xml $RPM_BUILD_ROOT/etc/ujetl
+
+%files
+/usr/share/ujetl/lib/CopyingApp.jar
+/usr/bin/run_copying_job
+/etc/ujetl/copying_defaults_log4j.xml