From f4ded0857c743aa3bdafafb323bc6aa9c84ff127 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Dec 2021 18:12:08 +0000 Subject: [PATCH 01/31] Bump log4j-api from 2.16.0 to 2.17.0 Bumps log4j-api from 2.16.0 to 2.17.0. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-api dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a1844b3..6ddc3a8 100644 --- a/pom.xml +++ b/pom.xml @@ -66,7 +66,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma org.apache.logging.log4j log4j-api - 2.16.0 + 2.17.0 org.apache.logging.log4j From 45c5900481507662e88d128d29a6bd0d1437814c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Dec 2021 18:20:05 +0000 Subject: [PATCH 02/31] Bump log4j-core from 2.16.0 to 2.17.0 Bumps log4j-core from 2.16.0 to 2.17.0. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-core dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a1844b3..339d15b 100644 --- a/pom.xml +++ b/pom.xml @@ -71,7 +71,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma org.apache.logging.log4j log4j-core - 2.16.0 + 2.17.0 org.postgresql From bc9849e3eed72c65c21594a9f84fce814ca0f096 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Dec 2021 21:30:36 +0000 Subject: [PATCH 03/31] Bump h2 from 1.4.199 to 2.0.202 Bumps [h2](https://github.com/h2database/h2database) from 1.4.199 to 2.0.202. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-1.4.199...version-2.0.202) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 339d15b..a9527f9 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma com.h2database h2 - 1.4.199 + 2.0.202 test From f42fa6550e105a9d305fc830e70681d29ac47244 Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Sat, 18 Dec 2021 21:42:39 +0000 Subject: [PATCH 04/31] version bump for security updstes --- pom.xml | 2 +- uJETL.spec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 9696a76..fbfd340 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma com.rasilon.ujetl CopyingApp jar - 2.3.2 + 2.3.3 uJETL https://github.com/rasilon/ujetl diff --git a/uJETL.spec b/uJETL.spec index 62936ab..d64abdf 100644 --- a/uJETL.spec +++ b/uJETL.spec @@ -1,6 +1,6 @@ Summary: Java app to facilitate moving data between databases. Name: uJETL -Version: 2.3.2 +Version: 2.3.3 Release: 1 Group: Applications/Database License: All rights reserved. From 7a22b6ddae9c41926f1c1c5bb44588070ffa31c6 Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Sat, 18 Dec 2021 21:49:35 +0000 Subject: [PATCH 05/31] cope with outdated source image --- docker/multistage/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/multistage/Dockerfile b/docker/multistage/Dockerfile index 6a7550d..355f1c7 100644 --- a/docker/multistage/Dockerfile +++ b/docker/multistage/Dockerfile @@ -13,7 +13,7 @@ RUN cd ujetl && JAVA_HOME=/usr/lib/jvm/java-11 mvn -e package FROM openjdk:11 as runner LABEL maintainer="Derry Hamilton " -RUN apt update && apt upgrade && apt install bash +RUN apt update && apt upgrade -y && apt install -y bash RUN mkdir -p /usr/share/ujetl/lib/ /var/ujetl /etc/ujetl From 79a3dbf499e13807dcf572f03ef8ed9019cd2064 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jan 2022 16:33:13 +0000 Subject: [PATCH 06/31] Bump log4j-api from 2.17.0 to 2.17.1 Bumps log4j-api from 2.17.0 to 2.17.1. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-api dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fbfd340..16bbe78 100644 --- a/pom.xml +++ b/pom.xml @@ -66,7 +66,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma org.apache.logging.log4j log4j-api - 2.17.0 + 2.17.1 org.apache.logging.log4j From 3235db7f6ac11b4570c9da877907f7871e4d2970 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Jan 2022 16:34:38 +0000 Subject: [PATCH 07/31] Bump log4j-core from 2.17.0 to 2.17.1 Bumps log4j-core from 2.17.0 to 2.17.1. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-core dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fbfd340..7e7978d 100644 --- a/pom.xml +++ b/pom.xml @@ -71,7 +71,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma org.apache.logging.log4j log4j-core - 2.17.0 + 2.17.1 org.postgresql From 36acc0ed23e875df460d623b77c73d325cb551ee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Jan 2022 00:11:15 +0000 Subject: [PATCH 08/31] Bump h2 from 2.0.202 to 2.0.206 Bumps [h2](https://github.com/h2database/h2database) from 2.0.202 to 2.0.206. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-2.0.202...version-2.0.206) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fbfd340..46446fe 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma com.h2database h2 - 2.0.202 + 2.0.206 test From d7bde365ed65d1fc5149edf11c25d116df3e7993 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Jan 2022 23:25:43 +0000 Subject: [PATCH 09/31] Bump h2 from 2.0.206 to 2.1.210 Bumps [h2](https://github.com/h2database/h2database) from 2.0.206 to 2.1.210. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-2.0.206...version-2.1.210) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d1385af..aa8b07e 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma com.h2database h2 - 2.0.206 + 2.1.210 test From 54a8cd631215950416e6264c0406c759e0bf8b40 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Feb 2022 00:56:38 +0000 Subject: [PATCH 10/31] Bump postgresql from 42.2.5 to 42.3.3 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.2.5 to 42.3.3. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.2.5...REL42.3.3) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d1385af..1a42734 100644 --- a/pom.xml +++ b/pom.xml @@ -76,7 +76,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma org.postgresql postgresql - 42.2.5 + 42.3.3 From 9554d3df63ae897866633ebcdd34b791816f72ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Jul 2022 16:58:42 +0000 Subject: [PATCH 11/31] Bump commons-configuration2 from 2.7 to 2.8.0 Bumps commons-configuration2 from 2.7 to 2.8.0. --- updated-dependencies: - dependency-name: org.apache.commons:commons-configuration2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index caaed58..307c3dc 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma org.apache.commons commons-configuration2 - 2.7 + 2.8.0 commons-beanutils From 67f6ec2dab997ac61fd9fe75f3a00ee2368645be Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Aug 2022 05:55:29 +0000 Subject: [PATCH 12/31] Bump postgresql from 42.3.3 to 42.4.1 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.3.3 to 42.4.1. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.3.3...REL42.4.1) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index caaed58..a6099ba 100644 --- a/pom.xml +++ b/pom.xml @@ -76,7 +76,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma org.postgresql postgresql - 42.3.3 + 42.4.1 From 17d33ec18b3894ec33dd8abbc7646d3e77d9ded4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Nov 2022 22:20:40 +0000 Subject: [PATCH 13/31] Bump postgresql from 42.4.1 to 42.4.3 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.4.1 to 42.4.3. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.4.1...REL42.4.3) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d7c7ec7..7fdfb89 100644 --- a/pom.xml +++ b/pom.xml @@ -76,7 +76,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma org.postgresql postgresql - 42.4.1 + 42.4.3 From 64743f430b9132b1af17cbd9d73832a362702de4 Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Wed, 7 Dec 2022 11:00:29 +0000 Subject: [PATCH 14/31] Update build host and postgres libs Bumping the postgres libs showed that the upstream build images were EOL, so we're now running a more modern version of Java and have migrated the build to ubuntu LTS --- build_util/build_rpm | 12 ------------ build_util/build_rpms_in_docker | 6 ------ docker/multistage/Dockerfile | 16 +++++----------- pom.xml | 2 +- uJETL.spec | 33 --------------------------------- 5 files changed, 6 insertions(+), 63 deletions(-) delete mode 100755 build_util/build_rpm delete mode 100755 build_util/build_rpms_in_docker delete mode 100644 uJETL.spec diff --git a/build_util/build_rpm b/build_util/build_rpm deleted file mode 100755 index 769915f..0000000 --- a/build_util/build_rpm +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -set -e - -cd /root -cp -Rv build build2 -cd build2 - -SPEC=$(ls *.spec) -VER=$(grep Version $SPEC | awk '{print $2}') -tar cvf $HOME/rpmbuild/SOURCES/uJETL-${VER}.tar.gz --show-transformed --transform="s/^\./uJETL-${VER}/" . -rpmbuild -ba $SPEC -cp /root/rpmbuild/RPMS/x86_64/* /root/build/ diff --git a/build_util/build_rpms_in_docker b/build_util/build_rpms_in_docker deleted file mode 100755 index acc6fee..0000000 --- a/build_util/build_rpms_in_docker +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -e - -docker build --rm -t local/c7-buildhost docker/build - -docker run -it --rm -v `pwd`:/root/build local/c7-buildhost /root/build/build_util/build_rpm diff --git a/docker/multistage/Dockerfile b/docker/multistage/Dockerfile index 355f1c7..dfc23a9 100644 --- a/docker/multistage/Dockerfile +++ b/docker/multistage/Dockerfile @@ -1,14 +1,8 @@ -FROM centos:centos8 as builder -RUN yum -y install epel-release java-11-openjdk-devel -RUN yum -y groupinstall 'Development Tools' -RUN yum -y install git maven -RUN alternatives --set javac /usr/lib/jvm/java-11-openjdk-11.*/bin/javac -RUN alternatives --set java /usr/lib/jvm/java-11-openjdk-11.*/bin/java -RUN git clone --single-branch --branch master https://github.com/rasilon/ujetl.git -RUN cd ujetl && JAVA_HOME=/usr/lib/jvm/java-11 mvn -e package - - - +FROM ubuntu:22.04 as builder +RUN apt-get update && apt-get -y upgrade +RUN apt-get -y install openjdk-19-jdk-headless maven git +RUN git clone --single-branch --branch main https://github.com/rasilon/ujetl.git +RUN cd ujetl && mvn -e package FROM openjdk:11 as runner LABEL maintainer="Derry Hamilton " diff --git a/pom.xml b/pom.xml index 7fdfb89..1ed75f7 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma com.rasilon.ujetl CopyingApp jar - 2.3.3 + 2.4.3 uJETL https://github.com/rasilon/ujetl diff --git a/uJETL.spec b/uJETL.spec deleted file mode 100644 index d64abdf..0000000 --- a/uJETL.spec +++ /dev/null @@ -1,33 +0,0 @@ -Summary: Java app to facilitate moving data between databases. -Name: uJETL -Version: 2.3.3 -Release: 1 -Group: Applications/Database -License: All rights reserved. -Source: uJETL-%{version}.tar.gz -URL: https://github.com/rasilon/ujetl.git -Distribution: derryh -Vendor: derryh -Packager: Derry Hamilton -#BuildRoot: . - -%description -A very small ETL app - -%prep -%setup - -%build -#mvn -Dmaven.test.skip=true clean package -true - -%install -mkdir -p $RPM_BUILD_ROOT/usr/share/ujetl/lib $RPM_BUILD_ROOT/etc/ujetl $RPM_BUILD_ROOT/usr/bin -cp target/CopyingApp-*-jar-with-dependencies.jar $RPM_BUILD_ROOT/usr/share/ujetl/lib/CopyingApp.jar -cp install_extra/run_copying_job $RPM_BUILD_ROOT/usr/bin -cp install_extra/copying_defaults_log4j.xml $RPM_BUILD_ROOT/etc/ujetl - -%files -/usr/share/ujetl/lib/CopyingApp.jar -/usr/bin/run_copying_job -/etc/ujetl/copying_defaults_log4j.xml From 2126553e5cba16859c4cc08e641a04a786d51c51 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Jul 2023 21:54:12 +0000 Subject: [PATCH 15/31] Bump h2 from 2.1.210 to 2.2.220 Bumps [h2](https://github.com/h2database/h2database) from 2.1.210 to 2.2.220. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-2.1.210...version-2.2.220) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1ed75f7..c364aa4 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma com.h2database h2 - 2.1.210 + 2.2.220 test From 49487a83afbc266cdce6c612843c95f55735cf93 Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Tue, 11 Jul 2023 12:33:48 +0100 Subject: [PATCH 16/31] Update unit test syntax to the current version of H2 --- src/test/java/com/rasilon/ujetl/TestJob.java | 2 +- src/test/java/com/rasilon/ujetl/TestPrePost.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/rasilon/ujetl/TestJob.java b/src/test/java/com/rasilon/ujetl/TestJob.java index 853aeb4..63ab4c8 100644 --- a/src/test/java/com/rasilon/ujetl/TestJob.java +++ b/src/test/java/com/rasilon/ujetl/TestJob.java @@ -42,7 +42,7 @@ public class TestJob { dest, "jUnit Test Config", "jUnit Test Job", - "SELECT -1 AS key", + "SELECT -1 AS \"key\"", "SELECT id,dat FROM src WHERE id > ?", "INSERT INTO dest VALUES(?,?)", null, diff --git a/src/test/java/com/rasilon/ujetl/TestPrePost.java b/src/test/java/com/rasilon/ujetl/TestPrePost.java index 77ff2a6..55f3ee1 100644 --- a/src/test/java/com/rasilon/ujetl/TestPrePost.java +++ b/src/test/java/com/rasilon/ujetl/TestPrePost.java @@ -42,7 +42,7 @@ public class TestPrePost { dest, "jUnit Test Config", "jUnit Test Job", - "SELECT -1 AS key", + "SELECT -1 AS \"key\"", "SELECT id,dat FROM src WHERE id > ?", "INSERT INTO tmp_dest VALUES(?,?)", "CREATE TEMP TABLE tmp_dest(id bigint not null primary key, dat varchar);", From b37818951234ea5d164cde53507551f71532378a Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Tue, 11 Jul 2023 12:36:44 +0100 Subject: [PATCH 17/31] Update default log config location --- src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java b/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java index d0d4d73..b700221 100644 --- a/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java +++ b/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java @@ -12,7 +12,7 @@ public class CopyingAppCommandParser { private String configFile; @Parameter(names = {"-log4j","--log4j"}, description = "Log4J config file for this run") - private String log4jConfigFile = "/etc/ppl/default_log4j_config.properties"; + private String log4jConfigFile = "/etc/ujetl/default_log4j_config.properties"; public CopyingAppCommandParser(String[] args) { super(); From 4d38679155011369390d61cc1a69cf8a121f92c2 Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Tue, 11 Jul 2023 12:49:34 +0100 Subject: [PATCH 18/31] bump minor version with H2 update --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c364aa4..304defa 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma com.rasilon.ujetl CopyingApp jar - 2.4.3 + 2.4.4 uJETL https://github.com/rasilon/ujetl From b81aedefb1bb91bc847dcc0981038b2186106338 Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Tue, 11 Jul 2023 13:21:04 +0100 Subject: [PATCH 19/31] Script te docker tagging --- build_util/create_run_docker | 1 + build_util/push_docker_images | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 build_util/push_docker_images diff --git a/build_util/create_run_docker b/build_util/create_run_docker index 1acbb42..864fdf6 100755 --- a/build_util/create_run_docker +++ b/build_util/create_run_docker @@ -1,2 +1,3 @@ #!/bin/bash docker build --target deploy -t rasilon/ujetl docker/multistage +docker tag rasilon/ujetl:latest rasilon/ujetl:$(xpath -q -e '/project/version/text()' pom.xml) diff --git a/build_util/push_docker_images b/build_util/push_docker_images new file mode 100644 index 0000000..14afd64 --- /dev/null +++ b/build_util/push_docker_images @@ -0,0 +1,4 @@ +#!/bin/bash +docker push rasilon/ujetl:latest +docker push rasilon/ujetl:$(xpath -q -e '/project/version/text()' pom.xml) + From 866d02fb525ad611f0dfdaddf1e561ba5a0c3b0e Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Tue, 11 Jul 2023 14:07:37 +0100 Subject: [PATCH 20/31] Change how log4j finds its config. log4j is reacting less well to reconfigurations, so this moves the config file location to earlier in the process, so that no class sees an unconfigured instance. --- install_extra/run_copying_job | 2 +- pom.xml | 2 +- src/main/java/com/rasilon/ujetl/CopyingApp.java | 4 ---- src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java | 4 ---- src/test/java/com/rasilon/ujetl/TestParser.java | 3 --- 5 files changed, 2 insertions(+), 13 deletions(-) diff --git a/install_extra/run_copying_job b/install_extra/run_copying_job index bf51414..5dfa2b0 100755 --- a/install_extra/run_copying_job +++ b/install_extra/run_copying_job @@ -30,9 +30,9 @@ fi /usr/bin/java \ -Xms1g \ -Xmx2g \ + -Dlog4j.configuration=file:"$LOG_PROPS" \ -cp /usr/share/ujetl/lib/CopyingApp.jar \ com.rasilon.ujetl.CopyingApp \ - --log4j "$LOG_PROPS" \ --config "/etc/ujetl/${JOBNAME}_config_live.xml" #rm -f $LOCKFILE diff --git a/pom.xml b/pom.xml index 304defa..043daad 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma com.rasilon.ujetl CopyingApp jar - 2.4.4 + 2.5.0 uJETL https://github.com/rasilon/ujetl diff --git a/src/main/java/com/rasilon/ujetl/CopyingApp.java b/src/main/java/com/rasilon/ujetl/CopyingApp.java index e1e47a5..1100d93 100644 --- a/src/main/java/com/rasilon/ujetl/CopyingApp.java +++ b/src/main/java/com/rasilon/ujetl/CopyingApp.java @@ -34,10 +34,6 @@ public class CopyingApp { public static void main(String[] args) { CopyingAppCommandParser cli = new CopyingAppCommandParser(args); LoggerContext context = (org.apache.logging.log4j.core.LoggerContext) LogManager.getContext(false); - String log4jConfigLocation = cli.getLog4jConfigFile(); - File file = new File(log4jConfigLocation); - context.setConfigLocation(file.toURI()); - System.out.println("Config set from "+file.toURI()); CopyingApp app = new CopyingApp(cli); try { diff --git a/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java b/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java index b700221..97d2dc8 100644 --- a/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java +++ b/src/main/java/com/rasilon/ujetl/CopyingAppCommandParser.java @@ -23,8 +23,4 @@ public class CopyingAppCommandParser { return configFile; } - public String getLog4jConfigFile() { - return log4jConfigFile; - } - } diff --git a/src/test/java/com/rasilon/ujetl/TestParser.java b/src/test/java/com/rasilon/ujetl/TestParser.java index 13366fd..398f046 100644 --- a/src/test/java/com/rasilon/ujetl/TestParser.java +++ b/src/test/java/com/rasilon/ujetl/TestParser.java @@ -13,15 +13,12 @@ public class TestParser { public void test001Parset() { try { String[] args = { - "--log4j", - "log4j_test_banana.xml", "--config", "config_test_banana.xml" }; CopyingAppCommandParser p = new CopyingAppCommandParser(args); assertEquals(p.getConfigFile(),"config_test_banana.xml"); - assertEquals(p.getLog4jConfigFile(),"log4j_test_banana.xml"); } catch(Exception e) { fail(e.toString()); From 1b1ba551c8bd47b319d60b7d208c9fe71d6c76c8 Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Tue, 11 Jul 2023 15:04:33 +0100 Subject: [PATCH 21/31] Add forced driver loading --- docker/multistage/TEST_config_live.xml | 5 +++ docker/multistage/small.csv | 4 ++ docker/test_db/setup.sql | 6 ++- pom.xml | 5 +++ .../java/com/rasilon/ujetl/CopyingApp.java | 8 ++++ .../java/com/rasilon/ujetl/TestConfig.java | 37 +++++++++++++++++++ src/test/resources/TEST_config_live.xml | 5 +++ 7 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 docker/multistage/small.csv create mode 100644 src/test/java/com/rasilon/ujetl/TestConfig.java diff --git a/docker/multistage/TEST_config_live.xml b/docker/multistage/TEST_config_live.xml index fa38eec..d1b7345 100644 --- a/docker/multistage/TEST_config_live.xml +++ b/docker/multistage/TEST_config_live.xml @@ -4,6 +4,11 @@ 10000 1000 500 + + org.postgresql.Driver + org.relique.jdbc.csv.CsvDriver + + jdbc:postgresql://testdb:5432/test test diff --git a/docker/multistage/small.csv b/docker/multistage/small.csv new file mode 100644 index 0000000..37ddeff --- /dev/null +++ b/docker/multistage/small.csv @@ -0,0 +1,4 @@ +id,dat +1,banana +2,potato +3,nugget diff --git a/docker/test_db/setup.sql b/docker/test_db/setup.sql index ff4a1b7..2469967 100644 --- a/docker/test_db/setup.sql +++ b/docker/test_db/setup.sql @@ -44,10 +44,14 @@ CREATE TABLE denormalised_personalia( lname text ); +CREATE TABLE test_csvjdbc( + id integer not null primary key, + dat text +); + GRANT SELECT ON ALL TABLES IN SCHEMA public TO test; GRANT SELECT,INSERT,UPDATE ON denormalised_personalia TO test; - \c postgres CREATE TABLE public.container_ready AS SELECT 1 FROM(VALUES(1)) AS a(a); GRANT SELECT ON public.container_ready TO TEST; diff --git a/pom.xml b/pom.xml index 043daad..3b55a96 100644 --- a/pom.xml +++ b/pom.xml @@ -78,6 +78,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma postgresql 42.4.3 + + net.sourceforge.csvjdbc + csvjdbc + 1.0.40 + diff --git a/src/main/java/com/rasilon/ujetl/CopyingApp.java b/src/main/java/com/rasilon/ujetl/CopyingApp.java index 1100d93..60dd65d 100644 --- a/src/main/java/com/rasilon/ujetl/CopyingApp.java +++ b/src/main/java/com/rasilon/ujetl/CopyingApp.java @@ -75,6 +75,7 @@ public class CopyingApp { Configuration config = configs.xml(cli.getConfigFile()); + loadDrivers(config); String hardLimitSeconds = config.getString("hardLimitSeconds"); if(hardLimitSeconds != null) { TimeLimiter hardLimit = new TimeLimiter(Integer.decode(hardLimitSeconds).intValue(),true); @@ -240,4 +241,11 @@ public class CopyingApp { return c; } + + private void loadDrivers(Configuration config){ + String[] drivers = config.get(String[].class, "drivers.driver"); + for(String d:drivers){ + log.info("Would load "+d); + } + } } diff --git a/src/test/java/com/rasilon/ujetl/TestConfig.java b/src/test/java/com/rasilon/ujetl/TestConfig.java new file mode 100644 index 0000000..bbf855b --- /dev/null +++ b/src/test/java/com/rasilon/ujetl/TestConfig.java @@ -0,0 +1,37 @@ +package com.rasilon.ujetl; + +import org.apache.commons.configuration2.Configuration; +import org.apache.commons.configuration2.builder.fluent.Configurations; +import org.apache.commons.configuration2.ex.ConfigurationException; + +import org.apache.commons.beanutils.PropertyUtils; // Why does config need this? + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.MethodOrderer.Alphanumeric; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + + +/** + * @author derryh + * + */ +public class TestConfig { + + @Test + public void test002verifyH2Works() { + try { + Configurations configs = new Configurations(); + Configuration config = configs.xml("TEST_config_live.xml"); + String[] drivers = config.get(String[].class, "drivers.driver"); + int ndrivers =drivers.length; + if(ndrivers != 3){ + fail("Expected 3 drivers, but found "+ndrivers); + } + } catch(Exception e) { + fail(e.toString()); + } + } + +} diff --git a/src/test/resources/TEST_config_live.xml b/src/test/resources/TEST_config_live.xml index 338faff..210d8f1 100644 --- a/src/test/resources/TEST_config_live.xml +++ b/src/test/resources/TEST_config_live.xml @@ -4,6 +4,11 @@ 10000 1000 500 + + org.postgresql.Driver + org.h2.Driver + org.relique.jdbc.csv.CsvDriver + jdbc:postgresql://localhost:5432/test test From 9a8716f33e7a4a6f0638b9c6beee5747518ccfe9 Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Wed, 12 Jul 2023 08:45:48 +0100 Subject: [PATCH 22/31] Add the real implementation --- .../java/com/rasilon/ujetl/CopyingApp.java | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/rasilon/ujetl/CopyingApp.java b/src/main/java/com/rasilon/ujetl/CopyingApp.java index 60dd65d..fa830c8 100644 --- a/src/main/java/com/rasilon/ujetl/CopyingApp.java +++ b/src/main/java/com/rasilon/ujetl/CopyingApp.java @@ -105,14 +105,14 @@ public class CopyingApp { log.info(String.format("%s - Setting Row count interval to default of 100 rows.",jobName)); } - Integer pollTimeout = null; - try { - pollTimeout = new Integer(config.getString("pollTimeout")); - log.info(String.format("%s - Setting Poll timeout to %s milliseconds", jobName, pollTimeout)); - } catch(Exception e) { - pollTimeout = new Integer(1000); // If we don't have a new setting, use the old default - log.info(String.format("%s - Setting poll timeout to default of 1 second.",jobName)); - } + Integer pollTimeout = null; + try { + pollTimeout = new Integer(config.getString("pollTimeout")); + log.info(String.format("%s - Setting Poll timeout to %s milliseconds", jobName, pollTimeout)); + } catch(Exception e) { + pollTimeout = new Integer(1000); // If we don't have a new setting, use the old default + log.info(String.format("%s - Setting poll timeout to default of 1 second.",jobName)); + } @@ -148,7 +148,7 @@ public class CopyingApp { pollTimeout, identifySourceSQL, identifyDestinationSQL - ); + ); j.start(); j.join(); @@ -179,7 +179,7 @@ public class CopyingApp { pollTimeout, identifySourceSQL, identifyDestinationSQL - ); + ); j.start(); j.join(); } else { @@ -242,10 +242,20 @@ public class CopyingApp { return c; } - private void loadDrivers(Configuration config){ + // Even with JDBC 4, some drivers don't play nicely with whatever + // the classloaders are up to. So this allows us to force it the + // old fashioned way, and works around the + // "But it works fine when it's the /only/ driver!" + // cross-database problem + private void loadDrivers(Configuration config) { String[] drivers = config.get(String[].class, "drivers.driver"); - for(String d:drivers){ - log.info("Would load "+d); + for(String d:drivers) { + try { + Class.forName(d); + log.info("Preloaded driver "+d); + } catch(ClassNotFoundException e) { + log.error("Could not preload driver "+d,e); + } } } } From a88ac568485aa46d3bb2de41eeb814076bdee7df Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Wed, 12 Jul 2023 08:48:21 +0100 Subject: [PATCH 23/31] Correct test name; CnP error --- src/test/java/com/rasilon/ujetl/TestConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/rasilon/ujetl/TestConfig.java b/src/test/java/com/rasilon/ujetl/TestConfig.java index bbf855b..05f9d82 100644 --- a/src/test/java/com/rasilon/ujetl/TestConfig.java +++ b/src/test/java/com/rasilon/ujetl/TestConfig.java @@ -20,7 +20,7 @@ import static org.junit.jupiter.api.Assertions.fail; public class TestConfig { @Test - public void test002verifyH2Works() { + public void test001VerifyArrayOfDrivers() { try { Configurations configs = new Configurations(); Configuration config = configs.xml("TEST_config_live.xml"); From f651fd720e2cca1c3738da7508854981c0c54e2e Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Wed, 12 Jul 2023 09:47:21 +0100 Subject: [PATCH 24/31] Fix log file param --- build_util/push_docker_images | 0 install_extra/run_copying_job | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 build_util/push_docker_images diff --git a/build_util/push_docker_images b/build_util/push_docker_images old mode 100644 new mode 100755 diff --git a/install_extra/run_copying_job b/install_extra/run_copying_job index 5dfa2b0..be923bd 100755 --- a/install_extra/run_copying_job +++ b/install_extra/run_copying_job @@ -30,7 +30,7 @@ fi /usr/bin/java \ -Xms1g \ -Xmx2g \ - -Dlog4j.configuration=file:"$LOG_PROPS" \ + -Dlog4j.configurationFile="$LOG_PROPS" \ -cp /usr/share/ujetl/lib/CopyingApp.jar \ com.rasilon.ujetl.CopyingApp \ --config "/etc/ujetl/${JOBNAME}_config_live.xml" From d0a5075191d4642301b54ad9fac2efccfadda531 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Feb 2024 00:12:25 +0000 Subject: [PATCH 25/31] Bump org.postgresql:postgresql from 42.4.3 to 42.7.2 Bumps [org.postgresql:postgresql](https://github.com/pgjdbc/pgjdbc) from 42.4.3 to 42.7.2. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/commits) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3b55a96..8de826d 100644 --- a/pom.xml +++ b/pom.xml @@ -76,7 +76,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma org.postgresql postgresql - 42.4.3 + 42.7.2 net.sourceforge.csvjdbc From 07f68922b39fd8056138410494d6013cb5bd1604 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Mar 2024 19:15:25 +0000 Subject: [PATCH 26/31] Bump org.apache.commons:commons-configuration2 from 2.8.0 to 2.10.1 Bumps org.apache.commons:commons-configuration2 from 2.8.0 to 2.10.1. --- updated-dependencies: - dependency-name: org.apache.commons:commons-configuration2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8de826d..d457914 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma org.apache.commons commons-configuration2 - 2.8.0 + 2.10.1 commons-beanutils From 1da5fabb052f139f248ee914ca223691d8961ffd Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Wed, 17 Apr 2024 14:22:03 +0100 Subject: [PATCH 27/31] Update version for dep updates --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3b55a96..3d96646 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma com.rasilon.ujetl CopyingApp jar - 2.5.0 + 2.5.1 uJETL https://github.com/rasilon/ujetl From d50951308471ffb943d0d0d00310058fdd92bf64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 May 2025 18:01:36 +0000 Subject: [PATCH 28/31] Bump commons-beanutils:commons-beanutils from 1.9.4 to 1.11.0 Bumps commons-beanutils:commons-beanutils from 1.9.4 to 1.11.0. --- updated-dependencies: - dependency-name: commons-beanutils:commons-beanutils dependency-version: 1.11.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d457914..9dd513e 100644 --- a/pom.xml +++ b/pom.xml @@ -56,7 +56,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma commons-beanutils commons-beanutils - 1.9.4 + 1.11.0 com.beust From 9f0fe6bb3b751236e52c0be30fc8045a1acfe89c Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Tue, 3 Jun 2025 09:23:29 +0100 Subject: [PATCH 29/31] Security; update dependencies --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index aa354a5..01e44ec 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma com.rasilon.ujetl CopyingApp jar - 2.5.1 + 2.5.2 uJETL https://github.com/rasilon/ujetl From 26d47f16c9744c0c4522190f6e967ac1ae79b50e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 12 Jul 2025 02:04:46 +0000 Subject: [PATCH 30/31] Bump org.apache.commons:commons-lang3 from 3.9 to 3.18.0 Bumps org.apache.commons:commons-lang3 from 3.9 to 3.18.0. --- updated-dependencies: - dependency-name: org.apache.commons:commons-lang3 dependency-version: 3.18.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 01e44ec..dc76cb4 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma org.apache.commons commons-lang3 - 3.9 + 3.18.0 commons-logging From d2257f1b8195840c97e59a299abf78524fd746f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Dec 2025 22:49:23 +0000 Subject: [PATCH 31/31] Bump org.apache.logging.log4j:log4j-core from 2.17.1 to 2.25.3 Bumps org.apache.logging.log4j:log4j-core from 2.17.1 to 2.25.3. --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-core dependency-version: 2.25.3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dc76cb4..6970fdb 100644 --- a/pom.xml +++ b/pom.xml @@ -71,7 +71,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/ma org.apache.logging.log4j log4j-core - 2.17.1 + 2.25.3 org.postgresql