From 5571bd1b4d41fdc885bf1fa0a2695bf44ea3a876 Mon Sep 17 00:00:00 2001 From: Derry Hamilton Date: Mon, 17 Jun 2019 12:53:31 +0100 Subject: [PATCH] Add dockerised tests --- .../{build_in_docker => build_rpms_in_docker} | 0 build_util/create_run_docker | 4 +- build_util/run_docker_tests | 5 ++ docker/build/Dockerfile | 31 ----------- docker/multistage/Dockerfile | 21 +++++--- docker/multistage/TEST_config_live.xml | 52 +++++++++++++++++++ docker/multistage/ujetl_entrypoint | 4 ++ docker/multistage/wait_for_postgres | 12 +++++ docker/run/Dockerfile | 18 ------- docker/run/ujetl_entrypoint | 17 ------ docker/test_compose/docker-compose.yml | 23 ++++++++ docker/test_db/Dockerfile | 3 ++ docker/test_db/is_ready | 3 ++ docker/test_db/setup.sql | 26 ++++++++++ 14 files changed, 143 insertions(+), 76 deletions(-) rename build_util/{build_in_docker => build_rpms_in_docker} (100%) create mode 100755 build_util/run_docker_tests delete mode 100644 docker/build/Dockerfile create mode 100644 docker/multistage/TEST_config_live.xml create mode 100755 docker/multistage/wait_for_postgres delete mode 100644 docker/run/Dockerfile delete mode 100644 docker/run/ujetl_entrypoint create mode 100644 docker/test_compose/docker-compose.yml create mode 100644 docker/test_db/Dockerfile create mode 100644 docker/test_db/is_ready create mode 100644 docker/test_db/setup.sql diff --git a/build_util/build_in_docker b/build_util/build_rpms_in_docker similarity index 100% rename from build_util/build_in_docker rename to build_util/build_rpms_in_docker diff --git a/build_util/create_run_docker b/build_util/create_run_docker index 0ccc0f6..1acbb42 100755 --- a/build_util/create_run_docker +++ b/build_util/create_run_docker @@ -1,4 +1,2 @@ #!/bin/bash -set -e -cp *.rpm docker/run/ -docker build --rm -t local/c7-runhost docker/run +docker build --target deploy -t rasilon/ujetl docker/multistage diff --git a/build_util/run_docker_tests b/build_util/run_docker_tests new file mode 100755 index 0000000..e8bd891 --- /dev/null +++ b/build_util/run_docker_tests @@ -0,0 +1,5 @@ +#!/bin/bash + +docker-compose -f docker/test_compose/docker-compose.yml run --rm tests +docker-compose -f docker/test_compose/docker-compose.yml down + diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile deleted file mode 100644 index c780b8f..0000000 --- a/docker/build/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -# General build RPM environment for CentOS 6.x -# -# VERSION 0.0.1 - -FROM centos:centos7 -MAINTAINER Derry Hamilton - -# Install up-to-date epel rpm repository -RUN yum -y install epel-release - -# Install java first, to get a sensible one. -RUN yum -y install java-1.8.0-openjdk-devel - -# Install various packages to get compile environment -RUN yum -y groupinstall 'Development Tools' - -# Install git command to access GitHub repository -RUN yum -y install git - -# Install rpm-build to use rpmrebuild command -RUN yum -y install rpm-build - -# Install yum-utils to use yumdownloader command -RUN yum -y install yum-utils - -# Install rpmdevtools to use rpmdev-setuptree command -RUN yum -y install rpmdevtools - -# Install rpmdevtools to use rpmdev-setuptree command -RUN yum -y install maven -RUN mkdir -p /root/rpmbuild/SOURCES diff --git a/docker/multistage/Dockerfile b/docker/multistage/Dockerfile index 4ff472a..f529941 100644 --- a/docker/multistage/Dockerfile +++ b/docker/multistage/Dockerfile @@ -8,17 +8,24 @@ RUN cd ujetl && mvn package -#FROM centos:centos7 -FROM openjdk:8-alpine -MAINTAINER Derry Hamilton +FROM openjdk:8-alpine as runner +LABEL maintainer="Derry Hamilton " -#RUN yum -y install epel-release -#RUN yum -y install java-1.8.0-openjdk-devel +RUN apk update && apk upgrade && apk add bash RUN mkdir -p /usr/share/ujetl/lib/ /var/ujetl /etc/ujetl COPY --from=builder /ujetl/target/CopyingApp-2.*-jar-with-dependencies.jar /usr/share/ujetl/lib/CopyingApp.jar COPY --from=builder /ujetl/install_extra/copying_defaults_log4j.xml /etc/ujetl/ -COPY docker/multistage/ujetl_entrypoint / -ENTRYPOINT ["/ujetl_entrypoint"] +COPY ujetl_entrypoint / +CMD ["/ujetl_entrypoint"] + +FROM runner as tester +COPY TEST_config_live.xml /var/ujetl/ +COPY wait_for_postgres / +RUN apk add postgresql-client + + +FROM runner as deploy +# Convice docker cloud to build the deploy image diff --git a/docker/multistage/TEST_config_live.xml b/docker/multistage/TEST_config_live.xml new file mode 100644 index 0000000..97f0500 --- /dev/null +++ b/docker/multistage/TEST_config_live.xml @@ -0,0 +1,52 @@ + + + 360000 + 10000 + 1000 + + jdbc:postgresql://testdb:5432/test + test + test + 600000 + + + jdbc:postgresql://testdb:5432/test + test + test + + + + test + select coalesce(-1,max(id),-1) as key from dest + + + insert into public.dest( + id, + test_int, + test_text, + test_ts + )values( + ?::bigint, + ?::integer, + ?::text, + ?::timestamp with time zone + )ON CONFLICT(id) DO UPDATE + set + test_int = EXCLUDED.test_int, + test_text = EXCLUDED.test_text, + test_ts = EXCLUDED.test_ts + WHERE + dest.test_int = EXCLUDED.test_int + OR dest.test_text = EXCLUDED.test_text + OR dest.test_ts = EXCLUDED.test_ts + + + + diff --git a/docker/multistage/ujetl_entrypoint b/docker/multistage/ujetl_entrypoint index b5c73fe..9979de6 100755 --- a/docker/multistage/ujetl_entrypoint +++ b/docker/multistage/ujetl_entrypoint @@ -4,6 +4,10 @@ set -e LOG_PROPS=/etc/ujetl/copying_defaults_log4j.xml cd /var/ujetl +echo Currently in `pwd` +echo processing files: +ls +echo Starting run loop for file in *.xml do /usr/bin/java \ diff --git a/docker/multistage/wait_for_postgres b/docker/multistage/wait_for_postgres new file mode 100755 index 0000000..d3dee95 --- /dev/null +++ b/docker/multistage/wait_for_postgres @@ -0,0 +1,12 @@ +#!/bin/bash +set -e + +cmd="$@" + +until PGPASSWORD=test psql -h "testdb" -U "test" -c 'SELECT 1 FROM public.container_ready' postgres; do + >&2 echo "Postgres is unavailable - sleeping" + sleep 1 +done + +>&2 echo "Postgres is up - executing command" +exec $cmd diff --git a/docker/run/Dockerfile b/docker/run/Dockerfile deleted file mode 100644 index bf6cc6d..0000000 --- a/docker/run/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# General build RPM environment for CentOS 6.x -# -# VERSION 0.0.1 - -FROM centos:centos7 -MAINTAINER Derry Hamilton - -# Install up-to-date epel rpm repository -RUN yum -y install epel-release - -# Install java first, to get a sensible one. -RUN yum -y install java-1.8.0-openjdk-devel - -COPY uJETL-2.*.x86_64.rpm /tmp/ -COPY ujetl_entrypoint / -RUN rpm -i /tmp/uJETL-2.*.x86_64.rpm -ENTRYPOINT ["/ujetl_entrypoint"] - diff --git a/docker/run/ujetl_entrypoint b/docker/run/ujetl_entrypoint deleted file mode 100644 index b5c73fe..0000000 --- a/docker/run/ujetl_entrypoint +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -set -e - -LOG_PROPS=/etc/ujetl/copying_defaults_log4j.xml - -cd /var/ujetl -for file in *.xml -do - /usr/bin/java \ - -Xms1g \ - -Xmx2g \ - -cp /usr/share/ujetl/lib/CopyingApp.jar \ - com.rasilon.ujetl.CopyingApp \ - --log4j "$LOG_PROPS" \ - --config "$file" -done - diff --git a/docker/test_compose/docker-compose.yml b/docker/test_compose/docker-compose.yml new file mode 100644 index 0000000..5048838 --- /dev/null +++ b/docker/test_compose/docker-compose.yml @@ -0,0 +1,23 @@ +# This is a sample to help put the full application together + +version: '3.3' + +services: + testdb: + image: rasilon/ujetl_testdb:latest + build: + context: ../test_db + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + POSTGRES_DB: postgres + tests: + image: rasilon/ujetl_test:latest + build: + context: ../multistage + links: + - "testdb" + command: ["/wait_for_postgres", "/ujetl_entrypoint"] + + + diff --git a/docker/test_db/Dockerfile b/docker/test_db/Dockerfile new file mode 100644 index 0000000..868ee9f --- /dev/null +++ b/docker/test_db/Dockerfile @@ -0,0 +1,3 @@ +FROM postgres:11 +COPY setup.sql /docker-entrypoint-initdb.d/ +COPY is_ready / diff --git a/docker/test_db/is_ready b/docker/test_db/is_ready new file mode 100644 index 0000000..a92c259 --- /dev/null +++ b/docker/test_db/is_ready @@ -0,0 +1,3 @@ +#!/bin/bash +/usr/lib/postgresql/9.6/bin/psql -U postgres -c "SELECT 1 FROM public.container_ready" postgres + diff --git a/docker/test_db/setup.sql b/docker/test_db/setup.sql new file mode 100644 index 0000000..d476bde --- /dev/null +++ b/docker/test_db/setup.sql @@ -0,0 +1,26 @@ +CREATE DATABASE test; +\c test +CREATE ROLE test login password 'test'; +CREATE UNLOGGED TABLE source ( + id bigserial primary key, + test_int integer, + test_text text, + test_ts timestamp with time zone +); +CREATE UNLOGGED TABLE dest ( + id bigint primary key, + test_int integer, + test_text text, + test_ts timestamp with time zone +); + + +GRANT SELECT ON source to test; +GRANT SELECT,INSERT,UPDATE,DELETE ON dest TO test; + +INSERT INTO source(test_int,test_text,test_ts) SELECT 1,'banana',now() FROM generate_series(1,100000); + +\c postgres +CREATE TABLE public.container_ready AS SELECT 1 FROM(VALUES(1)) AS a(a); +GRANT SELECT ON public.container_ready TO TEST; +