derryh/ujetl
1
0
Fork 0
mirror of https://github.com/rasilon/ujetl.git synced 2026-04-11 10:29:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add more test data, and fix log ordering

Browse source
This commit is contained in:
Derry Hamilton 2019-07-11 09:13:15 +01:00
parent 97baf0d11a
commit 2b5f4f1dbe
4 changed files with 62 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

21
docker/multistage/TEST_config_live.xml
View file

@ -3,14 +3,15 @@
<hardLimitSeconds>360000</hardLimitSeconds>
<nRowsToLog>10000</nRowsToLog>
<blockSize>1000</blockSize>
<pollTimeout>500</pollTimeout>
<source>
<dsn>jdbc:postgresql://testdb:5432/test</dsn>
<dsn>jdbc:postgresql://localhost:5432/test</dsn>
<username>test</username>
<password>test</password>
<networkTimeout>600000</networkTimeout>
</source>
<dest>
<dsn>jdbc:postgresql://testdb:5432/test</dsn>
<dsn>jdbc:postgresql://localhost:5432/test</dsn>
<username>test</username>
<password>test</password>
</dest>
@ -48,5 +49,21 @@
OR dest.test_ts = EXCLUDED.test_ts
</insert>
</job>
<job>
<name>denormalise</name>
<key>select -1 as key</key>
<select>select person_id,fname,lname from normalised_personalia p join normalised_first_names f using(fid) join normalised_last_names l using(lid);</select>
<insert>
INSERT INTO denormalised_personalia(person_id,fname,lname)
values(?::integer,?::text,?::text)
ON CONFLICT (person_id) DO UPDATE
SET
fname = EXCLUDED.fname,
lname = EXCLUDED.lname
WHERE
denormalised_personalia.fname is distinct from EXCLUDED.fname
OR denormalised_personalia.lname is distinct from EXCLUDED.lname
</insert>
</job>
</jobs>
</CopyingApp>

25
docker/test_db/setup.sql
View file

@ -20,6 +20,31 @@ GRANT SELECT,INSERT,UPDATE,DELETE ON dest TO test;
INSERT INTO source(test_int,test_text,test_ts) SELECT 1,'banana',now() FROM generate_series(1,100000);
CREATE TABLE normalised_first_names(
fid smallserial not null primary key,
fname text not null unique
);
CREATE TABLE normalised_last_names(
lid smallserial not null primary key,
lname text not null unique
);
INSERT INTO normalised_first_names (fname) values ('Abigail'), ('Adam'), ('Beatrice'), ('Bruce'), ('Claire'), ('Clive'), ('Deborah'), ('Dave');
INSERT INTO normalised_last_names (lname) values ('Adams'), ('Bellamy'), ('Clark'), ('Dabrowski');
CREATE TABLE normalised_personalia (
person_id serial not null primary key,
fid smallint not null references normalised_first_names(fid),
lid smallint not null references normalised_last_names(lid)
);
insert into normalised_personalia(fid,lid) values (1,1), (1,2), (1,3), (1,4), (2,1), (2,2), (2,3), (2,4), (3,1), (3,2), (3,3), (3,4), (4,1), (4,2), (4,3), (4,4);
CREATE TABLE denormalised_personalia(
person_id integer not null primary key,
fname text,
lname text
);
\c postgres
CREATE TABLE public.container_ready AS SELECT 1 FROM(VALUES(1)) AS a(a);
GRANT SELECT ON public.container_ready TO TEST;

4
src/main/java/com/rasilon/ujetl/Job.java
View file

@ -154,10 +154,10 @@ public class Job extends Thread {
if(rowNum % nRowsToLog == 0) {
rowsInserted += arraySum(insertStatement.executeBatch());
dConn.commit();
log.info(String.format("%s - Inserted %s of %s notified rows into %s so far",
log.info(String.format("%s - Inserted %s of %s notified rows into %s",
jobName,
rowNum,
rowsInserted,
rowNum,
name));
}
}

16
src/test/resources/TEST_config_live.xml
View file

@ -49,5 +49,21 @@
OR dest.test_ts = EXCLUDED.test_ts
</insert>
</job>
<job>
<name>denormalise</name>
<key>select -1 as key</key>
<select>select person_id,fname,lname from normalised_personalia p join normalised_first_names f using(fid) join normalised_last_names l using(lid);</select>
<insert>
INSERT INTO denormalised_personalia(person_id,fname,lname)
values(?::integer,?::text,?::text)
ON CONFLICT (person_id) DO UPDATE
SET
fname = EXCLUDED.fname,
lname = EXCLUDED.lname
WHERE
denormalised_personalia.fname is distinct from EXCLUDED.fname
OR denormalised_personalia.lname is distinct from EXCLUDED.lname
</insert>
</job>
</jobs>
</CopyingApp>