Add more test data, and fix log ordering

docker/multistage/TEST_config_live.xml

@@ -3,14 +3,15 @@
   <hardLimitSeconds>360000</hardLimitSeconds>
   <nRowsToLog>10000</nRowsToLog>
   <blockSize>1000</blockSize>
+  <pollTimeout>500</pollTimeout>
   <source>
-    <dsn>jdbc:postgresql://testdb:5432/test</dsn>
+    <dsn>jdbc:postgresql://localhost:5432/test</dsn>
     <username>test</username>
     <password>test</password>
     <networkTimeout>600000</networkTimeout>
   </source>
   <dest>
-    <dsn>jdbc:postgresql://testdb:5432/test</dsn>
+    <dsn>jdbc:postgresql://localhost:5432/test</dsn>
     <username>test</username>
     <password>test</password>
   </dest>
@@ -48,5 +49,21 @@
             OR dest.test_ts = EXCLUDED.test_ts
       </insert>
     </job>
+    <job>
+        <name>denormalise</name>
+        <key>select -1 as key</key>
+        <select>select person_id,fname,lname from normalised_personalia p join normalised_first_names f using(fid) join normalised_last_names l using(lid);</select>
+        <insert>
+            INSERT INTO denormalised_personalia(person_id,fname,lname)
+            values(?::integer,?::text,?::text)
+            ON CONFLICT (person_id) DO UPDATE
+            SET
+                fname = EXCLUDED.fname,
+                lname = EXCLUDED.lname
+            WHERE
+                denormalised_personalia.fname is distinct from EXCLUDED.fname
+                OR denormalised_personalia.lname is distinct from EXCLUDED.lname
+        </insert>
+    </job>
   </jobs>
 </CopyingApp>

docker/test_db/setup.sql

@@ -20,6 +20,31 @@ GRANT SELECT,INSERT,UPDATE,DELETE ON dest TO test;
 
 INSERT INTO source(test_int,test_text,test_ts) SELECT 1,'banana',now() FROM generate_series(1,100000);
 
+CREATE TABLE normalised_first_names(
+    fid smallserial not null primary key,
+    fname text not null unique
+);
+CREATE TABLE normalised_last_names(
+    lid smallserial not null primary key,
+    lname text not null unique
+);
+INSERT INTO normalised_first_names (fname) values ('Abigail'), ('Adam'), ('Beatrice'), ('Bruce'), ('Claire'), ('Clive'), ('Deborah'), ('Dave');
+INSERT INTO normalised_last_names (lname)  values ('Adams'), ('Bellamy'), ('Clark'), ('Dabrowski');
+
+CREATE TABLE normalised_personalia (
+    person_id serial not null primary key,
+    fid smallint not null references normalised_first_names(fid),
+    lid smallint not null references normalised_last_names(lid)
+);
+insert into normalised_personalia(fid,lid) values (1,1), (1,2), (1,3), (1,4), (2,1), (2,2), (2,3), (2,4), (3,1), (3,2), (3,3), (3,4), (4,1), (4,2), (4,3), (4,4);
+
+CREATE TABLE denormalised_personalia(
+    person_id integer not null primary key,
+    fname text,
+    lname text
+);
+
+
 \c postgres
 CREATE TABLE public.container_ready AS SELECT 1 FROM(VALUES(1)) AS a(a);
 GRANT SELECT ON public.container_ready TO TEST;

src/main/java/com/rasilon/ujetl/Job.java

@@ -154,10 +154,10 @@ public class Job extends Thread {
                     if(rowNum % nRowsToLog == 0) {
                         rowsInserted += arraySum(insertStatement.executeBatch());
                         dConn.commit();
-                        log.info(String.format("%s - Inserted %s of %s notified rows into %s so far",
+                        log.info(String.format("%s - Inserted %s of %s notified rows into %s",
                             jobName,
-                            rowNum,
                             rowsInserted,
+                            rowNum,
                             name));
                     }
                 }

src/test/resources/TEST_config_live.xml

@@ -49,5 +49,21 @@
             OR dest.test_ts = EXCLUDED.test_ts
       </insert>
     </job>
+    <job>
+        <name>denormalise</name>
+        <key>select -1 as key</key>
+        <select>select person_id,fname,lname from normalised_personalia p join normalised_first_names f using(fid) join normalised_last_names l using(lid);</select>
+        <insert>
+            INSERT INTO denormalised_personalia(person_id,fname,lname)
+            values(?::integer,?::text,?::text)
+            ON CONFLICT (person_id) DO UPDATE
+            SET
+                fname = EXCLUDED.fname,
+                lname = EXCLUDED.lname
+            WHERE
+                denormalised_personalia.fname is distinct from EXCLUDED.fname
+                OR denormalised_personalia.lname is distinct from EXCLUDED.lname
+        </insert>
+    </job>
   </jobs>
 </CopyingApp>