diff --git a/archives/archives_test.go b/archives/archives_test.go index 78485ed..7ec5032 100644 --- a/archives/archives_test.go +++ b/archives/archives_test.go @@ -225,9 +225,9 @@ func TestCreateRunArchive(t *testing.T) { assert.NoError(t, err) // should have two record - assert.Equal(t, 2, task.RecordCount) - assert.Equal(t, int64(458), task.Size) - assert.Equal(t, "7220a13c19f5b6065e7d4c419c114635", task.Hash) + assert.Equal(t, 3, task.RecordCount) + assert.Equal(t, int64(578), task.Size) + assert.Equal(t, "cd8ce82019986ac1f4ec1482aac7bca0", task.Hash) assertArchiveFile(t, task, "runs1.jsonl") DeleteArchiveFile(task) @@ -459,7 +459,7 @@ func TestArchiveOrgRuns(t *testing.T) { time.Date(2020, 2, 1, 0, 0, 0, 0, time.UTC), ) assert.NoError(t, err) - assert.Equal(t, 3, count) + assert.Equal(t, 4, count) // more recent run unaffected (even though it was parent) count, err = getCountInRange( @@ -511,7 +511,7 @@ func TestArchiveActiveOrgs(t *testing.T) { "archiver.msgs_archives_failed": {0}, "archiver.msgs_rollups_created": {3}, "archiver.msgs_rollups_failed": {0}, - "archiver.runs_records_archived": {4}, + "archiver.runs_records_archived": {5}, "archiver.runs_archives_created": {41}, "archiver.runs_archives_failed": {1}, "archiver.runs_rollups_created": {3}, diff --git a/archives/runs.go b/archives/runs.go index 2764a02..a17e513 100644 --- a/archives/runs.go +++ b/archives/runs.go @@ -24,36 +24,47 @@ const ( const sqlLookupRuns = ` SELECT rec.uuid, rec.exited_on, row_to_json(rec) FROM ( - SELECT - fr.id as id, - fr.uuid as uuid, - row_to_json(flow_struct) AS flow, - row_to_json(contact_struct) AS contact, - fr.responded, - (SELECT coalesce(jsonb_agg(path_data), '[]'::jsonb) from ( - SELECT path_row ->> 'node_uuid' AS node, (path_row ->> 'arrived_on')::timestamptz as time - FROM jsonb_array_elements(fr.path::jsonb) AS path_row LIMIT 500) as path_data - ) as path, - (SELECT coalesce(jsonb_object_agg(values_data.key, values_data.value), '{}'::jsonb) from ( - SELECT key, jsonb_build_object('name', value -> 'name', 'value', value -> 'value', 'input', value -> 'input', 'time', (value -> 'created_on')::text::timestamptz, 'category', value -> 'category', 'node', value -> 'node_uuid') as value - FROM jsonb_each(fr.results::jsonb)) AS values_data - ) as values, - fr.created_on, - fr.modified_on, - fr.exited_on, - CASE - WHEN status = 'C' THEN 'completed' - WHEN status = 'I' THEN 'interrupted' - WHEN status = 'X' THEN 'expired' - WHEN status = 'F' THEN 'failed' - ELSE NULL - END as exit_type - - FROM flows_flowrun fr - JOIN LATERAL (SELECT uuid, name FROM flows_flow WHERE flows_flow.id = fr.flow_id) AS flow_struct ON True - JOIN LATERAL (SELECT uuid, name FROM contacts_contact cc WHERE cc.id = fr.contact_id) AS contact_struct ON True - WHERE fr.org_id = $1 AND fr.modified_on >= $2 AND fr.modified_on < $3 - ORDER BY fr.modified_on ASC, id ASC + SELECT + fr.id as id, + fr.uuid as uuid, + row_to_json(flow_struct) AS flow, + row_to_json(contact_struct) AS contact, + fr.responded, + (SELECT CASE + WHEN (fr.path_nodes IS NOT NULL AND fr.path_times IS NOT NULL) + THEN ( + SELECT coalesce(jsonb_agg(path_data), '[]'::jsonb) + FROM ( + SELECT node, time + FROM unnest(fr.path_nodes::text[] , fr.path_times::timestamptz[]) x(node, time) LIMIT 500) + as path_data) + ELSE ( + SELECT coalesce(jsonb_agg(path_data), '[]'::jsonb) + FROM ( + SELECT path_row ->> 'node_uuid' AS node, (path_row ->> 'arrived_on')::timestamptz as time + FROM jsonb_array_elements(fr.path::jsonb) AS path_row LIMIT 500) + as path_data) + END as path), + (SELECT coalesce(jsonb_object_agg(values_data.key, values_data.value), '{}'::jsonb) from ( + SELECT key, jsonb_build_object('name', value -> 'name', 'value', value -> 'value', 'input', value -> 'input', 'time', (value -> 'created_on')::text::timestamptz, 'category', value -> 'category', 'node', value -> 'node_uuid') as value + FROM jsonb_each(fr.results::jsonb)) AS values_data + ) as values, + fr.created_on, + fr.modified_on, + fr.exited_on, + CASE + WHEN status = 'C' THEN 'completed' + WHEN status = 'I' THEN 'interrupted' + WHEN status = 'X' THEN 'expired' + WHEN status = 'F' THEN 'failed' + ELSE NULL + END as exit_type + + FROM flows_flowrun fr + JOIN LATERAL (SELECT uuid, name FROM flows_flow WHERE flows_flow.id = fr.flow_id) AS flow_struct ON True + JOIN LATERAL (SELECT uuid, name FROM contacts_contact cc WHERE cc.id = fr.contact_id) AS contact_struct ON True + WHERE fr.org_id = $1 AND fr.modified_on >= $2 AND fr.modified_on < $3 + ORDER BY fr.modified_on ASC, id ASC ) as rec;` // writeRunRecords writes the runs in the archive's date range to the passed in writer diff --git a/archives/testdata/runs1.jsonl b/archives/testdata/runs1.jsonl index 51f81dd..534023e 100644 --- a/archives/testdata/runs1.jsonl +++ b/archives/testdata/runs1.jsonl @@ -1,2 +1,3 @@ {"id":1,"uuid":"4ced1260-9cfe-4b7f-81dd-b637108f15b9","flow":{"uuid":"6639286a-9120-45d4-aa39-03ae3942a4a6","name":"Flow 1"},"contact":{"uuid":"3e814add-e614-41f7-8b5d-a07f670a698f","name":"Ajodinabiff Dane"},"responded":true,"path":[],"values":{},"created_on":"2017-08-12T19:11:59.890662+00:00","modified_on":"2017-08-12T19:11:59.890662+00:00","exited_on":"2017-08-12T19:11:59.890662+00:00","exit_type":"completed"} {"id":2,"uuid":"7d68469c-0494-498a-bdf3-bac68321fd6d","flow":{"uuid":"6639286a-9120-45d4-aa39-03ae3942a4a6","name":"Flow 1"},"contact":{"uuid":"3e814add-e614-41f7-8b5d-a07f670a698f","name":"Ajodinabiff Dane"},"responded":true,"path":[{"node": "10896d63-8df7-4022-88dd-a9d93edf355b", "time": "2017-08-12T13:07:24.049815+00:00"}],"values":{"agree": {"name": "Do you agree?", "node": "a0434c54-3e26-4eb0-bafc-46cdeaf435ac", "time": "2017-05-03T12:25:21.714339+00:00", "input": "A", "value": "A", "category": "Strongly agree"}},"created_on":"2017-08-12T19:11:59.890662+00:00","modified_on":"2017-08-12T19:11:59.890662+00:00","exited_on":"2017-08-12T19:11:59.890662+00:00","exit_type":"completed"} +{"id":8,"uuid":"0c54f7b9-875b-4385-ae85-fb9e84f4b3d6","flow":{"uuid":"6639286a-9120-45d4-aa39-03ae3942a4a6","name":"Flow 1"},"contact":{"uuid":"3e814add-e614-41f7-8b5d-a07f670a698f","name":"Ajodinabiff Dane"},"responded":true,"path":[{"node": "d1a55403-83a3-42f1-b24c-6446bb18e6a6", "time": "2017-08-12T13:07:25.049815+00:00"}, {"node": "ce1bdc68-5c16-452e-a0ce-52440fc7bb9a", "time": "2017-08-12T13:07:26.049815+00:00"}, {"node": "1640b40d-63ed-43b0-a443-097ce8bb8710", "time": "2017-08-12T13:07:27.049815+00:00"}],"values":{"agree": {"name": "Do you agree?", "node": "a0434c54-3e26-4eb0-bafc-46cdeaf435ac", "time": "2017-05-03T12:25:21.714339+00:00", "input": "A", "value": "A", "category": "Strongly agree"}},"created_on":"2017-08-12T19:11:59.890662+00:00","modified_on":"2017-08-12T19:11:59.890662+00:00","exited_on":"2017-08-12T19:11:59.890662+00:00","exit_type":"completed"} diff --git a/testdb.sql b/testdb.sql index a0513ca..4471a11 100644 --- a/testdb.sql +++ b/testdb.sql @@ -209,6 +209,8 @@ CREATE TABLE flows_flowrun ( start_id integer NULL REFERENCES flows_flowstart(id), results text NOT NULL, path text NOT NULL, + path_nodes uuid[] NULL, + path_times timestamp with time zone[] NULL, created_on timestamp with time zone NOT NULL, modified_on timestamp with time zone NOT NULL, exited_on timestamp with time zone NULL, @@ -335,23 +337,27 @@ INSERT INTO flows_flowstart_groups(flowstart_id, contactgroup_id) VALUES INSERT INTO flows_flowstart_calls(flowstart_id, call_id) VALUES (1, 1); -INSERT INTO flows_flowrun(id, uuid, org_id, responded, contact_id, flow_id, results, path, created_on, modified_on, exited_on, status, start_id) VALUES -(1, '4ced1260-9cfe-4b7f-81dd-b637108f15b9', 2, TRUE, 6, 1, '{}', '[]', '2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00', 'C', 1), +INSERT INTO flows_flowrun(id, uuid, org_id, responded, contact_id, flow_id, results, path, path_nodes, path_times, created_on, modified_on, exited_on, status, start_id) VALUES +(1, '4ced1260-9cfe-4b7f-81dd-b637108f15b9', 2, TRUE, 6, 1, '{}', '[]', NULL, NULL, '2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00', 'C', 1), (2, '7d68469c-0494-498a-bdf3-bac68321fd6d', 2, TRUE, 6, 1, '{"agree": {"category": "Strongly agree", "node_uuid": "a0434c54-3e26-4eb0-bafc-46cdeaf435ac", "name": "Do you agree?", "value": "A", "created_on": "2017-05-03T12:25:21.714339+00:00", "input": "A"}}', '[{"uuid": "c3d0b417-db75-417c-8050-33776ec8f620", "node_uuid": "10896d63-8df7-4022-88dd-a9d93edf355b", "arrived_on": "2017-08-12T15:07:24.049815+02:00", "exit_uuid": "2f890507-2ad2-4bd1-92fc-0ca031155fca"}]', -'2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00', 'C', NULL), +NULL, NULL, '2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00', 'C', NULL), (3, 'de782b35-a398-46ed-8550-34c66053841b', 3, TRUE, 7, 2, '{"agree": {"category": "Strongly agree", "node_uuid": "084c8cf1-715d-4d0a-b38d-a616ed74e638", "name": "Agree", "value": "A", "created_on": "2017-05-03T12:25:21.714339+00:00", "input": "A"}, "confirm_agree": {"category": "Confirmed Strongly agree", "node_uuid": "a0434c54-3e26-4eb0-bafc-46cdeaf435ab", "name": "Do you agree?", "value": "A", "created_on": "2017-05-03T12:25:21.714339+00:00", "input": "A"}}', '[{"uuid": "600ac5b4-4895-4161-ad97-6e2f1bb48bcb", "node_uuid": "accbc6e2-b0df-46cd-9a76-bff0fdf4d753", "arrived_on": "2017-08-12T15:07:24.049815+02:00", "exit_uuid": "8249e2dc-c893-4200-b6d2-398d07a459bc"}]', -'2017-08-10 21:11:59.890662+02:00','2017-08-10 21:11:59.890662+02:00','2017-08-10 21:11:59.890662+02:00', 'C', NULL), +NULL, NULL, '2017-08-10 21:11:59.890662+02:00','2017-08-10 21:11:59.890662+02:00','2017-08-10 21:11:59.890662+02:00', 'C', NULL), (4, '329a5d24-64fc-479c-8d24-9674c9b46530', 3, TRUE, 7, 2, '{"agree": {"category": "Disagree", "node_uuid": "084c8cf1-715d-4d0a-b38d-a616ed74e638", "name": "Agree", "value": "B", "created_on": "2017-10-10T12:25:21.714339+00:00", "input": "B"}}', '[{"uuid": "babf4fc8-e12c-4bb9-a9dd-61178a118b5a", "node_uuid": "accbc6e2-b0df-46cd-9a76-bff0fdf4d753", "arrived_on": "2017-10-12T15:07:24.049815+02:00", "exit_uuid": "8249e2dc-c893-4200-b6d2-398d07a459bc"}]', -'2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00', 'C', NULL), -(5, 'abed67d2-06b8-4749-8bb9-ecda037b673b', 3, TRUE, 7, 2, '{}', '[]', '2017-10-10 21:11:59.890663+02:00','2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00', 'C', NULL), -(6, '6262eefe-a6e9-4201-9b76-a7f25e3b7f29', 3, TRUE, 7, 2, '{}', '[]', '2017-12-12 21:11:59.890662+02:00','2017-12-12 21:11:59.890662+02:00','2017-12-12 21:11:59.890662+02:00', 'C', NULL), -(7, '6c0d7db9-076b-4edc-ab4b-38576ae394fc', 2, TRUE, 7, 2, '{}', '[]', '2017-08-13 13:11:59.890662+02:00','2017-08-14 16:11:59.890662+02:00', NULL, 'W', NULL); +NULL, NULL, '2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00', 'C', NULL), +(5, 'abed67d2-06b8-4749-8bb9-ecda037b673b', 3, TRUE, 7, 2, '{}', '[]', NULL, NULL, '2017-10-10 21:11:59.890663+02:00','2017-10-10 21:11:59.890662+02:00','2017-10-10 21:11:59.890662+02:00', 'C', NULL), +(6, '6262eefe-a6e9-4201-9b76-a7f25e3b7f29', 3, TRUE, 7, 2, '{}', '[]', NULL, NULL, '2017-12-12 21:11:59.890662+02:00','2017-12-12 21:11:59.890662+02:00','2017-12-12 21:11:59.890662+02:00', 'C', NULL), +(7, '6c0d7db9-076b-4edc-ab4b-38576ae394fc', 2, TRUE, 7, 2, '{}', '[]', NULL, NULL, '2017-08-13 13:11:59.890662+02:00','2017-08-14 16:11:59.890662+02:00', NULL, 'W', NULL), +(8, '0c54f7b9-875b-4385-ae85-fb9e84f4b3d6', 2, TRUE, 6, 1, +'{"agree": {"category": "Strongly agree", "node_uuid": "a0434c54-3e26-4eb0-bafc-46cdeaf435ac", "name": "Do you agree?", "value": "A", "created_on": "2017-05-03T12:25:21.714339+00:00", "input": "A"}}', +'[]', +'{"d1a55403-83a3-42f1-b24c-6446bb18e6a6","ce1bdc68-5c16-452e-a0ce-52440fc7bb9a","1640b40d-63ed-43b0-a443-097ce8bb8710"}', '{"2017-08-12T15:07:25.049815+02:00","2017-08-12T15:07:26.049815+02:00","2017-08-12T15:07:27.049815+02:00"}', '2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00','2017-08-12 21:11:59.890662+02:00', 'C', NULL); -- update run #5 to have a path longer than 500 steps UPDATE flows_flowrun SET path = s.path FROM (