From 4cdfd60ce74b9a6918b2bfe6e6050b19ad403cad Mon Sep 17 00:00:00 2001 From: Fusionshh Date: Thu, 16 Oct 2025 12:52:09 +0300 Subject: [PATCH 1/4] Child + citizen review --- .../child/parallel/child_flow.hpl | 21 +++++++++++++++++- .../child/parallel/child_flow_delta.hpl | 21 +++++++++++++++++- .../child/parallel/child_flow_repeat.hpl | 21 +++++++++++++++++- .../citizen/parallel/citizen_flow.hpl | 21 ++++++++++++++++++ .../citizen/parallel/citizen_flow_delta.hpl | 22 ++++++++++++++++++- .../citizen/parallel/citizen_flow_repeat.hpl | 22 ++++++++++++++++++- 6 files changed, 123 insertions(+), 5 deletions(-) diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl index 10b6a4b..21a7984 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl @@ -18,6 +18,26 @@ 2025/08/05 14:31:22.799 + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 26 + 576 + 944 + 1. table output, update 2 указан лишний error_code для error handling + 387 + @@ -333,7 +353,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - SELECT ri.recruit_id AS recruit_id, (ch->>'id')::uuid AS child_external_id, diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl index 838fe3f..520525a 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl @@ -18,6 +18,26 @@ 2025/08/11 13:22:56.048 + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 26 + 574 + 358 + 1. insert/update, update указан лишний error_code для error handling + 384 + @@ -429,7 +449,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - SELECT ri.recruit_id AS recruit_id, (ch->>'id')::uuid AS child_external_id, diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl index f18761f..c35551d 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl @@ -18,6 +18,26 @@ 2025/08/08 13:25:52.319 + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 26 + 368 + 592 + 1.Insert/update не указан error_description для error handling + 338 + @@ -445,7 +465,6 @@ DO UPDATE SET ervu-dashboard N - SELECT ri.recruit_id AS recruit_id, (ch->>'id')::uuid AS child_external_id, diff --git a/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow.hpl b/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow.hpl index bb9d4b9..5a38fa7 100644 --- a/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow.hpl @@ -18,6 +18,27 @@ 2025/08/05 14:31:22.799 + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 42 + 192 + 432 + 1. Убрать лишнее из input (Закоментированные строки запроса, незакоменченная переменная лимит в конце) +2. Убрать error_code из output в части error handling + 624 + diff --git a/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow_delta.hpl index 680ab2c..bf6bc4e 100644 --- a/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow_delta.hpl @@ -18,6 +18,27 @@ 2025/08/11 13:22:56.048 + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 41 + 326 + 407 + 1. Insert/update отсутсвует error_description + + 246 + @@ -439,7 +460,6 @@ and recruitment_id = '${IDM_ID}'; ervu_person_registry N - /* with adresses as ( -- по одному ли адресу тут должно быть?? diff --git a/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow_repeat.hpl index a1d15c2..99816af 100644 --- a/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow_repeat.hpl @@ -18,6 +18,27 @@ 2025/08/08 13:25:52.319 + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 41 + 432 + 352 + 1. Insert/update отсутсвует error_description + + 246 + @@ -453,7 +474,6 @@ DO UPDATE SET ervu_person_registry N - /* with adresses as ( -- по одному ли адресу тут должно быть?? From e053440f30fa04cf41c1c5f351d3c87a615a7aad Mon Sep 17 00:00:00 2001 From: "adel.ka" Date: Thu, 16 Oct 2025 12:53:17 +0300 Subject: [PATCH 2/4] =?UTF-8?q?=D1=84=D0=B8=D0=BA=D1=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../child/parallel/child_flow.hpl | 1 - .../child/parallel/child_flow_delta.hpl | 3 +- .../child/parallel/child_flow_repeat.hpl | 1 - .../child/recruitment_five_flow_delta.hpl | 38 +++-- .../parallel/citizen_address_flow.hpl | 1 - .../parallel/citizen_address_flow_delta.hpl | 3 +- .../parallel/citizen_address_flow_repeat.hpl | 1 - .../recruitment_five_flow_delta.hpl | 39 +++-- .../parallel/citizen_guardianship_flow.hpl | 149 +++++++---------- .../citizen_guardianship_flow_delta.hpl | 151 +++++++----------- .../citizen_guardianship_flow_repeat.hpl | 149 +++++++---------- .../recruitment_five_flow_delta.hpl | 38 +++-- .../citizen_information_search_flow.hpl | 46 +----- .../citizen_information_search_flow_delta.hpl | 49 ++---- ...citizen_information_search_flow_repeat.hpl | 46 +----- .../recruitment_five_flow_delta.hpl | 39 +++-- .../parallel/citizen_spouse_flow.hpl | 45 +++--- .../parallel/citizen_spouse_flow_delta.hpl | 43 ++--- .../parallel/citizen_spouse_flow_repeat.hpl | 43 ++--- .../recruitment_five_flow_delta.hpl | 39 +++-- .../passport/parallel/passport_flow.hpl | 1 - .../passport/parallel/passport_flow_delta.hpl | 3 +- .../parallel/passport_flow_repeat.hpl | 1 - .../passport/recruitment_five_flow_delta.hpl | 39 +++-- .../prosecution/parallel/prosecution_flow.hpl | 134 ++++++---------- .../parallel/prosecution_flow_delta.hpl | 132 ++++++--------- .../parallel/prosecution_flow_repeat.hpl | 134 ++++++---------- .../recruitment_five_flow_delta.hpl | 39 +++-- .../punishment/parallel/punishment_flow.hpl | 141 ++++++---------- .../parallel/punishment_flow_delta.hpl | 143 +++++++---------- .../parallel/punishment_flow_repeat.hpl | 131 ++++++--------- .../recruitment_five_flow_delta.hpl | 39 +++-- 32 files changed, 811 insertions(+), 1050 deletions(-) diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl index 10b6a4b..7f3c39f 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl @@ -380,7 +380,6 @@ CROSS JOIN LATERAL ( WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array'; Y diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl index 838fe3f..93b2a0f 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl @@ -476,8 +476,7 @@ CROSS JOIN LATERAL ( WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date - AND c.update_date >= '${M_R_UP_DATE}'::timestamp + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp, AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array'; Y diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl index f18761f..251e180 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl @@ -492,7 +492,6 @@ CROSS JOIN LATERAL ( WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array'; Y diff --git a/mappings/info_recruits/citizen_tables/child/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/child/recruitment_five_flow_delta.hpl index 6de3c73..3a20589 100644 --- a/mappings/info_recruits/citizen_tables/child/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/child/recruitment_five_flow_delta.hpl @@ -60,14 +60,28 @@ ervu-dashboard N 0 - SELECT - r.idm_id as recruitment_id + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'child_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN ervu_dashboard.recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from ervu_dashboard.recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -95,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -146,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -197,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -248,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -299,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl index c43604b..b957c88 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl @@ -292,7 +292,6 @@ FROM ervu_dashboard.recruits_info ri WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date GROUP BY ri.recruit_id; Y diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl index e460d7a..5f43412 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl @@ -379,8 +379,7 @@ FROM ervu_dashboard.recruits_info ri WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date - AND c.update_date >= '${M_R_UP_DATE}'::timestamp + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp GROUP BY ri.recruit_id; Y diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl index 8b0eb5a..b151211 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl @@ -393,7 +393,6 @@ FROM ervu_dashboard.recruits_info ri WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date GROUP BY ri.recruit_id; Y diff --git a/mappings/info_recruits/citizen_tables/citizen_address/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_address/recruitment_five_flow_delta.hpl index 366d5b1..529eb65 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/recruitment_five_flow_delta.hpl @@ -59,14 +59,29 @@ ervu-dashboard N - SELECT - r.idm_id as recruitment_id + + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'citizen_address_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN ervu_dashboard.recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from ervu_dashboard.recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -94,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -145,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -196,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -247,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -298,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl index e00ac54..13e3a9e 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl @@ -35,36 +35,16 @@ Change job status on error Y - - Add constants - Update - Y - Update Change job status on error Y - - Table output - Group by - Y - Change job status on error Abort Y - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - Add constants - Y - Update Filter rows @@ -76,13 +56,28 @@ Y - Update - Detect empty stream + Detect empty stream + Change job status on success 2 Y - Detect empty stream - Change job status on success 2 + Identify last row in a stream + Update + Y + + + Table output + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + + + Table output + Detect empty stream Y @@ -106,33 +101,6 @@ 544 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - is_guardian - true - -1 - N - Boolean - - - - - 1104 - 0 - - Change job status on error ExecSql @@ -244,8 +212,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 1536 - 160 + 1280 + 688 @@ -284,36 +252,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 0 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - N - - - recruit_id - - - N - grp - - - 720 - 0 - - Identify last row in a stream DetectLastRow @@ -328,7 +266,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); last_row - 912 + 1008 0 @@ -345,7 +283,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - SELECT ri.recruit_id AS recruit_id, (ch->>'id')::uuid AS guardianship_external_id, @@ -363,15 +300,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, NULLIF(ch->'dataRozhdDok'->>'den', '')::int ) AS birth_date, - REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils + REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, + true AS is_guardian FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' - AND '${M_R_CR_DATE}'::timestamp >= c.recruit_create_date; + AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array'; Y @@ -383,7 +320,7 @@ WHERE Table output TableOutput - Y + N 1 @@ -489,6 +426,40 @@ WHERE 0 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 720 + 0 + + Table output diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl index 9d188e4..950066d 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl @@ -25,11 +25,6 @@ Table input Y - - Add constants - Update - Y - Insert / update Change job status on error @@ -40,11 +35,6 @@ Insert / update Y - - Insert / update - Group by - Y - Update Change job status on error @@ -55,21 +45,6 @@ Abort Y - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - Add constants - Y - - - Update - Detect empty stream - Y - Detect empty stream Change job status on success @@ -85,6 +60,26 @@ Change job status on success Y + + Identify last row in a stream + Update + Y + + + Insert / update + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + + + Insert / update + Detect empty stream + Y + Abort @@ -106,33 +101,6 @@ 480 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - is_guardian - true - -1 - N - Boolean - - - - - 1024 - 0 - - Change job status on error ExecSql @@ -249,8 +217,8 @@ and recruitment_id = '${IDM_ID}'; - 1456 - 224 + 1216 + 608 @@ -289,36 +257,6 @@ and recruitment_id = '${IDM_ID}'; 0 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - N - - - recruit_id - - - N - grp - - - 704 - 0 - - Identify last row in a stream DetectLastRow @@ -333,7 +271,7 @@ and recruitment_id = '${IDM_ID}'; last_row - 864 + 944 0 @@ -341,7 +279,7 @@ and recruitment_id = '${IDM_ID}'; Insert / update InsertUpdate - Y + N 1 @@ -429,7 +367,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - SELECT ri.recruit_id AS recruit_id, (ch->>'id')::uuid AS guardianship_external_id, @@ -447,7 +384,8 @@ and recruitment_id = '${IDM_ID}'; NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, NULLIF(ch->'dataRozhdDok'->>'den', '')::int ) AS birth_date, - REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils + REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, + true AS is_guardian FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch @@ -455,8 +393,7 @@ WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' - AND '${M_R_CR_DATE}'::timestamp >= c.recruit_create_date - AND c.update_date >= '${M_R_UP_DATE}'::timestamp; + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp; Y @@ -504,6 +441,40 @@ WHERE 0 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 704 + 0 + + Insert / update diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl index 0ead75e..23b84b5 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl @@ -35,31 +35,11 @@ Change job status on error Y - - Add constants - Update - Y - Update Change job status on error Y - - Insert / update - Group by - Y - - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - Add constants - Y - Change job status on error Abort @@ -76,13 +56,28 @@ Y - Update - Detect empty stream + Detect empty stream + Change job status on success Y - Detect empty stream - Change job status on success + Identify last row in a stream + Update + Y + + + Insert / update + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + + + Insert / update + Detect empty stream Y @@ -106,33 +101,6 @@ 384 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - is_guardian - true - -1 - N - Boolean - - - - - 1040 - 48 - - Change job status on error ExecSql @@ -265,8 +233,8 @@ DO UPDATE SET - 1488 - 192 + 1232 + 576 @@ -305,36 +273,6 @@ DO UPDATE SET 48 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - N - - - recruit_id - - - N - grp - - - 704 - 48 - - Identify last row in a stream DetectLastRow @@ -349,7 +287,7 @@ DO UPDATE SET last_row - 896 + 992 48 @@ -357,7 +295,7 @@ DO UPDATE SET Insert / update InsertUpdate - Y + N 1 @@ -445,7 +383,6 @@ DO UPDATE SET ervu-dashboard N - SELECT ri.recruit_id AS recruit_id, (ch->>'id')::uuid AS guardianship_external_id, @@ -463,15 +400,15 @@ DO UPDATE SET NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, NULLIF(ch->'dataRozhdDok'->>'den', '')::int ) AS birth_date, - REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils + REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, + true AS is_guardian FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' - AND '${M_R_CR_DATE}'::timestamp >= c.recruit_create_date; + AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array'; Y @@ -519,6 +456,40 @@ WHERE 48 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 704 + 48 + + Insert / update diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/recruitment_five_flow_delta.hpl index 0231baa..6ca2fe7 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/recruitment_five_flow_delta.hpl @@ -60,14 +60,28 @@ ervu-dashboard N 0 - SELECT - r.idm_id as recruitment_id + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'citizen_guardianship_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN ervu_dashboard.recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from ervu_dashboard.recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -95,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -146,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -197,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -248,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -299,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl index 3e8198a..c4c6987 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl @@ -55,16 +55,6 @@ Table output Y - - Identify last row in a stream - Add constants - Y - - - Add constants - Update - Y - Update Filter rows @@ -80,6 +70,11 @@ Identify last row in a stream Y + + Identify last row in a stream + Update + Y + Abort @@ -101,33 +96,6 @@ 448 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - is_wanted - true - -1 - N - Boolean - - - - - 880 - 624 - - Change job status on error ExecSql @@ -319,12 +287,12 @@ SELECT TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, + true AS is_wanted FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id WHERE'${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' Y diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl index 9d2a38d..ae9a2ef 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl @@ -55,16 +55,6 @@ Insert / update Y - - Identify last row in a stream - Add constants - Y - - - Add constants - Update - Y - Update Filter rows @@ -80,6 +70,11 @@ Identify last row in a stream Y + + Identify last row in a stream + Update + Y + Abort @@ -101,33 +96,6 @@ 384 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - is_wanted - true - -1 - N - Boolean - - - - - 864 - 608 - - Change job status on error ExecSql @@ -386,12 +354,12 @@ SELECT TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, + true AS is_wanted FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id WHERE'${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' @@ -416,16 +384,19 @@ WHERE'${IDM_ID}' != '' 10000 ervu-dashboard N + = recruit_id recruit_id + <> is_wanted is_wanted + ervu_dashboard citizen
diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl index 182c8e0..3526f1b 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl @@ -25,11 +25,6 @@ Table input Y - - Add constants - Update - Y - Change job status on error Abort @@ -45,11 +40,6 @@ Change job status on success Y - - Identify last row in a stream 2 - Add constants - Y - Update Change job status on error @@ -80,6 +70,11 @@ Identify last row in a stream 2 Y + + Identify last row in a stream 2 + Update + Y + Abort @@ -101,33 +96,6 @@ 432 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - is_wanted - true - -1 - N - Boolean - - - - - 880 - 640 - - Change job status on error ExecSql @@ -400,12 +368,12 @@ SELECT TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, + true AS is_wanted // Наличие информации о розыске FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id WHERE'${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' Y diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/recruitment_five_flow_delta.hpl index 6f730ba..b072e72 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/recruitment_five_flow_delta.hpl @@ -59,14 +59,29 @@ ervu-dashboard N - SELECT - r.idm_id as recruitment_id + + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'citizen_information_search_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN ervu_dashboard.recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from ervu_dashboard.recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -94,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -145,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -196,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -247,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -298,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl index a675cfb..656e10e 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl @@ -60,16 +60,6 @@ Change job status on error Y - - Table output - Unique rows - Y - - - Unique rows - Identify last row in a stream - Y - Identify last row in a stream Table output 2 @@ -80,6 +70,16 @@ Table output Y + + Table output + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + Abort @@ -283,7 +283,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - SELECT ri.recruit_id AS recruit_id, (supr->>'id')::uuid AS spouse_external_id, @@ -321,8 +320,7 @@ CROSS JOIN LATERAL ( ) AS supr(supr) WHERE '${IDM_ID}' != '' - AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date; + AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')); Y @@ -467,8 +465,8 @@ WHERE - Unique rows - Unique + Sort rows + SortRows Y @@ -477,18 +475,27 @@ WHERE none - N + ${java.io.tmpdir} + out + 1000000 + + N + + Y - N recruit_id + Y + N + N + 0 + N - N 736 - 400 + 416 diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl index 2f01364..d3d609c 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl @@ -50,11 +50,6 @@ Change job status on success Y - - Insert / update - Unique rows - Y - Insert / update 2 Filter rows @@ -65,11 +60,6 @@ Change job status on error Y - - Unique rows - Identify last row in a stream - Y - Identify last row in a stream Insert / update 2 @@ -80,6 +70,16 @@ Insert / update Y + + Insert / update + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + Abort @@ -436,7 +436,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - SELECT ri.recruit_id AS recruit_id, (supr->>'id')::uuid AS spouse_external_id, @@ -475,8 +474,7 @@ CROSS JOIN LATERAL ( WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date - AND r.update_date >= '${M_R_UP_DATE}'::timestamp; + AND ri.update_date >= '${M_R_UP_DATE}'::timestamp; Y @@ -486,8 +484,8 @@ WHERE - Unique rows - Unique + Sort rows + SortRows Y @@ -496,14 +494,23 @@ WHERE none - N + ${java.io.tmpdir} + out + 1000000 + + N + + Y - N recruit_id + Y + N + N + 0 + N - N 768 diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl index 65727f6..5c07301 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl @@ -82,21 +82,11 @@ M_R_CR_DATE Filter rows Y - - Insert / update - Unique rows - Y - Insert / update 2 Change job status on error Y - - Unique rows - Identify last row in a stream - Y - Identify last row in a stream Insert / update 2 @@ -107,6 +97,16 @@ M_R_CR_DATE Insert / update Y + + Insert / update + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + Abort @@ -480,7 +480,6 @@ DO UPDATE SET ervu-dashboard N - SELECT ri.recruit_id AS recruit_id, (supr->>'id')::uuid AS spouse_external_id, @@ -518,8 +517,7 @@ CROSS JOIN LATERAL ( ) AS supr(supr) WHERE '${IDM_ID}' != '' - AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date; + AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')); Y @@ -528,8 +526,8 @@ WHERE - Unique rows - Unique + Sort rows + SortRows Y @@ -538,14 +536,23 @@ WHERE none - N + ${java.io.tmpdir} + out + 1000000 + + N + + Y - N recruit_id + Y + N + N + 0 + N - N 704 diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/recruitment_five_flow_delta.hpl index 2cc4f14..89fee31 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/recruitment_five_flow_delta.hpl @@ -59,14 +59,29 @@ ervu-dashboard N - SELECT - r.idm_id as recruitment_id + + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'citizen_spouse_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN ervu_dashboard.recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from ervu_dashboard.recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -94,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -145,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -196,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -247,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -298,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y diff --git a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl index 8b735be..33b5478 100644 --- a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl @@ -285,7 +285,6 @@ JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date JOIN LATERAL ( SELECT passport_data FROM UNNEST(ARRAY[ diff --git a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl index d284859..c3e59e7 100644 --- a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl @@ -357,8 +357,7 @@ JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date - AND r.update_date >= '${M_R_UP_DATE}'::timestamp + AND ri.update_date >= '${M_R_UP_DATE}'::timestamp JOIN LATERAL ( SELECT passport_data FROM UNNEST(ARRAY[ diff --git a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl index a11e7ff..e11050f 100644 --- a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl @@ -371,7 +371,6 @@ JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date JOIN LATERAL ( SELECT passport_data FROM UNNEST(ARRAY[ diff --git a/mappings/info_recruits/citizen_tables/passport/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/passport/recruitment_five_flow_delta.hpl index 3aff0ef..bb45170 100644 --- a/mappings/info_recruits/citizen_tables/passport/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/passport/recruitment_five_flow_delta.hpl @@ -59,14 +59,29 @@ ervu-dashboard N - SELECT - r.idm_id as recruitment_id + + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'passport_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN ervu_dashboard.recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from ervu_dashboard.recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -94,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -145,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -196,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -247,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -298,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y diff --git a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl index 40f73b5..f678d40 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl @@ -55,26 +55,6 @@ Table output Y - - Table output - Group by - Y - - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - Add constants - Y - - - Add constants - Update - Y - Update Filter rows @@ -85,6 +65,21 @@ Change job status on error Y + + Identify last row in a stream + Update + Y + + + Table output + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + Abort @@ -106,33 +101,6 @@ 224 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - has_criminal_prosecution - true - -1 - N - Boolean - - - - - 832 - 32 - - Change job status on error ExecSql @@ -284,36 +252,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 32 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - N - - - recruit_id - - - N - grp - - - 832 - 320 - - Identify last row in a stream DetectLastRow @@ -329,7 +267,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 832 - 176 + 32 @@ -355,7 +293,6 @@ ugolov_presl AS ( JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at WHERE jsonb_typeof(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') = 'array' ) SELECT @@ -381,12 +318,13 @@ SELECT NULLIF(up->'sledSud'->>'statusNaim', '') AS conviction_status, TO_DATE(up->'sledSud'->>'dataPrekrUD', 'YYYY-MM-DD') AS close_date, NULLIF(up->'sledSudTekst'->>'svedTekst' , '') AS sled_sud_tekst_sved_tekst, - NULLIF(up->'sledSudTekst'->>'statusNaim', '') AS sled_sud_tekst_status_naim, + NULLIF(up->'sledSudTekst'->>'statusNaim', '') AS sled_sud_tekst_status_naim, CASE WHEN (up->>'sledSud') IS NOT NULL AND up->>'sledSud' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в структурированном формате' WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде' ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей' - END AS type_info + END AS type_info, + true AS has_criminal_prosecution FROM ervu_dashboard.citizen r JOIN ugolov_presl up ON r.recruit_id = up.recruit_id LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true @@ -556,6 +494,40 @@ GROUP BY r.recruit_id, up; 32 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 832 + 224 + + Table output diff --git a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl index ad5f0be..bb1ceca 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl @@ -50,26 +50,6 @@ Abort Y - - Identify last row in a stream 2 - Add constants - Y - - - Group by - Identify last row in a stream 2 - Y - - - Insert / update - Group by - Y - - - Add constants - Update - Y - Update Filter rows @@ -85,6 +65,21 @@ Insert / update Y + + Insert / update + Sort rows + Y + + + Sort rows + Identify last row in a stream 2 + Y + + + Identify last row in a stream 2 + Update + Y + Abort @@ -106,33 +101,6 @@ 592 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - has_criminal_prosecution - true - -1 - N - Boolean - - - - - 736 - 416 - - Change job status on error ExecSql @@ -287,36 +255,6 @@ and recruitment_id = '${IDM_ID}'; 416 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - N - - - recruit_id - - - N - grp - - - 736 - 640 - - Identify last row in a stream 2 DetectLastRow @@ -332,7 +270,7 @@ and recruitment_id = '${IDM_ID}'; 736 - 528 + 416 @@ -474,6 +412,40 @@ and recruitment_id = '${IDM_ID}'; 768 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 736 + 560 + + Table input TableInput @@ -498,7 +470,6 @@ ugolov_presl AS ( join ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp WHERE jsonb_typeof(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') = 'array' ) @@ -530,7 +501,8 @@ SELECT WHEN (up->>'sledSud') IS NOT NULL AND up->>'sledSud' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в структурированном формате' WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде' ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей' - END AS type_info + END AS type_info, + true AS has_criminal_prosecution FROM ervu_dashboard.citizen r JOIN ugolov_presl up ON r.recruit_id = up.recruit_id LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true diff --git a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl index 39d554c..931465b 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl @@ -55,26 +55,6 @@ Insert / update Y - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - Add constants - Y - - - Insert / update - Group by - Y - - - Add constants - Update - Y - Update Filter rows @@ -85,6 +65,21 @@ Change job status on error Y + + Insert / update + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + + + Identify last row in a stream + Update + Y + Abort @@ -106,33 +101,6 @@ 736 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - has_criminal_prosecution - true - -1 - N - Boolean - - - - - 768 - 576 - - Change job status on error ExecSql @@ -301,36 +269,6 @@ DO UPDATE SET 576 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - N - - - recruit_id - - - N - grp - - - 768 - 800 - - Identify last row in a stream DetectLastRow @@ -346,7 +284,7 @@ DO UPDATE SET 768 - 704 + 576 @@ -511,7 +449,6 @@ ugolov_presl AS ( JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at WHERE jsonb_typeof(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') = 'array' ) SELECT @@ -537,12 +474,13 @@ SELECT NULLIF(up->'sledSud'->>'statusNaim', '') AS conviction_status, TO_DATE(up->'sledSud'->>'dataPrekrUD', 'YYYY-MM-DD') AS close_date, NULLIF(up->'sledSudTekst'->>'svedTekst' , '') AS sled_sud_tekst_sved_tekst, - NULLIF(up->'sledSudTekst'->>'statusNaim', '') AS sled_sud_tekst_status_naim, + NULLIF(up->'sledSudTekst'->>'statusNaim', '') AS sled_sud_tekst_status_naim, CASE WHEN (up->>'sledSud') IS NOT NULL AND up->>'sledSud' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в структурированном формате' WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде' ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей' - END AS type_info + END AS type_info, + true AS has_criminal_prosecution FROM ervu_dashboard.citizen r JOIN ugolov_presl up ON r.recruit_id = up.recruit_id LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true @@ -594,6 +532,40 @@ GROUP BY r.recruit_id, up; 576 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 768 + 768 + + Insert / update diff --git a/mappings/info_recruits/citizen_tables/prosecution/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/prosecution/recruitment_five_flow_delta.hpl index 4ef7d06..ddd29ef 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/recruitment_five_flow_delta.hpl @@ -59,14 +59,29 @@ ervu-dashboard N - SELECT - r.idm_id as recruitment_id + + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'prosecution_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN ervu_dashboard.recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from ervu_dashboard.recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -94,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSIN IDM_ID - recruitment_id + idm_id Y @@ -145,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSIN IDM_ID - recruitment_id + idm_id Y @@ -196,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSIN IDM_ID - recruitment_id + idm_id Y @@ -247,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSIN IDM_ID - recruitment_id + idm_id Y @@ -298,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSIN IDM_ID - recruitment_id + idm_id Y diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl index 468d997..57a77d6 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl @@ -55,26 +55,6 @@ Table output Y - - Table output - Group by - Y - - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - Add constants - Y - - - Add constants - Update - Y - Update Filter rows @@ -85,6 +65,21 @@ Change job status on error Y + + Table output + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + + + Identify last row in a stream + Update + Y + Abort @@ -106,41 +101,6 @@ 448 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - has_punishment_execution_info - true - -1 - N - Boolean - - - -1 - has_sentence_info - true - -1 - N - Boolean - - - - - 880 - 624 - - Change job status on error ExecSql @@ -292,36 +252,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 624 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - N - - - recruit_id - - - N - grp - - - 720 - 464 - - Identify last row in a stream DetectLastRow @@ -340,6 +270,40 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 624 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 720 + 464 + + Table input TableInput @@ -361,7 +325,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null' ) SELECT recruit_id, @@ -383,7 +346,7 @@ SELECT recruit_id, punishment_info ->> 'naimSubekt' AS region_name, -- Наименование субъекта punishment_info ->> 'naimOrg' AS facility_name, -- Название учреждения punishment_info ->> 'adresOrg' AS facility_address, -- Адрес учреждения - punishment_info ->> 'nomerUD' AS criminal_case_number, -- Номер уголовного дела + punishment_info ->> 'nomerUD' AS criminal_case_number, -- Номер уголовного дела TO_DATE(punishment_info ->> 'dataNachSrok', 'YYYY-MM-DD') AS start_date, -- Дата начала срока TO_DATE(punishment_info ->> 'dataOkonchSrok', 'YYYY-MM-DD') AS planned_date, -- Планируемая дата освобождения CASE @@ -392,7 +355,9 @@ SELECT recruit_id, ELSE punishment_info ->> 'kodPerekvalif' END AS article_modification_flag, -- Признак переквалификации punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения - TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date -- Фактическая дата освобождения + TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения + true AS has_punishment_execution_info, + true AS has_sentence_info FROM punishment_data LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true GROUP BY recruit_id, punishment_info; diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl index c9c4bc9..e6579d3 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl @@ -32,8 +32,8 @@ N -1 26 - 987 - 654 + 976 + 736 has_punishment_execution_info 180 @@ -74,26 +74,6 @@ Insert / update Y - - Insert / update - Group by - Y - - - Identify last row in a stream - Add constants - Y - - - Group by - Identify last row in a stream - Y - - - Add constants - Update - Y - Update Filter rows @@ -104,6 +84,21 @@ Change job status on error Y + + Insert / update + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + + + Identify last row in a stream + Update + Y + Abort @@ -125,41 +120,6 @@ 384 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - has_punishment_execution_info - true - -1 - N - Boolean - - - -1 - has_sentence_info - true - -1 - N - Boolean - - - - - 864 - 608 - - Change job status on error ExecSql @@ -314,36 +274,6 @@ and recruitment_id = '${IDM_ID}'; 608 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - N - - - recruit_id - - - N - grp - - - 752 - 400 - - Identify last row in a stream DetectLastRow @@ -512,7 +442,6 @@ and recruitment_id = '${IDM_ID}'; JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null' ) @@ -544,7 +473,9 @@ SELECT recruit_id, ELSE punishment_info ->> 'kodPerekvalif' END AS article_modification_flag, -- Признак переквалификации punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения - TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date -- Фактическая дата освобождения + TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения + true AS has_punishment_execution_info, + true AS has_sentence_info FROM punishment_data LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true GROUP BY recruit_id, punishment_info; @@ -604,6 +535,40 @@ GROUP BY recruit_id, punishment_info; 608 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 752 + 384 + + Insert / update diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl index aa1aedf..2e7b75d 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl @@ -25,11 +25,6 @@ Table input Y - - Add constants - Update - Y - Change job status on error Abort @@ -45,16 +40,6 @@ Change job status on success Y - - Group by - Identify last row in a stream 2 - Y - - - Identify last row in a stream 2 - Add constants - Y - Update Change job status on error @@ -82,7 +67,17 @@ Insert / update - Group by + Sort rows + Y + + + Sort rows + Identify last row in a stream 2 + Y + + + Identify last row in a stream 2 + Update Y @@ -106,41 +101,6 @@ 432 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - has_punishment_execution_info - true - -1 - N - Boolean - - - -1 - has_sentence_info - true - -1 - N - Boolean - - - - - 880 - 640 - - Change job status on error ExecSql @@ -309,36 +269,6 @@ DO UPDATE SET 640 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - N - - - recruit_id - - - N - grp - - - 688 - 432 - - Identify last row in a stream 2 DetectLastRow @@ -486,6 +416,40 @@ DO UPDATE SET 272 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 688 + 448 + + Table input TableInput @@ -507,7 +471,6 @@ DO UPDATE SET JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null' ) SELECT recruit_id, @@ -538,7 +501,9 @@ SELECT recruit_id, ELSE punishment_info ->> 'kodPerekvalif' END AS article_modification_flag, -- Признак переквалификации punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения - TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date -- Фактическая дата освобождения + TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения + true AS has_punishment_execution_info, + true AS has_sentence_info FROM punishment_data LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true GROUP BY recruit_id, punishment_info; diff --git a/mappings/info_recruits/citizen_tables/punishment/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/punishment/recruitment_five_flow_delta.hpl index 63648bb..6ff5027 100644 --- a/mappings/info_recruits/citizen_tables/punishment/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/recruitment_five_flow_delta.hpl @@ -59,14 +59,29 @@ ervu-dashboard N - SELECT - r.idm_id as recruitment_id + + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'punishment_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN ervu_dashboard.recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from ervu_dashboard.recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -94,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -145,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -196,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -247,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -298,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y From 4bb3f8ce0d742e0defdb82b764b2d9dc627e01d2 Mon Sep 17 00:00:00 2001 From: "adel.ka" Date: Thu, 16 Oct 2025 13:02:08 +0300 Subject: [PATCH 3/4] =?UTF-8?q?=D1=84=D0=B8=D0=BA=D1=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../child/parallel/child_flow.hpl | 24 +----- .../child/parallel/child_flow_delta.hpl | 2 +- .../child/parallel/child_flow_repeat.hpl | 4 +- .../parallel/citizen_address_flow.hpl | 3 +- .../parallel/citizen_address_flow_delta.hpl | 3 +- .../parallel/citizen_address_flow_repeat.hpl | 3 +- .../parallel/citizen_guardianship_flow.hpl | 70 +++++++++--------- .../citizen_guardianship_flow_delta.hpl | 72 +++++++++--------- .../citizen_guardianship_flow_repeat.hpl | 72 +++++++++--------- .../citizen_information_search_flow.hpl | 5 +- .../citizen_information_search_flow_delta.hpl | 6 +- ...citizen_information_search_flow_repeat.hpl | 3 +- .../parallel/citizen_spouse_flow.hpl | 72 +++++++++--------- .../parallel/citizen_spouse_flow_delta.hpl | 72 +++++++++--------- .../parallel/citizen_spouse_flow_repeat.hpl | 72 +++++++++--------- .../passport/parallel/passport_flow.hpl | 2 +- .../passport/parallel/passport_flow_delta.hpl | 6 +- .../parallel/passport_flow_repeat.hpl | 2 +- .../prosecution/parallel/prosecution_flow.hpl | 4 +- .../parallel/prosecution_flow_delta.hpl | 4 +- .../parallel/prosecution_flow_repeat.hpl | 73 +++++++++---------- .../punishment/parallel/punishment_flow.hpl | 4 +- .../parallel/punishment_flow_delta.hpl | 70 +++++++++--------- .../parallel/punishment_flow_repeat.hpl | 3 +- 24 files changed, 310 insertions(+), 341 deletions(-) diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl index a5dfaff..058c647 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl @@ -18,26 +18,6 @@ 2025/08/05 14:31:22.799 - - 251 - 232 - 201 - 90 - 58 - 14 - N - 90 - 58 - 14 - N - Segoe UI - 9 - 26 - 576 - 944 - 1. table output, update 2 указан лишний error_code для error handling - 387 - @@ -593,7 +573,7 @@ WHERE error_description - error_code + @@ -605,7 +585,7 @@ WHERE error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl index 0bcc0b5..5348e19 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl @@ -596,7 +596,7 @@ WHERE error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl index 16fbb7f..2bd1039 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl @@ -609,7 +609,7 @@ WHERE Change job status on error 2 Y - + error_description @@ -623,7 +623,7 @@ WHERE error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl index b957c88..7a11073 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl @@ -268,7 +268,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - SELECT ri.recruit_id::uuid, MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, @@ -386,7 +385,7 @@ GROUP BY ri.recruit_id; error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl index 5f43412..daceb10 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl @@ -355,7 +355,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - SELECT ri.recruit_id::uuid, MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, @@ -396,7 +395,7 @@ GROUP BY ri.recruit_id; error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl index b151211..665afe3 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl @@ -369,7 +369,6 @@ DO UPDATE SET ervu-dashboard N - SELECT ri.recruit_id::uuid, MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, @@ -409,7 +408,7 @@ GROUP BY ri.recruit_id; error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl index 13e3a9e..40ad7cd 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl @@ -270,6 +270,40 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 0 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 720 + 0 + + Table input TableInput @@ -426,40 +460,6 @@ WHERE 0 - - Sort rows - SortRows - - Y - - 1 - - none - - - ${java.io.tmpdir} - out - 1000000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - - - - 720 - 0 - - Table output @@ -468,7 +468,7 @@ WHERE error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl index 950066d..dc435bc 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl @@ -354,6 +354,40 @@ and recruitment_id = '${IDM_ID}'; 224 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 704 + 0 + + Table input TableInput @@ -441,40 +475,6 @@ WHERE 0 - - Sort rows - SortRows - - Y - - 1 - - none - - - ${java.io.tmpdir} - out - 1000000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - - - - 704 - 0 - - Insert / update @@ -483,7 +483,7 @@ WHERE error_description - error_code + @@ -495,7 +495,7 @@ WHERE error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl index 23b84b5..3d25f03 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl @@ -370,6 +370,40 @@ DO UPDATE SET 208 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 704 + 48 + + Table input TableInput @@ -456,40 +490,6 @@ WHERE 48 - - Sort rows - SortRows - - Y - - 1 - - none - - - ${java.io.tmpdir} - out - 1000000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - - - - 704 - 48 - - Insert / update @@ -498,7 +498,7 @@ WHERE error_description - error_code + @@ -510,7 +510,7 @@ WHERE error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl index c4c6987..142ffd5 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl @@ -278,7 +278,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - SELECT ri.recruit_id, @@ -412,7 +411,7 @@ WHERE'${IDM_ID}' != '' error_description - error_code + @@ -424,7 +423,7 @@ WHERE'${IDM_ID}' != '' error_decription - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl index ae9a2ef..0c9f107 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl @@ -345,7 +345,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - SELECT ri.recruit_id, @@ -384,19 +383,16 @@ WHERE'${IDM_ID}' != '' 10000 ervu-dashboard N - = recruit_id recruit_id - <> is_wanted is_wanted - ervu_dashboard citizen
@@ -421,7 +417,7 @@ WHERE'${IDM_ID}' != '' error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl index 3526f1b..b3c67b1 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl @@ -359,7 +359,6 @@ DO UPDATE SET ervu-dashboard N - SELECT ri.recruit_id, @@ -431,7 +430,7 @@ WHERE'${IDM_ID}' != '' error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl index 656e10e..940fed6 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl @@ -270,6 +270,40 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 288 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 736 + 416 + + Table input TableInput @@ -464,40 +498,6 @@ WHERE 192 - - Sort rows - SortRows - - Y - - 1 - - none - - - ${java.io.tmpdir} - out - 1000000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - - - - 736 - 416 - - Table output @@ -506,7 +506,7 @@ WHERE error_description - error_code + @@ -518,7 +518,7 @@ WHERE error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl index d3d609c..de73175 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl @@ -423,6 +423,40 @@ and recruitment_id = '${IDM_ID}'; 176 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 768 + 384 + + Table input TableInput @@ -483,40 +517,6 @@ WHERE 496 - - Sort rows - SortRows - - Y - - 1 - - none - - - ${java.io.tmpdir} - out - 1000000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - - - - 768 - 384 - - Insert / update @@ -525,7 +525,7 @@ WHERE error_description - error_code + @@ -537,7 +537,7 @@ WHERE error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl index 5c07301..b8e2649 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl @@ -467,6 +467,40 @@ DO UPDATE SET 224 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 704 + 416 + + Table input TableInput @@ -525,40 +559,6 @@ WHERE 528 - - Sort rows - SortRows - - Y - - 1 - - none - - - ${java.io.tmpdir} - out - 1000000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - - - - 704 - 416 - - Insert / update @@ -567,7 +567,7 @@ WHERE error_description - error_code + @@ -579,7 +579,7 @@ WHERE error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl index 33b5478..5606ea7 100644 --- a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl @@ -373,7 +373,7 @@ LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6 error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl index c3e59e7..b07ce7b 100644 --- a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl @@ -340,7 +340,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - SELECT ri.recruit_id, NULLIF(p.passport_data->>'nomDok', '') AS passport_number, @@ -357,7 +356,8 @@ JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND ri.update_date >= '${M_R_UP_DATE}'::timestamp + AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date + AND r.update_date >= '${M_R_UP_DATE}'::timestamp JOIN LATERAL ( SELECT passport_data FROM UNNEST(ARRAY[ @@ -385,7 +385,7 @@ LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6 error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl index e11050f..94504a9 100644 --- a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl @@ -398,7 +398,7 @@ LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6 error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl index f678d40..fc569e2 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl @@ -536,7 +536,7 @@ GROUP BY r.recruit_id, up; error_description - error_code + @@ -548,7 +548,7 @@ GROUP BY r.recruit_id, up; error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl index bb1ceca..89fe31e 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl @@ -562,7 +562,7 @@ GROUP BY r.recruit_id, up; error_description - error_code + @@ -574,7 +574,7 @@ GROUP BY r.recruit_id, up; error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl index 931465b..b038a37 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl @@ -426,6 +426,40 @@ DO UPDATE SET 944 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 768 + 768 + + Table input TableInput @@ -439,7 +473,6 @@ DO UPDATE SET ervu-dashboard N - WITH ugolov_presl AS ( SELECT @@ -532,40 +565,6 @@ GROUP BY r.recruit_id, up; 576 - - Sort rows - SortRows - - Y - - 1 - - none - - - ${java.io.tmpdir} - out - 1000000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - - - - 768 - 768 - - Insert / update @@ -574,7 +573,7 @@ GROUP BY r.recruit_id, up; error_description - error_code + @@ -586,7 +585,7 @@ GROUP BY r.recruit_id, up; error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl index 57a77d6..43bd0ec 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl @@ -535,7 +535,7 @@ GROUP BY recruit_id, punishment_info; error_description - error_code + @@ -547,7 +547,7 @@ GROUP BY recruit_id, punishment_info; error_decription - error_code + diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl index e6579d3..a6736e2 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl @@ -421,6 +421,40 @@ and recruitment_id = '${IDM_ID}'; 224 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 752 + 384 + + Table input TableInput @@ -535,40 +569,6 @@ GROUP BY recruit_id, punishment_info; 608 - - Sort rows - SortRows - - Y - - 1 - - none - - - ${java.io.tmpdir} - out - 1000000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - - - - 752 - 384 - - Insert / update @@ -577,7 +577,7 @@ GROUP BY recruit_id, punishment_info; error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl index 2e7b75d..5a2ad11 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl @@ -463,7 +463,6 @@ DO UPDATE SET ervu-dashboard N - WITH punishment_data AS ( SELECT ri.recruit_id, jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info @@ -571,7 +570,7 @@ GROUP BY recruit_id, punishment_info; error_description - error_code + From 4d9551f9d604f3a1fb5ac8f2ef8c67cc00d80d0b Mon Sep 17 00:00:00 2001 From: Fusionshh Date: Thu, 16 Oct 2025 13:04:23 +0300 Subject: [PATCH 4/4] citizen_appealing_violations_fz53 job review --- .../job_last_update_date_appeal_document.hwf | 22 ++++++++++++++++-- ...citizen_appealing_violations_fz53_flow.hpl | 23 ++++++++++++++++++- .../recruitments_five_flow.hpl | 21 +++++++++++++++++ 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/job_last_update_date_appeal_document.hwf b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/job_last_update_date_appeal_document.hwf index fb20613..3b23c49 100644 --- a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/job_last_update_date_appeal_document.hwf +++ b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/job_last_update_date_appeal_document.hwf @@ -68,8 +68,6 @@ N N ${PROJECT_HOME}/info_recruits/raw_data/recruits_info/row_last_recruit_update_date.hpl - - Basic Y @@ -119,6 +117,26 @@ + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 26 + 581 + 546 + А где испольуезтся джоба? + 162 + diff --git a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow.hpl index 8878d67..1bcb6cb 100644 --- a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow.hpl @@ -38,6 +38,28 @@ в таблице нет апдейт даты 154 + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 58 + 576 + 752 + 1. is_appealing_violations_fz53 case when можно заменить на not ad.hidden и без java expression передавать в апдейт +2. Убрать лишнее из input +3. Зачем группировка нужна если исходные данные не массив? + 642 + @@ -342,7 +364,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); postgres.decision-document-service N - SELECT ad.recruit_id, ad.create_date, -- дата создания обжалования diff --git a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/recruitments_five_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/recruitments_five_flow.hpl index 41aa770..b993317 100644 --- a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/recruitments_five_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/recruitments_five_flow.hpl @@ -18,6 +18,27 @@ 2025/04/18 09:48:01.970 + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 41 + 235 + 489 + убрать лишнее из input + + 141 +