diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl index 10b6a4b..058c647 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl @@ -333,7 +333,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - SELECT ri.recruit_id AS recruit_id, (ch->>'id')::uuid AS child_external_id, @@ -380,7 +379,6 @@ CROSS JOIN LATERAL ( WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array'; Y @@ -575,7 +573,7 @@ WHERE error_description - error_code + @@ -587,7 +585,7 @@ WHERE error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl index 838fe3f..5348e19 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl @@ -18,6 +18,26 @@ 2025/08/11 13:22:56.048 + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 26 + 574 + 358 + 1. insert/update, update указан лишний error_code для error handling + 384 + @@ -429,7 +449,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - SELECT ri.recruit_id AS recruit_id, (ch->>'id')::uuid AS child_external_id, @@ -476,8 +495,7 @@ CROSS JOIN LATERAL ( WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date - AND c.update_date >= '${M_R_UP_DATE}'::timestamp + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp, AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array'; Y @@ -578,7 +596,7 @@ WHERE error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl index f18761f..2bd1039 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl @@ -18,6 +18,26 @@ 2025/08/08 13:25:52.319 + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 26 + 368 + 592 + 1.Insert/update не указан error_description для error handling + 338 + @@ -445,7 +465,6 @@ DO UPDATE SET ervu-dashboard N - SELECT ri.recruit_id AS recruit_id, (ch->>'id')::uuid AS child_external_id, @@ -492,7 +511,6 @@ CROSS JOIN LATERAL ( WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array'; Y @@ -591,7 +609,7 @@ WHERE Change job status on error 2 Y - + error_description @@ -605,7 +623,7 @@ WHERE error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/child/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/child/recruitment_five_flow_delta.hpl index 6de3c73..3a20589 100644 --- a/mappings/info_recruits/citizen_tables/child/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/child/recruitment_five_flow_delta.hpl @@ -60,14 +60,28 @@ ervu-dashboard N 0 - SELECT - r.idm_id as recruitment_id + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'child_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN ervu_dashboard.recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from ervu_dashboard.recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -95,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -146,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -197,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -248,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -299,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y diff --git a/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow.hpl b/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow.hpl index bb9d4b9..5a38fa7 100644 --- a/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow.hpl @@ -18,6 +18,27 @@ 2025/08/05 14:31:22.799 + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 42 + 192 + 432 + 1. Убрать лишнее из input (Закоментированные строки запроса, незакоменченная переменная лимит в конце) +2. Убрать error_code из output в части error handling + 624 + diff --git a/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow_delta.hpl index 680ab2c..bf6bc4e 100644 --- a/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow_delta.hpl @@ -18,6 +18,27 @@ 2025/08/11 13:22:56.048 + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 41 + 326 + 407 + 1. Insert/update отсутсвует error_description + + 246 + @@ -439,7 +460,6 @@ and recruitment_id = '${IDM_ID}'; ervu_person_registry N - /* with adresses as ( -- по одному ли адресу тут должно быть?? diff --git a/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow_repeat.hpl index a1d15c2..99816af 100644 --- a/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen/parallel/citizen_flow_repeat.hpl @@ -18,6 +18,27 @@ 2025/08/08 13:25:52.319 + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 41 + 432 + 352 + 1. Insert/update отсутсвует error_description + + 246 + @@ -453,7 +474,6 @@ DO UPDATE SET ervu_person_registry N - /* with adresses as ( -- по одному ли адресу тут должно быть?? diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl index c43604b..7a11073 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl @@ -268,7 +268,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - SELECT ri.recruit_id::uuid, MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, @@ -292,7 +291,6 @@ FROM ervu_dashboard.recruits_info ri WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date GROUP BY ri.recruit_id; Y @@ -387,7 +385,7 @@ GROUP BY ri.recruit_id; error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl index e460d7a..daceb10 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl @@ -355,7 +355,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - SELECT ri.recruit_id::uuid, MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, @@ -379,8 +378,7 @@ FROM ervu_dashboard.recruits_info ri WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date - AND c.update_date >= '${M_R_UP_DATE}'::timestamp + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp GROUP BY ri.recruit_id; Y @@ -397,7 +395,7 @@ GROUP BY ri.recruit_id; error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl index 8b0eb5a..665afe3 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl @@ -369,7 +369,6 @@ DO UPDATE SET ervu-dashboard N - SELECT ri.recruit_id::uuid, MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, @@ -393,7 +392,6 @@ FROM ervu_dashboard.recruits_info ri WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date GROUP BY ri.recruit_id; Y @@ -410,7 +408,7 @@ GROUP BY ri.recruit_id; error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_address/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_address/recruitment_five_flow_delta.hpl index 366d5b1..529eb65 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/recruitment_five_flow_delta.hpl @@ -59,14 +59,29 @@ ervu-dashboard N - SELECT - r.idm_id as recruitment_id + + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'citizen_address_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN ervu_dashboard.recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from ervu_dashboard.recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -94,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -145,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -196,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -247,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -298,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y diff --git a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/job_last_update_date_appeal_document.hwf b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/job_last_update_date_appeal_document.hwf index fb20613..3b23c49 100644 --- a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/job_last_update_date_appeal_document.hwf +++ b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/job_last_update_date_appeal_document.hwf @@ -68,8 +68,6 @@ N N ${PROJECT_HOME}/info_recruits/raw_data/recruits_info/row_last_recruit_update_date.hpl - - Basic Y @@ -119,6 +117,26 @@ + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 26 + 581 + 546 + А где испольуезтся джоба? + 162 + diff --git a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow.hpl index 8878d67..1bcb6cb 100644 --- a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow.hpl @@ -38,6 +38,28 @@ в таблице нет апдейт даты 154 + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 58 + 576 + 752 + 1. is_appealing_violations_fz53 case when можно заменить на not ad.hidden и без java expression передавать в апдейт +2. Убрать лишнее из input +3. Зачем группировка нужна если исходные данные не массив? + 642 + @@ -342,7 +364,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); postgres.decision-document-service N - SELECT ad.recruit_id, ad.create_date, -- дата создания обжалования diff --git a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/recruitments_five_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/recruitments_five_flow.hpl index 41aa770..b993317 100644 --- a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/recruitments_five_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/recruitments_five_flow.hpl @@ -18,6 +18,27 @@ 2025/04/18 09:48:01.970 + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 41 + 235 + 489 + убрать лишнее из input + + 141 + diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl index e00ac54..40ad7cd 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl @@ -35,36 +35,16 @@ Change job status on error Y - - Add constants - Update - Y - Update Change job status on error Y - - Table output - Group by - Y - Change job status on error Abort Y - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - Add constants - Y - Update Filter rows @@ -76,13 +56,28 @@ Y - Update - Detect empty stream + Detect empty stream + Change job status on success 2 Y - Detect empty stream - Change job status on success 2 + Identify last row in a stream + Update + Y + + + Table output + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + + + Table output + Detect empty stream Y @@ -106,33 +101,6 @@ 544 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - is_guardian - true - -1 - N - Boolean - - - - - 1104 - 0 - - Change job status on error ExecSql @@ -244,8 +212,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 1536 - 160 + 1280 + 688 @@ -284,36 +252,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 0 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - N - - - recruit_id - - - N - grp - - - 720 - 0 - - Identify last row in a stream DetectLastRow @@ -328,7 +266,41 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); last_row - 912 + 1008 + 0 + + + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 720 0 @@ -345,7 +317,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - SELECT ri.recruit_id AS recruit_id, (ch->>'id')::uuid AS guardianship_external_id, @@ -363,15 +334,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, NULLIF(ch->'dataRozhdDok'->>'den', '')::int ) AS birth_date, - REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils + REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, + true AS is_guardian FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' - AND '${M_R_CR_DATE}'::timestamp >= c.recruit_create_date; + AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array'; Y @@ -383,7 +354,7 @@ WHERE Table output TableOutput - Y + N 1 @@ -497,7 +468,7 @@ WHERE error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl index 9d188e4..dc435bc 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl @@ -25,11 +25,6 @@ Table input Y - - Add constants - Update - Y - Insert / update Change job status on error @@ -40,11 +35,6 @@ Insert / update Y - - Insert / update - Group by - Y - Update Change job status on error @@ -55,21 +45,6 @@ Abort Y - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - Add constants - Y - - - Update - Detect empty stream - Y - Detect empty stream Change job status on success @@ -85,6 +60,26 @@ Change job status on success Y + + Identify last row in a stream + Update + Y + + + Insert / update + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + + + Insert / update + Detect empty stream + Y + Abort @@ -106,33 +101,6 @@ 480 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - is_guardian - true - -1 - N - Boolean - - - - - 1024 - 0 - - Change job status on error ExecSql @@ -249,8 +217,8 @@ and recruitment_id = '${IDM_ID}'; - 1456 - 224 + 1216 + 608 @@ -289,36 +257,6 @@ and recruitment_id = '${IDM_ID}'; 0 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - N - - - recruit_id - - - N - grp - - - 704 - 0 - - Identify last row in a stream DetectLastRow @@ -333,7 +271,7 @@ and recruitment_id = '${IDM_ID}'; last_row - 864 + 944 0 @@ -341,7 +279,7 @@ and recruitment_id = '${IDM_ID}'; Insert / update InsertUpdate - Y + N 1 @@ -416,6 +354,40 @@ and recruitment_id = '${IDM_ID}'; 224 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 704 + 0 + + Table input TableInput @@ -429,7 +401,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - SELECT ri.recruit_id AS recruit_id, (ch->>'id')::uuid AS guardianship_external_id, @@ -447,7 +418,8 @@ and recruitment_id = '${IDM_ID}'; NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, NULLIF(ch->'dataRozhdDok'->>'den', '')::int ) AS birth_date, - REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils + REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, + true AS is_guardian FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch @@ -455,8 +427,7 @@ WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' - AND '${M_R_CR_DATE}'::timestamp >= c.recruit_create_date - AND c.update_date >= '${M_R_UP_DATE}'::timestamp; + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp; Y @@ -512,7 +483,7 @@ WHERE error_description - error_code + @@ -524,7 +495,7 @@ WHERE error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl index 0ead75e..3d25f03 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl @@ -35,31 +35,11 @@ Change job status on error Y - - Add constants - Update - Y - Update Change job status on error Y - - Insert / update - Group by - Y - - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - Add constants - Y - Change job status on error Abort @@ -76,13 +56,28 @@ Y - Update - Detect empty stream + Detect empty stream + Change job status on success Y - Detect empty stream - Change job status on success + Identify last row in a stream + Update + Y + + + Insert / update + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + + + Insert / update + Detect empty stream Y @@ -106,33 +101,6 @@ 384 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - is_guardian - true - -1 - N - Boolean - - - - - 1040 - 48 - - Change job status on error ExecSql @@ -265,8 +233,8 @@ DO UPDATE SET - 1488 - 192 + 1232 + 576 @@ -305,36 +273,6 @@ DO UPDATE SET 48 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - N - - - recruit_id - - - N - grp - - - 704 - 48 - - Identify last row in a stream DetectLastRow @@ -349,7 +287,7 @@ DO UPDATE SET last_row - 896 + 992 48 @@ -357,7 +295,7 @@ DO UPDATE SET Insert / update InsertUpdate - Y + N 1 @@ -432,6 +370,40 @@ DO UPDATE SET 208 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 704 + 48 + + Table input TableInput @@ -445,7 +417,6 @@ DO UPDATE SET ervu-dashboard N - SELECT ri.recruit_id AS recruit_id, (ch->>'id')::uuid AS guardianship_external_id, @@ -463,15 +434,15 @@ DO UPDATE SET NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, NULLIF(ch->'dataRozhdDok'->>'den', '')::int ) AS birth_date, - REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils + REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, + true AS is_guardian FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' - AND '${M_R_CR_DATE}'::timestamp >= c.recruit_create_date; + AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array'; Y @@ -527,7 +498,7 @@ WHERE error_description - error_code + @@ -539,7 +510,7 @@ WHERE error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/recruitment_five_flow_delta.hpl index 0231baa..6ca2fe7 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/recruitment_five_flow_delta.hpl @@ -60,14 +60,28 @@ ervu-dashboard N 0 - SELECT - r.idm_id as recruitment_id + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'citizen_guardianship_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN ervu_dashboard.recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from ervu_dashboard.recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -95,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -146,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -197,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -248,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -299,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl index 3e8198a..142ffd5 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl @@ -55,16 +55,6 @@ Table output Y - - Identify last row in a stream - Add constants - Y - - - Add constants - Update - Y - Update Filter rows @@ -80,6 +70,11 @@ Identify last row in a stream Y + + Identify last row in a stream + Update + Y + Abort @@ -101,33 +96,6 @@ 448 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - is_wanted - true - -1 - N - Boolean - - - - - 880 - 624 - - Change job status on error ExecSql @@ -310,7 +278,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - SELECT ri.recruit_id, @@ -319,12 +286,12 @@ SELECT TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, + true AS is_wanted FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id WHERE'${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' Y @@ -444,7 +411,7 @@ WHERE'${IDM_ID}' != '' error_description - error_code + @@ -456,7 +423,7 @@ WHERE'${IDM_ID}' != '' error_decription - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl index 9d2a38d..0c9f107 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl @@ -55,16 +55,6 @@ Insert / update Y - - Identify last row in a stream - Add constants - Y - - - Add constants - Update - Y - Update Filter rows @@ -80,6 +70,11 @@ Identify last row in a stream Y + + Identify last row in a stream + Update + Y + Abort @@ -101,33 +96,6 @@ 384 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - is_wanted - true - -1 - N - Boolean - - - - - 864 - 608 - - Change job status on error ExecSql @@ -377,7 +345,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - SELECT ri.recruit_id, @@ -386,12 +353,12 @@ SELECT TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, + true AS is_wanted FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id WHERE'${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' @@ -450,7 +417,7 @@ WHERE'${IDM_ID}' != '' error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl index 182c8e0..b3c67b1 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl @@ -25,11 +25,6 @@ Table input Y - - Add constants - Update - Y - Change job status on error Abort @@ -45,11 +40,6 @@ Change job status on success Y - - Identify last row in a stream 2 - Add constants - Y - Update Change job status on error @@ -80,6 +70,11 @@ Identify last row in a stream 2 Y + + Identify last row in a stream 2 + Update + Y + Abort @@ -101,33 +96,6 @@ 432 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - is_wanted - true - -1 - N - Boolean - - - - - 880 - 640 - - Change job status on error ExecSql @@ -391,7 +359,6 @@ DO UPDATE SET ervu-dashboard N - SELECT ri.recruit_id, @@ -400,12 +367,12 @@ SELECT TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, + true AS is_wanted // Наличие информации о розыске FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id WHERE'${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' Y @@ -463,7 +430,7 @@ WHERE'${IDM_ID}' != '' error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/recruitment_five_flow_delta.hpl index 6f730ba..b072e72 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/recruitment_five_flow_delta.hpl @@ -59,14 +59,29 @@ ervu-dashboard N - SELECT - r.idm_id as recruitment_id + + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'citizen_information_search_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN ervu_dashboard.recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from ervu_dashboard.recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -94,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -145,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -196,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -247,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -298,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl index a675cfb..940fed6 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl @@ -60,16 +60,6 @@ Change job status on error Y - - Table output - Unique rows - Y - - - Unique rows - Identify last row in a stream - Y - Identify last row in a stream Table output 2 @@ -80,6 +70,16 @@ Table output Y + + Table output + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + Abort @@ -270,6 +270,40 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 288 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 736 + 416 + + Table input TableInput @@ -283,7 +317,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - SELECT ri.recruit_id AS recruit_id, (supr->>'id')::uuid AS spouse_external_id, @@ -321,8 +354,7 @@ CROSS JOIN LATERAL ( ) AS supr(supr) WHERE '${IDM_ID}' != '' - AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date; + AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')); Y @@ -466,31 +498,6 @@ WHERE 192 - - Unique rows - Unique - - Y - - 1 - - none - - - N - - - N - recruit_id - - - N - - - 736 - 400 - - Table output @@ -499,7 +506,7 @@ WHERE error_description - error_code + @@ -511,7 +518,7 @@ WHERE error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl index 2f01364..de73175 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl @@ -50,11 +50,6 @@ Change job status on success Y - - Insert / update - Unique rows - Y - Insert / update 2 Filter rows @@ -65,11 +60,6 @@ Change job status on error Y - - Unique rows - Identify last row in a stream - Y - Identify last row in a stream Insert / update 2 @@ -80,6 +70,16 @@ Insert / update Y + + Insert / update + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + Abort @@ -423,6 +423,40 @@ and recruitment_id = '${IDM_ID}'; 176 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 768 + 384 + + Table input TableInput @@ -436,7 +470,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - SELECT ri.recruit_id AS recruit_id, (supr->>'id')::uuid AS spouse_external_id, @@ -475,8 +508,7 @@ CROSS JOIN LATERAL ( WHERE '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date - AND r.update_date >= '${M_R_UP_DATE}'::timestamp; + AND ri.update_date >= '${M_R_UP_DATE}'::timestamp; Y @@ -485,31 +517,6 @@ WHERE 496 - - Unique rows - Unique - - Y - - 1 - - none - - - N - - - N - recruit_id - - - N - - - 768 - 384 - - Insert / update @@ -518,7 +525,7 @@ WHERE error_description - error_code + @@ -530,7 +537,7 @@ WHERE error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl index 65727f6..b8e2649 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl @@ -82,21 +82,11 @@ M_R_CR_DATE Filter rows Y - - Insert / update - Unique rows - Y - Insert / update 2 Change job status on error Y - - Unique rows - Identify last row in a stream - Y - Identify last row in a stream Insert / update 2 @@ -107,6 +97,16 @@ M_R_CR_DATE Insert / update Y + + Insert / update + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + Abort @@ -467,6 +467,40 @@ DO UPDATE SET 224 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 704 + 416 + + Table input TableInput @@ -480,7 +514,6 @@ DO UPDATE SET ervu-dashboard N - SELECT ri.recruit_id AS recruit_id, (supr->>'id')::uuid AS spouse_external_id, @@ -518,8 +551,7 @@ CROSS JOIN LATERAL ( ) AS supr(supr) WHERE '${IDM_ID}' != '' - AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date; + AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')); Y @@ -527,31 +559,6 @@ WHERE 528 - - Unique rows - Unique - - Y - - 1 - - none - - - N - - - N - recruit_id - - - N - - - 704 - 416 - - Insert / update @@ -560,7 +567,7 @@ WHERE error_description - error_code + @@ -572,7 +579,7 @@ WHERE error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/recruitment_five_flow_delta.hpl index 2cc4f14..89fee31 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/recruitment_five_flow_delta.hpl @@ -59,14 +59,29 @@ ervu-dashboard N - SELECT - r.idm_id as recruitment_id + + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'citizen_spouse_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN ervu_dashboard.recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from ervu_dashboard.recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -94,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -145,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -196,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -247,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -298,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y diff --git a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl index 8b735be..5606ea7 100644 --- a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl @@ -285,7 +285,6 @@ JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date JOIN LATERAL ( SELECT passport_data FROM UNNEST(ARRAY[ @@ -374,7 +373,7 @@ LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6 error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl index d284859..b07ce7b 100644 --- a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl @@ -340,7 +340,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - SELECT ri.recruit_id, NULLIF(p.passport_data->>'nomDok', '') AS passport_number, @@ -386,7 +385,7 @@ LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6 error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl index a11e7ff..94504a9 100644 --- a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl @@ -371,7 +371,6 @@ JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date JOIN LATERAL ( SELECT passport_data FROM UNNEST(ARRAY[ @@ -399,7 +398,7 @@ LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6 error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/passport/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/passport/recruitment_five_flow_delta.hpl index 3aff0ef..bb45170 100644 --- a/mappings/info_recruits/citizen_tables/passport/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/passport/recruitment_five_flow_delta.hpl @@ -59,14 +59,29 @@ ervu-dashboard N - SELECT - r.idm_id as recruitment_id + + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'passport_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN ervu_dashboard.recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from ervu_dashboard.recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -94,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -145,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -196,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -247,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -298,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y diff --git a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl index 40f73b5..fc569e2 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl @@ -55,26 +55,6 @@ Table output Y - - Table output - Group by - Y - - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - Add constants - Y - - - Add constants - Update - Y - Update Filter rows @@ -85,6 +65,21 @@ Change job status on error Y + + Identify last row in a stream + Update + Y + + + Table output + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + Abort @@ -106,33 +101,6 @@ 224 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - has_criminal_prosecution - true - -1 - N - Boolean - - - - - 832 - 32 - - Change job status on error ExecSql @@ -284,36 +252,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 32 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - N - - - recruit_id - - - N - grp - - - 832 - 320 - - Identify last row in a stream DetectLastRow @@ -329,7 +267,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 832 - 176 + 32 @@ -355,7 +293,6 @@ ugolov_presl AS ( JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at WHERE jsonb_typeof(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') = 'array' ) SELECT @@ -381,12 +318,13 @@ SELECT NULLIF(up->'sledSud'->>'statusNaim', '') AS conviction_status, TO_DATE(up->'sledSud'->>'dataPrekrUD', 'YYYY-MM-DD') AS close_date, NULLIF(up->'sledSudTekst'->>'svedTekst' , '') AS sled_sud_tekst_sved_tekst, - NULLIF(up->'sledSudTekst'->>'statusNaim', '') AS sled_sud_tekst_status_naim, + NULLIF(up->'sledSudTekst'->>'statusNaim', '') AS sled_sud_tekst_status_naim, CASE WHEN (up->>'sledSud') IS NOT NULL AND up->>'sledSud' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в структурированном формате' WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде' ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей' - END AS type_info + END AS type_info, + true AS has_criminal_prosecution FROM ervu_dashboard.citizen r JOIN ugolov_presl up ON r.recruit_id = up.recruit_id LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true @@ -556,6 +494,40 @@ GROUP BY r.recruit_id, up; 32 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 832 + 224 + + Table output @@ -564,7 +536,7 @@ GROUP BY r.recruit_id, up; error_description - error_code + @@ -576,7 +548,7 @@ GROUP BY r.recruit_id, up; error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl index ad5f0be..89fe31e 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl @@ -50,26 +50,6 @@ Abort Y - - Identify last row in a stream 2 - Add constants - Y - - - Group by - Identify last row in a stream 2 - Y - - - Insert / update - Group by - Y - - - Add constants - Update - Y - Update Filter rows @@ -85,6 +65,21 @@ Insert / update Y + + Insert / update + Sort rows + Y + + + Sort rows + Identify last row in a stream 2 + Y + + + Identify last row in a stream 2 + Update + Y + Abort @@ -106,33 +101,6 @@ 592 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - has_criminal_prosecution - true - -1 - N - Boolean - - - - - 736 - 416 - - Change job status on error ExecSql @@ -287,36 +255,6 @@ and recruitment_id = '${IDM_ID}'; 416 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - N - - - recruit_id - - - N - grp - - - 736 - 640 - - Identify last row in a stream 2 DetectLastRow @@ -332,7 +270,7 @@ and recruitment_id = '${IDM_ID}'; 736 - 528 + 416 @@ -474,6 +412,40 @@ and recruitment_id = '${IDM_ID}'; 768 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 736 + 560 + + Table input TableInput @@ -498,7 +470,6 @@ ugolov_presl AS ( join ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp WHERE jsonb_typeof(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') = 'array' ) @@ -530,7 +501,8 @@ SELECT WHEN (up->>'sledSud') IS NOT NULL AND up->>'sledSud' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в структурированном формате' WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде' ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей' - END AS type_info + END AS type_info, + true AS has_criminal_prosecution FROM ervu_dashboard.citizen r JOIN ugolov_presl up ON r.recruit_id = up.recruit_id LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true @@ -590,7 +562,7 @@ GROUP BY r.recruit_id, up; error_description - error_code + @@ -602,7 +574,7 @@ GROUP BY r.recruit_id, up; error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl index 39d554c..b038a37 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl @@ -55,26 +55,6 @@ Insert / update Y - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - Add constants - Y - - - Insert / update - Group by - Y - - - Add constants - Update - Y - Update Filter rows @@ -85,6 +65,21 @@ Change job status on error Y + + Insert / update + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + + + Identify last row in a stream + Update + Y + Abort @@ -106,33 +101,6 @@ 736 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - has_criminal_prosecution - true - -1 - N - Boolean - - - - - 768 - 576 - - Change job status on error ExecSql @@ -301,36 +269,6 @@ DO UPDATE SET 576 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - N - - - recruit_id - - - N - grp - - - 768 - 800 - - Identify last row in a stream DetectLastRow @@ -346,7 +284,7 @@ DO UPDATE SET 768 - 704 + 576 @@ -488,6 +426,40 @@ DO UPDATE SET 944 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 768 + 768 + + Table input TableInput @@ -501,7 +473,6 @@ DO UPDATE SET ervu-dashboard N - WITH ugolov_presl AS ( SELECT @@ -511,7 +482,6 @@ ugolov_presl AS ( JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at WHERE jsonb_typeof(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') = 'array' ) SELECT @@ -537,12 +507,13 @@ SELECT NULLIF(up->'sledSud'->>'statusNaim', '') AS conviction_status, TO_DATE(up->'sledSud'->>'dataPrekrUD', 'YYYY-MM-DD') AS close_date, NULLIF(up->'sledSudTekst'->>'svedTekst' , '') AS sled_sud_tekst_sved_tekst, - NULLIF(up->'sledSudTekst'->>'statusNaim', '') AS sled_sud_tekst_status_naim, + NULLIF(up->'sledSudTekst'->>'statusNaim', '') AS sled_sud_tekst_status_naim, CASE WHEN (up->>'sledSud') IS NOT NULL AND up->>'sledSud' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в структурированном формате' WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде' ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей' - END AS type_info + END AS type_info, + true AS has_criminal_prosecution FROM ervu_dashboard.citizen r JOIN ugolov_presl up ON r.recruit_id = up.recruit_id LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true @@ -602,7 +573,7 @@ GROUP BY r.recruit_id, up; error_description - error_code + @@ -614,7 +585,7 @@ GROUP BY r.recruit_id, up; error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/prosecution/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/prosecution/recruitment_five_flow_delta.hpl index 4ef7d06..ddd29ef 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/recruitment_five_flow_delta.hpl @@ -59,14 +59,29 @@ ervu-dashboard N - SELECT - r.idm_id as recruitment_id + + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'prosecution_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN ervu_dashboard.recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from ervu_dashboard.recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -94,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSIN IDM_ID - recruitment_id + idm_id Y @@ -145,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSIN IDM_ID - recruitment_id + idm_id Y @@ -196,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSIN IDM_ID - recruitment_id + idm_id Y @@ -247,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSIN IDM_ID - recruitment_id + idm_id Y @@ -298,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSIN IDM_ID - recruitment_id + idm_id Y diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl index 468d997..43bd0ec 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl @@ -55,26 +55,6 @@ Table output Y - - Table output - Group by - Y - - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - Add constants - Y - - - Add constants - Update - Y - Update Filter rows @@ -85,6 +65,21 @@ Change job status on error Y + + Table output + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + + + Identify last row in a stream + Update + Y + Abort @@ -106,41 +101,6 @@ 448 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - has_punishment_execution_info - true - -1 - N - Boolean - - - -1 - has_sentence_info - true - -1 - N - Boolean - - - - - 880 - 624 - - Change job status on error ExecSql @@ -292,36 +252,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 624 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - N - - - recruit_id - - - N - grp - - - 720 - 464 - - Identify last row in a stream DetectLastRow @@ -340,6 +270,40 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 624 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 720 + 464 + + Table input TableInput @@ -361,7 +325,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null' ) SELECT recruit_id, @@ -383,7 +346,7 @@ SELECT recruit_id, punishment_info ->> 'naimSubekt' AS region_name, -- Наименование субъекта punishment_info ->> 'naimOrg' AS facility_name, -- Название учреждения punishment_info ->> 'adresOrg' AS facility_address, -- Адрес учреждения - punishment_info ->> 'nomerUD' AS criminal_case_number, -- Номер уголовного дела + punishment_info ->> 'nomerUD' AS criminal_case_number, -- Номер уголовного дела TO_DATE(punishment_info ->> 'dataNachSrok', 'YYYY-MM-DD') AS start_date, -- Дата начала срока TO_DATE(punishment_info ->> 'dataOkonchSrok', 'YYYY-MM-DD') AS planned_date, -- Планируемая дата освобождения CASE @@ -392,7 +355,9 @@ SELECT recruit_id, ELSE punishment_info ->> 'kodPerekvalif' END AS article_modification_flag, -- Признак переквалификации punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения - TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date -- Фактическая дата освобождения + TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения + true AS has_punishment_execution_info, + true AS has_sentence_info FROM punishment_data LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true GROUP BY recruit_id, punishment_info; @@ -570,7 +535,7 @@ GROUP BY recruit_id, punishment_info; error_description - error_code + @@ -582,7 +547,7 @@ GROUP BY recruit_id, punishment_info; error_decription - error_code + diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl index c9c4bc9..a6736e2 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl @@ -32,8 +32,8 @@ N -1 26 - 987 - 654 + 976 + 736 has_punishment_execution_info 180 @@ -74,26 +74,6 @@ Insert / update Y - - Insert / update - Group by - Y - - - Identify last row in a stream - Add constants - Y - - - Group by - Identify last row in a stream - Y - - - Add constants - Update - Y - Update Filter rows @@ -104,6 +84,21 @@ Change job status on error Y + + Insert / update + Sort rows + Y + + + Sort rows + Identify last row in a stream + Y + + + Identify last row in a stream + Update + Y + Abort @@ -125,41 +120,6 @@ 384 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - has_punishment_execution_info - true - -1 - N - Boolean - - - -1 - has_sentence_info - true - -1 - N - Boolean - - - - - 864 - 608 - - Change job status on error ExecSql @@ -314,36 +274,6 @@ and recruitment_id = '${IDM_ID}'; 608 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - N - - - recruit_id - - - N - grp - - - 752 - 400 - - Identify last row in a stream DetectLastRow @@ -491,6 +421,40 @@ and recruitment_id = '${IDM_ID}'; 224 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 752 + 384 + + Table input TableInput @@ -512,7 +476,6 @@ and recruitment_id = '${IDM_ID}'; JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null' ) @@ -544,7 +507,9 @@ SELECT recruit_id, ELSE punishment_info ->> 'kodPerekvalif' END AS article_modification_flag, -- Признак переквалификации punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения - TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date -- Фактическая дата освобождения + TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения + true AS has_punishment_execution_info, + true AS has_sentence_info FROM punishment_data LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true GROUP BY recruit_id, punishment_info; @@ -612,7 +577,7 @@ GROUP BY recruit_id, punishment_info; error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl index aa1aedf..5a2ad11 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl @@ -25,11 +25,6 @@ Table input Y - - Add constants - Update - Y - Change job status on error Abort @@ -45,16 +40,6 @@ Change job status on success Y - - Group by - Identify last row in a stream 2 - Y - - - Identify last row in a stream 2 - Add constants - Y - Update Change job status on error @@ -82,7 +67,17 @@ Insert / update - Group by + Sort rows + Y + + + Sort rows + Identify last row in a stream 2 + Y + + + Identify last row in a stream 2 + Update Y @@ -106,41 +101,6 @@ 432 - - Add constants - Constant - - Y - - 1 - - none - - - - - -1 - has_punishment_execution_info - true - -1 - N - Boolean - - - -1 - has_sentence_info - true - -1 - N - Boolean - - - - - 880 - 640 - - Change job status on error ExecSql @@ -309,36 +269,6 @@ DO UPDATE SET 640 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - N - - - recruit_id - - - N - grp - - - 688 - 432 - - Identify last row in a stream 2 DetectLastRow @@ -486,6 +416,40 @@ DO UPDATE SET 272 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 688 + 448 + + Table input TableInput @@ -499,7 +463,6 @@ DO UPDATE SET ervu-dashboard N - WITH punishment_data AS ( SELECT ri.recruit_id, jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info @@ -507,7 +470,6 @@ DO UPDATE SET JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null' ) SELECT recruit_id, @@ -538,7 +500,9 @@ SELECT recruit_id, ELSE punishment_info ->> 'kodPerekvalif' END AS article_modification_flag, -- Признак переквалификации punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения - TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date -- Фактическая дата освобождения + TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения + true AS has_punishment_execution_info, + true AS has_sentence_info FROM punishment_data LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true GROUP BY recruit_id, punishment_info; @@ -606,7 +570,7 @@ GROUP BY recruit_id, punishment_info; error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/punishment/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/punishment/recruitment_five_flow_delta.hpl index 63648bb..6ff5027 100644 --- a/mappings/info_recruits/citizen_tables/punishment/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/recruitment_five_flow_delta.hpl @@ -59,14 +59,29 @@ ervu-dashboard N - SELECT - r.idm_id as recruitment_id + + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'punishment_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN ervu_dashboard.recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from ervu_dashboard.recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -94,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -145,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -196,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -247,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -298,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y