From e89c18bd20dd4afb4bcfba265edcc06738569a50 Mon Sep 17 00:00:00 2001 From: "adel.ka" Date: Wed, 12 Nov 2025 17:38:09 +0300 Subject: [PATCH] fixes --- .../child/parallel/child_flow.hpl | 388 ++++++++---------- .../child/parallel/child_flow_delta.hpl | 213 +++++----- .../child/parallel/child_flow_repeat.hpl | 172 ++++---- .../parallel/citizen_address_flow.hpl | 235 +++++------ .../parallel/citizen_address_flow_delta.hpl | 274 ++++++------- .../parallel/citizen_address_flow_repeat.hpl | 112 +++-- .../parallel/citizen_guardianship_flow.hpl | 135 +++--- .../citizen_guardianship_flow_delta.hpl | 90 ++-- .../citizen_guardianship_flow_repeat.hpl | 119 +++--- .../citizen_information_search_flow.hpl | 226 +++++----- .../citizen_information_search_flow_delta.hpl | 197 ++++----- ...citizen_information_search_flow_repeat.hpl | 103 ++--- .../parallel/citizen_spouse_flow.hpl | 165 ++++---- .../parallel/citizen_spouse_flow_delta.hpl | 165 ++++---- .../parallel/citizen_spouse_flow_repeat.hpl | 242 +++++------ .../passport/parallel/passport_flow.hpl | 206 ++++------ .../passport/parallel/passport_flow_delta.hpl | 292 ++++++------- .../parallel/passport_flow_repeat.hpl | 54 +-- .../prosecution/parallel/prosecution_flow.hpl | 60 ++- .../parallel/prosecution_flow_delta.hpl | 115 +++--- .../parallel/prosecution_flow_repeat.hpl | 37 +- .../punishment/parallel/punishment_flow.hpl | 108 +++-- .../parallel/punishment_flow_delta.hpl | 113 +++-- .../parallel/punishment_flow_repeat.hpl | 50 +-- 24 files changed, 1755 insertions(+), 2116 deletions(-) diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl index 3bcfdba..4a1d686 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl @@ -45,11 +45,6 @@ child_success_job_status Y - - child_input - child_output - Y - child_output child_error_job_status @@ -65,11 +60,6 @@ User defined Java expression 2 Y - - child_output - Filter rows - Y - child_input Detect empty stream @@ -77,12 +67,22 @@ Filter rows - Unique rows + child_success_job_status Y - Unique rows - child_success_job_status + child_output + Identify last row in a stream + Y + + + Identify last row in a stream + Filter rows + Y + + + child_input + child_output Y @@ -106,83 +106,6 @@ 1024 - - child_error_job_status - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - - Y - - N - - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - - 1216 - 800 - - - - child_success_job_status - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - - Y - - N - - Y - N - N - UPDATE etl.job_execution -SET status = 'SUCCESS', - error_description = null, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - - 576 - 1040 - - Create job execution record ExecSql @@ -209,7 +132,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 112 - 800 + 768 @@ -258,13 +181,90 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Unique rows + child_success_job_status 896 1040 + + User defined Java expression 2 + Janino + + Y + + 1 + + none + + + + minors_count + minors_count_raw == 1 ? "1" : minors_count_raw == 2 ? "2" : minors_count_raw >= 3 ? "3 и более" : "0" + String + 10 + -1 + + + + has_minor_child + has_minor_child_raw == 1 + Boolean + -1 + -1 + + + + has_dead_child + has_dead_child_raw == 1 + Boolean + -1 + -1 + + + + + 688 + 480 + + + + child_error_job_status + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1216 + 768 + + child_group_by GroupBy @@ -284,19 +284,16 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); minors_count_raw is_minor SUM - has_minor_child_raw is_minor MAX - has_dead_child_raw is_dead MAX - N @@ -306,7 +303,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); N - grp @@ -328,65 +324,59 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - WITH children_data AS ( + SELECT + ri.recruit_id AS recruit_id, + (ch->>'id')::uuid AS child_external_id, + NULLIF(ch->>'idERN', '') AS child_id_ern, + CASE NULLIF(ch->>'rodstvSvyazReb', '')::int + WHEN 1 THEN 3 + WHEN 2 THEN 4 + END AS kinship_type, + ch->'svedFLBS'->'fio'->>'familiya' AS last_name, + ch->'svedFLBS'->'fio'->>'imya' AS first_name, + ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + ch->'svedFLBS'->'fio'->>'familiya', + ch->'svedFLBS'->'fio'->>'imya', + ch->'svedFLBS'->'fio'->>'otchestvo' + ) AS full_name, + dates.birth_date, + dates.death_date, + ch->'svedSmert'->>'nomerZapis' AS death_az_number, + ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number, + CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead, + CASE + WHEN dates.death_date IS NOT NULL THEN 0 + WHEN dates.birth_date IS NULL THEN 0 + WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1 + ELSE 0 + END AS is_minor +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id +CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch +CROSS JOIN LATERAL ( SELECT - ri.recruit_id AS recruit_id, - (ch->>'id')::uuid AS child_external_id, - NULLIF(ch->>'idERN', '') AS child_id_ern, - CASE NULLIF(ch->>'rodstvSvyazReb', '')::int - WHEN 1 THEN 3 - WHEN 2 THEN 4 - END AS kinship_type, - ch->'svedFLBS'->'fio'->>'familiya' AS last_name, - ch->'svedFLBS'->'fio'->>'imya' AS first_name, - ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - ch->'svedFLBS'->'fio'->>'familiya', - ch->'svedFLBS'->'fio'->>'imya', - ch->'svedFLBS'->'fio'->>'otchestvo' - ) AS full_name, - dates.birth_date, - dates.death_date, - ch->'svedSmert'->>'nomerZapis' AS death_az_number, - ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number, - CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead, - CASE - WHEN dates.death_date IS NOT NULL THEN 0 - WHEN dates.birth_date IS NULL THEN 0 - WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1 - ELSE 0 - END AS is_minor, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch - CROSS JOIN LATERAL ( - SELECT - MAKE_DATE( - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - MAKE_DATE( - NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int, - NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int, - NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int - ) AS death_date - ) AS dates - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array' -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM children_data) as last_row -FROM children_data; + MAKE_DATE( + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + MAKE_DATE( + NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int, + NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int, + NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int + ) AS death_date +) AS dates +WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array' +ORDER BY recruit_id; Y 304 - 800 + 768 @@ -456,14 +446,11 @@ FROM children_data; N N N - Y - N ervu_dashboard Y citizen_child
- N Y N @@ -471,7 +458,40 @@ FROM children_data; 896 - 800 + 768 + + + + child_success_job_status + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS', + error_description = null, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 592 + 1040 @@ -488,13 +508,11 @@ FROM children_data; 10000 ervu-dashboard Y - = recruit_id recruit_id - ervu_dashboard citizen
@@ -520,8 +538,8 @@ FROM children_data; - User defined Java expression 2 - Janino + Identify last row in a stream + DetectLastRow Y @@ -530,59 +548,11 @@ FROM children_data; none - - minors_count - minors_count_raw == 1 ? "1" : minors_count_raw == 2 ? "2" : minors_count_raw >= 3 ? "3 и более" : "0" - String - 10 - -1 - - - - has_minor_child - has_minor_child_raw == 1 - Boolean - -1 - -1 - - - - has_dead_child - has_dead_child_raw == 1 - Boolean - -1 - -1 - - + last_row - 688 - 480 - - - - Unique rows - Unique - - Y - - 1 - - none - - - N - - - N - recruit_id - - - N - - - 736 - 1040 + 896 + 912 diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl index e97744e..e90262a 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl @@ -65,23 +65,23 @@ Detect empty stream Y - - child_insert_or_update - Filter rows - Y - child_input child_group_by Y - Filter rows - Unique rows + child_insert_or_update + Identify last row in a stream Y - Unique rows + Identify last row in a stream + Filter rows + Y + + + Filter rows child_success_job_status Y @@ -103,7 +103,7 @@ 1376 - 400 + 416 @@ -142,43 +142,6 @@ and recruitment_id = '${IDM_ID}'; 224 - - child_success_job_status - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - - Y - - N - - Y - N - N - UPDATE etl.job_execution -SET status = 'DELTA_SUCCESS', - error_description = null, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' - and recruitment_id = '${IDM_ID}' -and status = 'DELTA_PROCESSING'; - - - - - 784 - 496 - - Create job execution record ExecSql @@ -224,7 +187,7 @@ and recruitment_id = '${IDM_ID}'; 496 - 496 + 512 @@ -256,16 +219,16 @@ and recruitment_id = '${IDM_ID}'; - Unique rows + child_success_job_status 1072 - 496 + 512 - Unique rows - Unique + Identify last row in a stream + DetectLastRow Y @@ -274,18 +237,11 @@ and recruitment_id = '${IDM_ID}'; none - N - - - N - recruit_id - - - N + last_row - 960 - 496 + 1072 + 368 @@ -387,61 +343,55 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH children_data AS ( + SELECT + ri.recruit_id AS recruit_id, + (ch->>'id')::uuid AS child_external_id, + NULLIF(ch->>'idERN', '') AS child_id_ern, + CASE NULLIF(ch->>'rodstvSvyazReb', '')::int + WHEN 1 THEN 3 + WHEN 2 THEN 4 + END AS kinship_type, + ch->'svedFLBS'->'fio'->>'familiya' AS last_name, + ch->'svedFLBS'->'fio'->>'imya' AS first_name, + ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + ch->'svedFLBS'->'fio'->>'familiya', + ch->'svedFLBS'->'fio'->>'imya', + ch->'svedFLBS'->'fio'->>'otchestvo' + ) AS full_name, + dates.birth_date, + dates.death_date, + ch->'svedSmert'->>'nomerZapis' AS death_az_number, + ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number, + CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead, + CASE + WHEN dates.death_date IS NOT NULL THEN 0 + WHEN dates.birth_date IS NULL THEN 0 + WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1 + ELSE 0 + END AS is_minor +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id +CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch +CROSS JOIN LATERAL ( SELECT - ri.recruit_id AS recruit_id, - (ch->>'id')::uuid AS child_external_id, - NULLIF(ch->>'idERN', '') AS child_id_ern, - CASE NULLIF(ch->>'rodstvSvyazReb', '')::int - WHEN 1 THEN 3 - WHEN 2 THEN 4 - END AS kinship_type, - ch->'svedFLBS'->'fio'->>'familiya' AS last_name, - ch->'svedFLBS'->'fio'->>'imya' AS first_name, - ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - ch->'svedFLBS'->'fio'->>'familiya', - ch->'svedFLBS'->'fio'->>'imya', - ch->'svedFLBS'->'fio'->>'otchestvo' - ) AS full_name, - dates.birth_date, - dates.death_date, - ch->'svedSmert'->>'nomerZapis' AS death_az_number, - ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number, - CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead, - CASE - WHEN dates.death_date IS NOT NULL THEN 0 - WHEN dates.birth_date IS NULL THEN 0 - WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1 - ELSE 0 - END AS is_minor, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch - CROSS JOIN LATERAL ( - SELECT - MAKE_DATE( - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - MAKE_DATE( - NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int, - NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int, - NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int - ) AS death_date - ) AS dates - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp - AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array' -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM children_data) as last_row -FROM children_data; + MAKE_DATE( + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + MAKE_DATE( + NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int, + NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int, + NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int + ) AS death_date +) AS dates +WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp + AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array' +ORDER BY recruit_id; Y @@ -543,6 +493,39 @@ FROM children_data; 224 + + child_success_job_status + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'DELTA_SUCCESS', + error_description = null, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' + and recruitment_id = '${IDM_ID}' +and status = 'DELTA_PROCESSING'; + + + + 784 + 512 + + child_update_flags Update diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl index 8404d15..2b6aa0f 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl @@ -65,23 +65,23 @@ Detect empty stream Y - - child_insert_or_update - Filter rows - Y - child_input child_group_by Y - Filter rows - Unique rows + child_insert_or_update + Identify last row in a stream Y - Unique rows + Identify last row in a stream + Filter rows + Y + + + Filter rows child_success_job_status Y @@ -149,7 +149,7 @@ DO UPDATE SET 0 - 432 + 400 @@ -198,38 +198,13 @@ DO UPDATE SET - Unique rows + child_success_job_status 752 608 - - Unique rows - Unique - - Y - - 1 - - none - - - N - - - N - recruit_id - - - N - - - 608 - 608 - - User defined Java expression Janino @@ -304,7 +279,7 @@ and recruitment_id = '${IDM_ID}'; 976 - 432 + 400 @@ -365,65 +340,60 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH children_data AS ( + + SELECT + ri.recruit_id AS recruit_id, + (ch->>'id')::uuid AS child_external_id, + NULLIF(ch->>'idERN', '') AS child_id_ern, + CASE NULLIF(ch->>'rodstvSvyazReb', '')::int + WHEN 1 THEN 3 + WHEN 2 THEN 4 + END AS kinship_type, + ch->'svedFLBS'->'fio'->>'familiya' AS last_name, + ch->'svedFLBS'->'fio'->>'imya' AS first_name, + ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + ch->'svedFLBS'->'fio'->>'familiya', + ch->'svedFLBS'->'fio'->>'imya', + ch->'svedFLBS'->'fio'->>'otchestvo' + ) AS full_name, + dates.birth_date, + dates.death_date, + ch->'svedSmert'->>'nomerZapis' AS death_az_number, + ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number, + CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead, + CASE + WHEN dates.death_date IS NOT NULL THEN 0 + WHEN dates.birth_date IS NULL THEN 0 + WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1 + ELSE 0 + END AS is_minor +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id +CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch +CROSS JOIN LATERAL ( SELECT - ri.recruit_id AS recruit_id, - (ch->>'id')::uuid AS child_external_id, - NULLIF(ch->>'idERN', '') AS child_id_ern, - CASE NULLIF(ch->>'rodstvSvyazReb', '')::int - WHEN 1 THEN 3 - WHEN 2 THEN 4 - END AS kinship_type, - ch->'svedFLBS'->'fio'->>'familiya' AS last_name, - ch->'svedFLBS'->'fio'->>'imya' AS first_name, - ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - ch->'svedFLBS'->'fio'->>'familiya', - ch->'svedFLBS'->'fio'->>'imya', - ch->'svedFLBS'->'fio'->>'otchestvo' - ) AS full_name, - dates.birth_date, - dates.death_date, - ch->'svedSmert'->>'nomerZapis' AS death_az_number, - ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number, - CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead, - CASE - WHEN dates.death_date IS NOT NULL THEN 0 - WHEN dates.birth_date IS NULL THEN 0 - WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1 - ELSE 0 - END AS is_minor, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch - CROSS JOIN LATERAL ( - SELECT - MAKE_DATE( - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - MAKE_DATE( - NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int, - NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int, - NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int - ) AS death_date - ) AS dates - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array' -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM children_data) as last_row -FROM children_data; + MAKE_DATE( + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + MAKE_DATE( + NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int, + NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int, + NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int + ) AS death_date +) AS dates +WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array' +ORDER BY recruit_id; Y 224 - 432 + 400 @@ -444,13 +414,11 @@ FROM children_data; = recruit_id recruit_id - = child_external_id child_external_id - ervu_dashboard citizen_child
@@ -519,7 +487,7 @@ FROM children_data; 752 - 432 + 400 @@ -599,6 +567,24 @@ and status = 'PROCESSING'; 176 + + Identify last row in a stream + DetectLastRow + + Y + + 1 + + none + + + last_row + + + 752 + 512 + + child_insert_or_update diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl index f1f72b4..97e7bf6 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl @@ -40,11 +40,6 @@ citizen_address_success_job_status Y - - citizen_address_output - Filter rows - Y - citizen_address_input citizen_address_output @@ -56,12 +51,17 @@ Y - Filter rows - Unique rows + citizen_address_output + Identify last row in a stream Y - Unique rows + Identify last row in a stream + Filter rows + Y + + + Filter rows citizen_address_success_job_status Y @@ -86,83 +86,6 @@ 704 - - citizen_address_error_job_status - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - - Y - - N - - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - - 928 - 704 - - - - citizen_address_success_job_status - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - - Y - - N - - Y - N - N - UPDATE etl.job_execution -SET status = 'SUCCESS', - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'PROCESSING'; - - - - - - 1328 - 96 - - Create job execution record ExecSql @@ -238,7 +161,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Unique rows + citizen_address_success_job_status 1328 @@ -246,8 +169,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Unique rows - Unique + citizen_address_error_job_status + ExecSql Y @@ -256,18 +179,29 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); none - N - - - N - recruit_id - - - N + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + - 1328 - 208 + 928 + 704 @@ -284,36 +218,29 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - WITH address_data AS ( - SELECT - ri.recruit_id::uuid, - ri.ctid as source_ctid, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, + SELECT + ri.recruit_id::uuid, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, - CASE - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' - END AS place_stay_registration_type, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - GROUP BY ri.recruit_id, ri.ctid -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM address_data) as last_row -FROM address_data; + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, + CASE + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' + END AS place_stay_registration_type, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id +CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr +WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' +GROUP BY ri.recruit_id Y @@ -384,14 +311,11 @@ FROM address_data; N N N - Y - N ervu_dashboard Y citizen_address
- N Y N @@ -402,6 +326,57 @@ FROM address_data; 320 + + citizen_address_success_job_status + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS', + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}' +and status = 'PROCESSING'; + + + + + 1328 + 96 + + + + Identify last row in a stream + DetectLastRow + + Y + + 1 + + none + + + last_row + + + 1136 + 320 + + citizen_address_output diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl index 318a1d5..6c0a99d 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl @@ -52,16 +52,16 @@ citizen_address_insert_or_update + Identify last row in a stream + Y + + + Identify last row in a stream Filter rows Y Filter rows - Unique rows - Y - - - Unique rows citizen_address_success_job_status Y @@ -86,83 +86,6 @@ 464 - - citizen_address_error_job_status - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - - Y - - N - - Y - Y - N - UPDATE etl.job_execution -SET status = 'DELTA_ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - - 752 - 464 - - - - citizen_address_success_job_status - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - - Y - - N - - Y - N - N - UPDATE etl.job_execution -SET status = 'DELTA_SUCCESS', - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'DELTA_PROCESSING'; - - - - - - 1120 - 16 - - Create job execution record ExecSql @@ -241,10 +164,108 @@ and recruitment_id = '${IDM_ID}'; - Unique rows + citizen_address_success_job_status - 1120 + 1184 + 224 + + + + Identify last row in a stream + DetectLastRow + + Y + + 1 + + none + + + last_row + + + 1024 + 224 + + + + citizen_address_error_job_status + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'DELTA_ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 752 + 464 + + + + citizen_address_input + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + SELECT + ri.recruit_id::uuid, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, + + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, + CASE + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' + END AS place_stay_registration_type, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id +CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr +WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp +GROUP BY ri.recruit_id + Y + + + 400 224 @@ -266,7 +287,6 @@ and recruitment_id = '${IDM_ID}'; = recruit_id recruit_id - ervu_dashboard citizen_address
@@ -334,60 +354,8 @@ and recruitment_id = '${IDM_ID}'; - citizen_address_input - TableInput - - N - - 1 - - none - - - ervu-dashboard - N - - WITH address_data AS ( - SELECT - ri.recruit_id::uuid, - ri.ctid as source_ctid, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, - - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, - CASE - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' - END AS place_stay_registration_type, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp - GROUP BY ri.recruit_id, ri.ctid -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM address_data) as last_row -FROM address_data; - Y - - - 400 - 224 - - - - Unique rows - Unique + citizen_address_success_job_status + ExecSql Y @@ -396,18 +364,26 @@ FROM address_data; none - N - - - N - recruit_id - - - N + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'DELTA_SUCCESS', + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}' +and status = 'DELTA_PROCESSING'; + + - 1120 - 112 + 1184 + 16 diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl index 257b9a2..067819b 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl @@ -52,16 +52,16 @@ citizen_address_insert_or_update + Identify last row in a stream + Y + + + Identify last row in a stream Filter rows Y Filter rows - Unique rows - Y - - - Unique rows citizen_address_success_job_status Y @@ -178,38 +178,13 @@ DO UPDATE SET - Unique rows + citizen_address_success_job_status - 1120 + 1200 272 - - Unique rows - Unique - - Y - - 1 - - none - - - N - - - N - recruit_id - - - N - - - 1120 - 144 - - citizen_address_error_job_status ExecSql @@ -259,36 +234,30 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH address_data AS ( - SELECT - ri.recruit_id::uuid, - ri.ctid as source_ctid, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, + + SELECT + ri.recruit_id::uuid, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, - CASE - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' - END AS place_stay_registration_type, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - GROUP BY ri.recruit_id, ri.ctid -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM address_data) as last_row -FROM address_data; + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, + CASE + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' + END AS place_stay_registration_type, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id +CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr +WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' +GROUP BY ri.recruit_id Y @@ -314,7 +283,6 @@ FROM address_data; = recruit_id recruit_id - ervu_dashboard citizen_address
@@ -410,10 +378,28 @@ and status = 'PROCESSING'; - 1120 + 1200 16 + + Identify last row in a stream + DetectLastRow + + Y + + 1 + + none + + + last_row + + + 1008 + 272 + + citizen_address_insert_or_update diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl index 056251b..042ac9d 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl @@ -60,23 +60,23 @@ update_guardianship_flags Y - - citizen_guardianship_output - Filter rows - Y - citizen_guardianship_input Detect empty stream Y - Filter rows - Unique rows + citizen_guardianship_output + Identify last row in a stream Y - Unique rows + Identify last row in a stream + Filter rows + Y + + + Filter rows citizen_guardianship_success_job_status Y @@ -144,7 +144,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 496 - 560 + 720 @@ -176,36 +176,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Unique rows + citizen_guardianship_success_job_status - 816 - 464 - - - - Unique rows - Unique - - Y - - 1 - - none - - - N - - - N - recruit_id - - - N - - - 816 - 576 + 880 + 592 @@ -225,11 +200,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard - Y - N - Y Y N @@ -241,7 +213,6 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - 1280 @@ -261,39 +232,33 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH guardianship_data AS ( - SELECT - ri.recruit_id AS recruit_id, - (ch->>'id')::uuid AS guardianship_external_id, - NULLIF(ch->>'idERN', '') AS guardian_id_ern, - ch->'fioOpek'->>'familiya' AS last_name, - ch->'fioOpek'->>'imya' AS first_name, - ch->'fioOpek'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - ch->'fioOpek'->>'familiya', - ch->'fioOpek'->>'imya', - ch->'fioOpek'->>'otchestvo' - ) AS full_name, - MAKE_DATE( - NULLIF(ch->'dataRozhdDok'->>'god', '')::int, - NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(ch->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, - true AS is_guardian, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM guardianship_data) as last_row -FROM guardianship_data; + + SELECT + ri.recruit_id AS recruit_id, + (ch->>'id')::uuid AS guardianship_external_id, + NULLIF(ch->>'idERN', '') AS guardian_id_ern, + ch->'fioOpek'->>'familiya' AS last_name, + ch->'fioOpek'->>'imya' AS first_name, + ch->'fioOpek'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + ch->'fioOpek'->>'familiya', + ch->'fioOpek'->>'imya', + ch->'fioOpek'->>'otchestvo' + ) AS full_name, + MAKE_DATE( + NULLIF(ch->'dataRozhdDok'->>'god', '')::int, + NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(ch->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, + true AS is_guardian +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id +CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch +WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' Y @@ -367,7 +332,7 @@ FROM guardianship_data; Y - 816 + 880 320 @@ -434,7 +399,7 @@ and status = 'PROCESSING'; - 816 + 880 720 @@ -478,6 +443,24 @@ and status = 'PROCESSING'; 0 + + Identify last row in a stream + DetectLastRow + + Y + + 1 + + none + + + last_row + + + 880 + 480 + + citizen_guardianship_output diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl index d2683b2..03735e4 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl @@ -67,16 +67,16 @@ citizen_guardianship_insert_or_update + Identify last row in a stream + Y + + + Identify last row in a stream Filter rows Y Filter rows - Unique rows - Y - - - Unique rows citizen_guardianship_success_job_status Y @@ -178,16 +178,16 @@ and recruitment_id = '${IDM_ID}'; - Unique rows + citizen_guardianship_success_job_status 880 - 384 + 480 - Unique rows - Unique + Identify last row in a stream + DetectLastRow Y @@ -196,18 +196,11 @@ and recruitment_id = '${IDM_ID}'; none - N - - - N - recruit_id - - - N + last_row 880 - 496 + 352 @@ -262,40 +255,33 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH guardianship_data AS ( - SELECT - ri.recruit_id AS recruit_id, - (ch->>'id')::uuid AS guardianship_external_id, - NULLIF(ch->>'idERN', '') AS guardian_id_ern, - ch->'fioOpek'->>'familiya' AS last_name, - ch->'fioOpek'->>'imya' AS first_name, - ch->'fioOpek'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - ch->'fioOpek'->>'familiya', - ch->'fioOpek'->>'imya', - ch->'fioOpek'->>'otchestvo' - ) AS full_name, - MAKE_DATE( - NULLIF(ch->'dataRozhdDok'->>'god', '')::int, - NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(ch->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, - true AS is_guardian, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' - AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM guardianship_data) as last_row -FROM guardianship_data; + SELECT + ri.recruit_id AS recruit_id, + (ch->>'id')::uuid AS guardianship_external_id, + NULLIF(ch->>'idERN', '') AS guardian_id_ern, + ch->'fioOpek'->>'familiya' AS last_name, + ch->'fioOpek'->>'imya' AS first_name, + ch->'fioOpek'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + ch->'fioOpek'->>'familiya', + ch->'fioOpek'->>'imya', + ch->'fioOpek'->>'otchestvo' + ) AS full_name, + MAKE_DATE( + NULLIF(ch->'dataRozhdDok'->>'god', '')::int, + NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(ch->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, + true AS is_guardian +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id +CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch +WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp Y diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl index 73b076a..ad8aebd 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl @@ -67,16 +67,16 @@ citizen_guardianship_insert_update + Identify last row in a stream + Y + + + Identify last row in a stream Filter rows Y Filter rows - Unique rows - Y - - - Unique rows citizen_guardianship_success_job_status Y @@ -193,36 +193,11 @@ DO UPDATE SET - Unique rows + citizen_guardianship_success_job_status 704 - 384 - - - - Unique rows - Unique - - Y - - 1 - - none - - - N - - - N - recruit_id - - - N - - - 704 - 496 + 448 @@ -278,39 +253,33 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH guardianship_data AS ( - SELECT - ri.recruit_id AS recruit_id, - (ch->>'id')::uuid AS guardianship_external_id, - NULLIF(ch->>'idERN', '') AS guardian_id_ern, - ch->'fioOpek'->>'familiya' AS last_name, - ch->'fioOpek'->>'imya' AS first_name, - ch->'fioOpek'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - ch->'fioOpek'->>'familiya', - ch->'fioOpek'->>'imya', - ch->'fioOpek'->>'otchestvo' - ) AS full_name, - MAKE_DATE( - NULLIF(ch->'dataRozhdDok'->>'god', '')::int, - NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(ch->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, - true AS is_guardian, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM guardianship_data) as last_row -FROM guardianship_data; + + SELECT + ri.recruit_id AS recruit_id, + (ch->>'id')::uuid AS guardianship_external_id, + NULLIF(ch->>'idERN', '') AS guardian_id_ern, + ch->'fioOpek'->>'familiya' AS last_name, + ch->'fioOpek'->>'imya' AS first_name, + ch->'fioOpek'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + ch->'fioOpek'->>'familiya', + ch->'fioOpek'->>'imya', + ch->'fioOpek'->>'otchestvo' + ) AS full_name, + MAKE_DATE( + NULLIF(ch->'dataRozhdDok'->>'god', '')::int, + NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(ch->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, + true AS is_guardian +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id +CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch +WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' Y @@ -336,13 +305,11 @@ FROM guardianship_data; = recruit_id recruit_id - = guardianship_external_id guardianship_external_id - ervu_dashboard citizen_guardianship
@@ -506,6 +473,24 @@ and status = 'PROCESSING'; 48 + + Identify last row in a stream + DetectLastRow + + Y + + 1 + + none + + + last_row + + + 704 + 352 + + citizen_guardianship_insert_update diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl index 138aef9..82eb904 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl @@ -62,16 +62,16 @@ citizen_information_search_output + Identify last row in a stream + Y + + + Identify last row in a stream Filter rows Y Filter rows - Unique rows - Y - - - Unique rows citizen_information_success_job_status Y @@ -93,84 +93,7 @@ 880 - 624 - - - - citizen_information_error_job_status - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - - Y - - N - - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - - 720 - 624 - - - - citizen_information_success_job_status - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - - Y - - N - - Y - N - N - UPDATE etl.job_execution -SET status = 'SUCCESS', - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'PROCESSING'; - - - - - - 1104 - 160 + 720 @@ -199,7 +122,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 80 - 320 + 416 @@ -248,16 +171,16 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Unique rows + citizen_information_success_job_status 1104 - 320 + 416 - Unique rows - Unique + Identify last row in a stream + DetectLastRow Y @@ -266,18 +189,47 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); none - N - - - N - recruit_id - - - N + last_row - 1104 - 240 + 960 + 416 + + + + citizen_information_error_job_status + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 720 + 720 @@ -293,32 +245,26 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - WITH wanted_data AS ( - SELECT - ri.recruit_id, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code, - TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, - TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, - true AS is_wanted, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - WHERE '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM wanted_data) as last_row -FROM wanted_data; + + SELECT + ri.recruit_id, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code, + TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, + TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, + true AS is_wanted +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id +WHERE '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' Y 336 - 320 + 416 @@ -368,14 +314,11 @@ FROM wanted_data; N N N - Y - N ervu_dashboard Y citizen_information_search
- N Y N @@ -383,7 +326,40 @@ FROM wanted_data; 720 - 320 + 416 + + + + citizen_information_success_job_status + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS', + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}' +and status = 'PROCESSING'; + + + + + 1104 + 160 @@ -423,7 +399,7 @@ FROM wanted_data; 336 - 624 + 720 diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl index 449cda0..459772f 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl @@ -55,23 +55,23 @@ Detect empty stream Y - - citizen_information_search_insert_or_update - Filter rows - Y - citizen_information_search_input update_citizen_information_search_flags Y - Filter rows - Unique rows + citizen_information_search_insert_or_update + Identify last row in a stream Y - Unique rows + Identify last row in a stream + Filter rows + Y + + + Filter rows citizen_information_success_job_status Y @@ -96,83 +96,6 @@ 480 - - citizen_information_error_job_status - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - - Y - - N - - Y - Y - N - UPDATE etl.job_execution -SET status = 'DELTA_ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - - 752 - 480 - - - - citizen_information_success_job_status - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - - Y - - N - - Y - N - N - UPDATE etl.job_execution -SET status = 'DELTA_SUCCESS', - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'DELTA_PROCESSING'; - - - - - - 1008 - 32 - - Create job execution record ExecSql @@ -251,16 +174,16 @@ and recruitment_id = '${IDM_ID}'; - Unique rows + citizen_information_success_job_status - 1008 + 1296 224 - Unique rows - Unique + Identify last row in a stream + DetectLastRow Y @@ -269,18 +192,47 @@ and recruitment_id = '${IDM_ID}'; none - N - - - N - recruit_id - - - N + last_row - 1008 - 112 + 1120 + 224 + + + + citizen_information_error_job_status + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'DELTA_ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 752 + 480 @@ -296,8 +248,8 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH wanted_data AS ( - SELECT + + SELECT ri.recruit_id, ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search, ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code, @@ -305,19 +257,13 @@ and recruitment_id = '${IDM_ID}'; TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, - true AS is_wanted, - ri.ctid as source_ctid + true AS is_wanted FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id WHERE '${IDM_ID}' != '' AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp - AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM wanted_data) as last_row -FROM wanted_data; + AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' Y @@ -389,6 +335,39 @@ FROM wanted_data; 224 + + citizen_information_success_job_status + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'DELTA_SUCCESS', + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}' +and status = 'DELTA_PROCESSING'; + + + + + 1296 + 32 + + update_citizen_information_search_flags Update diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl index f24a90f..958485c 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl @@ -50,11 +50,6 @@ citizen_information_search_insert_or_update Y - - citizen_information_search_insert_or_update - Filter rows - Y - citizen_information_search_input update_citizen_information_search_flags @@ -66,12 +61,17 @@ Y - Filter rows - Unique rows + citizen_information_search_insert_or_update + Identify last row in a stream Y - Unique rows + Identify last row in a stream + Filter rows + Y + + + Filter rows citizen_information_success_job_status Y @@ -188,38 +188,13 @@ DO UPDATE SET - Unique rows + citizen_information_success_job_status - 1024 + 1216 272 - - Unique rows - Unique - - Y - - 1 - - none - - - N - - - N - recruit_id - - - N - - - 1024 - 160 - - citizen_information_error_job_status ExecSql @@ -269,27 +244,22 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH wanted_data AS ( - SELECT - ri.recruit_id, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code, - TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, - TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, - true AS is_wanted, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - WHERE '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM wanted_data) as last_row -FROM wanted_data; + + SELECT + ri.recruit_id, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code, + TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, + TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, + true AS is_wanted +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id +WHERE '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' + Y @@ -315,7 +285,6 @@ FROM wanted_data; = recruit_id recruit_id - ervu_dashboard citizen_information_search
@@ -391,7 +360,7 @@ and status = 'PROCESSING'; - 1024 + 1216 80 @@ -435,6 +404,24 @@ and status = 'PROCESSING'; 416 + + Identify last row in a stream + DetectLastRow + + Y + + 1 + + none + + + last_row + + + 1008 + 272 + + citizen_information_search_insert_or_update diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl index 84f7c73..b3d9a95 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl @@ -65,21 +65,6 @@ spouse_detect_empty_stream Y - - citizen_spouse_output - Filter rows - Y - - - Filter rows - Unique rows - Y - - - Unique rows - spouse_success_job_status - Y - spouse_sort_rows spouse_update_flags @@ -90,6 +75,21 @@ spouse_error_job_status Y + + citizen_spouse_output + Identify last row in a stream + Y + + + Identify last row in a stream + Filter rows + Y + + + Filter rows + spouse_success_job_status + Y + Abort @@ -169,32 +169,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Unique rows - - - 800 - 912 - - - - Unique rows - Unique - - Y - - 1 - - none - - - N - - - N - recruit_id - - - N + spouse_success_job_status 592 @@ -403,53 +378,46 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH spouse_data AS ( - SELECT - ri.recruit_id AS recruit_id, - (supr->>'id')::uuid AS spouse_external_id, - NULLIF(supr->>'idERN', '') AS spouse_id_ern, - NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, - supr->'svedFLBS'->'fio'->>'familiya' AS last_name, - supr->'svedFLBS'->'fio'->>'imya' AS first_name, - supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - supr->'svedFLBS'->'fio'->>'familiya', - supr->'svedFLBS'->'fio'->>'imya', - supr->'svedFLBS'->'fio'->>'otchestvo' - ) AS full_name, - MAKE_DATE( - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, - supr->'svedSmert'->>'nomerZapis' AS death_az_number, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, - NULLIF(ri.info->'svedFL'->'svedSemPolozh'->>'semPolozh','')::int AS marital_status, - CASE - WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true - ELSE false - END AS information_excluded, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - CROSS JOIN LATERAL ( - SELECT supri - FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri - WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' - ) AS supr(supr) - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM spouse_data) as last_row -FROM spouse_data; - + + SELECT + ri.recruit_id AS recruit_id, + (supr->>'id')::uuid AS spouse_external_id, + NULLIF(supr->>'idERN', '') AS spouse_id_ern, + NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, + supr->'svedFLBS'->'fio'->>'familiya' AS last_name, + supr->'svedFLBS'->'fio'->>'imya' AS first_name, + supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + supr->'svedFLBS'->'fio'->>'familiya', + supr->'svedFLBS'->'fio'->>'imya', + supr->'svedFLBS'->'fio'->>'otchestvo' + ) AS full_name, + MAKE_DATE( + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, + supr->'svedSmert'->>'nomerZapis' AS death_az_number, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, + NULLIF(ri.info->'svedFL'->'svedSemPolozh'->>'semPolozh','')::int AS marital_status, + CASE + WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true + ELSE false + END AS information_excluded +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id +CROSS JOIN LATERAL ( + SELECT supri + FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri + WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' + ) AS supr(supr) +WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' Y @@ -538,19 +506,16 @@ and status = 'PROCESSING'; 10000 ervu-dashboard Y - = recruit_id recruit_id - <> marital_status marital_status - ervu_dashboard citizen
@@ -567,6 +532,24 @@ and status = 'PROCESSING'; 32 + + Identify last row in a stream + DetectLastRow + + Y + + 1 + + none + + + last_row + + + 800 + 912 + + citizen_spouse_output diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl index 7ffae33..d213721 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl @@ -60,26 +60,11 @@ Detect empty stream Y - - spouse_update_or_insert - Filter rows - Y - Sort rows marriage_registry_update_or_insert Y - - Filter rows - Unique rows - Y - - - Unique rows - spouse_success_job_status - Y - Sort rows spouse_update_flags @@ -90,6 +75,21 @@ spouse_error_job_status Y + + spouse_update_or_insert + Identify last row in a stream + Y + + + Identify last row in a stream + Filter rows + Y + + + Filter rows + spouse_success_job_status + Y + Abort @@ -188,11 +188,11 @@ and recruitment_id = '${IDM_ID}'; - Unique rows + spouse_success_job_status 1072 - 496 + 592 @@ -229,31 +229,6 @@ and recruitment_id = '${IDM_ID}'; 320 - - Unique rows - Unique - - Y - - 1 - - none - - - N - - - N - recruit_id - - - N - - - 1072 - 592 - - marriage_registry_update_or_insert InsertUpdate @@ -361,54 +336,46 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH spouse_data AS ( - SELECT - ri.recruit_id AS recruit_id, - (supr->>'id')::uuid AS spouse_external_id, - NULLIF(supr->>'idERN', '') AS spouse_id_ern, - NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, - supr->'svedFLBS'->'fio'->>'familiya' AS last_name, - supr->'svedFLBS'->'fio'->>'imya' AS first_name, - supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - supr->'svedFLBS'->'fio'->>'familiya', - supr->'svedFLBS'->'fio'->>'imya', - supr->'svedFLBS'->'fio'->>'otchestvo' - ) AS full_name, - MAKE_DATE( - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, - supr->'svedSmert'->>'nomerZapis' AS death_az_number, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, - NULLIF(ri.info->'svedFL'->'svedSemPolozh'->>'semPolozh','')::int AS marital_status, - CASE - WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true - ELSE false - END AS information_excluded, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - CROSS JOIN LATERAL ( - SELECT supri - FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri - WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' - ) AS supr(supr) - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND ri.update_date >= '${M_R_UP_DATE}'::timestamp -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM spouse_data) as last_row -FROM spouse_data; - + SELECT + ri.recruit_id AS recruit_id, + (supr->>'id')::uuid AS spouse_external_id, + NULLIF(supr->>'idERN', '') AS spouse_id_ern, + NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, + supr->'svedFLBS'->'fio'->>'familiya' AS last_name, + supr->'svedFLBS'->'fio'->>'imya' AS first_name, + supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + supr->'svedFLBS'->'fio'->>'familiya', + supr->'svedFLBS'->'fio'->>'imya', + supr->'svedFLBS'->'fio'->>'otchestvo' + ) AS full_name, + MAKE_DATE( + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, + supr->'svedSmert'->>'nomerZapis' AS death_az_number, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, + NULLIF(ri.info->'svedFL'->'svedSemPolozh'->>'semPolozh','')::int AS marital_status, + CASE + WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true + ELSE false + END AS information_excluded +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id +CROSS JOIN LATERAL ( + SELECT supri + FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri + WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' + ) AS supr(supr) +WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND ri.update_date >= '${M_R_UP_DATE}'::timestamp Y @@ -583,6 +550,24 @@ WHERE job_name = '${JOB_NAME}' 496 + + Identify last row in a stream + DetectLastRow + + Y + + 1 + + none + + + last_row + + + 1072 + 496 + + marriage_registry_update_or_insert diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl index f917e7f..2342cbb 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl @@ -87,26 +87,11 @@ M_R_CR_DATE marriage_registry_update_or_insert Y - - spouse_update_or_insert - Filter rows - Y - spouse_input Detect empty stream Y - - Filter rows - Unique rows - Y - - - Unique rows - spouse_success_job_status - Y - Sort rows spouse_update_flags @@ -117,6 +102,21 @@ M_R_CR_DATE spouse_error_job_status Y + + spouse_update_or_insert + Identify last row in a stream + Y + + + Identify last row in a stream + Filter rows + Y + + + Filter rows + spouse_success_job_status + Y + Abort @@ -230,11 +230,11 @@ DO UPDATE SET - Unique rows + spouse_success_job_status 944 - 528 + 784 @@ -271,31 +271,6 @@ DO UPDATE SET 368 - - Unique rows - Unique - - Y - - 1 - - none - - - N - - - N - recruit_id - - - N - - - 944 - 784 - - marriage_registry_update_or_insert InsertUpdate @@ -402,53 +377,46 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH spouse_data AS ( - SELECT - ri.recruit_id AS recruit_id, - (supr->>'id')::uuid AS spouse_external_id, - NULLIF(supr->>'idERN', '') AS spouse_id_ern, - NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, - supr->'svedFLBS'->'fio'->>'familiya' AS last_name, - supr->'svedFLBS'->'fio'->>'imya' AS first_name, - supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - supr->'svedFLBS'->'fio'->>'familiya', - supr->'svedFLBS'->'fio'->>'imya', - supr->'svedFLBS'->'fio'->>'otchestvo' - ) AS full_name, - MAKE_DATE( - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, - supr->'svedSmert'->>'nomerZapis' AS death_az_number, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, - NULLIF(ri.info->'svedFL'->'svedSemPolozh'->>'semPolozh','')::int AS marital_status, - CASE - WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true - ELSE false - END AS information_excluded, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - CROSS JOIN LATERAL ( - SELECT supri - FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri - WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' - ) AS supr(supr) - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM spouse_data) as last_row -FROM spouse_data; - + + SELECT + ri.recruit_id AS recruit_id, + (supr->>'id')::uuid AS spouse_external_id, + NULLIF(supr->>'idERN', '') AS spouse_id_ern, + NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, + supr->'svedFLBS'->'fio'->>'familiya' AS last_name, + supr->'svedFLBS'->'fio'->>'imya' AS first_name, + supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + supr->'svedFLBS'->'fio'->>'familiya', + supr->'svedFLBS'->'fio'->>'imya', + supr->'svedFLBS'->'fio'->>'otchestvo' + ) AS full_name, + MAKE_DATE( + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, + supr->'svedSmert'->>'nomerZapis' AS death_az_number, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, + NULLIF(ri.info->'svedFL'->'svedSemPolozh'->>'semPolozh','')::int AS marital_status, + CASE + WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true + ELSE false + END AS information_excluded +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id +CROSS JOIN LATERAL ( + SELECT supri + FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri + WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' + ) AS supr(supr) +WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' Y @@ -491,6 +459,46 @@ and status = 'PROCESSING'; 784 + + spouse_update_flags + Update + + Y + + 1 + + none + + + 10000 + ervu-dashboard + Y + + + = + recruit_id + recruit_id + + + <> + marital_status + marital_status + + ervu_dashboard + citizen
+ + marital_status + marital_status + + + N + Y + + + 512 + 368 + + spouse_update_or_insert InsertUpdate @@ -586,8 +594,8 @@ and status = 'PROCESSING'; - spouse_update_flags - Update + Identify last row in a stream + DetectLastRow Y @@ -596,33 +604,11 @@ and status = 'PROCESSING'; none - 10000 - ervu-dashboard - Y - - - = - recruit_id - recruit_id - - - <> - marital_status - marital_status - - ervu_dashboard - citizen
- - marital_status - marital_status - - - N - Y + last_row - 512 - 368 + 944 + 528 @@ -638,18 +624,6 @@ and status = 'PROCESSING'; - - spouse_update_or_insert - spouse_error_job_status - Y - - error_description - - - - - - spouse_update_flags spouse_error_job_status @@ -662,6 +636,18 @@ and status = 'PROCESSING'; + + spouse_update_or_insert + spouse_error_job_status + Y + + error_description + + + + + + diff --git a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl index 777f5e1..b8c1906 100644 --- a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl @@ -35,11 +35,6 @@ passport_success_job_status Y - - passport_output - Filter rows - Y - passport_error_job_status Abort @@ -56,12 +51,17 @@ Y - Filter rows - Unique rows + passport_output + Identify last row in a stream Y - Unique rows + Identify last row in a stream + Filter rows + Y + + + Filter rows passport_success_job_status Y @@ -83,84 +83,7 @@ 1264 - 608 - - - - passport_error_job_status - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - - Y - - N - - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - - 896 - 608 - - - - passport_success_job_status - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - - Y - - N - - Y - N - N - UPDATE etl.job_execution -SET status = 'SUCCESS', - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'PROCESSING'; - - - - - - 1264 - 112 + 640 @@ -189,7 +112,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 288 - 320 + 352 @@ -238,11 +161,47 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Unique rows + passport_success_job_status 1264 - 320 + 352 + + + + passport_error_job_status + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 896 + 640 @@ -258,9 +217,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - - WITH passport_data AS ( - SELECT + SELECT ri.recruit_id, NULLIF(p.passport_data->>'nomDok', '') AS passport_number, NULLIF(p.passport_data->>'serDok', '') AS passport_series, @@ -270,8 +227,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); CASE WHEN NULLIF(p.passport_data->>'kodStatus', '') = '1' THEN TRUE ELSE FALSE - END AS actual, - ri.ctid as source_ctid + END AS actual FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id @@ -286,17 +242,12 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); WHERE passport_data IS NOT NULL AND passport_data::text <> 'null' ) AS p ON TRUE WHERE LENGTH(REGEXP_REPLACE(p.passport_data->>'serDok', '[^0-9]', '', 'g')) = 4 - AND LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6 -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM passport_data) as last_row -FROM passport_data; + AND LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6 Y 496 - 320 + 352 @@ -346,14 +297,11 @@ FROM passport_data; N N N - Y - N ervu_dashboard Y passport
- N Y N @@ -361,12 +309,12 @@ FROM passport_data; 896 - 320 + 352 - Unique rows - Unique + passport_success_job_status + ExecSql Y @@ -375,18 +323,44 @@ FROM passport_data; none - N - - - N - recruit_id - - - N + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS', + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}' +and status = 'PROCESSING'; + + 1264 - 208 + 112 + + + + Identify last row in a stream + DetectLastRow + + Y + + 1 + + none + + + last_row + + + 1104 + 352 diff --git a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl index 69d0acb..84efa21 100644 --- a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl @@ -35,11 +35,6 @@ passport_success_job_status Y - - passport_insert_or_update - Filter rows - Y - passport_error_job_status Abort @@ -56,12 +51,17 @@ Y - Filter rows - Unique rows + passport_insert_or_update + Identify last row in a stream Y - Unique rows + Identify last row in a stream + Filter rows + Y + + + Filter rows passport_success_job_status Y @@ -86,83 +86,6 @@ 464 - - passport_error_job_status - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - - Y - - N - - Y - Y - N - UPDATE etl.job_execution -SET status = 'DELTA_ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - - 752 - 464 - - - - passport_success_job_status - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - - Y - - N - - Y - N - N - UPDATE etl.job_execution -SET status = 'DELTA_SUCCESS', - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'DELTA_PROCESSING'; - - - - - - 1120 - 16 - - Create job execution record ExecSql @@ -241,10 +164,116 @@ and recruitment_id = '${IDM_ID}'; - Unique rows + passport_success_job_status - 1120 + 1248 + 224 + + + + Identify last row in a stream + DetectLastRow + + Y + + 1 + + none + + + last_row + + + 960 + 224 + + + + passport_error_job_status + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'DELTA_ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 752 + 464 + + + + passport_input + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + SELECT + ri.recruit_id, + NULLIF(p.passport_data->>'nomDok', '') AS passport_number, + NULLIF(p.passport_data->>'serDok', '') AS passport_series, + NULLIF(p.passport_data->>'vydDok', '') AS organization_name, + NULLIF(p.passport_data->>'kodVydDok', '') AS unit_code, + TO_DATE(NULLIF(p.passport_data->>'dataDok', ''), 'YYYY-MM-DD') AS issue_date, + CASE + WHEN NULLIF(p.passport_data->>'kodStatus', '') = '1' THEN TRUE + ELSE FALSE + END AS actual + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r + ON r.recruit_id = ri.recruit_id + AND '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND r.update_date >= '${M_R_UP_DATE}'::timestamp + JOIN LATERAL ( + SELECT passport_data + FROM UNNEST(ARRAY[ + ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF', + ri.info->'svedFL'->'svedBS'->'pasportRF'->'predPasportRF' + ]) AS passport_data + WHERE passport_data IS NOT NULL AND passport_data::text <> 'null' + ) AS p ON TRUE + WHERE LENGTH(REGEXP_REPLACE(p.passport_data->>'serDok', '[^0-9]', '', 'g')) = 4 + AND LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6 + + + + + + Y + + + 400 224 @@ -266,13 +295,11 @@ and recruitment_id = '${IDM_ID}'; = recruit_id recruit_id - = actual actual - ervu_dashboard passport
@@ -320,63 +347,8 @@ and recruitment_id = '${IDM_ID}'; - passport_input - TableInput - - N - - 1 - - none - - - ervu-dashboard - N - - WITH passport_data AS ( - SELECT - ri.recruit_id, - NULLIF(p.passport_data->>'nomDok', '') AS passport_number, - NULLIF(p.passport_data->>'serDok', '') AS passport_series, - NULLIF(p.passport_data->>'vydDok', '') AS organization_name, - NULLIF(p.passport_data->>'kodVydDok', '') AS unit_code, - TO_DATE(NULLIF(p.passport_data->>'dataDok', ''), 'YYYY-MM-DD') AS issue_date, - CASE - WHEN NULLIF(p.passport_data->>'kodStatus', '') = '1' THEN TRUE - ELSE FALSE - END AS actual, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r - ON r.recruit_id = ri.recruit_id - AND '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND r.update_date >= '${M_R_UP_DATE}'::timestamp - JOIN LATERAL ( - SELECT passport_data - FROM UNNEST(ARRAY[ - ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF', - ri.info->'svedFL'->'svedBS'->'pasportRF'->'predPasportRF' - ]) AS passport_data - WHERE passport_data IS NOT NULL AND passport_data::text <> 'null' - ) AS p ON TRUE - WHERE LENGTH(REGEXP_REPLACE(p.passport_data->>'serDok', '[^0-9]', '', 'g')) = 4 - AND LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6 -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM passport_data) as last_row -FROM passport_data; - Y - - - 400 - 224 - - - - Unique rows - Unique + passport_success_job_status + ExecSql Y @@ -385,18 +357,26 @@ FROM passport_data; none - N - - - N - recruit_id - - - N + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'DELTA_SUCCESS', + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}' +and status = 'DELTA_PROCESSING'; + + - 1120 - 112 + 1248 + 16 diff --git a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl index 8b44cba..06957fb 100644 --- a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl @@ -35,11 +35,6 @@ passport_success_job_status Y - - passport_insert_or_update - Filter rows - Y - passport_error_job_status Abort @@ -56,12 +51,17 @@ Y - Filter rows - Unique rows + passport_insert_or_update + Identify last row in a stream Y - Unique rows + Identify last row in a stream + Filter rows + Y + + + Filter rows passport_success_job_status Y @@ -178,16 +178,16 @@ DO UPDATE SET - Unique rows + passport_success_job_status - 1040 + 1312 272 - Unique rows - Unique + Identify last row in a stream + DetectLastRow Y @@ -196,18 +196,11 @@ DO UPDATE SET none - N - - - N - recruit_id - - - N + last_row - 1040 - 144 + 1088 + 272 @@ -259,8 +252,7 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH passport_data AS ( - SELECT + SELECT ri.recruit_id, NULLIF(p.passport_data->>'nomDok', '') AS passport_number, NULLIF(p.passport_data->>'serDok', '') AS passport_series, @@ -270,8 +262,7 @@ and recruitment_id = '${IDM_ID}'; CASE WHEN NULLIF(p.passport_data->>'kodStatus', '') = '1' THEN TRUE ELSE FALSE - END AS actual, - ri.ctid as source_ctid + END AS actual FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id @@ -286,12 +277,7 @@ and recruitment_id = '${IDM_ID}'; WHERE passport_data IS NOT NULL AND passport_data::text <> 'null' ) AS p ON TRUE WHERE LENGTH(REGEXP_REPLACE(p.passport_data->>'serDok', '[^0-9]', '', 'g')) = 4 - AND LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6 -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM passport_data) as last_row -FROM passport_data; + AND LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6 Y @@ -317,13 +303,11 @@ FROM passport_data; = recruit_id recruit_id - = actual actual - ervu_dashboard passport
@@ -399,7 +383,7 @@ and status = 'PROCESSING'; - 1040 + 1312 16 diff --git a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl index 68e2e28..e1b0fa1 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl @@ -65,19 +65,19 @@ Detect empty stream Y - - prosecution_output - Filter rows - Y - Filter rows - Unique rows + prosecution_success_job_status Y - Unique rows - prosecution_success_job_status + prosecution_output + Identify last row in a stream + Y + + + Identify last row in a stream + Filter rows Y @@ -127,7 +127,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 272 - 464 + 512 @@ -144,7 +144,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 480 - 640 + 688 @@ -176,16 +176,16 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Unique rows + prosecution_success_job_status - 1072 - 464 + 1248 + 512 - Unique rows - Unique + Identify last row in a stream + DetectLastRow Y @@ -194,18 +194,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); none - N - - - N - recruit_id - - - N + last_row 1072 - 544 + 512 @@ -257,11 +250,11 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N + WITH ugolov_presl AS ( SELECT jsonb_array_elements(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') AS up, - ri.recruit_id, - ri.ctid as source_ctid + ri.recruit_id FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' @@ -286,7 +279,7 @@ SELECT NULLIF(up->'sledSud'->'svedSnyat'->>'osnSnyatSud', '') AS conviction_removal_basis, TO_DATE(up->'sledSud'->'svedSnyat'->>'dataSnyatSud', 'YYYY-MM-DD') AS conviction_removal_date, TO_DATE(up->'sledSud'->>'dataOsuzhd', 'YYYY-MM-DD') AS conviction_date, - TO_DATE(up->'sledSud'->>'dataVozbUD', 'YYYY-MM-DD') AS create_date, + TO_DATE(up->'sledSud'->>'dataVozbUD', 'YYYY-MM-DD') AS create_date, NULLIF(up->'sledSud'->>'osnPrekrUD', '') AS termination_basis, NULLIF(up->'sledSud'->>'statusNaim', '') AS conviction_status, TO_DATE(up->'sledSud'->>'dataPrekrUD', 'YYYY-MM-DD') AS close_date, @@ -297,17 +290,16 @@ SELECT WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде' ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей' END AS type_info, - true AS has_criminal_prosecution, - up.source_ctid = (SELECT MAX(source_ctid) FROM ugolov_presl) as last_row + true AS has_criminal_prosecution FROM ervu_dashboard.citizen r JOIN ugolov_presl up ON r.recruit_id = up.recruit_id LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true -GROUP BY r.recruit_id, up.up, up.source_ctid; +GROUP BY r.recruit_id, up.up; Y 480 - 464 + 512 @@ -425,7 +417,7 @@ GROUP BY r.recruit_id, up.up, up.source_ctid; 848 - 464 + 512 @@ -491,8 +483,8 @@ and status = 'PROCESSING'; - 1072 - 640 + 1248 + 688 diff --git a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl index 174aafc..f6891dc 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl @@ -60,23 +60,23 @@ prosecution_sort_rows Y - - prosecution_insert_or_update - Filter rows - Y - prosecution_input Detect empty stream Y - Filter rows - Unique rows + prosecution_insert_or_update + Identify last row in a stream Y - Unique rows + Identify last row in a stream + Filter rows + Y + + + Filter rows prosecution_success_job_status Y @@ -179,16 +179,16 @@ and recruitment_id = '${IDM_ID}'; - Unique rows + prosecution_success_job_status - 896 + 1008 768 - prosecution_sort_rows - SortRows + Identify last row in a stream + DetectLastRow Y @@ -197,52 +197,11 @@ and recruitment_id = '${IDM_ID}'; none - ${java.io.tmpdir} - out - 100000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - + last_row - 384 - 416 - - - - Unique rows - Unique - - Y - - 1 - - none - - - N - - - N - recruit_id - - - N - - - 896 - 864 + 880 + 768 @@ -297,8 +256,7 @@ and recruitment_id = '${IDM_ID}'; WITH ugolov_presl AS ( SELECT jsonb_array_elements(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') AS up, - ri.recruit_id, - ri.ctid as source_ctid + ri.recruit_id FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' @@ -335,12 +293,11 @@ SELECT WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде' ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей' END AS type_info, - true AS has_criminal_prosecution, - up.source_ctid = (SELECT MAX(source_ctid) FROM ugolov_presl) as last_row + true AS has_criminal_prosecution FROM ervu_dashboard.citizen r JOIN ugolov_presl up ON r.recruit_id = up.recruit_id LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true -GROUP BY r.recruit_id, up.up, up.source_ctid; +GROUP BY r.recruit_id, up.up; Y @@ -487,6 +444,40 @@ GROUP BY r.recruit_id, up.up, up.source_ctid; 768 + + prosecution_sort_rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 384 + 416 + + prosecution_success_job_status ExecSql @@ -516,7 +507,7 @@ and status = 'DELTA_PROCESSING'; - 896 + 1008 944 diff --git a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl index 7ca9352..41f80d0 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl @@ -72,6 +72,11 @@ Insert / update + Identify last row in a stream + Y + + + Identify last row in a stream Filter rows Y @@ -190,6 +195,24 @@ DO UPDATE SET prosecution_success_job_status + + 1072 + 944 + + + + Identify last row in a stream + DetectLastRow + + Y + + 1 + + none + + + last_row + 928 944 @@ -213,13 +236,11 @@ DO UPDATE SET = recruit_id recruit_id - = prosecution_external_id prosecution_external_id - ervu_dashboard prosecution
@@ -352,8 +373,7 @@ DO UPDATE SET WITH ugolov_presl AS ( SELECT jsonb_array_elements(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') AS up, - ri.recruit_id, - ri.ctid as source_ctid + ri.recruit_id FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' @@ -378,7 +398,7 @@ SELECT NULLIF(up->'sledSud'->'svedSnyat'->>'osnSnyatSud', '') AS conviction_removal_basis, TO_DATE(up->'sledSud'->'svedSnyat'->>'dataSnyatSud', 'YYYY-MM-DD') AS conviction_removal_date, TO_DATE(up->'sledSud'->>'dataOsuzhd', 'YYYY-MM-DD') AS conviction_date, - TO_DATE(up->'sledSud'->>'dataVozbUD', 'YYYY-MM-DD') AS create_date, + TO_DATE(up->'sledSud'->>'dataVozbUD', 'YYYY-MM-DD') AS create_date, NULLIF(up->'sledSud'->>'osnPrekrUD', '') AS termination_basis, NULLIF(up->'sledSud'->>'statusNaim', '') AS conviction_status, TO_DATE(up->'sledSud'->>'dataPrekrUD', 'YYYY-MM-DD') AS close_date, @@ -389,12 +409,11 @@ SELECT WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде' ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей' END AS type_info, - true AS has_criminal_prosecution, - up.source_ctid = (SELECT MAX(source_ctid) FROM ugolov_presl) as last_row + true AS has_criminal_prosecution FROM ervu_dashboard.citizen r JOIN ugolov_presl up ON r.recruit_id = up.recruit_id LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true -GROUP BY r.recruit_id, up.up, up.source_ctid; +GROUP BY r.recruit_id, up.up; Y @@ -501,7 +520,7 @@ and status = 'PROCESSING'; - 928 + 1072 1088 diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl index 597755c..693b56f 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl @@ -67,16 +67,16 @@ punishment_output + Identify last row in a stream + Y + + + Identify last row in a stream Filter rows Y Filter rows - Unique rows - Y - - - Unique rows punishment_success_job_status Y @@ -176,7 +176,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Unique rows + punishment_success_job_status 720 @@ -184,8 +184,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - punishment_sort_rows - SortRows + Identify last row in a stream + DetectLastRow Y @@ -194,52 +194,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); none - ${java.io.tmpdir} - out - 100000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - + last_row - 336 - 624 - - - - Unique rows - Unique - - Y - - 1 - - none - - - N - - - N - recruit_id - - - N - - - 640 - 144 + 720 + 224 @@ -291,11 +250,11 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N + WITH punishment_data AS ( SELECT ri.recruit_id, - jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info, - ri.ctid as source_ctid + jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' @@ -332,11 +291,10 @@ SELECT recruit_id, punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения true AS has_punishment_execution_info, - true AS has_sentence_info, - source_ctid = (SELECT MAX(source_ctid) FROM punishment_data) as last_row + true AS has_sentence_info FROM punishment_data LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true -GROUP BY recruit_id, punishment_info,source_ctid; +GROUP BY recruit_id, punishment_info; Y @@ -454,6 +412,40 @@ GROUP BY recruit_id, punishment_info,source_ctid; 320 + + punishment_sort_rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 336 + 624 + + punishment_success_job_status ExecSql @@ -484,7 +476,7 @@ and status = 'PROCESSING'; 512 - 256 + 144 diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl index b802798..b8801b1 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl @@ -86,16 +86,16 @@ punishment_insert_or_update + Identify last row in a stream + Y + + + Identify last row in a stream Filter rows Y Filter rows - Unique rows - Y - - - Unique rows punishment_success_job_status Y @@ -166,7 +166,7 @@ and recruitment_id = '${IDM_ID}'; 400 - 32 + 0 @@ -198,16 +198,16 @@ and recruitment_id = '${IDM_ID}'; - Unique rows + punishment_success_job_status 784 - 32 + 0 - punishment_sort_rows - SortRows + Identify last row in a stream + DetectLastRow Y @@ -216,52 +216,11 @@ and recruitment_id = '${IDM_ID}'; none - ${java.io.tmpdir} - out - 100000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - + last_row - 400 - 608 - - - - Unique rows - Unique - - Y - - 1 - - none - - - N - - - N - recruit_id - - - N - - - 704 - 32 + 784 + 96 @@ -316,8 +275,7 @@ and recruitment_id = '${IDM_ID}'; WITH punishment_data AS ( SELECT ri.recruit_id, - jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info, - ri.ctid as source_ctid + jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' @@ -355,11 +313,10 @@ SELECT recruit_id, punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения true AS has_punishment_execution_info, - true AS has_sentence_info, - source_ctid = (SELECT MAX(source_ctid) FROM punishment_data) as last_row + true AS has_sentence_info FROM punishment_data LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true -GROUP BY recruit_id, punishment_info,source_ctid; +GROUP BY recruit_id, punishment_info; Y @@ -496,6 +453,40 @@ GROUP BY recruit_id, punishment_info,source_ctid; 224 + + punishment_sort_rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 400 + 608 + + punishment_success_job_status ExecSql @@ -525,8 +516,8 @@ and status = 'DELTA_PROCESSING'; - 592 - 160 + 576 + 0 diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl index 463ff07..2295a8e 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl @@ -67,16 +67,16 @@ punishment_insert_or_update + Identify last row in a stream + Y + + + Identify last row in a stream Filter rows Y Filter rows - Unique rows - Y - - - Unique rows punishment_success_job_status Y @@ -161,7 +161,7 @@ DO UPDATE SET 416 - 112 + 64 @@ -193,16 +193,16 @@ DO UPDATE SET - Unique rows + punishment_success_job_status - 816 - 112 + 800 + 64 - Unique rows - Unique + Identify last row in a stream + DetectLastRow Y @@ -211,18 +211,11 @@ DO UPDATE SET none - N - - - N - recruit_id - - - N + last_row - 736 - 112 + 800 + 144 @@ -274,11 +267,11 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N + WITH punishment_data AS ( SELECT ri.recruit_id, - jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info, - ri.ctid as source_ctid + jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' @@ -315,11 +308,10 @@ SELECT recruit_id, punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения true AS has_punishment_execution_info, - true AS has_sentence_info, - source_ctid = (SELECT MAX(source_ctid) FROM punishment_data) as last_row + true AS has_sentence_info FROM punishment_data LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true -GROUP BY recruit_id, punishment_info,source_ctid; +GROUP BY recruit_id, punishment_info; Y @@ -345,13 +337,11 @@ GROUP BY recruit_id, punishment_info,source_ctid; = recruit_id recruit_id - = punishment_external_id punishment_external_id - ervu_dashboard punishment
@@ -521,8 +511,8 @@ and status = 'PROCESSING'; - 640 - 192 + 608 + 64