diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl index 4450844..3bcfdba 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl @@ -22,64 +22,69 @@ Create job execution record - Table input + child_input Y User defined Java expression 2 - Update 2 + child_update_flags Y - Update 2 - Change job status on error + child_update_flags + child_error_job_status Y - Change job status on error + child_error_job_status Abort Y - - Filter rows - Change job status on success - Y - Detect empty stream - Change job status on success + child_success_job_status Y - Table input - Table output + child_input + child_output Y - Table output - Change job status on error + child_output + child_error_job_status Y - Table input - Group by 2 + child_input + child_group_by Y - Group by 2 + child_group_by User defined Java expression 2 Y - Table output + child_output Filter rows Y - Table input + child_input Detect empty stream Y + + Filter rows + Unique rows + Y + + + Unique rows + child_success_job_status + Y + Abort @@ -102,7 +107,7 @@ - Change job status on error + child_error_job_status ExecSql Y @@ -118,8 +123,11 @@ ervu-dashboard + Y + N + Y Y N @@ -131,6 +139,7 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + 1216 @@ -138,7 +147,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + child_success_job_status ExecSql Y @@ -151,8 +160,11 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard + Y + N + Y N N @@ -164,6 +176,7 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + 576 @@ -245,7 +258,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Change job status on success + Unique rows 896 @@ -253,7 +266,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Group by 2 + child_group_by GroupBy Y @@ -271,16 +284,19 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); minors_count_raw is_minor SUM + has_minor_child_raw is_minor MAX + has_dead_child_raw is_dead MAX + N @@ -290,6 +306,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); N + grp @@ -298,7 +315,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Table input + child_input TableInput N @@ -310,6 +327,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N + WITH children_data AS ( SELECT ri.recruit_id AS recruit_id, @@ -372,7 +390,7 @@ FROM children_data; - Table output + child_output TableOutput N @@ -438,11 +456,14 @@ FROM children_data; N N N + Y + N ervu_dashboard Y citizen_child
+ N Y N @@ -454,7 +475,7 @@ FROM children_data; - Update 2 + child_update_flags Update N @@ -539,10 +560,35 @@ FROM children_data; 480 + + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 736 + 1040 + + Create job execution record - Change job status on error + child_error_job_status N @@ -553,8 +599,8 @@ FROM children_data; - Table output - Change job status on error + child_output + child_error_job_status Y error_description @@ -565,8 +611,8 @@ FROM children_data; - Update 2 - Change job status on error + child_update_flags + child_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl index 3ed095c..e97744e 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl @@ -21,33 +21,28 @@ - Insert / update + child_insert_or_update Change job status on error Y - Update + child_update_flags Change job status on error Y User defined Java expression - Update + child_update_flags Y Create job execution record - Table input 2 - Y - - - Filter rows - Change job status on success + child_input Y Detect empty stream - Change job status on success + child_success_job_status Y @@ -56,28 +51,38 @@ Y - Table input 2 - Insert / update + child_input + child_insert_or_update Y - Group by + child_group_by User defined Java expression Y - Table input 2 + child_input Detect empty stream Y - Insert / update + child_insert_or_update Filter rows Y - Table input 2 - Group by + child_input + child_group_by + Y + + + Filter rows + Unique rows + Y + + + Unique rows + child_success_job_status Y @@ -138,7 +143,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + child_success_job_status ExecSql Y @@ -151,8 +156,11 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard + Y + N + Y N N @@ -164,6 +172,7 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}' and status = 'DELTA_PROCESSING'; + 784 @@ -247,7 +256,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + Unique rows 1072 @@ -255,7 +264,73 @@ and recruitment_id = '${IDM_ID}'; - Group by + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 960 + 496 + + + + User defined Java expression + Janino + + Y + + 1 + + none + + + + minors_count + minors_count_raw == 1 ? "1" : minors_count_raw == 2 ? "2" : minors_count_raw >= 3 ? "3 и более" : "0" + String + 10 + -1 + + + + has_minor_child + has_minor_child_raw == 1 + Boolean + -1 + -1 + + + + has_dead_child + has_dead_child_raw == 1 + Boolean + -1 + -1 + + + + + 784 + 0 + + + + child_group_by GroupBy Y @@ -300,7 +375,82 @@ and recruitment_id = '${IDM_ID}'; - Insert / update + child_input + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + WITH children_data AS ( + SELECT + ri.recruit_id AS recruit_id, + (ch->>'id')::uuid AS child_external_id, + NULLIF(ch->>'idERN', '') AS child_id_ern, + CASE NULLIF(ch->>'rodstvSvyazReb', '')::int + WHEN 1 THEN 3 + WHEN 2 THEN 4 + END AS kinship_type, + ch->'svedFLBS'->'fio'->>'familiya' AS last_name, + ch->'svedFLBS'->'fio'->>'imya' AS first_name, + ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + ch->'svedFLBS'->'fio'->>'familiya', + ch->'svedFLBS'->'fio'->>'imya', + ch->'svedFLBS'->'fio'->>'otchestvo' + ) AS full_name, + dates.birth_date, + dates.death_date, + ch->'svedSmert'->>'nomerZapis' AS death_az_number, + ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number, + CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead, + CASE + WHEN dates.death_date IS NOT NULL THEN 0 + WHEN dates.birth_date IS NULL THEN 0 + WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1 + ELSE 0 + END AS is_minor, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id + CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch + CROSS JOIN LATERAL ( + SELECT + MAKE_DATE( + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + MAKE_DATE( + NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int, + NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int, + NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int + ) AS death_date + ) AS dates + WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp + AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array' +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM children_data) as last_row +FROM children_data; + Y + + + 496 + 224 + + + + child_insert_or_update InsertUpdate Y @@ -394,82 +544,7 @@ and recruitment_id = '${IDM_ID}'; - Table input 2 - TableInput - - N - - 1 - - none - - - ervu-dashboard - N - WITH children_data AS ( - SELECT - ri.recruit_id AS recruit_id, - (ch->>'id')::uuid AS child_external_id, - NULLIF(ch->>'idERN', '') AS child_id_ern, - CASE NULLIF(ch->>'rodstvSvyazReb', '')::int - WHEN 1 THEN 3 - WHEN 2 THEN 4 - END AS kinship_type, - ch->'svedFLBS'->'fio'->>'familiya' AS last_name, - ch->'svedFLBS'->'fio'->>'imya' AS first_name, - ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - ch->'svedFLBS'->'fio'->>'familiya', - ch->'svedFLBS'->'fio'->>'imya', - ch->'svedFLBS'->'fio'->>'otchestvo' - ) AS full_name, - dates.birth_date, - dates.death_date, - ch->'svedSmert'->>'nomerZapis' AS death_az_number, - ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number, - CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead, - CASE - WHEN dates.death_date IS NOT NULL THEN 0 - WHEN dates.birth_date IS NULL THEN 0 - WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1 - ELSE 0 - END AS is_minor, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch - CROSS JOIN LATERAL ( - SELECT - MAKE_DATE( - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - MAKE_DATE( - NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int, - NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int, - NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int - ) AS death_date - ) AS dates - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp - AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array' -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM children_data) as last_row -FROM children_data; - Y - - - 496 - 224 - - - - Update + child_update_flags Update N @@ -482,13 +557,11 @@ FROM children_data; 10000 ervu-dashboard Y - = recruit_id recruit_id - ervu_dashboard citizen
@@ -513,50 +586,9 @@ FROM children_data; 0 - - User defined Java expression - Janino - - Y - - 1 - - none - - - - minors_count - minors_count_raw == 1 ? "1" : minors_count_raw == 2 ? "2" : minors_count_raw >= 3 ? "3 и более" : "0" - String - 10 - -1 - - - - has_minor_child - has_minor_child_raw == 1 - Boolean - -1 - -1 - - - - has_dead_child - has_dead_child_raw == 1 - Boolean - -1 - -1 - - - - - 784 - 0 - - - Insert / update + child_insert_or_update Change job status on error Y @@ -568,7 +600,7 @@ FROM children_data; - Update + child_update_flags Change job status on error Y diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl index e761671..d06b736 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl @@ -22,62 +22,67 @@ User defined Java expression - Update + child_update_flags Y Create job execution record 2 - Table input 2 + child_input Y - Update - Change job status on error 2 + child_update_flags + child_error_job_status Y - Insert / update - Change job status on error 2 + child_insert_or_update + child_error_job_status Y Detect empty stream - Change job status on success + child_success_job_status Y - Filter rows - Change job status on success - Y - - - Change job status on error 2 + child_error_job_status Abort Y - Table input 2 - Insert / update + child_input + child_insert_or_update Y - Group by + child_group_by User defined Java expression Y - Table input 2 + child_input Detect empty stream Y - Insert / update + child_insert_or_update Filter rows Y - Table input 2 - Group by + child_input + child_group_by + Y + + + Filter rows + Unique rows + Y + + + Unique rows + child_success_job_status Y @@ -102,7 +107,7 @@ - Change job status on error 2 + child_error_job_status ExecSql Y @@ -118,8 +123,11 @@ ervu-dashboard + Y + N + Y Y N @@ -131,46 +139,13 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + 976 432 - - Change job status on success - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'SUCCESS', - error_description = null, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'PROCESSING'; - - - - - 416 - 608 - - Create job execution record 2 ExecSql @@ -263,7 +238,32 @@ DO UPDATE SET - Change job status on success + Unique rows + + + 752 + 608 + + + + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N 608 @@ -271,7 +271,48 @@ DO UPDATE SET - Group by + User defined Java expression + Janino + + Y + + 1 + + none + + + + minors_count + minors_count_raw == 1 ? "1" : minors_count_raw == 2 ? "2" : minors_count_raw >= 3 ? "3 и более" : "0" + String + 10 + -1 + + + + has_minor_child + has_minor_child_raw == 1 + Boolean + -1 + -1 + + + + has_dead_child + has_dead_child_raw == 1 + Boolean + -1 + -1 + + + + + 608 + 176 + + + + child_group_by GroupBy N @@ -316,7 +357,81 @@ DO UPDATE SET - Insert / update + child_input + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + WITH children_data AS ( + SELECT + ri.recruit_id AS recruit_id, + (ch->>'id')::uuid AS child_external_id, + NULLIF(ch->>'idERN', '') AS child_id_ern, + CASE NULLIF(ch->>'rodstvSvyazReb', '')::int + WHEN 1 THEN 3 + WHEN 2 THEN 4 + END AS kinship_type, + ch->'svedFLBS'->'fio'->>'familiya' AS last_name, + ch->'svedFLBS'->'fio'->>'imya' AS first_name, + ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + ch->'svedFLBS'->'fio'->>'familiya', + ch->'svedFLBS'->'fio'->>'imya', + ch->'svedFLBS'->'fio'->>'otchestvo' + ) AS full_name, + dates.birth_date, + dates.death_date, + ch->'svedSmert'->>'nomerZapis' AS death_az_number, + ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number, + CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead, + CASE + WHEN dates.death_date IS NOT NULL THEN 0 + WHEN dates.birth_date IS NULL THEN 0 + WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1 + ELSE 0 + END AS is_minor, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id + CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch + CROSS JOIN LATERAL ( + SELECT + MAKE_DATE( + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + MAKE_DATE( + NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int, + NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int, + NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int + ) AS death_date + ) AS dates + WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array' +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM children_data) as last_row +FROM children_data; + Y + + + 224 + 432 + + + + child_insert_or_update InsertUpdate N @@ -405,86 +520,46 @@ DO UPDATE SET N - 608 + 752 432 - Table input 2 - TableInput + child_success_job_status + ExecSql - N + Y 1 none + + ervu-dashboard - N - WITH children_data AS ( - SELECT - ri.recruit_id AS recruit_id, - (ch->>'id')::uuid AS child_external_id, - NULLIF(ch->>'idERN', '') AS child_id_ern, - CASE NULLIF(ch->>'rodstvSvyazReb', '')::int - WHEN 1 THEN 3 - WHEN 2 THEN 4 - END AS kinship_type, - ch->'svedFLBS'->'fio'->>'familiya' AS last_name, - ch->'svedFLBS'->'fio'->>'imya' AS first_name, - ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - ch->'svedFLBS'->'fio'->>'familiya', - ch->'svedFLBS'->'fio'->>'imya', - ch->'svedFLBS'->'fio'->>'otchestvo' - ) AS full_name, - dates.birth_date, - dates.death_date, - ch->'svedSmert'->>'nomerZapis' AS death_az_number, - ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number, - CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead, - CASE - WHEN dates.death_date IS NOT NULL THEN 0 - WHEN dates.birth_date IS NULL THEN 0 - WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1 - ELSE 0 - END AS is_minor, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch - CROSS JOIN LATERAL ( - SELECT - MAKE_DATE( - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - MAKE_DATE( - NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int, - NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int, - NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int - ) AS death_date - ) AS dates - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array' -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM children_data) as last_row -FROM children_data; - Y + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS', + error_description = null, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}' +and status = 'PROCESSING'; + + - 224 - 432 + 448 + 608 - Update + child_update_flags Update N @@ -497,13 +572,11 @@ FROM children_data; 10000 ervu-dashboard Y - = recruit_id recruit_id - ervu_dashboard citizen
@@ -528,51 +601,10 @@ FROM children_data; 176 - - User defined Java expression - Janino - - Y - - 1 - - none - - - - minors_count - minors_count_raw == 1 ? "1" : minors_count_raw == 2 ? "2" : minors_count_raw >= 3 ? "3 и более" : "0" - String - 10 - -1 - - - - has_minor_child - has_minor_child_raw == 1 - Boolean - -1 - -1 - - - - has_dead_child - has_dead_child_raw == 1 - Boolean - -1 - -1 - - - - - 608 - 176 - - - Insert / update - Change job status on error 2 + child_insert_or_update + child_error_job_status Y error_description @@ -583,8 +615,8 @@ FROM children_data; - Update - Change job status on error 2 + child_update_flags + child_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl index f8e78c4..f1f72b4 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl @@ -22,44 +22,49 @@ Create job execution record - Table input + citizen_address_input Y - Table output - Change job status on error + citizen_address_output + citizen_address_error_job_status Y - Change job status on error + citizen_address_error_job_status Abort Y Detect empty stream - Change job status on success + citizen_address_success_job_status Y - Filter rows - Change job status on success - Y - - - Table output + citizen_address_output Filter rows Y - Table input - Table output + citizen_address_input + citizen_address_output Y - Table input + citizen_address_input Detect empty stream Y + + Filter rows + Unique rows + Y + + + Unique rows + citizen_address_success_job_status + Y + Abort @@ -82,7 +87,7 @@ - Change job status on error + citizen_address_error_job_status ExecSql Y @@ -98,8 +103,11 @@ ervu-dashboard + Y + N + Y Y N @@ -111,6 +119,7 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + 928 @@ -118,7 +127,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + citizen_address_success_job_status ExecSql Y @@ -131,8 +140,11 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard + Y + N + Y N N @@ -144,6 +156,7 @@ and recruitment_id = '${IDM_ID}' and status = 'PROCESSING'; + 1328 @@ -225,7 +238,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Change job status on success + Unique rows 1328 @@ -233,7 +246,32 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Table input + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 1328 + 208 + + + + citizen_address_input TableInput N @@ -245,6 +283,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N + WITH address_data AS ( SELECT ri.recruit_id::uuid, @@ -283,7 +322,7 @@ FROM address_data; - Table output + citizen_address_output TableOutput N @@ -345,11 +384,14 @@ FROM address_data; N N N + Y + N ervu_dashboard Y citizen_address
+ N Y N @@ -362,8 +404,8 @@ FROM address_data; - Table output - Change job status on error + citizen_address_output + citizen_address_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl index a1694cb..318a1d5 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl @@ -22,44 +22,49 @@ Create job execution record - Table input + citizen_address_input Y - Insert / update - Change job status on error + citizen_address_insert_or_update + citizen_address_error_job_status Y Detect empty stream - Change job status on success + citizen_address_success_job_status Y - Filter rows - Change job status on success - Y - - - Change job status on error + citizen_address_error_job_status Abort Y - Table input - Insert / update + citizen_address_input + citizen_address_insert_or_update Y - Table input + citizen_address_input Detect empty stream Y - Insert / update + citizen_address_insert_or_update Filter rows Y + + Filter rows + Unique rows + Y + + + Unique rows + citizen_address_success_job_status + Y + Abort @@ -82,7 +87,7 @@ - Change job status on error + citizen_address_error_job_status ExecSql Y @@ -98,8 +103,11 @@ ervu-dashboard + Y + N + Y Y N @@ -111,6 +119,7 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + 752 @@ -118,7 +127,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + citizen_address_success_job_status ExecSql Y @@ -131,8 +140,11 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard + Y + N + Y N N @@ -144,6 +156,7 @@ and recruitment_id = '${IDM_ID}' and status = 'DELTA_PROCESSING'; + 1120 @@ -228,7 +241,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + Unique rows 1120 @@ -236,7 +249,7 @@ and recruitment_id = '${IDM_ID}'; - Insert / update + citizen_address_insert_or_update InsertUpdate N @@ -253,6 +266,7 @@ and recruitment_id = '${IDM_ID}'; = recruit_id recruit_id + ervu_dashboard citizen_address
@@ -320,7 +334,7 @@ and recruitment_id = '${IDM_ID}'; - Table input + citizen_address_input TableInput N @@ -371,10 +385,35 @@ FROM address_data; 224 + + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 1120 + 112 + + - Insert / update - Change job status on error + citizen_address_insert_or_update + citizen_address_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl index 170fa7d..69540d9 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl @@ -22,44 +22,49 @@ Create job execution record - Table input + citizen_address_input Y - Insert / update - Change job status on error + citizen_address_insert_or_update + citizen_address_error_job_status Y Detect empty stream - Change job status on success + citizen_address_success_job_status Y - Filter rows - Change job status on success - Y - - - Change job status on error + citizen_address_error_job_status Abort Y - Table input - Insert / update + citizen_address_input + citizen_address_insert_or_update Y - Table input + citizen_address_input Detect empty stream Y - Insert / update + citizen_address_insert_or_update Filter rows Y + + Filter rows + Unique rows + Y + + + Unique rows + citizen_address_success_job_status + Y + Abort @@ -82,7 +87,7 @@ - Change job status on error + citizen_address_error_job_status ExecSql Y @@ -98,8 +103,11 @@ ervu-dashboard + Y + N + Y Y N @@ -111,6 +119,7 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + 768 @@ -118,7 +127,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + citizen_address_success_job_status ExecSql Y @@ -131,8 +140,11 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard + Y + N + Y N N @@ -144,6 +156,7 @@ and recruitment_id = '${IDM_ID}' and status = 'PROCESSING'; + 1120 @@ -242,7 +255,7 @@ DO UPDATE SET - Change job status on success + Unique rows 1120 @@ -250,7 +263,82 @@ DO UPDATE SET - Insert / update + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 1120 + 144 + + + + citizen_address_input + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + WITH address_data AS ( + SELECT + ri.recruit_id::uuid, + ri.ctid as source_ctid, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, + + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, + CASE + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' + END AS place_stay_registration_type, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id + CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr + WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + GROUP BY ri.recruit_id, ri.ctid +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM address_data) as last_row +FROM address_data; + Y + + + 480 + 272 + + + + citizen_address_insert_or_update InsertUpdate N @@ -333,60 +421,10 @@ DO UPDATE SET 272 - - Table input - TableInput - - N - - 1 - - none - - - ervu-dashboard - N - WITH address_data AS ( - SELECT - ri.recruit_id::uuid, - ri.ctid as source_ctid, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, - - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, - CASE - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' - END AS place_stay_registration_type, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - GROUP BY ri.recruit_id, ri.ctid -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM address_data) as last_row -FROM address_data; - Y - - - 480 - 272 - - - Insert / update - Change job status on error + citizen_address_insert_or_update + citizen_address_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl index 0e5a74d..056251b 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl @@ -21,60 +21,65 @@ - Table input - Table output + citizen_guardianship_input + citizen_guardianship_output Y Create job execution record - Table input + citizen_guardianship_input Y - Table output - Change job status on error + citizen_guardianship_output + citizen_guardianship_error_job_status Y - Update - Change job status on error + update_guardianship_flags + citizen_guardianship_error_job_status Y - Change job status on error + citizen_guardianship_error_job_status Abort Y - - Filter rows - Change job status on success 2 - Y - Detect empty stream - Change job status on success 2 + citizen_guardianship_success_job_status Y - Table input - Sort rows + citizen_guardianship_input + citizen_guardianship_sort_rows Y - Sort rows - Update + citizen_guardianship_sort_rows + update_guardianship_flags Y - Table output + citizen_guardianship_output Filter rows Y - Table input + citizen_guardianship_input Detect empty stream Y + + Filter rows + Unique rows + Y + + + Unique rows + citizen_guardianship_success_job_status + Y + Abort @@ -96,75 +101,6 @@ 544 - - Change job status on error - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 1280 - 320 - - - - Change job status on success 2 - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'SUCCESS', - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'PROCESSING'; - - - - - 816 - 720 - - Create job execution record ExecSql @@ -240,7 +176,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Change job status on success 2 + Unique rows 816 @@ -248,8 +184,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Sort rows - SortRows + Unique rows + Unique Y @@ -258,31 +194,62 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); none - ${java.io.tmpdir} - out - 1000000 - - N - - Y + N + N recruit_id - Y - N - N - 0 - N + N - 496 - 0 + 816 + 576 - Table input + citizen_guardianship_error_job_status + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + + Y + + N + + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + + 1280 + 320 + + + + citizen_guardianship_input TableInput N @@ -335,7 +302,7 @@ FROM guardianship_data; - Table output + citizen_guardianship_output TableOutput N @@ -405,7 +372,74 @@ FROM guardianship_data; - Update + citizen_guardianship_sort_rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 496 + 0 + + + + citizen_guardianship_success_job_status + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS', + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}' +and status = 'PROCESSING'; + + + + + 816 + 720 + + + + update_guardianship_flags Update N @@ -418,19 +452,16 @@ FROM guardianship_data; 10000 ervu-dashboard Y - = recruit_id recruit_id - <> is_guardian is_guardian - ervu_dashboard citizen
@@ -449,8 +480,8 @@ FROM guardianship_data; - Table output - Change job status on error + citizen_guardianship_output + citizen_guardianship_error_job_status Y error_description @@ -461,8 +492,8 @@ FROM guardianship_data; - Update - Change job status on error + update_guardianship_flags + citizen_guardianship_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl index 53b1676..d2683b2 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl @@ -22,59 +22,64 @@ Create job execution record - Table input + citizen_guardianship_input Y - Insert / update - Change job status on error + citizen_guardianship_insert_or_update + citizen_guardianship_error_job_status Y - Table input - Insert / update + citizen_guardianship_input + citizen_guardianship_insert_or_update Y - Update - Change job status on error + update_guardianship_flags + citizen_guardianship_error_job_status Y - Change job status on error + citizen_guardianship_error_job_status Abort Y Detect empty stream - Change job status on success + citizen_guardianship_success_job_status Y - Filter rows - Change job status on success - Y - - - Table input + citizen_guardianship_input Detect empty stream Y - Table input - Sort rows + citizen_guardianship_input + citizen_guardianship_sort_rows Y - Sort rows - Update + citizen_guardianship_sort_rows + update_guardianship_flags Y - Insert / update + citizen_guardianship_insert_or_update Filter rows Y + + Filter rows + Unique rows + Y + + + Unique rows + citizen_guardianship_success_job_status + Y + Abort @@ -96,78 +101,6 @@ 480 - - Change job status on error - ExecSql - - Y - - 1 - - none - - - - - error_description - - - error_code - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'DELTA_ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 1200 - 224 - - - - Change job status on success - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'DELTA_SUCCESS', - error_description = null, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' - and recruitment_id = '${IDM_ID}' - and status = 'DELTA_PROCESSING'; - - - - 880 - 624 - - Create job execution record ExecSql @@ -245,7 +178,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + Unique rows 880 @@ -253,7 +186,125 @@ and recruitment_id = '${IDM_ID}'; - Insert / update + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 880 + 496 + + + + citizen_guardianship_error_job_status + ExecSql + + Y + + 1 + + none + + + + + error_description + + + error_code + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'DELTA_ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1200 + 224 + + + + citizen_guardianship_input + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + WITH guardianship_data AS ( + SELECT + ri.recruit_id AS recruit_id, + (ch->>'id')::uuid AS guardianship_external_id, + NULLIF(ch->>'idERN', '') AS guardian_id_ern, + ch->'fioOpek'->>'familiya' AS last_name, + ch->'fioOpek'->>'imya' AS first_name, + ch->'fioOpek'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + ch->'fioOpek'->>'familiya', + ch->'fioOpek'->>'imya', + ch->'fioOpek'->>'otchestvo' + ) AS full_name, + MAKE_DATE( + NULLIF(ch->'dataRozhdDok'->>'god', '')::int, + NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(ch->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, + true AS is_guardian, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id + CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch + WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM guardianship_data) as last_row +FROM guardianship_data; + Y + + + 544 + 224 + + + + citizen_guardianship_insert_or_update InsertUpdate N @@ -332,7 +383,7 @@ and recruitment_id = '${IDM_ID}'; - Sort rows + citizen_guardianship_sort_rows SortRows Y @@ -344,7 +395,7 @@ and recruitment_id = '${IDM_ID}'; ${java.io.tmpdir} out - 1000000 + 100000 N @@ -366,61 +417,40 @@ and recruitment_id = '${IDM_ID}'; - Table input - TableInput + citizen_guardianship_success_job_status + ExecSql - N + Y 1 none + + ervu-dashboard - N - WITH guardianship_data AS ( - SELECT - ri.recruit_id AS recruit_id, - (ch->>'id')::uuid AS guardianship_external_id, - NULLIF(ch->>'idERN', '') AS guardian_id_ern, - ch->'fioOpek'->>'familiya' AS last_name, - ch->'fioOpek'->>'imya' AS first_name, - ch->'fioOpek'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - ch->'fioOpek'->>'familiya', - ch->'fioOpek'->>'imya', - ch->'fioOpek'->>'otchestvo' - ) AS full_name, - MAKE_DATE( - NULLIF(ch->'dataRozhdDok'->>'god', '')::int, - NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(ch->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, - true AS is_guardian, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' - AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM guardianship_data) as last_row -FROM guardianship_data; - Y + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'DELTA_SUCCESS', + error_description = null, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' + and recruitment_id = '${IDM_ID}' + and status = 'DELTA_PROCESSING'; + - 544 - 224 + 880 + 624 - Update + update_guardianship_flags Update N @@ -433,19 +463,16 @@ FROM guardianship_data; 10000 ervu-dashboard Y - = recruit_id recruit_id - <> is_guardian is_guardian - ervu_dashboard citizen
@@ -464,8 +491,8 @@ FROM guardianship_data; - Insert / update - Change job status on error + citizen_guardianship_insert_or_update + citizen_guardianship_error_job_status Y error_description @@ -476,8 +503,8 @@ FROM guardianship_data; - Update - Change job status on error + update_guardianship_flags + citizen_guardianship_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl index e3b6359..308ebf6 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl @@ -22,59 +22,64 @@ Create job execution record - Table input + citizen_guardianship_input Y - Table input - Insert / update + citizen_guardianship_input + citizen_guardianship_insert_update Y - Insert / update - Change job status on error + citizen_guardianship_insert_update + citizen_guardianship_error_job_status Y - Update - Change job status on error + update_guardianship_flags + citizen_guardianship_error_job_status Y - Change job status on error + citizen_guardianship_error_job_status Abort Y - - Filter rows - Change job status on success - Y - Detect empty stream - Change job status on success + citizen_guardianship_success_job_status Y - Sort rows - Update + citizen_guardianship_sort_rows + update_guardianship_flags Y - Table input - Sort rows + citizen_guardianship_input + citizen_guardianship_sort_rows Y - Table input + citizen_guardianship_input Detect empty stream Y - Insert / update + citizen_guardianship_insert_update Filter rows Y + + Filter rows + Unique rows + Y + + + Unique rows + citizen_guardianship_success_job_status + Y + Abort @@ -96,79 +101,6 @@ 384 - - Change job status on error - ExecSql - - Y - - 1 - - none - - - - - error_description - - - error_code - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - - 1232 - 208 - - - - Change job status on success - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'SUCCESS', - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'PROCESSING'; - - - - - 704 - 624 - - Create job execution record ExecSql @@ -261,7 +193,7 @@ DO UPDATE SET - Change job status on success + Unique rows 704 @@ -269,7 +201,159 @@ DO UPDATE SET - Insert / update + citizen_guardianship_sort_rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 416 + 48 + + + + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 704 + 496 + + + + citizen_guardianship_error_job_status + ExecSql + + Y + + 1 + + none + + + + + error_description + + + error_code + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + + 1232 + 208 + + + + citizen_guardianship_input + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + WITH guardianship_data AS ( + SELECT + ri.recruit_id AS recruit_id, + (ch->>'id')::uuid AS guardianship_external_id, + NULLIF(ch->>'idERN', '') AS guardian_id_ern, + ch->'fioOpek'->>'familiya' AS last_name, + ch->'fioOpek'->>'imya' AS first_name, + ch->'fioOpek'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + ch->'fioOpek'->>'familiya', + ch->'fioOpek'->>'imya', + ch->'fioOpek'->>'otchestvo' + ) AS full_name, + MAKE_DATE( + NULLIF(ch->'dataRozhdDok'->>'god', '')::int, + NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(ch->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, + true AS is_guardian, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id + CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch + WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM guardianship_data) as last_row +FROM guardianship_data; + Y + + + 416 + 208 + + + + citizen_guardianship_insert_update InsertUpdate N @@ -348,8 +432,8 @@ DO UPDATE SET - Sort rows - SortRows + citizen_guardianship_success_job_status + ExecSql Y @@ -358,84 +442,30 @@ DO UPDATE SET none - ${java.io.tmpdir} - out - 1000000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - - - - 416 - 48 - - - - Table input - TableInput - - N - - 1 - - none - - + + ervu-dashboard - N - WITH guardianship_data AS ( - SELECT - ri.recruit_id AS recruit_id, - (ch->>'id')::uuid AS guardianship_external_id, - NULLIF(ch->>'idERN', '') AS guardian_id_ern, - ch->'fioOpek'->>'familiya' AS last_name, - ch->'fioOpek'->>'imya' AS first_name, - ch->'fioOpek'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - ch->'fioOpek'->>'familiya', - ch->'fioOpek'->>'imya', - ch->'fioOpek'->>'otchestvo' - ) AS full_name, - MAKE_DATE( - NULLIF(ch->'dataRozhdDok'->>'god', '')::int, - NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(ch->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, - true AS is_guardian, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM guardianship_data) as last_row -FROM guardianship_data; - Y + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS', + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}' +and status = 'PROCESSING'; + + - 416 - 208 + 704 + 624 - Update + update_guardianship_flags Update N @@ -448,19 +478,16 @@ FROM guardianship_data; 10000 ervu-dashboard Y - = recruit_id recruit_id - <> is_guardian is_guardian - ervu_dashboard citizen
@@ -479,8 +506,8 @@ FROM guardianship_data; - Insert / update - Change job status on error + citizen_guardianship_insert_update + citizen_guardianship_error_job_status Y error_description @@ -491,8 +518,8 @@ FROM guardianship_data; - Update - Change job status on error + update_guardianship_flags + citizen_guardianship_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl index c214d29..138aef9 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl @@ -22,54 +22,59 @@ Create job execution record - Table input + citizen_information_search_input Y - Table output - Change job status on error + citizen_information_search_output + citizen_information_error_job_status Y - Change job status on error + citizen_information_error_job_status Abort Y - - Filter rows - Change job status on success - Y - Detect empty stream - Change job status on success + citizen_information_success_job_status Y - Table input - Table output + citizen_information_search_input + citizen_information_search_output Y - Update - Change job status on error + update_citizen_information_search_flags + citizen_information_error_job_status Y - Table input - Update + citizen_information_search_input + update_citizen_information_search_flags Y - Table input + citizen_information_search_input Detect empty stream Y - Table output + citizen_information_search_output Filter rows Y + + Filter rows + Unique rows + Y + + + Unique rows + citizen_information_success_job_status + Y + Abort @@ -92,7 +97,7 @@ - Change job status on error + citizen_information_error_job_status ExecSql Y @@ -108,8 +113,11 @@ ervu-dashboard + Y + N + Y Y N @@ -121,6 +129,7 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + 720 @@ -128,7 +137,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + citizen_information_success_job_status ExecSql Y @@ -141,8 +150,11 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard + Y + N + Y N N @@ -154,6 +166,7 @@ and recruitment_id = '${IDM_ID}' and status = 'PROCESSING'; + 1104 @@ -235,7 +248,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Change job status on success + Unique rows 1104 @@ -243,7 +256,32 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Table input + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 1104 + 240 + + + + citizen_information_search_input TableInput N @@ -284,7 +322,7 @@ FROM wanted_data; - Table output + citizen_information_search_output TableOutput N @@ -330,11 +368,14 @@ FROM wanted_data; N N N + Y + N ervu_dashboard Y citizen_information_search
+ N Y N @@ -346,7 +387,7 @@ FROM wanted_data; - Update + update_citizen_information_search_flags Update N @@ -359,19 +400,16 @@ FROM wanted_data; 10000 ervu-dashboard Y - = recruit_id recruit_id - <> is_wanted is_wanted - ervu_dashboard citizen
@@ -390,8 +428,8 @@ FROM wanted_data; - Table output - Change job status on error + citizen_information_search_output + citizen_information_error_job_status Y error_description @@ -402,8 +440,8 @@ FROM wanted_data; - Update - Change job status on error + update_citizen_information_search_flags + citizen_information_error_job_status Y error_decription diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl index 334833f..449cda0 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl @@ -22,52 +22,57 @@ Create job execution record - Table input + citizen_information_search_input Y - Insert / update - Change job status on error + citizen_information_search_insert_or_update + citizen_information_error_job_status Y Detect empty stream - Change job status on success + citizen_information_success_job_status Y - Filter rows - Change job status on success - Y - - - Change job status on error + citizen_information_error_job_status Abort Y - Table input - Insert / update + citizen_information_search_input + citizen_information_search_insert_or_update Y - Update - Change job status on error + update_citizen_information_search_flags + citizen_information_error_job_status Y - Table input + citizen_information_search_input Detect empty stream Y - Insert / update + citizen_information_search_insert_or_update Filter rows Y - Table input - Update + citizen_information_search_input + update_citizen_information_search_flags + Y + + + Filter rows + Unique rows + Y + + + Unique rows + citizen_information_success_job_status Y @@ -92,7 +97,7 @@ - Change job status on error + citizen_information_error_job_status ExecSql Y @@ -108,8 +113,11 @@ ervu-dashboard + Y + N + Y Y N @@ -121,6 +129,7 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + 752 @@ -128,7 +137,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + citizen_information_success_job_status ExecSql Y @@ -141,8 +150,11 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard + Y + N + Y N N @@ -154,6 +166,7 @@ and recruitment_id = '${IDM_ID}' and status = 'DELTA_PROCESSING'; + 1008 @@ -238,7 +251,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + Unique rows 1008 @@ -246,7 +259,74 @@ and recruitment_id = '${IDM_ID}'; - Insert / update + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 1008 + 112 + + + + citizen_information_search_input + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + WITH wanted_data AS ( + SELECT + ri.recruit_id, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code, + TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, + TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, + true AS is_wanted, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id + WHERE '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp + AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM wanted_data) as last_row +FROM wanted_data; + Y + + + 400 + 224 + + + + citizen_information_search_insert_or_update InsertUpdate N @@ -310,49 +390,7 @@ and recruitment_id = '${IDM_ID}'; - Table input - TableInput - - N - - 1 - - none - - - ervu-dashboard - N - WITH wanted_data AS ( - SELECT - ri.recruit_id, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code, - TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, - TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, - true AS is_wanted, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - WHERE '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp - AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM wanted_data) as last_row -FROM wanted_data; - Y - - - 400 - 224 - - - - Update + update_citizen_information_search_flags Update N @@ -365,19 +403,16 @@ FROM wanted_data; 10000 ervu-dashboard Y - = recruit_id recruit_id - <> is_wanted is_wanted - ervu_dashboard citizen
@@ -396,8 +431,8 @@ FROM wanted_data; - Insert / update - Change job status on error + citizen_information_search_insert_or_update + citizen_information_error_job_status Y error_description @@ -408,8 +443,8 @@ FROM wanted_data; - Update - Change job status on error + update_citizen_information_search_flags + citizen_information_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl index 1ceef00..b803213 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl @@ -22,54 +22,59 @@ Create job execution record - Table input + citizen_information_search_input Y - Change job status on error + citizen_information_error_job_status Abort Y Detect empty stream - Change job status on success + citizen_information_success_job_status Y - Filter rows - Change job status on success + update_citizen_information_search_flags + citizen_information_error_job_status Y - Update - Change job status on error + citizen_information_search_insert_or_update + citizen_information_error_job_status Y - Insert / update - Change job status on error + citizen_information_search_input + citizen_information_search_insert_or_update Y - Table input - Insert / update - Y - - - Insert / update + citizen_information_search_insert_or_update Filter rows Y - Table input - Update + citizen_information_search_input + update_citizen_information_search_flags Y - Table input + citizen_information_search_input Detect empty stream Y + + Filter rows + Unique rows + Y + + + Unique rows + citizen_information_success_job_status + Y + Abort @@ -92,7 +97,7 @@ - Change job status on error + citizen_information_error_job_status ExecSql Y @@ -108,8 +113,11 @@ ervu-dashboard + Y + N + Y Y N @@ -121,6 +129,7 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + 688 @@ -128,7 +137,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + citizen_information_success_job_status ExecSql Y @@ -141,8 +150,11 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard + Y + N + Y N N @@ -154,6 +166,7 @@ and recruitment_id = '${IDM_ID}' and status = 'PROCESSING'; + 1024 @@ -252,7 +265,7 @@ DO UPDATE SET - Change job status on success + Unique rows 1024 @@ -260,7 +273,73 @@ DO UPDATE SET - Insert / update + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 1024 + 160 + + + + citizen_information_search_input + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + WITH wanted_data AS ( + SELECT + ri.recruit_id, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code, + TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, + TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, + true AS is_wanted, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id + WHERE '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM wanted_data) as last_row +FROM wanted_data; + Y + + + 416 + 272 + + + + citizen_information_search_insert_or_update InsertUpdate N @@ -324,48 +403,7 @@ DO UPDATE SET - Table input - TableInput - - N - - 1 - - none - - - ervu-dashboard - N - WITH wanted_data AS ( - SELECT - ri.recruit_id, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code, - TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, - TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, - true AS is_wanted, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - WHERE '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM wanted_data) as last_row -FROM wanted_data; - Y - - - 416 - 272 - - - - Update + update_citizen_information_search_flags Update N @@ -378,19 +416,16 @@ FROM wanted_data; 10000 ervu-dashboard Y - = recruit_id recruit_id - <> is_wanted is_wanted - ervu_dashboard citizen
@@ -409,8 +444,8 @@ FROM wanted_data; - Insert / update - Change job status on error + citizen_information_search_insert_or_update + citizen_information_error_job_status Y error_description @@ -421,8 +456,8 @@ FROM wanted_data; - Update - Change job status on error + update_citizen_information_search_flags + citizen_information_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl index d1eae63..23f11b2 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl @@ -22,59 +22,64 @@ Create job execution record - Table input + spouse_input Y - Table output - Change job status on error + citizen_spouse_output + spouse_error_job_status Y - Filter rows - Change job status on success + spouse_detect_empty_stream + spouse_success_job_status Y - Detect empty stream - Change job status on success - Y - - - Change job status on error + spouse_error_job_status Abort Y - Table output 2 - Change job status on error + marriage_registry_output + spouse_error_job_status Y - Table input - Table output + spouse_input + citizen_spouse_output Y - Table input - Sort rows + spouse_input + spouse_sort_rows Y - Sort rows - Table output 2 + spouse_sort_rows + marriage_registry_output Y - Table input - Detect empty stream + spouse_input + spouse_detect_empty_stream Y - Table output + citizen_spouse_output Filter rows Y + + Filter rows + Unique rows + Y + + + Unique rows + spouse_success_job_status + Y + Abort @@ -97,7 +102,7 @@ - Change job status on error + spouse_error_job_status ExecSql Y @@ -113,8 +118,11 @@ ervu-dashboard + Y + N + Y Y N @@ -126,14 +134,15 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + - 736 + 800 192 - Change job status on success + spouse_success_job_status ExecSql Y @@ -146,8 +155,11 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard + Y + N + Y N N @@ -159,10 +171,11 @@ and recruitment_id = '${IDM_ID}' and status = 'PROCESSING'; + - 576 - 912 + 592 + 720 @@ -195,7 +208,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Detect empty stream + spouse_detect_empty_stream DetectEmptyStream Y @@ -208,7 +221,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 352 - 912 + 720 @@ -240,15 +253,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Change job status on success + Unique rows - 736 + 800 912 - Sort rows + spouse_sort_rows SortRows Y @@ -260,7 +273,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ${java.io.tmpdir} out - 1000000 + 100000 N @@ -282,7 +295,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Table input + spouse_input TableInput N @@ -294,6 +307,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N + WITH spouse_data AS ( SELECT ri.recruit_id AS recruit_id, @@ -348,7 +362,7 @@ FROM spouse_data; - Table output + citizen_spouse_output TableOutput N @@ -414,23 +428,26 @@ FROM spouse_data; N N N + Y + N ervu_dashboard Y citizen_spouse
+ N Y N Y - 736 + 800 576 - Table output 2 + marriage_registry_output TableOutput N @@ -468,11 +485,14 @@ FROM spouse_data; N N N + Y + N ervu_dashboard Y marriage_registry
+ N Y N @@ -483,10 +503,35 @@ FROM spouse_data; 192 + + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 592 + 912 + + - Table output - Change job status on error + citizen_spouse_output + spouse_error_job_status Y error_description @@ -497,8 +542,8 @@ FROM spouse_data; - Table output 2 - Change job status on error + marriage_registry_output + spouse_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl index 1610236..e288ad6 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl @@ -22,57 +22,62 @@ Create job execution record - Table input + spouse_input Y - Insert / update - Change job status on error + spouse_update_or_insert + spouse_error_job_status Y - Change job status on error + spouse_error_job_status Abort Y - - Filter rows - Change job status on success - Y - Detect empty stream - Change job status on success + spouse_success_job_status Y - Insert / update 2 - Change job status on error + marriage_registry_update_or_insert + spouse_error_job_status Y - Table input - Insert / update + spouse_input + spouse_update_or_insert Y - Table input + spouse_input Sort rows Y - Table input + spouse_input Detect empty stream Y - Insert / update + spouse_update_or_insert Filter rows Y Sort rows - Insert / update 2 + marriage_registry_update_or_insert + Y + + + Filter rows + Unique rows + Y + + + Unique rows + spouse_success_job_status Y @@ -96,78 +101,6 @@ 176 - - Change job status on error - ExecSql - - Y - - 1 - - none - - - - - error_description - - - error_code - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'DELTA_ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 768 - 176 - - - - Change job status on success - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'DELTA_SUCCESS', - error_description = null, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' - and recruitment_id = '${IDM_ID}' - and status = 'DELTA_PROCESSING'; - - - - 1072 - 688 - - Create job execution record ExecSql @@ -245,7 +178,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + Unique rows 1072 @@ -253,7 +186,259 @@ and recruitment_id = '${IDM_ID}'; - Insert / update + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 496 + 320 + + + + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 1072 + 592 + + + + marriage_registry_update_or_insert + InsertUpdate + + N + + 1 + + none + + + 10000 + ervu-dashboard + + + = + recruit_id + recruit_id + + ervu_dashboard + marriage_registry
+ + recruit_id + recruit_id + N + + + marriage_az_number + marriage_az_number + Y + + + marriage_date + marriage_date + Y + + + divorce_date + divorce_date + Y + + + divorce_az_number + divorce_az_number + Y + + + N + + + 496 + 176 + + + + spouse_error_job_status + ExecSql + + Y + + 1 + + none + + + + + error_description + + + error_code + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'DELTA_ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 768 + 176 + + + + spouse_input + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + WITH spouse_data AS ( + SELECT + ri.recruit_id AS recruit_id, + (supr->>'id')::uuid AS spouse_external_id, + NULLIF(supr->>'idERN', '') AS spouse_id_ern, + NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, + supr->'svedFLBS'->'fio'->>'familiya' AS last_name, + supr->'svedFLBS'->'fio'->>'imya' AS first_name, + supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + supr->'svedFLBS'->'fio'->>'familiya', + supr->'svedFLBS'->'fio'->>'imya', + supr->'svedFLBS'->'fio'->>'otchestvo' + ) AS full_name, + MAKE_DATE( + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, + supr->'svedSmert'->>'nomerZapis' AS death_az_number, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, + CASE + WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true + ELSE false + END AS information_excluded, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id + CROSS JOIN LATERAL ( + SELECT supri + FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri + WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' + ) AS supr(supr) + WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND ri.update_date >= '${M_R_UP_DATE}'::timestamp +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM spouse_data) as last_row +FROM spouse_data; + + Y + + + 496 + 496 + + + + spouse_success_job_status + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'DELTA_SUCCESS', + error_description = null, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' + and recruitment_id = '${IDM_ID}' + and status = 'DELTA_PROCESSING'; + + + + 1072 + 688 + + + + spouse_update_or_insert InsertUpdate N @@ -346,166 +531,10 @@ and recruitment_id = '${IDM_ID}'; 496 - - Insert / update 2 - InsertUpdate - - N - - 1 - - none - - - 10000 - ervu-dashboard - - - = - recruit_id - recruit_id - - ervu_dashboard - marriage_registry
- - recruit_id - recruit_id - N - - - marriage_az_number - marriage_az_number - Y - - - marriage_date - marriage_date - Y - - - divorce_date - divorce_date - Y - - - divorce_az_number - divorce_az_number - Y - - - N - - - 496 - 176 - - - - Sort rows - SortRows - - Y - - 1 - - none - - - ${java.io.tmpdir} - out - 1000000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - - - - 496 - 320 - - - - Table input - TableInput - - N - - 1 - - none - - - ervu-dashboard - N - - WITH spouse_data AS ( - SELECT - ri.recruit_id AS recruit_id, - (supr->>'id')::uuid AS spouse_external_id, - NULLIF(supr->>'idERN', '') AS spouse_id_ern, - NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, - supr->'svedFLBS'->'fio'->>'familiya' AS last_name, - supr->'svedFLBS'->'fio'->>'imya' AS first_name, - supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - supr->'svedFLBS'->'fio'->>'familiya', - supr->'svedFLBS'->'fio'->>'imya', - supr->'svedFLBS'->'fio'->>'otchestvo' - ) AS full_name, - MAKE_DATE( - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, - supr->'svedSmert'->>'nomerZapis' AS death_az_number, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, - CASE - WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true - ELSE false - END AS information_excluded, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - CROSS JOIN LATERAL ( - SELECT supri - FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri - WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' - ) AS supr(supr) - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND ri.update_date >= '${M_R_UP_DATE}'::timestamp -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM spouse_data) as last_row -FROM spouse_data; - - Y - - - 496 - 496 - - - Insert / update - Change job status on error + marriage_registry_update_or_insert + spouse_error_job_status Y error_description @@ -516,8 +545,8 @@ FROM spouse_data; - Insert / update 2 - Change job status on error + spouse_update_or_insert + spouse_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl index 59cdc6c..1c98ee2 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl @@ -49,59 +49,64 @@ M_R_CR_DATE Create job execution record - Table input + spouse_input Y - Insert / update - Change job status on error + spouse_update_or_insert + spouse_error_job_status Y - Change job status on error + spouse_error_job_status Abort Y - - Filter rows - Change job status on success - Y - Detect empty stream - Change job status on success + spouse_success_job_status Y - Insert / update 2 - Change job status on error + marriage_registry_update_or_insert + spouse_error_job_status Y - Table input - Insert / update + spouse_input + spouse_update_or_insert Y - Table input + spouse_input Sort rows Y Sort rows - Insert / update 2 + marriage_registry_update_or_insert Y - Insert / update + spouse_update_or_insert Filter rows Y - Table input + spouse_input Detect empty stream Y + + Filter rows + Unique rows + Y + + + Unique rows + spouse_success_job_status + Y + Abort @@ -124,7 +129,7 @@ M_R_CR_DATE - Change job status on error + spouse_error_job_status ExecSql Y @@ -143,8 +148,11 @@ M_R_CR_DATE ervu-dashboard + Y + N + Y Y N @@ -156,47 +164,13 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + 704 224 - - Change job status on success - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'SUCCESS', - error_description = null, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'PROCESSING'; - - - - - - 944 - 752 - - Create job execution record ExecSql @@ -257,7 +231,7 @@ DO UPDATE SET 352 - 752 + 784 @@ -289,7 +263,7 @@ DO UPDATE SET - Change job status on success + Unique rows 944 @@ -297,7 +271,221 @@ DO UPDATE SET - Insert / update + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 352 + 368 + + + + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 944 + 784 + + + + marriage_registry_update_or_insert + InsertUpdate + + N + + 1 + + none + + + 10000 + ervu-dashboard + + + = + recruit_id + recruit_id + + ervu_dashboard + marriage_registry
+ + recruit_id + recruit_id + N + + + marriage_az_number + marriage_az_number + Y + + + marriage_date + marriage_date + Y + + + divorce_date + divorce_date + Y + + + divorce_az_number + divorce_az_number + Y + + + N + + + 352 + 224 + + + + spouse_input + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + WITH spouse_data AS ( + SELECT + ri.recruit_id AS recruit_id, + (supr->>'id')::uuid AS spouse_external_id, + NULLIF(supr->>'idERN', '') AS spouse_id_ern, + NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, + supr->'svedFLBS'->'fio'->>'familiya' AS last_name, + supr->'svedFLBS'->'fio'->>'imya' AS first_name, + supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + supr->'svedFLBS'->'fio'->>'familiya', + supr->'svedFLBS'->'fio'->>'imya', + supr->'svedFLBS'->'fio'->>'otchestvo' + ) AS full_name, + MAKE_DATE( + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, + supr->'svedSmert'->>'nomerZapis' AS death_az_number, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, + CASE + WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true + ELSE false + END AS information_excluded, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id + CROSS JOIN LATERAL ( + SELECT supri + FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri + WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' + ) AS supr(supr) + WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM spouse_data) as last_row +FROM spouse_data; + + Y + + + 352 + 528 + + + + spouse_success_job_status + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS', + error_description = null, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}' +and status = 'PROCESSING'; + + + + + + 672 + 784 + + + + spouse_update_or_insert InsertUpdate N @@ -390,165 +578,10 @@ DO UPDATE SET 528 - - Insert / update 2 - InsertUpdate - - N - - 1 - - none - - - 10000 - ervu-dashboard - - - = - recruit_id - recruit_id - - ervu_dashboard - marriage_registry
- - recruit_id - recruit_id - N - - - marriage_az_number - marriage_az_number - Y - - - marriage_date - marriage_date - Y - - - divorce_date - divorce_date - Y - - - divorce_az_number - divorce_az_number - Y - - - N - - - 352 - 224 - - - - Sort rows - SortRows - - Y - - 1 - - none - - - ${java.io.tmpdir} - out - 1000000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - - - - 352 - 368 - - - - Table input - TableInput - - N - - 1 - - none - - - ervu-dashboard - N - - WITH spouse_data AS ( - SELECT - ri.recruit_id AS recruit_id, - (supr->>'id')::uuid AS spouse_external_id, - NULLIF(supr->>'idERN', '') AS spouse_id_ern, - NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, - supr->'svedFLBS'->'fio'->>'familiya' AS last_name, - supr->'svedFLBS'->'fio'->>'imya' AS first_name, - supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - supr->'svedFLBS'->'fio'->>'familiya', - supr->'svedFLBS'->'fio'->>'imya', - supr->'svedFLBS'->'fio'->>'otchestvo' - ) AS full_name, - MAKE_DATE( - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, - supr->'svedSmert'->>'nomerZapis' AS death_az_number, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, - CASE - WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true - ELSE false - END AS information_excluded, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - CROSS JOIN LATERAL ( - SELECT supri - FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri - WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' - ) AS supr(supr) - WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' -) -SELECT - *, - source_ctid = (SELECT MAX(source_ctid) FROM spouse_data) as last_row -FROM spouse_data; - - Y - - - 352 - 528 - - - Insert / update - Change job status on error + marriage_registry_update_or_insert + spouse_error_job_status Y error_description @@ -559,8 +592,8 @@ FROM spouse_data; - Insert / update 2 - Change job status on error + spouse_update_or_insert + spouse_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl index 6a3b20d..777f5e1 100644 --- a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl @@ -22,44 +22,49 @@ Create job execution record - Table input + passport_input Y - Table output - Change job status on error + passport_output + passport_error_job_status Y Detect empty stream - Change job status on success + passport_success_job_status Y - Table output + passport_output Filter rows Y - Filter rows - Change job status on success - Y - - - Change job status on error + passport_error_job_status Abort Y - Table input - Table output + passport_input + passport_output Y - Table input + passport_input Detect empty stream Y + + Filter rows + Unique rows + Y + + + Unique rows + passport_success_job_status + Y + Abort @@ -82,7 +87,7 @@ - Change job status on error + passport_error_job_status ExecSql Y @@ -98,8 +103,11 @@ ervu-dashboard + Y + N + Y Y N @@ -111,6 +119,7 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + 896 @@ -118,7 +127,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + passport_success_job_status ExecSql Y @@ -131,8 +140,11 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard + Y + N + Y N N @@ -144,10 +156,11 @@ and recruitment_id = '${IDM_ID}' and status = 'PROCESSING'; + 1264 - 176 + 112 @@ -193,7 +206,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 496 - 176 + 112 @@ -225,7 +238,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Change job status on success + Unique rows 1264 @@ -233,7 +246,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Table input + passport_input TableInput N @@ -287,7 +300,7 @@ FROM passport_data; - Table output + passport_output TableOutput N @@ -333,11 +346,14 @@ FROM passport_data; N N N + Y + N ervu_dashboard Y passport
+ N Y N @@ -348,10 +364,35 @@ FROM passport_data; 320 + + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 1264 + 208 + + - Table output - Change job status on error + passport_output + passport_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl index da8b0ca..69d0acb 100644 --- a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl @@ -22,42 +22,47 @@ Create job execution record - Table input + passport_input Y - Insert / update - Change job status on error + passport_insert_or_update + passport_error_job_status Y Detect empty stream - Change job status on success + passport_success_job_status Y - Insert / update + passport_insert_or_update Filter rows Y - Filter rows - Change job status on success - Y - - - Change job status on error + passport_error_job_status Abort Y - Table input + passport_input Detect empty stream Y - Table input - Insert / update + passport_input + passport_insert_or_update + Y + + + Filter rows + Unique rows + Y + + + Unique rows + passport_success_job_status Y @@ -82,7 +87,7 @@ - Change job status on error + passport_error_job_status ExecSql Y @@ -98,8 +103,11 @@ ervu-dashboard + Y + N + Y Y N @@ -111,6 +119,7 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + 752 @@ -118,7 +127,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + passport_success_job_status ExecSql Y @@ -131,8 +140,11 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard + Y + N + Y N N @@ -144,10 +156,11 @@ and recruitment_id = '${IDM_ID}' and status = 'DELTA_PROCESSING'; + 1120 - 80 + 16 @@ -196,7 +209,7 @@ and recruitment_id = '${IDM_ID}'; 400 - 80 + 16 @@ -228,7 +241,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + Unique rows 1120 @@ -236,7 +249,7 @@ and recruitment_id = '${IDM_ID}'; - Insert / update + passport_insert_or_update InsertUpdate N @@ -253,11 +266,13 @@ and recruitment_id = '${IDM_ID}'; = recruit_id recruit_id + = actual actual + ervu_dashboard passport
@@ -305,7 +320,7 @@ and recruitment_id = '${IDM_ID}'; - Table input + passport_input TableInput N @@ -359,10 +374,35 @@ FROM passport_data; 224 + + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 1120 + 112 + + - Insert / update - Change job status on error + passport_insert_or_update + passport_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl index ae8a3b5..29a0d25 100644 --- a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl @@ -22,42 +22,47 @@ Create job execution record - Table input + passport_input Y - Insert / update - Change job status on error + passport_insert_or_update + passport_error_job_status Y Detect empty stream - Change job status on success + passport_success_job_status Y - Insert / update + passport_insert_or_update Filter rows Y - Filter rows - Change job status on success - Y - - - Change job status on error + passport_error_job_status Abort Y - Table input + passport_input Detect empty stream Y - Table input - Insert / update + passport_input + passport_insert_or_update + Y + + + Filter rows + Unique rows + Y + + + Unique rows + passport_success_job_status Y @@ -82,7 +87,7 @@ - Change job status on error + passport_error_job_status ExecSql Y @@ -98,8 +103,11 @@ ervu-dashboard + Y + N + Y Y N @@ -111,6 +119,7 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + 768 @@ -118,7 +127,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + passport_success_job_status ExecSql Y @@ -131,8 +140,11 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard + Y + N + Y N N @@ -144,6 +156,7 @@ and recruitment_id = '${IDM_ID}' and status = 'PROCESSING'; + 1040 @@ -242,7 +255,7 @@ DO UPDATE SET - Change job status on success + Unique rows 1040 @@ -250,7 +263,7 @@ DO UPDATE SET - Insert / update + passport_insert_or_update InsertUpdate N @@ -267,11 +280,13 @@ DO UPDATE SET = recruit_id recruit_id + = actual actual + ervu_dashboard passport
@@ -319,7 +334,7 @@ DO UPDATE SET - Table input + passport_input TableInput N @@ -372,10 +387,35 @@ FROM passport_data; 272 + + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 1040 + 144 + + - Insert / update - Change job status on error + passport_insert_or_update + passport_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl index 02eb149..68e2e28 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl @@ -22,59 +22,64 @@ Create job execution record - Table input + prosecution_input Y - Table output - Change job status on error + prosecution_output + prosecution_error_job_status Y - Change job status on error + prosecution_error_job_status Abort Y - - Filter rows - Change job status on success - Y - Detect empty stream - Change job status on success + prosecution_success_job_status Y - Table input - Table output + prosecution_input + prosecution_output Y - Update - Change job status on error + prosecution_update_flags + prosecution_error_job_status Y - Table input - Sort rows + prosecution_input + prosecution_sort_rows Y - Sort rows - Update + prosecution_sort_rows + prosecution_update_flags Y - Table input + prosecution_input Detect empty stream Y - Table output + prosecution_output Filter rows Y + + Filter rows + Unique rows + Y + + + Unique rows + prosecution_success_job_status + Y + Abort @@ -96,75 +101,6 @@ 224 - - Change job status on error - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 848 - 224 - - - - Change job status on success - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'SUCCESS', - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'PROCESSING'; - - - - - 1072 - 640 - - Create job execution record ExecSql @@ -240,7 +176,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Change job status on success + Unique rows 1072 @@ -248,8 +184,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Sort rows - SortRows + Unique rows + Unique Y @@ -258,31 +194,58 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); none - ${java.io.tmpdir} - out - 1000000 - - N - - Y + N + N recruit_id - Y - N - N - 0 - N + N - 480 - 32 + 1072 + 544 - Table input + prosecution_error_job_status + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 848 + 224 + + + + prosecution_input TableInput N @@ -348,7 +311,7 @@ GROUP BY r.recruit_id, up.up, up.source_ctid; - Table output + prosecution_output TableOutput N @@ -466,7 +429,74 @@ GROUP BY r.recruit_id, up.up, up.source_ctid; - Update + prosecution_sort_rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 480 + 32 + + + + prosecution_success_job_status + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS', + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}' +and status = 'PROCESSING'; + + + + + 1072 + 640 + + + + prosecution_update_flags Update N @@ -507,8 +537,8 @@ GROUP BY r.recruit_id, up.up, up.source_ctid; - Table output - Change job status on error + prosecution_output + prosecution_error_job_status Y error_description @@ -519,8 +549,8 @@ GROUP BY r.recruit_id, up.up, up.source_ctid; - Update - Change job status on error + prosecution_update_flags + prosecution_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl index 1fd14db..174aafc 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl @@ -22,59 +22,64 @@ Create job execution record - Table input + prosecution_input Y - Insert / update - Change job status on error + prosecution_insert_or_update + prosecution_error_job_status Y Detect empty stream - Change job status on success + prosecution_success_job_status Y - Filter rows - Change job status on success - Y - - - Change job status on error + prosecution_error_job_status Abort Y - Update - Change job status on error + prosecution_update_flags + prosecution_error_job_status Y - Table input - Insert / update + prosecution_input + prosecution_insert_or_update Y - Sort rows - Update + prosecution_sort_rows + prosecution_update_flags Y - Table input - Sort rows + prosecution_input + prosecution_sort_rows Y - Insert / update + prosecution_insert_or_update Filter rows Y - Table input + prosecution_input Detect empty stream Y + + Filter rows + Unique rows + Y + + + Unique rows + prosecution_success_job_status + Y + Abort @@ -96,75 +101,6 @@ 592 - - Change job status on error - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'DELTA_ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 736 - 592 - - - - Change job status on success - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'DELTA_SUCCESS', - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'DELTA_PROCESSING'; - - - - - 896 - 944 - - Create job execution record ExecSql @@ -243,7 +179,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + Unique rows 896 @@ -251,7 +187,169 @@ and recruitment_id = '${IDM_ID}'; - Insert / update + prosecution_sort_rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 384 + 416 + + + + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 896 + 864 + + + + prosecution_error_job_status + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'DELTA_ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 736 + 592 + + + + prosecution_input + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + WITH ugolov_presl AS ( + SELECT + jsonb_array_elements(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') AS up, + ri.recruit_id, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id + AND '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp + WHERE jsonb_typeof(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') = 'array' +) +SELECT + r.recruit_id AS recruit_id, + (up->>'id')::uuid AS prosecution_external_id, + NULLIF(up->'sledSud'->>'uk', '') AS uk_name, + NULLIF(up->'sledSud'->'extend'->>'nomerUDPredv', '') AS case_number, + NULLIF(up->'sledSud'->'extend'->>'svedPerekvalif', '') AS reclassification, + NULLIF(up->'sledSud'->'extend'->>'naimVidUgolovPresl', '') AS kind, + CASE + WHEN up->'sledSud'->'extend'->>'priznakPeredachSud' = '1' THEN true + ELSE false -- нет или нет инфы + END AS transfer_to_court, + NULLIF(up->'sledSud'->>'naimSud','') AS court_name, + NULLIF(up->'sledSud'->>'nomerUD','') AS case_number_court, + STRING_AGG(TRIM(BOTH ' "' FROM law.f), ', ') AS law_point, + NULLIF(up->'sledSud'->>'naimOrgan', '') AS authority_name, + NULLIF(up->'sledSud'->'svedSnyat'->>'osnSnyatSud', '') AS conviction_removal_basis, + TO_DATE(up->'sledSud'->'svedSnyat'->>'dataSnyatSud', 'YYYY-MM-DD') AS conviction_removal_date, + TO_DATE(up->'sledSud'->>'dataOsuzhd', 'YYYY-MM-DD') AS conviction_date, + TO_DATE(up->'sledSud'->>'dataVozbUD', 'YYYY-MM-DD') AS create_date, + NULLIF(up->'sledSud'->>'osnPrekrUD', '') AS termination_basis, + NULLIF(up->'sledSud'->>'statusNaim', '') AS conviction_status, + TO_DATE(up->'sledSud'->>'dataPrekrUD', 'YYYY-MM-DD') AS close_date, + NULLIF(up->'sledSudTekst'->>'svedTekst' , '') AS sled_sud_tekst_sved_tekst, + NULLIF(up->'sledSudTekst'->>'statusNaim', '') AS sled_sud_tekst_status_naim, + CASE + WHEN (up->>'sledSud') IS NOT NULL AND up->>'sledSud' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в структурированном формате' + WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде' + ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей' + END AS type_info, + true AS has_criminal_prosecution, + up.source_ctid = (SELECT MAX(source_ctid) FROM ugolov_presl) as last_row +FROM ervu_dashboard.citizen r +JOIN ugolov_presl up ON r.recruit_id = up.recruit_id +LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true +GROUP BY r.recruit_id, up.up, up.source_ctid; + Y + + + 384 + 768 + + + + prosecution_insert_or_update InsertUpdate N @@ -390,8 +488,8 @@ and recruitment_id = '${IDM_ID}'; - Sort rows - SortRows + prosecution_success_job_status + ExecSql Y @@ -400,98 +498,30 @@ and recruitment_id = '${IDM_ID}'; none - ${java.io.tmpdir} - out - 1000000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - - - - 384 - 416 - - - - Table input - TableInput - - N - - 1 - - none - - + + ervu-dashboard - N - WITH ugolov_presl AS ( - SELECT - jsonb_array_elements(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') AS up, - ri.recruit_id, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - AND '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp - WHERE jsonb_typeof(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') = 'array' -) -SELECT - r.recruit_id AS recruit_id, - (up->>'id')::uuid AS prosecution_external_id, - NULLIF(up->'sledSud'->>'uk', '') AS uk_name, - NULLIF(up->'sledSud'->'extend'->>'nomerUDPredv', '') AS case_number, - NULLIF(up->'sledSud'->'extend'->>'svedPerekvalif', '') AS reclassification, - NULLIF(up->'sledSud'->'extend'->>'naimVidUgolovPresl', '') AS kind, - CASE - WHEN up->'sledSud'->'extend'->>'priznakPeredachSud' = '1' THEN true - ELSE false -- нет или нет инфы - END AS transfer_to_court, - NULLIF(up->'sledSud'->>'naimSud','') AS court_name, - NULLIF(up->'sledSud'->>'nomerUD','') AS case_number_court, - STRING_AGG(TRIM(BOTH ' "' FROM law.f), ', ') AS law_point, - NULLIF(up->'sledSud'->>'naimOrgan', '') AS authority_name, - NULLIF(up->'sledSud'->'svedSnyat'->>'osnSnyatSud', '') AS conviction_removal_basis, - TO_DATE(up->'sledSud'->'svedSnyat'->>'dataSnyatSud', 'YYYY-MM-DD') AS conviction_removal_date, - TO_DATE(up->'sledSud'->>'dataOsuzhd', 'YYYY-MM-DD') AS conviction_date, - TO_DATE(up->'sledSud'->>'dataVozbUD', 'YYYY-MM-DD') AS create_date, - NULLIF(up->'sledSud'->>'osnPrekrUD', '') AS termination_basis, - NULLIF(up->'sledSud'->>'statusNaim', '') AS conviction_status, - TO_DATE(up->'sledSud'->>'dataPrekrUD', 'YYYY-MM-DD') AS close_date, - NULLIF(up->'sledSudTekst'->>'svedTekst' , '') AS sled_sud_tekst_sved_tekst, - NULLIF(up->'sledSudTekst'->>'statusNaim', '') AS sled_sud_tekst_status_naim, - CASE - WHEN (up->>'sledSud') IS NOT NULL AND up->>'sledSud' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в структурированном формате' - WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде' - ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей' - END AS type_info, - true AS has_criminal_prosecution, - up.source_ctid = (SELECT MAX(source_ctid) FROM ugolov_presl) as last_row -FROM ervu_dashboard.citizen r -JOIN ugolov_presl up ON r.recruit_id = up.recruit_id -LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true -GROUP BY r.recruit_id, up.up, up.source_ctid; - Y + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'DELTA_SUCCESS', + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}' +and status = 'DELTA_PROCESSING'; + + - 384 - 768 + 896 + 944 - Update + prosecution_update_flags Update N @@ -504,19 +534,16 @@ GROUP BY r.recruit_id, up.up, up.source_ctid; 10000 ervu-dashboard Y - = recruit_id recruit_id - <> has_criminal_prosecution has_criminal_prosecution - ervu_dashboard citizen
@@ -535,8 +562,8 @@ GROUP BY r.recruit_id, up.up, up.source_ctid; - Insert / update - Change job status on error + prosecution_insert_or_update + prosecution_error_job_status Y error_description @@ -547,8 +574,8 @@ GROUP BY r.recruit_id, up.up, up.source_ctid; - Update - Change job status on error + prosecution_update_flags + prosecution_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl index ee5e098..83aab04 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl @@ -27,21 +27,21 @@ Insert / update - Change job status on error + prosecution_error_job_status Y Detect empty stream - Change job status on success + prosecution_success_job_status Y Filter rows - Change job status on success + prosecution_success_job_status Y - Change job status on error + prosecution_error_job_status Abort Y @@ -51,18 +51,18 @@ Y - Update - Change job status on error + prosecution_update_flags + prosecution_error_job_status Y - Sort rows - Update + prosecution_sort_rows + prosecution_update_flags Y Table input - Sort rows + prosecution_sort_rows Y @@ -96,75 +96,6 @@ 736 - - Change job status on error - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 768 - 736 - - - - Change job status on success - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'SUCCESS', - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'PROCESSING'; - - - - - 928 - 1088 - - Create job execution record ExecSql @@ -257,7 +188,7 @@ DO UPDATE SET - Change job status on success + prosecution_success_job_status 928 @@ -404,7 +335,7 @@ DO UPDATE SET - Sort rows + prosecution_sort_rows SortRows Y @@ -416,7 +347,7 @@ DO UPDATE SET ${java.io.tmpdir} out - 1000000 + 100000 N @@ -504,7 +435,76 @@ GROUP BY r.recruit_id, up.up, up.source_ctid; - Update + prosecution_error_job_status + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 768 + 736 + + + + prosecution_success_job_status + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS', + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}' +and status = 'PROCESSING'; + + + + + 928 + 1088 + + + + prosecution_update_flags Update N @@ -517,19 +517,16 @@ GROUP BY r.recruit_id, up.up, up.source_ctid; 10000 ervu-dashboard Y - = recruit_id recruit_id - <> has_criminal_prosecution has_criminal_prosecution - ervu_dashboard citizen
@@ -549,7 +546,7 @@ GROUP BY r.recruit_id, up.up, up.source_ctid; Insert / update - Change job status on error + prosecution_error_job_status Y error_description @@ -560,8 +557,8 @@ GROUP BY r.recruit_id, up.up, up.source_ctid; - Update - Change job status on error + prosecution_update_flags + prosecution_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl index 1f60b8c..597755c 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl @@ -22,59 +22,64 @@ Create job execution record - Table input + punishment_input Y - Table output - Change job status on error + punishment_output + punishment_error_job_status Y - Change job status on error + punishment_error_job_status Abort Y - - Filter rows - Change job status on success - Y - Detect empty stream - Change job status on success + punishment_success_job_status Y - Table input - Table output + punishment_input + punishment_output Y - Update - Change job status on error + punishment_update_flags + punishment_error_job_status Y - Sort rows - Update + punishment_sort_rows + punishment_update_flags Y - Table input - Sort rows + punishment_input + punishment_sort_rows Y - Table input + punishment_input Detect empty stream Y - Table output + punishment_output Filter rows Y + + Filter rows + Unique rows + Y + + + Unique rows + punishment_success_job_status + Y + Abort @@ -96,75 +101,6 @@ 448 - - Change job status on error - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 1008 - 448 - - - - Change job status on success - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'SUCCESS', - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'PROCESSING'; - - - - - 544 - 144 - - Create job execution record ExecSql @@ -240,7 +176,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Change job status on success + Unique rows 720 @@ -248,7 +184,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Sort rows + punishment_sort_rows SortRows Y @@ -260,7 +196,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ${java.io.tmpdir} out - 1000000 + 100000 N @@ -282,7 +218,68 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Table input + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 640 + 144 + + + + punishment_error_job_status + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1008 + 448 + + + + punishment_input TableInput N @@ -348,7 +345,7 @@ GROUP BY recruit_id, punishment_info,source_ctid; - Table output + punishment_output TableOutput N @@ -458,7 +455,40 @@ GROUP BY recruit_id, punishment_info,source_ctid; - Update + punishment_success_job_status + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS', + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}' +and status = 'PROCESSING'; + + + + + 512 + 256 + + + + punishment_update_flags Update N @@ -471,25 +501,21 @@ GROUP BY recruit_id, punishment_info,source_ctid; 10000 ervu-dashboard Y - = recruit_id recruit_id - <> has_punishment_execution_info has_punishment_execution_info - <> has_sentence_info has_sentence_info - ervu_dashboard citizen
@@ -512,8 +538,8 @@ GROUP BY recruit_id, punishment_info,source_ctid; - Table output - Change job status on error + punishment_output + punishment_error_job_status Y error_description @@ -524,8 +550,8 @@ GROUP BY recruit_id, punishment_info,source_ctid; - Update - Change job status on error + punishment_update_flags + punishment_error_job_status Y error_decription diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl index d53639f..b802798 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl @@ -41,59 +41,64 @@ Create job execution record - Table input + punishment_input Y - Insert / update - Change job status on error + punishment_insert_or_update + punishment_error_job_status Y Detect empty stream - Change job status on success + punishment_success_job_status Y - Filter rows - Change job status on success - Y - - - Change job status on error + punishment_error_job_status Abort Y - Table input - Insert / update + punishment_input + punishment_insert_or_update Y - Update - Change job status on error + punishment_update_flags + punishment_error_job_status Y - Table input - Sort rows + punishment_input + punishment_sort_rows Y - Table input + punishment_input Detect empty stream Y - Sort rows - Update + punishment_sort_rows + punishment_update_flags Y - Insert / update + punishment_insert_or_update Filter rows Y + + Filter rows + Unique rows + Y + + + Unique rows + punishment_success_job_status + Y + Abort @@ -115,75 +120,6 @@ 384 - - Change job status on error - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'DELTA_ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 960 - 384 - - - - Change job status on success - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'DELTA_SUCCESS', - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'DELTA_PROCESSING'; - - - - - 608 - 32 - - Create job execution record ExecSql @@ -262,7 +198,7 @@ and recruitment_id = '${IDM_ID}'; - Change job status on success + Unique rows 784 @@ -270,7 +206,169 @@ and recruitment_id = '${IDM_ID}'; - Insert / update + punishment_sort_rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 400 + 608 + + + + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 704 + 32 + + + + punishment_error_job_status + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'DELTA_ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 960 + 384 + + + + punishment_input + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + WITH punishment_data AS ( + SELECT + ri.recruit_id, + jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id + AND '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp + WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null' +) +SELECT recruit_id, + (punishment_info ->> 'id')::uuid AS punishment_external_id, + -- Блок "Сведения о вынесенном приговоре" + punishment_info ->> 'vidNakaz' AS punishment_type, -- Вид наказания + punishment_info ->> 'kodVidNakaz' AS punishment_code, -- Код вида наказания + TO_DATE(punishment_info ->> 'dataOsuzhd', 'YYYY-MM-DD') AS sentense_date, -- Дата осуждения + punishment_info ->> 'naimSud' AS authority_name, -- Наименование суда + punishment_info ->> 'srokNakaz' AS sentence, -- Срок наказания + CASE + WHEN punishment_info ->> 'kodIzmSrokNakaz' = '1' THEN 'Изменений срока наказания не было' + WHEN punishment_info ->> 'kodIzmSrokNakaz' = '2' THEN 'Были изменения срока наказания' + ELSE punishment_info ->> 'kodIzmSrokNakaz' + END AS sentence_changed, -- Признак изменения срока + STRING_AGG(TRIM(BOTH ' "' FROM law_articles), ', ') AS law_point, -- Статьи УК + -- Блок "Сведения об отбывании наказания" + punishment_info ->> 'katUcheta' AS registration_category, -- Категория учета + punishment_info ->> 'naimSubekt' AS region_name, -- Наименование субъекта + punishment_info ->> 'naimOrg' AS facility_name, -- Название учреждения + punishment_info ->> 'adresOrg' AS facility_address, -- Адрес учреждения + punishment_info ->> 'nomerUD' AS criminal_case_number, -- Номер уголовного дела + TO_DATE(punishment_info ->> 'dataNachSrok', 'YYYY-MM-DD') AS start_date, -- Дата начала срока + TO_DATE(punishment_info ->> 'dataOkonchSrok', 'YYYY-MM-DD') AS planned_date, -- Планируемая дата освобождения + CASE + WHEN punishment_info ->> 'kodPerekvalif' = '1' THEN 'Изменений статьи не было' + WHEN punishment_info ->> 'kodPerekvalif' = '2' THEN 'Были изменения статьи' + ELSE punishment_info ->> 'kodPerekvalif' + END AS article_modification_flag, -- Признак переквалификации + punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения + TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения + true AS has_punishment_execution_info, + true AS has_sentence_info, + source_ctid = (SELECT MAX(source_ctid) FROM punishment_data) as last_row +FROM punishment_data +LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true +GROUP BY recruit_id, punishment_info,source_ctid; + Y + + + 400 + 224 + + + + punishment_insert_or_update InsertUpdate N @@ -399,8 +497,8 @@ and recruitment_id = '${IDM_ID}'; - Sort rows - SortRows + punishment_success_job_status + ExecSql Y @@ -409,98 +507,30 @@ and recruitment_id = '${IDM_ID}'; none - ${java.io.tmpdir} - out - 1000000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - - - - 400 - 608 - - - - Table input - TableInput - - N - - 1 - - none - - + + ervu-dashboard - N - WITH punishment_data AS ( - SELECT - ri.recruit_id, - jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - AND '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp - WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null' -) -SELECT recruit_id, - (punishment_info ->> 'id')::uuid AS punishment_external_id, - -- Блок "Сведения о вынесенном приговоре" - punishment_info ->> 'vidNakaz' AS punishment_type, -- Вид наказания - punishment_info ->> 'kodVidNakaz' AS punishment_code, -- Код вида наказания - TO_DATE(punishment_info ->> 'dataOsuzhd', 'YYYY-MM-DD') AS sentense_date, -- Дата осуждения - punishment_info ->> 'naimSud' AS authority_name, -- Наименование суда - punishment_info ->> 'srokNakaz' AS sentence, -- Срок наказания - CASE - WHEN punishment_info ->> 'kodIzmSrokNakaz' = '1' THEN 'Изменений срока наказания не было' - WHEN punishment_info ->> 'kodIzmSrokNakaz' = '2' THEN 'Были изменения срока наказания' - ELSE punishment_info ->> 'kodIzmSrokNakaz' - END AS sentence_changed, -- Признак изменения срока - STRING_AGG(TRIM(BOTH ' "' FROM law_articles), ', ') AS law_point, -- Статьи УК - -- Блок "Сведения об отбывании наказания" - punishment_info ->> 'katUcheta' AS registration_category, -- Категория учета - punishment_info ->> 'naimSubekt' AS region_name, -- Наименование субъекта - punishment_info ->> 'naimOrg' AS facility_name, -- Название учреждения - punishment_info ->> 'adresOrg' AS facility_address, -- Адрес учреждения - punishment_info ->> 'nomerUD' AS criminal_case_number, -- Номер уголовного дела - TO_DATE(punishment_info ->> 'dataNachSrok', 'YYYY-MM-DD') AS start_date, -- Дата начала срока - TO_DATE(punishment_info ->> 'dataOkonchSrok', 'YYYY-MM-DD') AS planned_date, -- Планируемая дата освобождения - CASE - WHEN punishment_info ->> 'kodPerekvalif' = '1' THEN 'Изменений статьи не было' - WHEN punishment_info ->> 'kodPerekvalif' = '2' THEN 'Были изменения статьи' - ELSE punishment_info ->> 'kodPerekvalif' - END AS article_modification_flag, -- Признак переквалификации - punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения - TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения - true AS has_punishment_execution_info, - true AS has_sentence_info, - source_ctid = (SELECT MAX(source_ctid) FROM punishment_data) as last_row -FROM punishment_data -LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true -GROUP BY recruit_id, punishment_info,source_ctid; - Y + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'DELTA_SUCCESS', + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}' +and status = 'DELTA_PROCESSING'; + + - 400 - 224 + 592 + 160 - Update + punishment_update_flags Update N @@ -513,25 +543,21 @@ GROUP BY recruit_id, punishment_info,source_ctid; 10000 ervu-dashboard Y - = recruit_id recruit_id - <> has_punishment_execution_info has_punishment_execution_info - <> has_sentence_info has_sentence_info - ervu_dashboard citizen
@@ -554,8 +580,8 @@ GROUP BY recruit_id, punishment_info,source_ctid; - Insert / update - Change job status on error + punishment_insert_or_update + punishment_error_job_status Y error_description @@ -566,8 +592,8 @@ GROUP BY recruit_id, punishment_info,source_ctid; - Update - Change job status on error + punishment_update_flags + punishment_error_job_status Y error_description diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl index 8868c14..15b5812 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl @@ -22,59 +22,64 @@ Create job execution record - Table input + punishment_input Y - Change job status on error + punishment_error_job_status Abort Y Detect empty stream - Change job status on success + punishment_success_job_status Y - Filter rows - Change job status on success + punishment_update_flags + punishment_error_job_status Y - Update - Change job status on error + punishment_insert_or_update + punishment_error_job_status Y - Insert / update - Change job status on error + punishment_input + punishment_insert_or_update Y - Table input - Insert / update + punishment_input + punishment_sort_rows Y - Table input - Sort rows + punishment_sort_rows + punishment_update_flags Y - Sort rows - Update - Y - - - Table input + punishment_input Detect empty stream Y - Insert / update + punishment_insert_or_update Filter rows Y + + Filter rows + Unique rows + Y + + + Unique rows + punishment_success_job_status + Y + Abort @@ -96,75 +101,6 @@ 432 - - Change job status on error - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 992 - 432 - - - - Change job status on success - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'SUCCESS', - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'PROCESSING'; - - - - - 640 - 112 - - Create job execution record ExecSql @@ -257,7 +193,7 @@ DO UPDATE SET - Change job status on success + Unique rows 816 @@ -265,7 +201,168 @@ DO UPDATE SET - Insert / update + punishment_sort_rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 416 + 640 + + + + Unique rows + Unique + + Y + + 1 + + none + + + N + + + N + recruit_id + + + N + + + 736 + 112 + + + + punishment_error_job_status + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 992 + 432 + + + + punishment_input + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + WITH punishment_data AS ( + SELECT + ri.recruit_id, + jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id + AND '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null' +) +SELECT recruit_id, + (punishment_info ->> 'id')::uuid AS punishment_external_id, + -- Блок "Сведения о вынесенном приговоре" + punishment_info ->> 'vidNakaz' AS punishment_type, -- Вид наказания + punishment_info ->> 'kodVidNakaz' AS punishment_code, -- Код вида наказания + TO_DATE(punishment_info ->> 'dataOsuzhd', 'YYYY-MM-DD') AS sentense_date, -- Дата осуждения + punishment_info ->> 'naimSud' AS authority_name, -- Наименование суда + punishment_info ->> 'srokNakaz' AS sentence, -- Срок наказания + CASE + WHEN punishment_info ->> 'kodIzmSrokNakaz' = '1' THEN 'Изменений срока наказания не было' + WHEN punishment_info ->> 'kodIzmSrokNakaz' = '2' THEN 'Были изменения срока наказания' + ELSE punishment_info ->> 'kodIzmSrokNakaz' + END AS sentence_changed, -- Признак изменения срока + STRING_AGG(TRIM(BOTH ' "' FROM law_articles), ', ') AS law_point, -- Статьи УК + -- Блок "Сведения об отбывании наказания" + punishment_info ->> 'katUcheta' AS registration_category, -- Категория учета + punishment_info ->> 'naimSubekt' AS region_name, -- Наименование субъекта + punishment_info ->> 'naimOrg' AS facility_name, -- Название учреждения + punishment_info ->> 'adresOrg' AS facility_address, -- Адрес учреждения + punishment_info ->> 'nomerUD' AS criminal_case_number, -- Номер уголовного дела + TO_DATE(punishment_info ->> 'dataNachSrok', 'YYYY-MM-DD') AS start_date, -- Дата начала срока + TO_DATE(punishment_info ->> 'dataOkonchSrok', 'YYYY-MM-DD') AS planned_date, -- Планируемая дата освобождения + CASE + WHEN punishment_info ->> 'kodPerekvalif' = '1' THEN 'Изменений статьи не было' + WHEN punishment_info ->> 'kodPerekvalif' = '2' THEN 'Были изменения статьи' + ELSE punishment_info ->> 'kodPerekvalif' + END AS article_modification_flag, -- Признак переквалификации + punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения + TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения + true AS has_punishment_execution_info, + true AS has_sentence_info, + source_ctid = (SELECT MAX(source_ctid) FROM punishment_data) as last_row +FROM punishment_data +LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true +GROUP BY recruit_id, punishment_info,source_ctid; + Y + + + 416 + 272 + + + + punishment_insert_or_update InsertUpdate N @@ -389,13 +486,13 @@ DO UPDATE SET N - 816 + 800 272 - Sort rows - SortRows + punishment_success_job_status + ExecSql Y @@ -404,97 +501,30 @@ DO UPDATE SET none - ${java.io.tmpdir} - out - 1000000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - - - - 416 - 640 - - - - Table input - TableInput - - N - - 1 - - none - - + + ervu-dashboard - N - WITH punishment_data AS ( - SELECT - ri.recruit_id, - jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info, - ri.ctid as source_ctid - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - AND '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null' -) -SELECT recruit_id, - (punishment_info ->> 'id')::uuid AS punishment_external_id, - -- Блок "Сведения о вынесенном приговоре" - punishment_info ->> 'vidNakaz' AS punishment_type, -- Вид наказания - punishment_info ->> 'kodVidNakaz' AS punishment_code, -- Код вида наказания - TO_DATE(punishment_info ->> 'dataOsuzhd', 'YYYY-MM-DD') AS sentense_date, -- Дата осуждения - punishment_info ->> 'naimSud' AS authority_name, -- Наименование суда - punishment_info ->> 'srokNakaz' AS sentence, -- Срок наказания - CASE - WHEN punishment_info ->> 'kodIzmSrokNakaz' = '1' THEN 'Изменений срока наказания не было' - WHEN punishment_info ->> 'kodIzmSrokNakaz' = '2' THEN 'Были изменения срока наказания' - ELSE punishment_info ->> 'kodIzmSrokNakaz' - END AS sentence_changed, -- Признак изменения срока - STRING_AGG(TRIM(BOTH ' "' FROM law_articles), ', ') AS law_point, -- Статьи УК - -- Блок "Сведения об отбывании наказания" - punishment_info ->> 'katUcheta' AS registration_category, -- Категория учета - punishment_info ->> 'naimSubekt' AS region_name, -- Наименование субъекта - punishment_info ->> 'naimOrg' AS facility_name, -- Название учреждения - punishment_info ->> 'adresOrg' AS facility_address, -- Адрес учреждения - punishment_info ->> 'nomerUD' AS criminal_case_number, -- Номер уголовного дела - TO_DATE(punishment_info ->> 'dataNachSrok', 'YYYY-MM-DD') AS start_date, -- Дата начала срока - TO_DATE(punishment_info ->> 'dataOkonchSrok', 'YYYY-MM-DD') AS planned_date, -- Планируемая дата освобождения - CASE - WHEN punishment_info ->> 'kodPerekvalif' = '1' THEN 'Изменений статьи не было' - WHEN punishment_info ->> 'kodPerekvalif' = '2' THEN 'Были изменения статьи' - ELSE punishment_info ->> 'kodPerekvalif' - END AS article_modification_flag, -- Признак переквалификации - punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения - TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения - true AS has_punishment_execution_info, - true AS has_sentence_info, - source_ctid = (SELECT MAX(source_ctid) FROM punishment_data) as last_row -FROM punishment_data -LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true -GROUP BY recruit_id, punishment_info,source_ctid; - Y + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS', + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}' +and status = 'PROCESSING'; + + - 416 - 272 + 640 + 192 - Update + punishment_update_flags Update N @@ -507,25 +537,21 @@ GROUP BY recruit_id, punishment_info,source_ctid; 10000 ervu-dashboard Y - = recruit_id recruit_id - <> has_punishment_execution_info has_punishment_execution_info - <> has_sentence_info has_sentence_info - ervu_dashboard citizen
@@ -548,8 +574,8 @@ GROUP BY recruit_id, punishment_info,source_ctid; - Insert / update - Change job status on error + punishment_insert_or_update + punishment_error_job_status Y error_description @@ -560,8 +586,8 @@ GROUP BY recruit_id, punishment_info,source_ctid; - Update - Change job status on error + punishment_update_flags + punishment_error_job_status Y error_description