diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl index b83ef0c..5ea7327 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl @@ -40,21 +40,11 @@ Abort Y - - Update 2 - Filter rows - Y - Filter rows Change job status on success Y - - Update 2 - Detect empty stream - Y - Detect empty stream Change job status on success @@ -65,26 +55,31 @@ Table output Y - - Group by 2 - Identify last row in a stream - Y - - - Identify last row in a stream - User defined Java expression 2 - Y - Table output Change job status on error Y - Table output + Table input Group by 2 Y + + Group by 2 + User defined Java expression 2 + Y + + + Table output + Filter rows + Y + + + Table input + Detect empty stream + Y + Abort @@ -171,8 +166,8 @@ and recruitment_id = '${IDM_ID}'; - 1744 - 608 + 576 + 1040 @@ -217,8 +212,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 1568 - 480 + 304 + 1040 @@ -253,8 +248,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1392 - 608 + 896 + 1040 @@ -298,25 +293,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); grp - 560 - 480 - - - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 784 + 304 480 @@ -324,7 +301,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Table input TableInput - Y + N 1 @@ -334,53 +311,60 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - SELECT - ri.recruit_id AS recruit_id, - (ch->>'id')::uuid AS child_external_id, - NULLIF(ch->>'idERN', '') AS child_id_ern, - CASE NULLIF(ch->>'rodstvSvyazReb', '')::int - WHEN 1 THEN 3 - WHEN 2 THEN 4 - END AS kinship_type, - ch->'svedFLBS'->'fio'->>'familiya' AS last_name, - ch->'svedFLBS'->'fio'->>'imya' AS first_name, - ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - ch->'svedFLBS'->'fio'->>'familiya', - ch->'svedFLBS'->'fio'->>'imya', - ch->'svedFLBS'->'fio'->>'otchestvo' - ) AS full_name, - dates.birth_date, - dates.death_date, - ch->'svedSmert'->>'nomerZapis' AS death_az_number, - ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number, - CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead, - CASE - WHEN dates.death_date IS NOT NULL THEN 0 -- умершие не могут быть несовершеннолетними - WHEN dates.birth_date IS NULL THEN 0 -- если дата рождения неизвестна, считаем взрослым - WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1 - ELSE 0 - END AS is_minor -FROM ervu_dashboard.recruits_info ri -JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id -CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch -CROSS JOIN LATERAL ( + WITH children_data AS ( SELECT - MAKE_DATE( - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - MAKE_DATE( - NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int, - NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int, - NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int - ) AS death_date -) AS dates -WHERE + ri.recruit_id AS recruit_id, + (ch->>'id')::uuid AS child_external_id, + NULLIF(ch->>'idERN', '') AS child_id_ern, + CASE NULLIF(ch->>'rodstvSvyazReb', '')::int + WHEN 1 THEN 3 + WHEN 2 THEN 4 + END AS kinship_type, + ch->'svedFLBS'->'fio'->>'familiya' AS last_name, + ch->'svedFLBS'->'fio'->>'imya' AS first_name, + ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + ch->'svedFLBS'->'fio'->>'familiya', + ch->'svedFLBS'->'fio'->>'imya', + ch->'svedFLBS'->'fio'->>'otchestvo' + ) AS full_name, + dates.birth_date, + dates.death_date, + ch->'svedSmert'->>'nomerZapis' AS death_az_number, + ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number, + CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead, + CASE + WHEN dates.death_date IS NOT NULL THEN 0 + WHEN dates.birth_date IS NULL THEN 0 + WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1 + ELSE 0 + END AS is_minor, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id + CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch + CROSS JOIN LATERAL ( + SELECT + MAKE_DATE( + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + MAKE_DATE( + NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int, + NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int, + NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int + ) AS death_date + ) AS dates + WHERE '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array'; + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array' +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM children_data) as last_row +FROM children_data; Y @@ -466,7 +450,7 @@ WHERE Y - 560 + 896 800 @@ -550,7 +534,7 @@ WHERE - 1024 + 688 480 diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl index f4d95f8..742e83a 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl @@ -18,26 +18,6 @@ 2025/08/11 13:22:56.048 - - 251 - 232 - 201 - 90 - 58 - 14 - N - 90 - 58 - 14 - N - Segoe UI - 9 - 26 - 574 - 358 - 1. insert/update, update указан лишний error_code для error handling - 384 - @@ -45,11 +25,6 @@ Change job status on error Y - - Insert / update - Group by - Y - Update Change job status on error @@ -65,21 +40,11 @@ Table input 2 Y - - Update - Filter rows - Y - Filter rows Change job status on success Y - - Update - Detect empty stream - Y - Detect empty stream Change job status on success @@ -91,18 +56,28 @@ Y - Group by - Identify last row in a stream + Table input 2 + Insert / update Y - Identify last row in a stream + Group by User defined Java expression Y Table input 2 - Insert / update + Detect empty stream + Y + + + Insert / update + Filter rows + Y + + + Table input 2 + Group by Y @@ -191,8 +166,8 @@ and status = 'DELTA_PROCESSING'; - 1872 - 0 + 784 + 496 @@ -239,8 +214,8 @@ and recruitment_id = '${IDM_ID}'; - 1616 - 144 + 496 + 496 @@ -275,8 +250,8 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1616 - 0 + 1072 + 496 @@ -320,25 +295,7 @@ and recruitment_id = '${IDM_ID}'; grp - 784 - 0 - - - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 1024 + 496 0 @@ -432,7 +389,7 @@ and recruitment_id = '${IDM_ID}'; N - 784 + 1072 224 @@ -440,7 +397,7 @@ and recruitment_id = '${IDM_ID}'; Table input 2 TableInput - Y + N 1 @@ -450,54 +407,61 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - SELECT - ri.recruit_id AS recruit_id, - (ch->>'id')::uuid AS child_external_id, - NULLIF(ch->>'idERN', '') AS child_id_ern, - CASE NULLIF(ch->>'rodstvSvyazReb', '')::int - WHEN 1 THEN 3 - WHEN 2 THEN 4 - END AS kinship_type, - ch->'svedFLBS'->'fio'->>'familiya' AS last_name, - ch->'svedFLBS'->'fio'->>'imya' AS first_name, - ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - ch->'svedFLBS'->'fio'->>'familiya', - ch->'svedFLBS'->'fio'->>'imya', - ch->'svedFLBS'->'fio'->>'otchestvo' - ) AS full_name, - dates.birth_date, - dates.death_date, - ch->'svedSmert'->>'nomerZapis' AS death_az_number, - ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number, - CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead, - CASE - WHEN dates.death_date IS NOT NULL THEN 0 -- умершие не могут быть несовершеннолетними - WHEN dates.birth_date IS NULL THEN 0 -- если дата рождения неизвестна, считаем взрослым - WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1 - ELSE 0 - END AS is_minor -FROM ervu_dashboard.recruits_info ri -JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id -CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch -CROSS JOIN LATERAL ( + WITH children_data AS ( SELECT - MAKE_DATE( - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - MAKE_DATE( - NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int, - NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int, - NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int - ) AS death_date -) AS dates -WHERE + ri.recruit_id AS recruit_id, + (ch->>'id')::uuid AS child_external_id, + NULLIF(ch->>'idERN', '') AS child_id_ern, + CASE NULLIF(ch->>'rodstvSvyazReb', '')::int + WHEN 1 THEN 3 + WHEN 2 THEN 4 + END AS kinship_type, + ch->'svedFLBS'->'fio'->>'familiya' AS last_name, + ch->'svedFLBS'->'fio'->>'imya' AS first_name, + ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + ch->'svedFLBS'->'fio'->>'familiya', + ch->'svedFLBS'->'fio'->>'imya', + ch->'svedFLBS'->'fio'->>'otchestvo' + ) AS full_name, + dates.birth_date, + dates.death_date, + ch->'svedSmert'->>'nomerZapis' AS death_az_number, + ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number, + CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead, + CASE + WHEN dates.death_date IS NOT NULL THEN 0 + WHEN dates.birth_date IS NULL THEN 0 + WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1 + ELSE 0 + END AS is_minor, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id + CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch + CROSS JOIN LATERAL ( + SELECT + MAKE_DATE( + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + MAKE_DATE( + NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int, + NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int, + NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int + ) AS death_date + ) AS dates + WHERE '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp, - AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array'; + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp + AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array' +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM children_data) as last_row +FROM children_data; Y @@ -585,7 +549,7 @@ WHERE - 1232 + 784 0 diff --git a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl index 048056a..d568245 100644 --- a/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl @@ -18,26 +18,6 @@ 2025/08/08 13:25:52.319 - - 251 - 232 - 201 - 90 - 58 - 14 - N - 90 - 58 - 14 - N - Segoe UI - 9 - 26 - 368 - 592 - 1.Insert/update не указан error_description для error handling - 338 - @@ -60,26 +40,11 @@ Change job status on error 2 Y - - Insert / update - Group by - Y - - - Update - Detect empty stream - Y - Detect empty stream Change job status on success Y - - Update - Filter rows - Y - Filter rows Change job status on success @@ -91,18 +56,28 @@ Y - Group by - Identify last row in a stream + Table input 2 + Insert / update Y - Identify last row in a stream + Group by User defined Java expression Y Table input 2 - Insert / update + Detect empty stream + Y + + + Insert / update + Filter rows + Y + + + Table input 2 + Group by Y @@ -192,8 +167,8 @@ and status = 'PROCESSING'; - 1440 - 0 + 416 + 608 @@ -255,8 +230,8 @@ DO UPDATE SET - 1216 - 0 + 224 + 608 @@ -291,8 +266,8 @@ DO UPDATE SET Change job status on success - 1216 - 128 + 608 + 608 @@ -336,26 +311,8 @@ DO UPDATE SET grp - 608 - 192 - - - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 608 - 0 + 224 + 176 @@ -456,7 +413,7 @@ DO UPDATE SET Table input 2 TableInput - Y + N 1 @@ -465,58 +422,64 @@ DO UPDATE SET ervu-dashboard N - - SELECT - ri.recruit_id AS recruit_id, - (ch->>'id')::uuid AS child_external_id, - NULLIF(ch->>'idERN', '') AS child_id_ern, - CASE NULLIF(ch->>'rodstvSvyazReb', '')::int - WHEN 1 THEN 3 - WHEN 2 THEN 4 - END AS kinship_type, - ch->'svedFLBS'->'fio'->>'familiya' AS last_name, - ch->'svedFLBS'->'fio'->>'imya' AS first_name, - ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - ch->'svedFLBS'->'fio'->>'familiya', - ch->'svedFLBS'->'fio'->>'imya', - ch->'svedFLBS'->'fio'->>'otchestvo' - ) AS full_name, - dates.birth_date, - dates.death_date, - ch->'svedSmert'->>'nomerZapis' AS death_az_number, - ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number, - CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead, - CASE - WHEN dates.death_date IS NOT NULL THEN 0 -- умершие не могут быть несовершеннолетними - WHEN dates.birth_date IS NULL THEN 0 -- если дата рождения неизвестна, считаем взрослым - WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1 - ELSE 0 - END AS is_minor -FROM ervu_dashboard.recruits_info ri -JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id -CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch -CROSS JOIN LATERAL ( + WITH children_data AS ( SELECT - MAKE_DATE( - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - MAKE_DATE( - NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int, - NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int, - NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int - ) AS death_date -) AS dates -WHERE + ri.recruit_id AS recruit_id, + (ch->>'id')::uuid AS child_external_id, + NULLIF(ch->>'idERN', '') AS child_id_ern, + CASE NULLIF(ch->>'rodstvSvyazReb', '')::int + WHEN 1 THEN 3 + WHEN 2 THEN 4 + END AS kinship_type, + ch->'svedFLBS'->'fio'->>'familiya' AS last_name, + ch->'svedFLBS'->'fio'->>'imya' AS first_name, + ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + ch->'svedFLBS'->'fio'->>'familiya', + ch->'svedFLBS'->'fio'->>'imya', + ch->'svedFLBS'->'fio'->>'otchestvo' + ) AS full_name, + dates.birth_date, + dates.death_date, + ch->'svedSmert'->>'nomerZapis' AS death_az_number, + ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number, + CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead, + CASE + WHEN dates.death_date IS NOT NULL THEN 0 + WHEN dates.birth_date IS NULL THEN 0 + WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1 + ELSE 0 + END AS is_minor, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id + CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch + CROSS JOIN LATERAL ( + SELECT + MAKE_DATE( + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + MAKE_DATE( + NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int, + NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int, + NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int + ) AS death_date + ) AS dates + WHERE '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array'; + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array' +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM children_data) as last_row +FROM children_data; Y - 272 + 224 432 @@ -560,7 +523,7 @@ WHERE 976 - 0 + 176 @@ -600,8 +563,8 @@ WHERE - 800 - 0 + 608 + 176 diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl index 6e63a5a..f8e78c4 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl @@ -35,11 +35,6 @@ Abort Y - - Table output - Detect empty stream - Y - Detect empty stream Change job status on success @@ -51,18 +46,18 @@ Y - Table input - Identify last row in a stream + Table output + Filter rows Y - Identify last row in a stream + Table input Table output Y - Table output - Filter rows + Table input + Detect empty stream Y @@ -118,7 +113,7 @@ and recruitment_id = '${IDM_ID}'; - 832 + 928 704 @@ -197,7 +192,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 832 + 576 96 @@ -237,29 +232,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 320 - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 640 - 320 - - Table input TableInput - Y + N 1 @@ -268,35 +245,40 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - - SELECT - ri.recruit_id::uuid, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, + WITH address_data AS ( + SELECT + ri.recruit_id::uuid, + ri.ctid as source_ctid, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, - CASE - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' - ELSE NULL - END AS place_stay_registration_type, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date -FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr -WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' -GROUP BY ri.recruit_id; + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, + CASE + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' + END AS place_stay_registration_type, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id + CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr + WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + GROUP BY ri.recruit_id, ri.ctid +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM address_data) as last_row +FROM address_data; Y - 464 + 576 320 @@ -374,7 +356,7 @@ GROUP BY ri.recruit_id; Y - 832 + 928 320 diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl index a0e39b9..a1694cb 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl @@ -51,20 +51,15 @@ Y - Insert / update - Identify last row in a stream - Y - - - Identify last row in a stream - Filter rows - Y - - - Insert / update + Table input Detect empty stream Y + + Insert / update + Filter rows + Y + Abort @@ -200,7 +195,7 @@ and recruitment_id = '${IDM_ID}'; - 752 + 400 16 @@ -240,24 +235,6 @@ and recruitment_id = '${IDM_ID}'; 224 - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 944 - 224 - - Insert / update InsertUpdate @@ -346,7 +323,7 @@ and recruitment_id = '${IDM_ID}'; Table input TableInput - Y + N 1 @@ -356,31 +333,37 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - SELECT - ri.recruit_id::uuid, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, + WITH address_data AS ( + SELECT + ri.recruit_id::uuid, + ri.ctid as source_ctid, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, - CASE - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' - ELSE NULL - END AS place_stay_registration_type, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date -FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr -WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp -GROUP BY ri.recruit_id; + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, + CASE + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' + END AS place_stay_registration_type, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id + CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr + WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp + GROUP BY ri.recruit_id, ri.ctid +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM address_data) as last_row +FROM address_data; Y diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl index 2fe203d..170fa7d 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl @@ -30,11 +30,6 @@ Change job status on error Y - - Insert / update - Detect empty stream - Y - Detect empty stream Change job status on success @@ -51,18 +46,18 @@ Y - Insert / update - Identify last row in a stream - Y - - - Identify last row in a stream - Filter rows + Table input + Insert / update Y Table input - Insert / update + Detect empty stream + Y + + + Insert / update + Filter rows Y @@ -214,7 +209,7 @@ DO UPDATE SET - 768 + 480 16 @@ -254,24 +249,6 @@ DO UPDATE SET 272 - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 960 - 272 - - Insert / update InsertUpdate @@ -360,7 +337,7 @@ DO UPDATE SET Table input TableInput - Y + N 1 @@ -369,31 +346,36 @@ DO UPDATE SET ervu-dashboard N - - SELECT - ri.recruit_id::uuid, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, + WITH address_data AS ( + SELECT + ri.recruit_id::uuid, + ri.ctid as source_ctid, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, - CASE - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' - ELSE NULL - END AS place_stay_registration_type, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date -FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr -WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' -GROUP BY ri.recruit_id; + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, + CASE + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' + END AS place_stay_registration_type, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id + CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr + WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + GROUP BY ri.recruit_id, ri.ctid +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM address_data) as last_row +FROM address_data; Y diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl index c94b6f4..6ecf15b 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl @@ -45,11 +45,6 @@ Abort Y - - Update - Filter rows - Y - Filter rows Change job status on success 2 @@ -61,22 +56,22 @@ Y - Identify last row in a stream - Update - Y - - - Table output + Table input Sort rows Y Sort rows - Identify last row in a stream + Update Y Table output + Filter rows + Y + + + Table input Detect empty stream Y @@ -166,8 +161,8 @@ and status = 'PROCESSING'; - 1840 - 0 + 816 + 720 @@ -212,8 +207,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 1280 - 688 + 496 + 560 @@ -248,26 +243,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success 2 - 1520 - 0 - - - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 1008 - 0 + 816 + 464 @@ -300,7 +277,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 720 + 496 0 @@ -308,7 +285,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Table input TableInput - Y + N 1 @@ -317,33 +294,39 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - - SELECT - ri.recruit_id AS recruit_id, - (ch->>'id')::uuid AS guardianship_external_id, - NULLIF(ch->>'idERN', '') AS guardian_id_ern, - ch->'fioOpek'->>'familiya' AS last_name, - ch->'fioOpek'->>'imya' AS first_name, - ch->'fioOpek'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - ch->'fioOpek'->>'familiya', - ch->'fioOpek'->>'imya', - ch->'fioOpek'->>'otchestvo' - ) AS full_name, - MAKE_DATE( - NULLIF(ch->'dataRozhdDok'->>'god', '')::int, - NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(ch->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, - true AS is_guardian -FROM ervu_dashboard.recruits_info ri -JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id -CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch -WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array'; + WITH guardianship_data AS ( + SELECT + ri.recruit_id AS recruit_id, + (ch->>'id')::uuid AS guardianship_external_id, + NULLIF(ch->>'idERN', '') AS guardian_id_ern, + ch->'fioOpek'->>'familiya' AS last_name, + ch->'fioOpek'->>'imya' AS first_name, + ch->'fioOpek'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + ch->'fioOpek'->>'familiya', + ch->'fioOpek'->>'imya', + ch->'fioOpek'->>'otchestvo' + ) AS full_name, + MAKE_DATE( + NULLIF(ch->'dataRozhdDok'->>'god', '')::int, + NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(ch->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, + true AS is_guardian, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id + CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch + WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM guardianship_data) as last_row +FROM guardianship_data; Y @@ -417,7 +400,7 @@ WHERE Y - 720 + 816 320 diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl index 2a83f44..0677aed 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl @@ -50,34 +50,29 @@ Change job status on success Y - - Update - Filter rows - Y - Filter rows Change job status on success Y - Identify last row in a stream - Update + Table input + Detect empty stream Y - Insert / update + Table input Sort rows Y Sort rows - Identify last row in a stream + Update Y Insert / update - Detect empty stream + Filter rows Y @@ -169,8 +164,8 @@ WHERE job_name = '${JOB_NAME}' - 1712 - 0 + 880 + 624 @@ -217,8 +212,8 @@ and recruitment_id = '${IDM_ID}'; - 1216 - 608 + 544 + 624 @@ -253,26 +248,8 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1456 - 0 - - - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 944 - 0 + 880 + 384 @@ -350,7 +327,7 @@ and recruitment_id = '${IDM_ID}'; N - 704 + 880 224 @@ -384,7 +361,7 @@ and recruitment_id = '${IDM_ID}'; - 704 + 544 0 @@ -392,7 +369,7 @@ and recruitment_id = '${IDM_ID}'; Table input TableInput - Y + N 1 @@ -401,34 +378,40 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - - SELECT - ri.recruit_id AS recruit_id, - (ch->>'id')::uuid AS guardianship_external_id, - NULLIF(ch->>'idERN', '') AS guardian_id_ern, - ch->'fioOpek'->>'familiya' AS last_name, - ch->'fioOpek'->>'imya' AS first_name, - ch->'fioOpek'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - ch->'fioOpek'->>'familiya', - ch->'fioOpek'->>'imya', - ch->'fioOpek'->>'otchestvo' - ) AS full_name, - MAKE_DATE( - NULLIF(ch->'dataRozhdDok'->>'god', '')::int, - NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(ch->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, - true AS is_guardian -FROM ervu_dashboard.recruits_info ri -JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id -CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch -WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' - AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp; + WITH guardianship_data AS ( + SELECT + ri.recruit_id AS recruit_id, + (ch->>'id')::uuid AS guardianship_external_id, + NULLIF(ch->>'idERN', '') AS guardian_id_ern, + ch->'fioOpek'->>'familiya' AS last_name, + ch->'fioOpek'->>'imya' AS first_name, + ch->'fioOpek'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + ch->'fioOpek'->>'familiya', + ch->'fioOpek'->>'imya', + ch->'fioOpek'->>'otchestvo' + ) AS full_name, + MAKE_DATE( + NULLIF(ch->'dataRozhdDok'->>'god', '')::int, + NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(ch->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, + true AS is_guardian, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id + CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch + WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM guardianship_data) as last_row +FROM guardianship_data; Y diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl index cb62861..82a507f 100644 --- a/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl @@ -45,11 +45,6 @@ Abort Y - - Update - Filter rows - Y - Filter rows Change job status on success @@ -61,23 +56,23 @@ Y - Identify last row in a stream + Sort rows Update Y - Insert / update + Table input Sort rows Y - Sort rows - Identify last row in a stream + Table input + Detect empty stream Y Insert / update - Detect empty stream + Filter rows Y @@ -170,8 +165,8 @@ and status = 'PROCESSING'; - 1808 - 48 + 704 + 624 @@ -233,8 +228,8 @@ DO UPDATE SET - 1232 - 576 + 416 + 624 @@ -269,26 +264,8 @@ DO UPDATE SET Change job status on success - 1504 - 48 - - - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 992 - 48 + 704 + 384 @@ -400,7 +377,7 @@ DO UPDATE SET - 704 + 416 48 @@ -408,7 +385,7 @@ DO UPDATE SET Table input TableInput - Y + N 1 @@ -418,32 +395,39 @@ DO UPDATE SET ervu-dashboard N - SELECT - ri.recruit_id AS recruit_id, - (ch->>'id')::uuid AS guardianship_external_id, - NULLIF(ch->>'idERN', '') AS guardian_id_ern, - ch->'fioOpek'->>'familiya' AS last_name, - ch->'fioOpek'->>'imya' AS first_name, - ch->'fioOpek'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - ch->'fioOpek'->>'familiya', - ch->'fioOpek'->>'imya', - ch->'fioOpek'->>'otchestvo' - ) AS full_name, - MAKE_DATE( - NULLIF(ch->'dataRozhdDok'->>'god', '')::int, - NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(ch->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, - true AS is_guardian -FROM ervu_dashboard.recruits_info ri -JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id -CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch -WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array'; + WITH guardianship_data AS ( + SELECT + ri.recruit_id AS recruit_id, + (ch->>'id')::uuid AS guardianship_external_id, + NULLIF(ch->>'idERN', '') AS guardian_id_ern, + ch->'fioOpek'->>'familiya' AS last_name, + ch->'fioOpek'->>'imya' AS first_name, + ch->'fioOpek'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + ch->'fioOpek'->>'familiya', + ch->'fioOpek'->>'imya', + ch->'fioOpek'->>'otchestvo' + ) AS full_name, + MAKE_DATE( + NULLIF(ch->'dataRozhdDok'->>'god', '')::int, + NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(ch->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils, + true AS is_guardian, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id + CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch + WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array' +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM guardianship_data) as last_row +FROM guardianship_data; Y diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl index bb0c1b3..8d9cc59 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl @@ -40,11 +40,6 @@ Change job status on success Y - - Table output - Detect empty stream - Y - Detect empty stream Change job status on success @@ -55,24 +50,24 @@ Table output Y - - Update - Filter rows - Y - Update Change job status on error Y - Table output - Identify last row in a stream + Table input + Update Y - Identify last row in a stream - Update + Table input + Detect empty stream + Y + + + Table output + Filter rows Y @@ -92,8 +87,8 @@ 0 - 1168 - 448 + 880 + 624 @@ -128,8 +123,8 @@ and recruitment_id = '${IDM_ID}'; - 1008 - 448 + 720 + 624 @@ -161,8 +156,8 @@ and status = 'PROCESSING'; - 1488 - 624 + 1104 + 160 @@ -207,8 +202,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 1488 - 320 + 336 + 160 @@ -243,33 +238,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1200 - 624 - - - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 720 - 624 + 1104 + 320 Table input TableInput - Y + N 1 @@ -278,23 +255,27 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - - + WITH wanted_data AS ( + SELECT + ri.recruit_id, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code, + TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, + TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, + true AS is_wanted, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id + WHERE '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' +) SELECT - ri.recruit_id, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code, - TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, - TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, - true AS is_wanted -FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id -WHERE'${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' - + *, + source_ctid = (SELECT MAX(source_ctid) FROM wanted_data) as last_row +FROM wanted_data; Y @@ -400,7 +381,7 @@ WHERE'${IDM_ID}' != '' N - 1008 + 336 624 diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl index d4f588b..295d0ff 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl @@ -30,11 +30,6 @@ Change job status on error Y - - Insert / update - Detect empty stream - Y - Detect empty stream Change job status on success @@ -55,23 +50,23 @@ Insert / update Y - - Update - Filter rows - Y - Update Change job status on error Y - Insert / update - Identify last row in a stream + Table input + Detect empty stream Y - Identify last row in a stream + Insert / update + Filter rows + Y + + + Table input Update Y @@ -92,8 +87,8 @@ 0 - 1120 - 384 + 1008 + 480 @@ -128,8 +123,8 @@ and recruitment_id = '${IDM_ID}'; - 960 - 384 + 752 + 480 @@ -161,8 +156,8 @@ and status = 'DELTA_PROCESSING'; - 1280 - 608 + 1008 + 32 @@ -210,8 +205,8 @@ and recruitment_id = '${IDM_ID}'; - 1280 - 224 + 400 + 32 @@ -246,26 +241,8 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1072 - 608 - - - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 752 - 608 + 1008 + 224 @@ -336,7 +313,7 @@ and recruitment_id = '${IDM_ID}'; Table input TableInput - Y + N 1 @@ -345,24 +322,28 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - - + WITH wanted_data AS ( + SELECT + ri.recruit_id, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code, + TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, + TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, + true AS is_wanted, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id + WHERE '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp + AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' +) SELECT - ri.recruit_id, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code, - TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, - TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, - true AS is_wanted -FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id -WHERE'${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp - AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' - + *, + source_ctid = (SELECT MAX(source_ctid) FROM wanted_data) as last_row +FROM wanted_data; Y @@ -406,8 +387,8 @@ WHERE'${IDM_ID}' != '' N - 960 - 608 + 400 + 480 diff --git a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl index a27861e..3d49e52 100644 --- a/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl @@ -45,16 +45,6 @@ Change job status on error Y - - Update - Filter rows - Y - - - Insert / update - Detect empty stream - Y - Insert / update Change job status on error @@ -67,14 +57,19 @@ Insert / update - Identify last row in a stream 2 + Filter rows Y - Identify last row in a stream 2 + Table input Update Y + + Table input + Detect empty stream + Y + Abort @@ -92,8 +87,8 @@ 0 - 1152 - 432 + 1008 + 416 @@ -128,8 +123,8 @@ and recruitment_id = '${IDM_ID}'; - 992 - 432 + 688 + 416 @@ -161,8 +156,8 @@ and status = 'PROCESSING'; - 1312 - 640 + 1024 + 80 @@ -224,8 +219,8 @@ DO UPDATE SET - 1312 - 272 + 416 + 80 @@ -260,26 +255,8 @@ DO UPDATE SET Change job status on success - 1136 - 640 - - - - Identify last row in a stream 2 - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 688 - 640 + 1024 + 272 @@ -350,7 +327,7 @@ DO UPDATE SET Table input TableInput - Y + N 1 @@ -360,22 +337,27 @@ DO UPDATE SET ervu-dashboard N - + WITH wanted_data AS ( + SELECT + ri.recruit_id, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code, + TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, + TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, + true AS is_wanted, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id + WHERE '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' +) SELECT - ri.recruit_id, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code, - TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, - TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, - ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search, - true AS is_wanted // Наличие информации о розыске -FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id -WHERE'${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' - + *, + source_ctid = (SELECT MAX(source_ctid) FROM wanted_data) as last_row +FROM wanted_data; Y @@ -419,8 +401,8 @@ WHERE'${IDM_ID}' != '' N - 992 - 640 + 416 + 416 diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl index 73c9197..d1eae63 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl @@ -35,11 +35,6 @@ Change job status on success Y - - Table output - Detect empty stream - Y - Detect empty stream Change job status on success @@ -50,34 +45,34 @@ Abort Y - - Table output 2 - Filter rows - Y - Table output 2 Change job status on error Y - - Identify last row in a stream - Table output 2 - Y - Table input Table output Y - Table output + Table input Sort rows Y Sort rows - Identify last row in a stream + Table output 2 + Y + + + Table input + Detect empty stream + Y + + + Table output + Filter rows Y @@ -133,7 +128,7 @@ and recruitment_id = '${IDM_ID}'; - 912 + 736 192 @@ -166,8 +161,8 @@ and status = 'PROCESSING'; - 1120 - 64 + 576 + 912 @@ -212,8 +207,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 1120 - 576 + 352 + 912 @@ -249,25 +244,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 736 - 64 - - - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 736 - 288 + 912 @@ -300,15 +277,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 736 - 416 + 352 + 192 Table input TableInput - Y + N 1 @@ -317,45 +294,52 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - - SELECT - ri.recruit_id AS recruit_id, - (supr->>'id')::uuid AS spouse_external_id, - NULLIF(supr->>'idERN', '') AS spouse_id_ern, - NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, - supr->'svedFLBS'->'fio'->>'familiya' AS last_name, - supr->'svedFLBS'->'fio'->>'imya' AS first_name, - supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - supr->'svedFLBS'->'fio'->>'familiya', - supr->'svedFLBS'->'fio'->>'imya', - supr->'svedFLBS'->'fio'->>'otchestvo' - ) AS full_name, - MAKE_DATE( - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, - supr->'svedSmert'->>'nomerZapis' AS death_az_number, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, - CASE - WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true - ELSE false - END AS information_excluded -FROM ervu_dashboard.recruits_info ri -JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id -CROSS JOIN LATERAL ( - SELECT supri - FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri - WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' -) AS supr(supr) -WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'; + WITH spouse_data AS ( + SELECT + ri.recruit_id AS recruit_id, + (supr->>'id')::uuid AS spouse_external_id, + NULLIF(supr->>'idERN', '') AS spouse_id_ern, + NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, + supr->'svedFLBS'->'fio'->>'familiya' AS last_name, + supr->'svedFLBS'->'fio'->>'imya' AS first_name, + supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + supr->'svedFLBS'->'fio'->>'familiya', + supr->'svedFLBS'->'fio'->>'imya', + supr->'svedFLBS'->'fio'->>'otchestvo' + ) AS full_name, + MAKE_DATE( + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, + supr->'svedSmert'->>'nomerZapis' AS death_az_number, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, + CASE + WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true + ELSE false + END AS information_excluded, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id + CROSS JOIN LATERAL ( + SELECT supri + FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri + WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' + ) AS supr(supr) + WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM spouse_data) as last_row +FROM spouse_data; + Y @@ -495,7 +479,7 @@ WHERE Y - 736 + 544 192 diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl index 04adda6..1610236 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl @@ -40,44 +40,39 @@ Change job status on success Y - - Insert / update - Detect empty stream - Y - Detect empty stream Change job status on success Y - - Insert / update 2 - Filter rows - Y - Insert / update 2 Change job status on error Y - - Identify last row in a stream - Insert / update 2 - Y - Table input Insert / update Y - Insert / update + Table input Sort rows Y + + Table input + Detect empty stream + Y + + + Insert / update + Filter rows + Y + Sort rows - Identify last row in a stream + Insert / update 2 Y @@ -136,7 +131,7 @@ and recruitment_id = '${IDM_ID}'; - 960 + 768 176 @@ -169,8 +164,8 @@ WHERE job_name = '${JOB_NAME}' - 1200 - 80 + 1072 + 688 @@ -217,8 +212,8 @@ and recruitment_id = '${IDM_ID}'; - 1200 - 496 + 496 + 688 @@ -253,26 +248,8 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 768 - 80 - - - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 768 - 272 + 1072 + 496 @@ -419,7 +396,7 @@ and recruitment_id = '${IDM_ID}'; N - 768 + 496 176 @@ -453,8 +430,8 @@ and recruitment_id = '${IDM_ID}'; - 768 - 384 + 496 + 320 @@ -471,45 +448,52 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - SELECT - ri.recruit_id AS recruit_id, - (supr->>'id')::uuid AS spouse_external_id, - NULLIF(supr->>'idERN', '') AS spouse_id_ern, - NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, - supr->'svedFLBS'->'fio'->>'familiya' AS last_name, - supr->'svedFLBS'->'fio'->>'imya' AS first_name, - supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - supr->'svedFLBS'->'fio'->>'familiya', - supr->'svedFLBS'->'fio'->>'imya', - supr->'svedFLBS'->'fio'->>'otchestvo' - ) AS full_name, - MAKE_DATE( - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, - supr->'svedSmert'->>'nomerZapis' AS death_az_number, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, - CASE - WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true - ELSE false - END AS information_excluded -FROM ervu_dashboard.recruits_info ri -JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id -CROSS JOIN LATERAL ( - SELECT supri - FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri - WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' -) AS supr(supr) -WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND ri.update_date >= '${M_R_UP_DATE}'::timestamp; + WITH spouse_data AS ( + SELECT + ri.recruit_id AS recruit_id, + (supr->>'id')::uuid AS spouse_external_id, + NULLIF(supr->>'idERN', '') AS spouse_id_ern, + NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, + supr->'svedFLBS'->'fio'->>'familiya' AS last_name, + supr->'svedFLBS'->'fio'->>'imya' AS first_name, + supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + supr->'svedFLBS'->'fio'->>'familiya', + supr->'svedFLBS'->'fio'->>'imya', + supr->'svedFLBS'->'fio'->>'otchestvo' + ) AS full_name, + MAKE_DATE( + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, + supr->'svedSmert'->>'nomerZapis' AS death_az_number, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, + CASE + WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true + ELSE false + END AS information_excluded, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id + CROSS JOIN LATERAL ( + SELECT supri + FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri + WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' + ) AS supr(supr) + WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND ri.update_date >= '${M_R_UP_DATE}'::timestamp +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM spouse_data) as last_row +FROM spouse_data; Y diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl index c6bcf47..59cdc6c 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl @@ -32,8 +32,8 @@ N -1 149 - 528 - 608 + 0 + 64 M_R_UP_DATE @@ -67,44 +67,39 @@ M_R_CR_DATE Change job status on success Y - - Insert / update - Detect empty stream - Y - Detect empty stream Change job status on success Y - - Insert / update 2 - Filter rows - Y - Insert / update 2 Change job status on error Y - - Identify last row in a stream - Insert / update 2 - Y - Table input Insert / update Y - Insert / update + Table input Sort rows Y Sort rows - Identify last row in a stream + Insert / update 2 + Y + + + Insert / update + Filter rows + Y + + + Table input + Detect empty stream Y @@ -163,7 +158,7 @@ and recruitment_id = '${IDM_ID}'; - 912 + 704 224 @@ -198,8 +193,8 @@ and status = 'PROCESSING'; - 1152 - 64 + 944 + 752 @@ -261,8 +256,8 @@ DO UPDATE SET - 1152 - 528 + 352 + 752 @@ -297,26 +292,8 @@ DO UPDATE SET Change job status on success - 704 - 64 - - - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 704 - 320 + 944 + 528 @@ -463,7 +440,7 @@ DO UPDATE SET N - 704 + 352 224 @@ -497,8 +474,8 @@ DO UPDATE SET - 704 - 416 + 352 + 368 @@ -515,44 +492,52 @@ DO UPDATE SET ervu-dashboard N - SELECT - ri.recruit_id AS recruit_id, - (supr->>'id')::uuid AS spouse_external_id, - NULLIF(supr->>'idERN', '') AS spouse_id_ern, - NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, - supr->'svedFLBS'->'fio'->>'familiya' AS last_name, - supr->'svedFLBS'->'fio'->>'imya' AS first_name, - supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - supr->'svedFLBS'->'fio'->>'familiya', - supr->'svedFLBS'->'fio'->>'imya', - supr->'svedFLBS'->'fio'->>'otchestvo' - ) AS full_name, - MAKE_DATE( - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, - supr->'svedSmert'->>'nomerZapis' AS death_az_number, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, - CASE - WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true - ELSE false - END AS information_excluded -FROM ervu_dashboard.recruits_info ri -JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id -CROSS JOIN LATERAL ( - SELECT supri - FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri - WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' -) AS supr(supr) -WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'; + WITH spouse_data AS ( + SELECT + ri.recruit_id AS recruit_id, + (supr->>'id')::uuid AS spouse_external_id, + NULLIF(supr->>'idERN', '') AS spouse_id_ern, + NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, + supr->'svedFLBS'->'fio'->>'familiya' AS last_name, + supr->'svedFLBS'->'fio'->>'imya' AS first_name, + supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + supr->'svedFLBS'->'fio'->>'familiya', + supr->'svedFLBS'->'fio'->>'imya', + supr->'svedFLBS'->'fio'->>'otchestvo' + ) AS full_name, + MAKE_DATE( + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, + supr->'svedSmert'->>'nomerZapis' AS death_az_number, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, + CASE + WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true + ELSE false + END AS information_excluded, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id + CROSS JOIN LATERAL ( + SELECT supri + FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri + WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' + ) AS supr(supr) + WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM spouse_data) as last_row +FROM spouse_data; + Y diff --git a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl index 518830e..6a3b20d 100644 --- a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl @@ -30,26 +30,11 @@ Change job status on error Y - - Table output - Detect empty stream - Y - Detect empty stream Change job status on success Y - - Table input - Identify last row in a stream - Y - - - Identify last row in a stream - Table output - Y - Table output Filter rows @@ -65,6 +50,16 @@ Abort Y + + Table input + Table output + Y + + + Table input + Detect empty stream + Y + Abort @@ -197,7 +192,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 896 + 496 176 @@ -237,29 +232,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 320 - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 640 - 320 - - Table input TableInput - Y + N 1 @@ -269,33 +246,39 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - SELECT - ri.recruit_id, - NULLIF(p.passport_data->>'nomDok', '') AS passport_number, - NULLIF(p.passport_data->>'serDok', '') AS passport_series, - NULLIF(p.passport_data->>'vydDok', '') AS organization_name, - NULLIF(p.passport_data->>'kodVydDok', '') AS unit_code, - TO_DATE(NULLIF(p.passport_data->>'dataDok', ''), 'YYYY-MM-DD') AS issue_date, - CASE - WHEN NULLIF(p.passport_data->>'kodStatus', '') = '1' THEN TRUE - ELSE FALSE - END AS actual -FROM ervu_dashboard.recruits_info ri -JOIN ervu_dashboard.citizen r - ON r.recruit_id = ri.recruit_id - AND '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' -JOIN LATERAL ( - SELECT passport_data - FROM UNNEST(ARRAY[ - ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF', - ri.info->'svedFL'->'svedBS'->'pasportRF'->'predPasportRF' - ]) AS passport_data - WHERE passport_data IS NOT NULL AND passport_data::text <> 'null' -) AS p ON TRUE -WHERE LENGTH(REGEXP_REPLACE(p.passport_data->>'serDok', '[^0-9]', '', 'g')) = 4 and - -LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6 + WITH passport_data AS ( + SELECT + ri.recruit_id, + NULLIF(p.passport_data->>'nomDok', '') AS passport_number, + NULLIF(p.passport_data->>'serDok', '') AS passport_series, + NULLIF(p.passport_data->>'vydDok', '') AS organization_name, + NULLIF(p.passport_data->>'kodVydDok', '') AS unit_code, + TO_DATE(NULLIF(p.passport_data->>'dataDok', ''), 'YYYY-MM-DD') AS issue_date, + CASE + WHEN NULLIF(p.passport_data->>'kodStatus', '') = '1' THEN TRUE + ELSE FALSE + END AS actual, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r + ON r.recruit_id = ri.recruit_id + AND '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + JOIN LATERAL ( + SELECT passport_data + FROM UNNEST(ARRAY[ + ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF', + ri.info->'svedFL'->'svedBS'->'pasportRF'->'predPasportRF' + ]) AS passport_data + WHERE passport_data IS NOT NULL AND passport_data::text <> 'null' + ) AS p ON TRUE + WHERE LENGTH(REGEXP_REPLACE(p.passport_data->>'serDok', '[^0-9]', '', 'g')) = 4 + AND LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6 +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM passport_data) as last_row +FROM passport_data; Y diff --git a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl index 77eff60..da8b0ca 100644 --- a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl @@ -30,26 +30,11 @@ Change job status on error Y - - Insert / update - Detect empty stream - Y - Detect empty stream Change job status on success Y - - Table input - Identify last row in a stream - Y - - - Identify last row in a stream - Insert / update - Y - Insert / update Filter rows @@ -65,6 +50,16 @@ Abort Y + + Table input + Detect empty stream + Y + + + Table input + Insert / update + Y + Abort @@ -200,7 +195,7 @@ and recruitment_id = '${IDM_ID}'; - 752 + 400 80 @@ -240,24 +235,6 @@ and recruitment_id = '${IDM_ID}'; 224 - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 560 - 224 - - Insert / update InsertUpdate @@ -331,7 +308,7 @@ and recruitment_id = '${IDM_ID}'; Table input TableInput - Y + N 1 @@ -341,36 +318,40 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - SELECT - ri.recruit_id, - NULLIF(p.passport_data->>'nomDok', '') AS passport_number, - NULLIF(p.passport_data->>'serDok', '') AS passport_series, - NULLIF(p.passport_data->>'vydDok', '') AS organization_name, - NULLIF(p.passport_data->>'kodVydDok', '') AS unit_code, - TO_DATE(NULLIF(p.passport_data->>'dataDok', ''), 'YYYY-MM-DD') AS issue_date, - CASE - WHEN NULLIF(p.passport_data->>'kodStatus', '') = '1' THEN TRUE - ELSE FALSE - END AS actual -FROM ervu_dashboard.recruits_info ri -JOIN ervu_dashboard.citizen r - ON r.recruit_id = ri.recruit_id - AND '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date - AND r.update_date >= '${M_R_UP_DATE}'::timestamp -JOIN LATERAL ( - SELECT passport_data - FROM UNNEST(ARRAY[ - ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF', - ri.info->'svedFL'->'svedBS'->'pasportRF'->'predPasportRF' - ]) AS passport_data - WHERE passport_data IS NOT NULL AND passport_data::text <> 'null' -) AS p ON TRUE -WHERE LENGTH(REGEXP_REPLACE(p.passport_data->>'serDok', '[^0-9]', '', 'g')) = 4 and - -LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6 - + WITH passport_data AS ( + SELECT + ri.recruit_id, + NULLIF(p.passport_data->>'nomDok', '') AS passport_number, + NULLIF(p.passport_data->>'serDok', '') AS passport_series, + NULLIF(p.passport_data->>'vydDok', '') AS organization_name, + NULLIF(p.passport_data->>'kodVydDok', '') AS unit_code, + TO_DATE(NULLIF(p.passport_data->>'dataDok', ''), 'YYYY-MM-DD') AS issue_date, + CASE + WHEN NULLIF(p.passport_data->>'kodStatus', '') = '1' THEN TRUE + ELSE FALSE + END AS actual, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r + ON r.recruit_id = ri.recruit_id + AND '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND r.update_date >= '${M_R_UP_DATE}'::timestamp + JOIN LATERAL ( + SELECT passport_data + FROM UNNEST(ARRAY[ + ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF', + ri.info->'svedFL'->'svedBS'->'pasportRF'->'predPasportRF' + ]) AS passport_data + WHERE passport_data IS NOT NULL AND passport_data::text <> 'null' + ) AS p ON TRUE + WHERE LENGTH(REGEXP_REPLACE(p.passport_data->>'serDok', '[^0-9]', '', 'g')) = 4 + AND LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6 +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM passport_data) as last_row +FROM passport_data; Y diff --git a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl index 8631add..ae8a3b5 100644 --- a/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl @@ -30,11 +30,6 @@ Change job status on error Y - - Insert / update - Detect empty stream - Y - Detect empty stream Change job status on success @@ -50,21 +45,21 @@ Change job status on success Y - - Table input - Identify last row in a stream - Y - - - Identify last row in a stream - Insert / update - Y - Change job status on error Abort Y + + Table input + Detect empty stream + Y + + + Table input + Insert / update + Y + Abort @@ -214,7 +209,7 @@ DO UPDATE SET - 768 + 416 16 @@ -254,24 +249,6 @@ DO UPDATE SET 272 - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 608 - 272 - - Insert / update InsertUpdate @@ -345,7 +322,7 @@ DO UPDATE SET Table input TableInput - Y + N 1 @@ -355,34 +332,39 @@ DO UPDATE SET ervu-dashboard N - SELECT - ri.recruit_id, - NULLIF(p.passport_data->>'nomDok', '') AS passport_number, - NULLIF(p.passport_data->>'serDok', '') AS passport_series, - NULLIF(p.passport_data->>'vydDok', '') AS organization_name, - NULLIF(p.passport_data->>'kodVydDok', '') AS unit_code, - TO_DATE(NULLIF(p.passport_data->>'dataDok', ''), 'YYYY-MM-DD') AS issue_date, - CASE - WHEN NULLIF(p.passport_data->>'kodStatus', '') = '1' THEN TRUE - ELSE FALSE - END AS actual -FROM ervu_dashboard.recruits_info ri -JOIN ervu_dashboard.citizen r - ON r.recruit_id = ri.recruit_id - AND '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' -JOIN LATERAL ( - SELECT passport_data - FROM UNNEST(ARRAY[ - ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF', - ri.info->'svedFL'->'svedBS'->'pasportRF'->'predPasportRF' - ]) AS passport_data - WHERE passport_data IS NOT NULL AND passport_data::text <> 'null' -) AS p ON TRUE -WHERE LENGTH(REGEXP_REPLACE(p.passport_data->>'serDok', '[^0-9]', '', 'g')) = 4 and - -LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6 - + WITH passport_data AS ( + SELECT + ri.recruit_id, + NULLIF(p.passport_data->>'nomDok', '') AS passport_number, + NULLIF(p.passport_data->>'serDok', '') AS passport_series, + NULLIF(p.passport_data->>'vydDok', '') AS organization_name, + NULLIF(p.passport_data->>'kodVydDok', '') AS unit_code, + TO_DATE(NULLIF(p.passport_data->>'dataDok', ''), 'YYYY-MM-DD') AS issue_date, + CASE + WHEN NULLIF(p.passport_data->>'kodStatus', '') = '1' THEN TRUE + ELSE FALSE + END AS actual, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r + ON r.recruit_id = ri.recruit_id + AND '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + JOIN LATERAL ( + SELECT passport_data + FROM UNNEST(ARRAY[ + ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF', + ri.info->'svedFL'->'svedBS'->'pasportRF'->'predPasportRF' + ]) AS passport_data + WHERE passport_data IS NOT NULL AND passport_data::text <> 'null' + ) AS p ON TRUE + WHERE LENGTH(REGEXP_REPLACE(p.passport_data->>'serDok', '[^0-9]', '', 'g')) = 4 + AND LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6 +) +SELECT + *, + source_ctid = (SELECT MAX(source_ctid) FROM passport_data) as last_row +FROM passport_data; Y diff --git a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl index bc4ab1e..43d2f36 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl @@ -40,11 +40,6 @@ Change job status on success Y - - Table output - Detect empty stream - Y - Detect empty stream Change job status on success @@ -55,29 +50,29 @@ Table output Y - - Update - Filter rows - Y - Update Change job status on error Y - Identify last row in a stream - Update - Y - - - Table output + Table input Sort rows Y Sort rows - Identify last row in a stream + Update + Y + + + Table input + Detect empty stream + Y + + + Table output + Filter rows Y @@ -133,7 +128,7 @@ and recruitment_id = '${IDM_ID}'; - 1056 + 848 224 @@ -166,8 +161,8 @@ and status = 'PROCESSING'; - 1456 - 32 + 1072 + 640 @@ -212,8 +207,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 1456 - 464 + 480 + 640 @@ -248,26 +243,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1200 - 32 - - - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 832 - 32 + 1072 + 464 @@ -300,15 +277,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 832 - 224 + 480 + 32 Table input TableInput - Y + N 1 @@ -318,14 +295,14 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - WITH -ugolov_presl AS ( + WITH ugolov_presl AS ( SELECT jsonb_array_elements(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') AS up, - ri.recruit_id + ri.recruit_id, + ri.ctid as source_ctid FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - AND '${IDM_ID}' != '' -- Проверка на пустую строку + AND '${IDM_ID}' != '' AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' WHERE jsonb_typeof(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') = 'array' ) @@ -358,11 +335,12 @@ SELECT WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде' ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей' END AS type_info, - true AS has_criminal_prosecution + true AS has_criminal_prosecution, + up.source_ctid = (SELECT MAX(source_ctid) FROM ugolov_presl) as last_row FROM ervu_dashboard.citizen r - JOIN ugolov_presl up ON r.recruit_id = up.recruit_id - LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true -GROUP BY r.recruit_id, up; +JOIN ugolov_presl up ON r.recruit_id = up.recruit_id +LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true +GROUP BY r.recruit_id, up.up, up.source_ctid; Y @@ -484,7 +462,7 @@ GROUP BY r.recruit_id, up; Y - 832 + 848 464 @@ -524,7 +502,7 @@ GROUP BY r.recruit_id, up; Y - 1056 + 848 32 diff --git a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl index a5e2e1f..a912ec0 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl @@ -30,11 +30,6 @@ Change job status on error Y - - Insert / update - Detect empty stream - Y - Detect empty stream Change job status on success @@ -50,11 +45,6 @@ Abort Y - - Update - Filter rows - Y - Update Change job status on error @@ -66,18 +56,23 @@ Y - Insert / update + Sort rows + Update + Y + + + Table input Sort rows Y - Sort rows - Identify last row in a stream 2 + Insert / update + Filter rows Y - Identify last row in a stream 2 - Update + Table input + Detect empty stream Y @@ -133,7 +128,7 @@ and recruitment_id = '${IDM_ID}'; - 880 + 736 592 @@ -166,8 +161,8 @@ and status = 'DELTA_PROCESSING'; - 1152 - 416 + 896 + 944 @@ -215,8 +210,8 @@ and recruitment_id = '${IDM_ID}'; - 1152 - 768 + 384 + 944 @@ -251,26 +246,8 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1024 - 416 - - - - Identify last row in a stream 2 - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 736 - 416 + 896 + 768 @@ -442,15 +419,15 @@ and recruitment_id = '${IDM_ID}'; - 736 - 560 + 384 + 416 Table input TableInput - Y + N 1 @@ -460,53 +437,53 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH -ugolov_presl AS ( - SELECT - jsonb_array_elements(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') AS up, - ri.recruit_id - --'prOtsSvedUgolovPresl' - FROM ervu_dashboard.recruits_info ri - join ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - AND '${IDM_ID}' != '' -- Проверка на пустую строку - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp - WHERE jsonb_typeof(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') = 'array' + WITH ugolov_presl AS ( + SELECT + jsonb_array_elements(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') AS up, + ri.recruit_id, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id + AND '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp + WHERE jsonb_typeof(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') = 'array' ) SELECT r.recruit_id AS recruit_id, (up->>'id')::uuid AS prosecution_external_id, - NULLIF(up->'sledSud'->>'uk', '') AS uk_name, -- УК осуждения - NULLIF(up->'sledSud'->'extend'->>'nomerUDPredv', '') AS case_number, -- номер уголовного дела (предварительный) - NULLIF(up->'sledSud'->'extend'->>'svedPerekvalif', '') AS reclassification, -- сведения о переквалификации деяния - NULLIF(up->'sledSud'->'extend'->>'naimVidUgolovPresl', '') AS kind, -- наименование вида уголовного преследования + NULLIF(up->'sledSud'->>'uk', '') AS uk_name, + NULLIF(up->'sledSud'->'extend'->>'nomerUDPredv', '') AS case_number, + NULLIF(up->'sledSud'->'extend'->>'svedPerekvalif', '') AS reclassification, + NULLIF(up->'sledSud'->'extend'->>'naimVidUgolovPresl', '') AS kind, CASE - WHEN up->'sledSud'->'extend'->>'priznakPeredachSud' = '1' THEN true -- передано в суд + WHEN up->'sledSud'->'extend'->>'priznakPeredachSud' = '1' THEN true ELSE false -- нет или нет инфы - END AS transfer_to_court, -- признак передачи в суд уголовного дела в отношении гражданина - NULLIF(up->'sledSud'->>'naimSud','') AS court_name, -- Наименование суда, вынесшего приговор - NULLIF(up->'sledSud'->>'nomerUD','') AS case_number_court, -- номер уголовного дела (судебный) - STRING_AGG(TRIM(BOTH ' "' FROM law.f), ', ') AS law_point, -- статьи уголовного кодекса - NULLIF(up->'sledSud'->>'naimOrgan', '') AS authority_name, -- наименование органа, принявшего решение о возбуждении уголовного дела - NULLIF(up->'sledSud'->'svedSnyat'->>'osnSnyatSud', '') AS conviction_removal_basis, -- основание снятия судимости - TO_DATE(up->'sledSud'->'svedSnyat'->>'dataSnyatSud', 'YYYY-MM-DD') AS conviction_removal_date, -- дата снятия судимости - TO_DATE(up->'sledSud'->>'dataOsuzhd', 'YYYY-MM-DD') AS conviction_date, -- дата осуждения - TO_DATE(up->'sledSud'->>'dataVozbUD', 'YYYY-MM-DD') AS create_date, -- дата возбуждения уголовного дела - NULLIF(up->'sledSud'->>'osnPrekrUD', '') AS termination_basis, -- основания прекращения уголовного дела - NULLIF(up->'sledSud'->>'statusNaim', '') AS conviction_status, -- статус судимости - TO_DATE(up->'sledSud'->>'dataPrekrUD', 'YYYY-MM-DD') AS close_date, -- дата прекращения уголовного дела - NULLIF(up->'sledSudTekst'->>'svedTekst' , '') AS sled_sud_tekst_sved_tekst, -- текст судимости в текстовом формате - NULLIF(up->'sledSudTekst'->>'statusNaim', '') AS sled_sud_tekst_status_naim, -- статус судимости в текстовом формате + END AS transfer_to_court, + NULLIF(up->'sledSud'->>'naimSud','') AS court_name, + NULLIF(up->'sledSud'->>'nomerUD','') AS case_number_court, + STRING_AGG(TRIM(BOTH ' "' FROM law.f), ', ') AS law_point, + NULLIF(up->'sledSud'->>'naimOrgan', '') AS authority_name, + NULLIF(up->'sledSud'->'svedSnyat'->>'osnSnyatSud', '') AS conviction_removal_basis, + TO_DATE(up->'sledSud'->'svedSnyat'->>'dataSnyatSud', 'YYYY-MM-DD') AS conviction_removal_date, + TO_DATE(up->'sledSud'->>'dataOsuzhd', 'YYYY-MM-DD') AS conviction_date, + TO_DATE(up->'sledSud'->>'dataVozbUD', 'YYYY-MM-DD') AS create_date, + NULLIF(up->'sledSud'->>'osnPrekrUD', '') AS termination_basis, + NULLIF(up->'sledSud'->>'statusNaim', '') AS conviction_status, + TO_DATE(up->'sledSud'->>'dataPrekrUD', 'YYYY-MM-DD') AS close_date, + NULLIF(up->'sledSudTekst'->>'svedTekst' , '') AS sled_sud_tekst_sved_tekst, + NULLIF(up->'sledSudTekst'->>'statusNaim', '') AS sled_sud_tekst_status_naim, CASE WHEN (up->>'sledSud') IS NOT NULL AND up->>'sledSud' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в структурированном формате' WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде' ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей' END AS type_info, - true AS has_criminal_prosecution + true AS has_criminal_prosecution, + up.source_ctid = (SELECT MAX(source_ctid) FROM ugolov_presl) as last_row FROM ervu_dashboard.citizen r - JOIN ugolov_presl up ON r.recruit_id = up.recruit_id - LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true -GROUP BY r.recruit_id, up; +JOIN ugolov_presl up ON r.recruit_id = up.recruit_id +LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true +GROUP BY r.recruit_id, up.up, up.source_ctid; Y @@ -550,7 +527,7 @@ GROUP BY r.recruit_id, up; Y - 880 + 736 416 diff --git a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl index 7ac6a12..3b9f0a6 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl @@ -30,11 +30,6 @@ Change job status on error Y - - Insert / update - Detect empty stream - Y - Detect empty stream Change job status on success @@ -55,29 +50,29 @@ Insert / update Y - - Update - Filter rows - Y - Update Change job status on error Y - Insert / update + Sort rows + Update + Y + + + Table input Sort rows Y - Sort rows - Identify last row in a stream + Table input + Detect empty stream Y - Identify last row in a stream - Update + Insert / update + Filter rows Y @@ -133,7 +128,7 @@ and recruitment_id = '${IDM_ID}'; - 944 + 768 736 @@ -166,8 +161,8 @@ and status = 'PROCESSING'; - 1296 - 576 + 928 + 1088 @@ -229,8 +224,8 @@ DO UPDATE SET - 1296 - 944 + 432 + 1088 @@ -265,26 +260,8 @@ DO UPDATE SET Change job status on success - 1088 - 576 - - - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 768 - 576 + 928 + 944 @@ -456,15 +433,15 @@ DO UPDATE SET - 768 - 768 + 432 + 576 Table input TableInput - Y + N 1 @@ -474,20 +451,20 @@ DO UPDATE SET ervu-dashboard N - WITH -ugolov_presl AS ( + WITH ugolov_presl AS ( SELECT jsonb_array_elements(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') AS up, - ri.recruit_id + ri.recruit_id, + ri.ctid as source_ctid FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - AND '${IDM_ID}' != '' -- Проверка на пустую строку + AND '${IDM_ID}' != '' AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' WHERE jsonb_typeof(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') = 'array' ) SELECT r.recruit_id AS recruit_id, - (up->>'id')::uuid AS prosecution_external_id, + (up->>'id')::uuid AS prosecution_external_id, NULLIF(up->'sledSud'->>'uk', '') AS uk_name, NULLIF(up->'sledSud'->'extend'->>'nomerUDPredv', '') AS case_number, NULLIF(up->'sledSud'->'extend'->>'svedPerekvalif', '') AS reclassification, @@ -502,7 +479,7 @@ SELECT NULLIF(up->'sledSud'->>'naimOrgan', '') AS authority_name, NULLIF(up->'sledSud'->'svedSnyat'->>'osnSnyatSud', '') AS conviction_removal_basis, TO_DATE(up->'sledSud'->'svedSnyat'->>'dataSnyatSud', 'YYYY-MM-DD') AS conviction_removal_date, - TO_DATE(up->'sledSud'->>'dataOsuzhd', 'YYYY-MM-DD') AS conviction_date, + TO_DATE(up->'sledSud'->>'dataOsuzhd', 'YYYY-MM-DD') AS conviction_date, TO_DATE(up->'sledSud'->>'dataVozbUD', 'YYYY-MM-DD') AS create_date, NULLIF(up->'sledSud'->>'osnPrekrUD', '') AS termination_basis, NULLIF(up->'sledSud'->>'statusNaim', '') AS conviction_status, @@ -514,15 +491,16 @@ SELECT WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде' ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей' END AS type_info, - true AS has_criminal_prosecution + true AS has_criminal_prosecution, + up.source_ctid = (SELECT MAX(source_ctid) FROM ugolov_presl) as last_row FROM ervu_dashboard.citizen r - JOIN ugolov_presl up ON r.recruit_id = up.recruit_id - LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true -GROUP BY r.recruit_id, up; +JOIN ugolov_presl up ON r.recruit_id = up.recruit_id +LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true +GROUP BY r.recruit_id, up.up, up.source_ctid; Y - 416 + 432 944 @@ -562,7 +540,7 @@ GROUP BY r.recruit_id, up; Y - 976 + 768 576 diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl index fcb969c..4093e72 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl @@ -40,11 +40,6 @@ Change job status on success Y - - Table output - Detect empty stream - Y - Detect empty stream Change job status on success @@ -55,29 +50,29 @@ Table output Y - - Update - Filter rows - Y - Update Change job status on error Y - Table output + Sort rows + Update + Y + + + Table input Sort rows Y - Sort rows - Identify last row in a stream + Table input + Detect empty stream Y - Identify last row in a stream - Update + Table output + Filter rows Y @@ -166,8 +161,8 @@ and status = 'PROCESSING'; - 1488 - 624 + 544 + 144 @@ -212,8 +207,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 1488 - 320 + 336 + 144 @@ -247,27 +242,9 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - - 1200 - 624 - - - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - 720 - 624 + 144 @@ -300,15 +277,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 720 - 464 + 336 + 624 Table input TableInput - Y + N 1 @@ -319,16 +296,18 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); N WITH punishment_data AS ( - SELECT ri.recruit_id, - jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - AND '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null' + SELECT + ri.recruit_id, + jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id + AND '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null' ) SELECT recruit_id, - (punishment_info ->> 'id')::uuid AS punishment_external_id, + (punishment_info ->> 'id')::uuid AS punishment_external_id, -- Блок "Сведения о вынесенном приговоре" punishment_info ->> 'vidNakaz' AS punishment_type, -- Вид наказания punishment_info ->> 'kodVidNakaz' AS punishment_code, -- Код вида наказания @@ -347,7 +326,7 @@ SELECT recruit_id, punishment_info ->> 'naimOrg' AS facility_name, -- Название учреждения punishment_info ->> 'adresOrg' AS facility_address, -- Адрес учреждения punishment_info ->> 'nomerUD' AS criminal_case_number, -- Номер уголовного дела - TO_DATE(punishment_info ->> 'dataNachSrok', 'YYYY-MM-DD') AS start_date, -- Дата начала срока + TO_DATE(punishment_info ->> 'dataNachSrok', 'YYYY-MM-DD') AS start_date, -- Дата начала срока TO_DATE(punishment_info ->> 'dataOkonchSrok', 'YYYY-MM-DD') AS planned_date, -- Планируемая дата освобождения CASE WHEN punishment_info ->> 'kodPerekvalif' = '1' THEN 'Изменений статьи не было' @@ -357,10 +336,11 @@ SELECT recruit_id, punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения true AS has_punishment_execution_info, - true AS has_sentence_info + true AS has_sentence_info, + source_ctid = (SELECT MAX(source_ctid) FROM punishment_data) as last_row FROM punishment_data LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true -GROUP BY recruit_id, punishment_info; +GROUP BY recruit_id, punishment_info,source_ctid; Y diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl index 40a91fc..7d68e9d 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl @@ -49,11 +49,6 @@ Change job status on error Y - - Insert / update - Detect empty stream - Y - Detect empty stream Change job status on success @@ -74,31 +69,31 @@ Insert / update Y - - Update - Filter rows - Y - Update Change job status on error Y - Insert / update + Table input Sort rows Y - Sort rows - Identify last row in a stream + Table input + Detect empty stream Y - Identify last row in a stream + Sort rows Update Y + + Insert / update + Filter rows + Y + Abort @@ -185,8 +180,8 @@ and status = 'DELTA_PROCESSING'; - 1280 - 608 + 608 + 32 @@ -234,8 +229,8 @@ and recruitment_id = '${IDM_ID}'; - 1280 - 224 + 400 + 32 @@ -270,26 +265,8 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1072 - 608 - - - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 752 - 608 + 784 + 32 @@ -417,7 +394,7 @@ and recruitment_id = '${IDM_ID}'; N - 752 + 784 224 @@ -451,15 +428,15 @@ and recruitment_id = '${IDM_ID}'; - 752 - 384 + 400 + 608 Table input TableInput - Y + N 1 @@ -468,19 +445,20 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH punishment_data AS ( - SELECT ri.recruit_id, - jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - AND '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp - WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null' + SELECT + ri.recruit_id, + jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id + AND '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp + WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null' ) SELECT recruit_id, - (punishment_info ->> 'id')::uuid AS punishment_external_id, + (punishment_info ->> 'id')::uuid AS punishment_external_id, -- Блок "Сведения о вынесенном приговоре" punishment_info ->> 'vidNakaz' AS punishment_type, -- Вид наказания punishment_info ->> 'kodVidNakaz' AS punishment_code, -- Код вида наказания @@ -492,7 +470,7 @@ SELECT recruit_id, WHEN punishment_info ->> 'kodIzmSrokNakaz' = '2' THEN 'Были изменения срока наказания' ELSE punishment_info ->> 'kodIzmSrokNakaz' END AS sentence_changed, -- Признак изменения срока - STRING_AGG(TRIM(BOTH ' "' FROM law_articles), ', ') AS law_point, -- Статьи УК + STRING_AGG(TRIM(BOTH ' "' FROM law_articles), ', ') AS law_point, -- Статьи УК -- Блок "Сведения об отбывании наказания" punishment_info ->> 'katUcheta' AS registration_category, -- Категория учета punishment_info ->> 'naimSubekt' AS region_name, -- Наименование субъекта @@ -509,10 +487,11 @@ SELECT recruit_id, punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения true AS has_punishment_execution_info, - true AS has_sentence_info + true AS has_sentence_info, + source_ctid = (SELECT MAX(source_ctid) FROM punishment_data) as last_row FROM punishment_data LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true -GROUP BY recruit_id, punishment_info; +GROUP BY recruit_id, punishment_info,source_ctid; Y diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl index 3cc4020..2cea196 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl @@ -45,16 +45,6 @@ Change job status on error Y - - Update - Filter rows - Y - - - Insert / update - Detect empty stream - Y - Insert / update Change job status on error @@ -66,18 +56,23 @@ Y - Insert / update + Table input Sort rows Y Sort rows - Identify last row in a stream 2 + Update Y - Identify last row in a stream 2 - Update + Table input + Detect empty stream + Y + + + Insert / update + Filter rows Y @@ -166,8 +161,8 @@ and status = 'PROCESSING'; - 1312 - 640 + 640 + 112 @@ -229,8 +224,8 @@ DO UPDATE SET - 1312 - 272 + 416 + 112 @@ -265,26 +260,8 @@ DO UPDATE SET Change job status on success - 1136 - 640 - - - - Identify last row in a stream 2 - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 688 - 640 + 816 + 112 @@ -412,7 +389,7 @@ DO UPDATE SET N - 688 + 816 272 @@ -446,15 +423,15 @@ DO UPDATE SET - 688 - 448 + 416 + 640 Table input TableInput - Y + N 1 @@ -465,16 +442,18 @@ DO UPDATE SET N WITH punishment_data AS ( - SELECT ri.recruit_id, - jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - AND '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null' + SELECT + ri.recruit_id, + jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info, + ri.ctid as source_ctid + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id + AND '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null' ) SELECT recruit_id, - (punishment_info ->> 'id')::uuid AS punishment_external_id, + (punishment_info ->> 'id')::uuid AS punishment_external_id, -- Блок "Сведения о вынесенном приговоре" punishment_info ->> 'vidNakaz' AS punishment_type, -- Вид наказания punishment_info ->> 'kodVidNakaz' AS punishment_code, -- Код вида наказания @@ -486,7 +465,7 @@ SELECT recruit_id, WHEN punishment_info ->> 'kodIzmSrokNakaz' = '2' THEN 'Были изменения срока наказания' ELSE punishment_info ->> 'kodIzmSrokNakaz' END AS sentence_changed, -- Признак изменения срока - STRING_AGG(TRIM(BOTH ' "' FROM law_articles), ', ') AS law_point, -- Статьи УК + STRING_AGG(TRIM(BOTH ' "' FROM law_articles), ', ') AS law_point, -- Статьи УК -- Блок "Сведения об отбывании наказания" punishment_info ->> 'katUcheta' AS registration_category, -- Категория учета punishment_info ->> 'naimSubekt' AS region_name, -- Наименование субъекта @@ -503,10 +482,11 @@ SELECT recruit_id, punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения true AS has_punishment_execution_info, - true AS has_sentence_info + true AS has_sentence_info, + source_ctid = (SELECT MAX(source_ctid) FROM punishment_data) as last_row FROM punishment_data LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true -GROUP BY recruit_id, punishment_info; +GROUP BY recruit_id, punishment_info,source_ctid; Y