diff --git a/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow.hpl b/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow.hpl index 24fa6cb..516e4a2 100644 --- a/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow.hpl +++ b/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow.hpl @@ -50,26 +50,6 @@ Change job status on error Y - - Table input - Identify last row in a stream - Y - - - Identify last row in a stream - Table output - Y - - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - sort_by_recruit_id - Y - sort_by_recruit_id Update @@ -80,6 +60,21 @@ Filter rows Y + + Table input + Detect empty stream + Y + + + Table input + sort_by_recruit_id + Y + + + Table input + Table output + Y + Abort @@ -97,7 +92,7 @@ 0 - 1328 + 1056 496 @@ -133,7 +128,7 @@ and recruitment_id = '${IDM_ID}'; - 1168 + 880 496 @@ -165,7 +160,7 @@ and recruitment_id = '${IDM_ID}'; - 1392 + 1056 176 @@ -211,7 +206,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 688 + 496 176 @@ -247,13 +242,13 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1392 + 1056 320 - Identify last row in a stream - DetectLastRow + Table input + TableInput N @@ -262,63 +257,32 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); none - last_row - - - 688 - 320 - - - - Table input - TableInput - - Y - - 1 - - none - - ervu_person_registry N - WITH -part_recruits AS ( - SELECT - id - FROM public.recruits - WHERE '${IDM_ID}' != '' -- Проверка на пустую строку - AND (current_recruitment_id = '${IDM_ID}' or (current_recruitment_id is null and target_recruitment_id = '${IDM_ID}')) -), -p_recruits_deferment AS ( - SELECT - d.id AS deferment_id, - d.recruit_id, - rd.code AS base_deferment, -- коды основания права на отсрочку - d.postponement_date, -- Дата окончания действия права на отсрочку - d.date_of_information, -- Дата начала действия права на отсрочку - d.hidden, - d.system_create_date, - d.system_update_date - FROM public.recruits_deferment d - JOIN public.ervu_reason_for_deferment rd ON d.ervu_reason_for_deferment_id = rd.id -) -SELECT - d.deferment_id, + + with filtered_data as (SELECT + d.id AS deferment_id, d.recruit_id, - d.base_deferment, -- коды основания права на отсрочку - d.postponement_date, -- Дата окончания действия права на отсрочку - d.date_of_information, -- Дата начала действия права на отсрочку + rd.code AS base_deferment, -- коды основания права на отсрочку + d.postponement_date, -- Дата окончания действия права на отсрочку + d.date_of_information, -- Дата начала действия права на отсрочку d.hidden, d.system_create_date AS created_at, d.system_update_date AS updated_at, - 'permission' AS permission_fact, -- ключ принадлежности к праву на отсрочку - CASE - WHEN d.hidden is false THEN true - ELSE false - END AS defer_conscription -- наличие права на отсрочку от призыва -FROM part_recruits r -JOIN p_recruits_deferment d ON r.id = d.recruit_id + 'permission' AS permission_fact, -- ключ принадлежности к праву на отсрочку + NOT d.hidden AS defer_conscription, -- наличие права на отсрочку от призыва + CTID +FROM public.recruits r + JOIN public.recruits_deferment d + ON r.id = d.recruit_id + JOIN public.ervu_reason_for_deferment rd + ON d.ervu_reason_for_deferment_id = rd.id +WHERE (r.current_recruitment_id = '${IDM_ID}' + OR (r.current_recruitment_id IS NULL AND r.target_recruitment_id = '${IDM_ID}') + )) +select fd.*, + fd.ctid = (select max(ctid) from filtered_data) as last_row + from filtered_data fd Y @@ -392,7 +356,7 @@ JOIN p_recruits_deferment d ON r.id = d.recruit_id Y - 912 + 672 320 @@ -427,7 +391,7 @@ JOIN p_recruits_deferment d ON r.id = d.recruit_id Y - 928 + 672 496 @@ -461,7 +425,7 @@ JOIN p_recruits_deferment d ON r.id = d.recruit_id - 688 + 496 496 diff --git a/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_delta.hpl b/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_delta.hpl index 19d2b6a..58cba80 100644 --- a/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_delta.hpl @@ -50,26 +50,6 @@ Change job status on error Y - - Table input - Identify last row in a stream - Y - - - Identify last row in a stream - Insert / update - Y - - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - sort_by_recruit_id - Y - sort_by_recruit_id Update @@ -80,6 +60,21 @@ Filter rows Y + + Table input + Detect empty stream + Y + + + Table input + sort_by_recruit_id + Y + + + Table input + Insert / update + Y + Abort @@ -97,7 +92,7 @@ 0 - 1392 + 1200 528 @@ -133,7 +128,7 @@ and recruitment_id = '${IDM_ID}'; - 1200 + 1008 528 @@ -165,7 +160,7 @@ and recruitment_id = '${IDM_ID}'; - 1440 + 1008 208 @@ -214,7 +209,7 @@ and recruitment_id = '${IDM_ID}'; - 752 + 560 208 @@ -250,25 +245,7 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1440 - 352 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 752 + 1008 352 @@ -342,7 +319,7 @@ and recruitment_id = '${IDM_ID}'; N - 960 + 784 352 @@ -350,7 +327,7 @@ and recruitment_id = '${IDM_ID}'; Table input TableInput - Y + N 1 @@ -359,44 +336,33 @@ and recruitment_id = '${IDM_ID}'; ervu_person_registry N - WITH -part_recruits AS ( - SELECT - id - FROM public.recruits - WHERE '${IDM_ID}' != '' -- Проверка на пустую строку - AND (current_recruitment_id = '${IDM_ID}' or (current_recruitment_id is null and target_recruitment_id = '${IDM_ID}')) -), -p_recruits_deferment AS ( - SELECT - d.id AS deferment_id, - d.recruit_id, - rd.code AS base_deferment, -- коды основания права на отсрочку - d.postponement_date, -- Дата окончания действия права на отсрочку - d.date_of_information, -- Дата начала действия права на отсрочку - d.hidden, - d.system_create_date, - d.system_update_date - FROM public.recruits_deferment d - JOIN public.ervu_reason_for_deferment rd ON d.ervu_reason_for_deferment_id = rd.id - WHERE d.system_update_date >= '${M_RECDEF_UP_DATE}'::timestamp -) -SELECT - d.deferment_id, - d.recruit_id, - d.base_deferment, -- коды основания права на отсрочку - d.postponement_date, -- Дата окончания действия права на отсрочку - d.date_of_information, -- Дата начала действия права на отсрочку - d.hidden, - d.system_create_date AS created_at, - d.system_update_date AS updated_at, - 'permission' AS permission_fact, -- ключ принадлежности к праву на отсрочку - CASE - WHEN d.hidden is false THEN true - ELSE false - END AS defer_conscription -- наличие права на отсрочку от призыва -FROM part_recruits r -JOIN p_recruits_deferment d ON r.id = d.recruit_id + + with filtered_data as (SELECT + d.id AS deferment_id, + d.recruit_id, + rd.code AS base_deferment, -- коды основания права на отсрочку + d.postponement_date, -- Дата окончания действия права на отсрочку + d.date_of_information, -- Дата начала действия права на отсрочку + d.hidden, + d.system_create_date AS created_at, + d.system_update_date AS updated_at, + 'permission' AS permission_fact, -- ключ принадлежности к праву на отсрочку + NOT d.hidden AS defer_conscription, -- наличие права на отсрочку от призыва + CTID + FROM public.recruits r + JOIN public.recruits_deferment d + ON r.id = d.recruit_id + JOIN public.ervu_reason_for_deferment rd + ON d.ervu_reason_for_deferment_id = rd.id + WHERE (r.current_recruitment_id = '${IDM_ID}' + OR (r.current_recruitment_id IS NULL AND r.target_recruitment_id = '${IDM_ID}') + ) + and case when '${MAX_SOURCE_UPDATE_DATE}' <> '' + then d.system_update_date > to_date('${MAX_SOURCE_UPDATE_DATE}', 'YYYY-MM-DD') + else true end ) +select fd.*, + fd.ctid = (select max(ctid) from filtered_data) as last_row +from filtered_data fd Y @@ -440,7 +406,7 @@ JOIN p_recruits_deferment d ON r.id = d.recruit_id Y - 976 + 784 528 @@ -474,7 +440,7 @@ JOIN p_recruits_deferment d ON r.id = d.recruit_id - 752 + 560 528 diff --git a/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_repeat.hpl index ba88601..cec0779 100644 --- a/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_repeat.hpl @@ -50,26 +50,6 @@ Change job status on error Y - - Table input - Identify last row in a stream - Y - - - Identify last row in a stream - Insert / update - Y - - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - sort_by_recruit_id - Y - sort_by_recruit_id Update @@ -80,6 +60,21 @@ Filter rows Y + + Table input + Detect empty stream + Y + + + Table input + Insert / update + Y + + + Table input + sort_by_recruit_id + Y + Abort @@ -97,7 +92,7 @@ 0 - 1488 + 1120 400 @@ -133,7 +128,7 @@ and recruitment_id = '${IDM_ID}'; - 1184 + 912 400 @@ -165,8 +160,8 @@ and recruitment_id = '${IDM_ID}'; - 1488 - 48 + 1120 + 64 @@ -229,8 +224,8 @@ DO UPDATE SET - 704 - 48 + 480 + 64 @@ -265,25 +260,7 @@ DO UPDATE SET Change job status on success - 1488 - 208 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 704 + 1120 208 @@ -305,7 +282,6 @@ DO UPDATE SET = deferment_id deferment_id - ervu_dashboard deferment_liberation
@@ -358,7 +334,7 @@ DO UPDATE SET Y - 912 + 688 208 @@ -366,7 +342,7 @@ DO UPDATE SET Table input TableInput - Y + N 1 @@ -375,43 +351,30 @@ DO UPDATE SET ervu_person_registry N - WITH -part_recruits AS ( - SELECT - id - FROM public.recruits - WHERE '${IDM_ID}' != '' -- Проверка на пустую строку - AND (current_recruitment_id = '${IDM_ID}' or (current_recruitment_id is null and target_recruitment_id = '${IDM_ID}')) -), -p_recruits_deferment AS ( - SELECT - d.id AS deferment_id, - d.recruit_id, - rd.code AS base_deferment, -- коды основания права на отсрочку - d.postponement_date, -- Дата окончания действия права на отсрочку - d.date_of_information, -- Дата начала действия права на отсрочку - d.hidden, - d.system_create_date, - d.system_update_date - FROM public.recruits_deferment d - JOIN public.ervu_reason_for_deferment rd ON d.ervu_reason_for_deferment_id = rd.id -) -SELECT - d.deferment_id, + + with filtered_data as (SELECT + d.id AS deferment_id, d.recruit_id, - d.base_deferment, -- коды основания права на отсрочку - d.postponement_date, -- Дата окончания действия права на отсрочку - d.date_of_information, -- Дата начала действия права на отсрочку + rd.code AS base_deferment, -- коды основания права на отсрочку + d.postponement_date, -- Дата окончания действия права на отсрочку + d.date_of_information, -- Дата начала действия права на отсрочку d.hidden, d.system_create_date AS created_at, d.system_update_date AS updated_at, - 'permission' AS permission_fact, -- ключ принадлежности к праву на отсрочку - CASE - WHEN d.hidden is false THEN true - ELSE false - END AS defer_conscription -- наличие права на отсрочку от призыва -FROM part_recruits r -JOIN p_recruits_deferment d ON r.id = d.recruit_id + 'permission' AS permission_fact, -- ключ принадлежности к праву на отсрочку + NOT d.hidden AS defer_conscription, -- наличие права на отсрочку от призыва + CTID +FROM public.recruits r + JOIN public.recruits_deferment d + ON r.id = d.recruit_id + JOIN public.ervu_reason_for_deferment rd + ON d.ervu_reason_for_deferment_id = rd.id +WHERE (r.current_recruitment_id = '${IDM_ID}' + OR (r.current_recruitment_id IS NULL AND r.target_recruitment_id = '${IDM_ID}') + )) +select fd.*, + fd.ctid = (select max(ctid) from filtered_data) as last_row + from filtered_data fd Y @@ -455,7 +418,7 @@ JOIN p_recruits_deferment d ON r.id = d.recruit_id Y - 912 + 688 400 @@ -489,7 +452,7 @@ JOIN p_recruits_deferment d ON r.id = d.recruit_id - 704 + 480 400 diff --git a/mappings/info_recruits/citizen_tables/deferment_liberation_permission/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/deferment_liberation_permission/recruitment_five_flow_delta.hpl index 182b089..3404b4f 100644 --- a/mappings/info_recruits/citizen_tables/deferment_liberation_permission/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/deferment_liberation_permission/recruitment_five_flow_delta.hpl @@ -20,36 +20,164 @@ + + Table input 2 + Append streams + Y + + + Table input 3 + Append streams + Y + + + Append streams + Sort rows + Y + Table input + Table input 2 + Y + + + Sort rows deferment_liberation_permission_flow_delta.hpl Y Table input + Table input 3 + Y + + + Sort rows deferment_liberation_permission_flow_delta.hpl 2 Y - Table input + Sort rows deferment_liberation_permission_flow_delta.hpl 3 Y - Table input + Sort rows deferment_liberation_permission_flow_delta.hpl 4 Y - Table input + Sort rows deferment_liberation_permission_flow_delta.hpl 5 Y + + Append streams + Append + + Y + + 1 + + none + + + + + 592 + 288 + + + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruitment_id + Y + N + N + 0 + N + + + + + 784 + 288 + + Table input TableInput + N + + 1 + + none + + + ervu-dashboard + N + 0 + select max(updated_at) as max_update_date + from deferment_liberation; + N + + + 192 + 288 + + + + Table input 2 + TableInput + + Y + + 1 + + none + + + ervu_person_registry + N + 0 + Table input + WITH max_update_date(val) AS (SELECT CAST(? AS timestamp)) +SELECT DISTINCT COALESCE(r.current_recruitment_id, r.target_recruitment_id) AS recruitment_id, + ? as max_update_date +FROM recruits r +JOIN recruits_deferment rd ON rd.recruit_id = r.id +JOIN max_update_date mud ON TRUE +WHERE (mud.val IS NULL OR rd.system_update_date > mud.val); + N + + + 384 + 192 + + + + Table input 3 + TableInput + Y 1 @@ -60,32 +188,19 @@ ervu-dashboard N 0 - WITH mud AS (SELECT recruitment_id, - MAX(execution_datetime) AS max_upd_date - FROM etl.job_execution - WHERE job_name = '${JOB_NAME}' - AND status IN ('SUCCESS', 'DELTA_SUCCESS') - GROUP BY recruitment_id) -SELECT r.idm_id -FROM ervu_dashboard.recruitment r - JOIN mud ON mud.recruitment_id = r.idm_id - JOIN ervu_dashboard.recruits_info ri - ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id - AND ri.updated_at > mud.max_upd_date - -union - -select r2.idm_id -from ervu_dashboard.recruitment r2 + Table input + select r.idm_id as recruitment_id, + ? as max_update_date +from ervu_dashboard.recruitment r join etl.job_execution je - on r2.idm_id = je.recruitment_id + on r.idm_id = je.recruitment_id where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') and je.job_name = '${JOB_NAME}'; Y - 352 - 288 + 384 + 384 @@ -112,6 +227,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') recruitment_id + + MAX_UPDATE_DATE + max_update_date + + Y @@ -135,7 +255,7 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - 640 + 1072 128 @@ -163,6 +283,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') recruitment_id + + MAX_UPDATE_DATE + max_update_date + + Y @@ -186,7 +311,7 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - 640 + 1072 208 @@ -214,6 +339,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') recruitment_id + + MAX_UPDATE_DATE + max_update_date + + Y @@ -237,7 +367,7 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - 640 + 1072 288 @@ -265,6 +395,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') recruitment_id + + MAX_UPDATE_DATE + max_update_date + + Y @@ -288,8 +423,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - 640 - 384 + 1072 + 368 @@ -316,6 +451,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') recruitment_id + + MAX_UPDATE_DATE + max_update_date + + Y @@ -339,8 +479,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - 640 - 480 + 1072 + 448