From 2d5f59988cbc5a1b43043d7322a8d9801058a111 Mon Sep 17 00:00:00 2001 From: Ruslan Date: Fri, 17 Oct 2025 18:09:47 +0500 Subject: [PATCH] some fixes --- ...nfringement_update_date_ervu_dashboard.hpl | 150 ------ .../job_last_update_date_infringement.hwf | 126 ----- .../row_last_infringement_update_date.hpl | 28 +- .../parallel/drivers_licence_flow_delta.hpl | 1 - .../citizen_tables/inn/parallel/inn_flow.hpl | 3 +- .../inn/parallel/inn_flow_delta.hpl | 3 +- .../inn/parallel/inn_flow_repeat.hpl | 4 +- .../inn/recruitment_five_flow_delta.hpl | 35 +- .../inn/recruitment_five_flow_on_error.hpl | 4 +- .../inn/recruitments_five_flow.hpl | 20 +- .../property/parallel/property_flow.hpl | 266 ++++----- .../property/parallel/property_flow_delta.hpl | 507 ++++-------------- .../parallel/property_flow_repeat.hpl | 492 ++++------------- .../property/recruitment_five_flow_delta.hpl | 29 +- .../recruitment_five_flow_on_error.hpl | 4 +- .../property/recruitments_five_flow.hpl | 20 +- .../job_last_update_date_recruit_archive.hwf | 126 ----- .../parallel/recruit_archive_flow.hpl | 182 +++---- .../parallel/recruit_archive_flow_delta.hpl | 161 +++--- .../parallel/recruit_archive_flow_repeat.hpl | 184 +++---- ...uit_archive_update_date_ervu_dashboard.hpl | 93 ---- .../recruitment_five_flow_delta.hpl | 40 +- .../recruitment_five_flow_on_error.hpl | 4 +- .../recruitments_five_flow.hpl | 20 +- .../row_last_recruit_archive_update_date.hpl | 19 +- .../snils/parallel/snils_flow.hpl | 3 +- .../snils/parallel/snils_flow_delta.hpl | 4 +- .../snils/parallel/snils_flow_repeat.hpl | 4 +- .../snils/recruitment_five_flow_delta.hpl | 29 +- .../snils/recruitment_five_flow_on_error.hpl | 4 +- .../snils/recruitments_five_flow.hpl | 20 +- .../subpoena/parallel/subpoena_flow.hpl | 254 ++++----- .../subpoena/parallel/subpoena_flow_delta.hpl | 238 ++++---- .../parallel/subpoena_flow_repeat.hpl | 250 ++++----- .../subpoena/recruitment_five_flow_delta.hpl | 38 +- .../recruitment_five_flow_on_error.hpl | 4 +- .../subpoena/recruitments_five_flow.hpl | 20 +- ...allel_subpoena_decision_archive_person.hwf | 170 +++--- .../parallel/restriction_document_flow.hpl | 8 +- .../restriction_document_flow_delta.hpl | 5 +- .../restriction_document_flow_repeat.hpl | 7 +- .../recruitment_five_flow_delta.hpl | 38 +- .../recruitment_five_flow_on_error.hpl | 4 +- .../recruitments_five_flow.hpl | 20 +- .../restriction_document_item_flow.hpl | 191 ++++--- .../restriction_document_item_flow_delta.hpl | 187 +++---- .../restriction_document_item_flow_repeat.hpl | 194 ++++--- .../recruitment_five_flow_delta.hpl | 28 +- .../recruitment_five_flow_on_error.hpl | 4 +- .../recruitments_five_flow.hpl | 20 +- .../supportive_jobs/job2_info_from_json.hwf | 4 +- .../supportive_jobs/job3_info_from_json.hwf | 4 +- 52 files changed, 1429 insertions(+), 2844 deletions(-) delete mode 100644 mappings/info_recruits/citizen_tables/citizen_liability_fz53/infringement_update_date_ervu_dashboard.hpl delete mode 100644 mappings/info_recruits/citizen_tables/citizen_liability_fz53/job_last_update_date_infringement.hwf delete mode 100644 mappings/info_recruits/citizen_tables/recruit_archive/job_last_update_date_recruit_archive.hwf delete mode 100644 mappings/info_recruits/citizen_tables/recruit_archive/recruit_archive_update_date_ervu_dashboard.hpl diff --git a/mappings/info_recruits/citizen_tables/citizen_liability_fz53/infringement_update_date_ervu_dashboard.hpl b/mappings/info_recruits/citizen_tables/citizen_liability_fz53/infringement_update_date_ervu_dashboard.hpl deleted file mode 100644 index b258de3..0000000 --- a/mappings/info_recruits/citizen_tables/citizen_liability_fz53/infringement_update_date_ervu_dashboard.hpl +++ /dev/null @@ -1,150 +0,0 @@ - - - - infringement_update_date_ervu_dashboard - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/06/02 14:50:31.833 - - - 2025/06/02 14:50:31.833 - - - - - - Table input - Table output - Y - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT - system_create_date, - CASE - WHEN system_update_date IS NULL THEN '0001-01-01 00:00:00'::timestamp - ELSE system_update_date - END as system_update_date, - current_timestamp AS record_created, - 'infringement_table' AS workflow -FROM ( - SELECT - system_create_date, - system_update_date - FROM ( - SELECT - created_at AS system_create_date, - updated_at AS system_update_date, - ROW_NUMBER() OVER ( - ORDER BY COALESCE(updated_at, '1000-01-01'::timestamp) DESC - ) as rn - FROM ( - SELECT created_at, updated_at - FROM ervu_dashboard.citizen_criminal_liability_fz53 - UNION ALL - SELECT created_at, updated_at - FROM ervu_dashboard.citizen_liability_fz53 - ) AS all_data - ) AS ranked - WHERE rn = 1 - - -- Если нет данных, вернуть fallback строку - UNION ALL - - SELECT - current_timestamp AS system_create_date, - NULL AS system_update_date - WHERE NOT EXISTS ( - SELECT 1 - FROM ervu_dashboard.citizen_criminal_liability_fz53 - UNION ALL - SELECT 1 - FROM ervu_dashboard.citizen_liability_fz53 - ) -) AS result -LIMIT 1; - N - - - 896 - 304 - - - - Table output - TableOutput - - Y - - 1 - - none - - - 1000 - ervu-dashboard - - - system_create_date - system_create_date - - - system_update_date - system_update_date - - - workflow - workflow - - - record_created - record_created - - - N - N - N - N - - Y - - N - public - Y - recruit_create
- - N - Y - N - Y - - - 1216 - 304 - - - - - - diff --git a/mappings/info_recruits/citizen_tables/citizen_liability_fz53/job_last_update_date_infringement.hwf b/mappings/info_recruits/citizen_tables/citizen_liability_fz53/job_last_update_date_infringement.hwf deleted file mode 100644 index dccf147..0000000 --- a/mappings/info_recruits/citizen_tables/citizen_liability_fz53/job_last_update_date_infringement.hwf +++ /dev/null @@ -1,126 +0,0 @@ - - - job_last_update_date_infringement - Y - - - - - - 2025/06/02 14:48:05.666 - - - 2025/06/02 14:48:05.666 - - - - - Start - - SPECIAL - - 1 - 12 - 60 - 0 - 0 - N - 0 - 1 - N - 592 - 352 - - - - infringement_update_date_ervu_dashboard.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_liability_fz53/infringement_update_date_ervu_dashboard.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 784 - 352 - - - - row_last_infringement_update_date.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_liability_fz53/row_last_infringement_update_date.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 1088 - 352 - - - - Success - - SUCCESS - - N - 1264 - 352 - - - - - - Start - infringement_update_date_ervu_dashboard.hpl - Y - Y - Y - - - infringement_update_date_ervu_dashboard.hpl - row_last_infringement_update_date.hpl - Y - Y - N - - - row_last_infringement_update_date.hpl - Success - Y - Y - N - - - - - - diff --git a/mappings/info_recruits/citizen_tables/citizen_liability_fz53/row_last_infringement_update_date.hpl b/mappings/info_recruits/citizen_tables/citizen_liability_fz53/row_last_infringement_update_date.hpl index 7dbde99..17a0bcc 100644 --- a/mappings/info_recruits/citizen_tables/citizen_liability_fz53/row_last_infringement_update_date.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_liability_fz53/row_last_infringement_update_date.hpl @@ -76,7 +76,6 @@ - max_infringement_update_date M_INFR_UP_DATE GP_WORKFLOW @@ -103,13 +102,26 @@ ervu-dashboard N 0 - SELECT - system_update_date AS MAX_INFRINGEMENT_UPDATE_DATE--, ---record_created -FROM public.recruit_create -WHERE system_update_date = (select max(system_update_date) from public.recruit_create where workflow = 'infringement_table') -order by record_created desc -limit 1 + with +base as ( +select max(updated_at) AS max_infringement_update_date +from ervu_dashboard.citizen_criminal_liability_fz53 + +union all + +select max(updated_at) AS max_infringement_update_date +from ervu_dashboard.citizen_liability_fz53 +), +b AS ( +select max(max_infringement_update_date) AS max_infringement_update_date +from base +) +select + CASE + WHEN max_infringement_update_date is null THEN '0001-01-01 00:00:00'::timestamp + ELSE max_infringement_update_date + END AS max_infringement_update_date +from b N diff --git a/mappings/info_recruits/citizen_tables/drivers_licence/parallel/drivers_licence_flow_delta.hpl b/mappings/info_recruits/citizen_tables/drivers_licence/parallel/drivers_licence_flow_delta.hpl index a2a7386..6ffdc57 100644 --- a/mappings/info_recruits/citizen_tables/drivers_licence/parallel/drivers_licence_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/drivers_licence/parallel/drivers_licence_flow_delta.hpl @@ -401,7 +401,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH base AS ( SELECT diff --git a/mappings/info_recruits/citizen_tables/inn/parallel/inn_flow.hpl b/mappings/info_recruits/citizen_tables/inn/parallel/inn_flow.hpl index e32668a..93060a2 100644 --- a/mappings/info_recruits/citizen_tables/inn/parallel/inn_flow.hpl +++ b/mappings/info_recruits/citizen_tables/inn/parallel/inn_flow.hpl @@ -276,7 +276,6 @@ base as ( FROM ervu_dashboard.recruits_info ri join ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at ) SELECT recruit_id, @@ -364,7 +363,7 @@ WHERE LENGTH(REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'inn', Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/inn/parallel/inn_flow_delta.hpl b/mappings/info_recruits/citizen_tables/inn/parallel/inn_flow_delta.hpl index 59a7b99..27813c5 100644 --- a/mappings/info_recruits/citizen_tables/inn/parallel/inn_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/inn/parallel/inn_flow_delta.hpl @@ -322,7 +322,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH base AS ( SELECT ri.recruit_id, @@ -397,7 +396,7 @@ FROM inn_data; Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/inn/parallel/inn_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/inn/parallel/inn_flow_repeat.hpl index 70e4df6..bbf7bfa 100644 --- a/mappings/info_recruits/citizen_tables/inn/parallel/inn_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/inn/parallel/inn_flow_repeat.hpl @@ -337,7 +337,6 @@ DO UPDATE SET ervu-dashboard N - WITH base as ( SELECT @@ -346,7 +345,6 @@ base as ( FROM ervu_dashboard.recruits_info ri join ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at ) SELECT recruit_id, @@ -380,7 +378,7 @@ WHERE LENGTH(REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'inn', Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/inn/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/inn/recruitment_five_flow_delta.hpl index 7d382f8..9974181 100644 --- a/mappings/info_recruits/citizen_tables/inn/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/inn/recruitment_five_flow_delta.hpl @@ -65,22 +65,27 @@ ervu-dashboard N 0 - WITH mud AS ( - SELECT - recruitment_id, - MAX(execution_datetime) AS max_upd_date - FROM etl.job_execution - WHERE job_name = '${JOB_NAME}' - AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING') - GROUP BY recruitment_id -) -SELECT - r.idm_id + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r JOIN mud ON mud.recruitment_id = r.idm_id JOIN recruits_info ri ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id - AND ri.updated_at > mud.max_upd_date; + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; Y @@ -219,7 +224,7 @@ FROM ervu_dashboard.recruitment r - 592 + 608 208 @@ -331,7 +336,7 @@ FROM ervu_dashboard.recruitment r - 608 + 592 368 @@ -387,7 +392,7 @@ FROM ervu_dashboard.recruitment r - 608 + 592 448 diff --git a/mappings/info_recruits/citizen_tables/inn/recruitment_five_flow_on_error.hpl b/mappings/info_recruits/citizen_tables/inn/recruitment_five_flow_on_error.hpl index ea2a761..6170cd6 100644 --- a/mappings/info_recruits/citizen_tables/inn/recruitment_five_flow_on_error.hpl +++ b/mappings/info_recruits/citizen_tables/inn/recruitment_five_flow_on_error.hpl @@ -65,9 +65,9 @@ FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - and job_name = 'snils_inn_job' + and job_name = '${JOB_NAME}' where je.status is null or je.status in('ERROR', 'PROCESSING'); - N + Y 272 diff --git a/mappings/info_recruits/citizen_tables/inn/recruitments_five_flow.hpl b/mappings/info_recruits/citizen_tables/inn/recruitments_five_flow.hpl index e2e9e17..e5599c7 100644 --- a/mappings/info_recruits/citizen_tables/inn/recruitments_five_flow.hpl +++ b/mappings/info_recruits/citizen_tables/inn/recruitments_five_flow.hpl @@ -63,24 +63,8 @@ SELECT idm_id AS recruitment FROM ervu_dashboard.recruitment -${COMMENT}/* -where idm_id in ('110686b2-4a01-4976-885b-d26a2980f53a', -'cff66da8-2be0-4839-bd8e-332b1a150031', -'34c8685e-f08a-4877-afb1-a0589800a21c', -'0e10678c-f272-41c3-94dd-c15830321f9a', -'9703f108-761a-467a-aad3-89617e5539ed', -'55eae374-9bb5-4295-9e3a-5fbcdaba870d', -'dad6c101-34be-45c3-8173-57190cb69dfb', -'fc10523f-af31-4f6b-b0e2-00799758f634', -'61807cc2-9913-4060-8802-0672877c28c8', -'eba7fd7d-e3da-49a7-8e65-8fc85e47190e', -'f59afe6d-c166-4196-a489-6d17f02fc05e', -'22c64b79-f8e4-41e7-a2f3-1aa9df424fd9', -'0fe2bdf0-afbb-402d-93da-67e511799afa', -'6139de3b-2378-496f-9fca-7df7dab90832', -'50ba73dd-1726-48f1-8b7c-7a77ff965299') -${COMMENT}*/ - Y + + N 432 diff --git a/mappings/info_recruits/citizen_tables/property/parallel/property_flow.hpl b/mappings/info_recruits/citizen_tables/property/parallel/property_flow.hpl index 7916926..3023a31 100644 --- a/mappings/info_recruits/citizen_tables/property/parallel/property_flow.hpl +++ b/mappings/info_recruits/citizen_tables/property/parallel/property_flow.hpl @@ -45,43 +45,43 @@ Change job status on error Y - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - User defined Java expression - Y - - - User defined Java expression - Update - Y - - - Table input - Table output - Y - - - Table output - Group by - Y - Update Change job status on error Y - Update + Table input + Identify last row in a stream + Y + + + Identify last row in a stream + Table output + Y + + + Identify last row in a stream + sort_by_recruit_id + Y + + + sort_by_recruit_id + Update + Y + + + Table output + Block until transforms finish + Y + + + Identify last row in a stream Detect empty stream Y - Update + Block until transforms finish Filter rows Y @@ -102,8 +102,30 @@ 0 - 1328 - 496 + 1344 + 528 + + + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + Update + + + + + 1200 + 320 @@ -138,8 +160,8 @@ and recruitment_id = '${IDM_ID}'; - 1328 - 320 + 1120 + 528 @@ -170,8 +192,8 @@ and recruitment_id = '${IDM_ID}'; - 1856 - 176 + 1392 + 160 @@ -216,8 +238,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 1536 - 176 + 688 + 160 @@ -252,70 +274,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1536 - 304 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - has_property_raw - is_property - MAX - - - has_ground_transport_raw - is_ground_transport - MAX - - - has_air_transport_raw - is_air_transport - MAX - - - has_water_transport_raw - is_water_transport - MAX - - - has_possibility_reg_residence - is_possibility_reg_residence - MAX - - - N - - - recruit_id - - - N - grp - - - 688 - 176 + 1392 + 320 Identify last row in a stream DetectLastRow - Y + N 1 @@ -325,8 +292,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); last_row - 880 - 176 + 688 + 320 @@ -342,7 +309,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - WITH base as( SELECT @@ -351,7 +317,6 @@ base as( FROM ervu_dashboard.recruits_info ri join ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at ), prop AS ( SELECT @@ -432,7 +397,7 @@ all_data as( to_date(property_data->>'dataRegPrav', 'YYYY-MM-DD') AS start_date, to_date(property_data->>'dataPrekPrav', 'YYYY-MM-DD') AS close_date, 'realty' AS type_property, - 1 AS is_possibility_reg_residence + true AS has_possibility_reg_residence FROM base JOIN prop ON base.recruit_id = prop.recruit_id WHERE prop.property_type = 'realty' @@ -454,7 +419,7 @@ all_data as( to_date(property_data->>'dataRegPr', 'YYYY-MM-DD') AS start_date, to_date(property_data->>'dataPrekPr', 'YYYY-MM-DD') AS close_date, 'ground_transportation' AS type_property, - 0 AS is_possibility_reg_residence + false AS has_possibility_reg_residence FROM base JOIN prop ON base.recruit_id = prop.recruit_id WHERE prop.property_type = 'ground_transportation' @@ -476,7 +441,7 @@ all_data as( to_date(property_data->>'dataRegPr', 'YYYY-MM-DD') AS start_date, to_date(property_data->>'dataPrekPr', 'YYYY-MM-DD') AS close_date, 'air_vehicles' AS type_property, - 0 AS is_possibility_reg_residence + false AS has_possibility_reg_residence FROM base JOIN prop ON base.recruit_id = prop.recruit_id WHERE prop.property_type = 'air_vehicles' @@ -498,7 +463,7 @@ all_data as( to_date(property_data->>'dataRegPr', 'YYYY-MM-DD') AS start_date, to_date(property_data->>'dataPrekPr', 'YYYY-MM-DD') AS close_date, 'water_vehicles' AS type_property, - 0 AS is_possibility_reg_residence + false AS has_possibility_reg_residence FROM base JOIN prop ON base.recruit_id = prop.recruit_id WHERE prop.property_type = 'water_vehicles' @@ -517,23 +482,23 @@ select start_date, close_date, type_property, - is_possibility_reg_residence, + has_possibility_reg_residence, CASE - WHEN type_property = 'realty' THEN 1 - ELSE 0 - END AS is_property, -- недвиж//has_property + WHEN type_property = 'realty' THEN true + ELSE false + END AS has_property, -- недвиж//has_property CASE - WHEN type_property = 'ground_transportation'THEN 1 - ELSE 0 - END AS is_ground_transport, -- наземный транспорт//has_ground_transport + WHEN type_property = 'ground_transportation' THEN true + ELSE false + END AS has_ground_transport, -- наземный транспорт//has_ground_transport CASE - WHEN type_property = 'air_vehicles' THEN 1 - ELSE 0 - END AS is_air_transport, -- воздушный транспорт//has_air_transport + WHEN type_property = 'air_vehicles' THEN true + ELSE false + END AS has_air_transport, -- воздушный транспорт//has_air_transport CASE - WHEN type_property = 'water_vehicles' THEN 1 - ELSE 0 - END AS is_water_transport -- водный транспорт//has_water_transport + WHEN type_property = 'water_vehicles' THEN true + ELSE false + END AS has_water_transport -- водный транспорт//has_water_transport from all_data Y @@ -624,7 +589,7 @@ from all_data Y - 688 + 912 320 @@ -675,13 +640,13 @@ from all_data Y - 1328 - 176 + 912 + 528 - User defined Java expression - Janino + sort_by_recruit_id + SortRows Y @@ -690,50 +655,25 @@ from all_data none - - has_property - has_property_raw == 1 - Boolean - -1 - -1 - - - - has_ground_transport - has_ground_transport_raw == 1 - Boolean - -1 - -1 - - - - has_air_transport - has_air_transport_raw == 1 - Boolean - -1 - -1 - - - - has_water_transport - has_water_transport_raw == 1 - Boolean - -1 - -1 - - - - has_possibility_reg_residence - has_possibility_reg_residence_raw == 1 - Boolean - -1 - -1 - - + N + ${java.io.tmpdir} + + + Y + N + N + 0 + recruit_id + N + + + srt + 1000000 + Y - 1104 - 176 + 688 + 528 @@ -744,7 +684,7 @@ from all_data error_description - error_code + @@ -754,7 +694,7 @@ from all_data Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/property/parallel/property_flow_delta.hpl b/mappings/info_recruits/citizen_tables/property/parallel/property_flow_delta.hpl index f396ded..6242a83 100644 --- a/mappings/info_recruits/citizen_tables/property/parallel/property_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/property/parallel/property_flow_delta.hpl @@ -42,86 +42,46 @@ Table input - Insert / update - Y - - - Insert / update - Change job status on error - Y - - - Insert / update - Group by - Y - - - Group by Identify last row in a stream Y Identify last row in a stream - User defined Java expression + Insert / update Y - Update 2 - Update + Identify last row in a stream + sort_by_recruit_id Y - Update 3 - Update 2 - Y - - - Update 4 - Update 3 - Y - - - Update 5 - Update 4 - Y - - - User defined Java expression - Update 5 - Y - - - Update + Identify last row in a stream Detect empty stream Y Update + Change job status on error + Y + + + Insert / update + Block until transforms finish + Y + + + Block until transforms finish Filter rows Y - Update 5 - Change job status on error + sort_by_recruit_id + Update Y - Update 4 - Change job status on error - Y - - - Update 3 - Change job status on error - Y - - - Update 2 - Change job status on error - Y - - - Update + Insert / update Change job status on error Y @@ -142,8 +102,30 @@ 0 - 1648 - 528 + 1456 + 512 + + + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + Update + + + + + 1312 + 352 @@ -178,8 +160,8 @@ and recruitment_id = '${IDM_ID}'; - 1648 - 352 + 1200 + 512 @@ -210,7 +192,7 @@ and recruitment_id = '${IDM_ID}'; - 2128 + 1520 208 @@ -259,7 +241,7 @@ and recruitment_id = '${IDM_ID}'; - 1888 + 768 208 @@ -295,70 +277,15 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1888 - 320 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - has_property_raw - is_property - MAX - - - has_ground_transport_raw - is_ground_transport - MAX - - - has_air_transport_raw - is_air_transport - MAX - - - has_water_transport_raw - is_water_transport - MAX - - - has_possibility_reg_residence_raw - is_possibility_reg_residence - MAX - - - N - - - recruit_id - - - N - grp - - - 752 - 208 + 1520 + 352 Identify last row in a stream DetectLastRow - Y + N 1 @@ -368,8 +295,8 @@ and recruitment_id = '${IDM_ID}'; last_row - 944 - 208 + 768 + 352 @@ -472,7 +399,7 @@ and recruitment_id = '${IDM_ID}'; N - 752 + 960 352 @@ -582,7 +509,7 @@ all_data as( to_date(property_data->>'dataRegPrav', 'YYYY-MM-DD') AS start_date, to_date(property_data->>'dataPrekPrav', 'YYYY-MM-DD') AS close_date, 'realty' AS type_property, - 1 AS is_possibility_reg_residence + true AS has_possibility_reg_residence FROM prop WHERE prop.property_type = 'realty' @@ -603,7 +530,7 @@ all_data as( to_date(property_data->>'dataRegPr', 'YYYY-MM-DD') AS start_date, to_date(property_data->>'dataPrekPr', 'YYYY-MM-DD') AS close_date, 'ground_transportation' AS type_property, - 0 AS is_possibility_reg_residence + false AS has_possibility_reg_residence FROM prop WHERE prop.property_type = 'ground_transportation' @@ -624,7 +551,7 @@ all_data as( to_date(property_data->>'dataRegPr', 'YYYY-MM-DD') AS start_date, to_date(property_data->>'dataPrekPr', 'YYYY-MM-DD') AS close_date, 'air_vehicles' AS type_property, - 0 AS is_possibility_reg_residence + false AS has_possibility_reg_residence FROM prop WHERE prop.property_type = 'air_vehicles' @@ -645,7 +572,7 @@ all_data as( to_date(property_data->>'dataRegPr', 'YYYY-MM-DD') AS start_date, to_date(property_data->>'dataPrekPr', 'YYYY-MM-DD') AS close_date, 'water_vehicles' AS type_property, - 0 AS is_possibility_reg_residence + false AS has_possibility_reg_residence FROM prop WHERE prop.property_type = 'water_vehicles' ) @@ -663,23 +590,23 @@ SELECT start_date, close_date, type_property, - is_possibility_reg_residence, + has_possibility_reg_residence, CASE - WHEN type_property = 'realty' THEN 1 - ELSE 0 - END AS is_property, + WHEN type_property = 'realty' THEN true + ELSE false + END AS has_property, CASE - WHEN type_property = 'ground_transportation' THEN 1 - ELSE 0 - END AS is_ground_transport, + WHEN type_property = 'ground_transportation' THEN true + ELSE false + END AS has_ground_transport, CASE - WHEN type_property = 'air_vehicles' THEN 1 - ELSE 0 - END AS is_air_transport, + WHEN type_property = 'air_vehicles' THEN true + ELSE false + END AS has_air_transport, CASE - WHEN type_property = 'water_vehicles' THEN 1 - ELSE 0 - END AS is_water_transport + WHEN type_property = 'water_vehicles' THEN true + ELSE false + END AS has_water_transport FROM all_data Y @@ -708,13 +635,24 @@ FROM all_data recruit_id recruit_id - - <> - has_possibility_reg_residence - has_possibility_reg_residence - ervu_dashboard citizen
+ + has_property + has_property + + + has_ground_transport + has_ground_transport + + + has_air_transport + has_air_transport + + + has_water_transport + has_water_transport + has_possibility_reg_residence has_possibility_reg_residence @@ -724,13 +662,13 @@ FROM all_data Y - 1696 - 208 + 960 + 512 - Update 2 - Update + sort_by_recruit_id + SortRows Y @@ -739,210 +677,25 @@ FROM all_data none - 10000 - ervu-dashboard - Y - - - = - recruit_id + N + ${java.io.tmpdir} + + + Y + N + N + 0 recruit_id - - - <> - has_water_transport - has_water_transport - - ervu_dashboard - citizen
- - has_water_transport - has_water_transport - - - N - Y + N + + + srt + 1000000 + Y - 1600 - 208 - - - - Update 3 - Update - - Y - - 1 - - none - - - 10000 - ervu-dashboard - Y - - - = - recruit_id - recruit_id - - - <> - has_air_transport - has_air_transport - - ervu_dashboard - citizen
- - has_air_transport - has_air_transport - - - N - Y - - - 1504 - 208 - - - - Update 4 - Update - - Y - - 1 - - none - - - 10000 - ervu-dashboard - Y - - - = - recruit_id - recruit_id - - - <> - has_ground_transport - has_ground_transport - - ervu_dashboard - citizen
- - has_ground_transport - has_ground_transport - - - N - Y - - - 1408 - 208 - - - - Update 5 - Update - - Y - - 1 - - none - - - 10000 - ervu-dashboard - Y - - - = - recruit_id - recruit_id - - - <> - has_property - has_property - - ervu_dashboard - citizen
- - has_property - has_property - - - N - Y - - - 1312 - 208 - - - - User defined Java expression - Janino - - Y - - 1 - - none - - - - has_property - has_property_raw == 1 - Boolean - -1 - -1 - - - - has_ground_transport - has_ground_transport_raw == 1 - Boolean - -1 - -1 - - - - has_air_transport - has_air_transport_raw == 1 - Boolean - -1 - -1 - - - - has_water_transport - has_water_transport_raw == 1 - Boolean - -1 - -1 - - - - has_possibility_reg_residence - has_possibility_reg_residence_raw == 1 - Boolean - -1 - -1 - - - - - 1152 - 208 + 768 + 512 @@ -951,7 +704,7 @@ FROM all_data Change job status on error Y - + error_description @@ -963,55 +716,7 @@ FROM all_data Change job status on error Y - - - - - - - - - Update 2 - Change job status on error - Y - - - - - - - - - - Update 3 - Change job status on error - Y - - - - - - - - - - Update 4 - Change job status on error - Y - - - - - - - - - - Update 5 - Change job status on error - Y - - + error_description diff --git a/mappings/info_recruits/citizen_tables/property/parallel/property_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/property/parallel/property_flow_repeat.hpl index c53149c..b6e6a19 100644 --- a/mappings/info_recruits/citizen_tables/property/parallel/property_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/property/parallel/property_flow_repeat.hpl @@ -47,82 +47,42 @@ Table input - Insert / update - Y - - - Group by Identify last row in a stream Y Identify last row in a stream - User defined Java expression + Insert / update Y - Insert / update - Group by + Identify last row in a stream + sort_by_recruit_id Y - Update - Change job status on error - Y - - - Update + Identify last row in a stream Detect empty stream Y - Update - Filter rows - Y - - - Update 2 + sort_by_recruit_id Update Y - Update 3 - Update 2 - Y - - - Update 4 - Update 3 - Y - - - User defined Java expression - Update 5 - Y - - - Update 5 - Update 4 - Y - - - Update 2 + Update Change job status on error Y - Update 3 - Change job status on error + Insert / update + Block until transforms finish Y - Update 4 - Change job status on error - Y - - - Update 5 - Change job status on error + Block until transforms finish + Filter rows Y @@ -142,10 +102,32 @@ 0 - 1648 + 1744 688 + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + Update + + + + + 1472 + 496 + + Change job status on error ExecSql @@ -178,8 +160,8 @@ and recruitment_id = '${IDM_ID}'; - 1648 - 496 + 1328 + 688 @@ -210,7 +192,7 @@ and recruitment_id = '${IDM_ID}'; - 2112 + 1744 352 @@ -274,7 +256,7 @@ DO UPDATE SET - 1904 + 768 352 @@ -310,70 +292,15 @@ DO UPDATE SET Change job status on success - 1904 - 464 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - has_property_raw - is_property - MAX - - - has_ground_transport_raw - is_ground_transport - MAX - - - has_air_transport_raw - is_air_transport - MAX - - - has_water_transport_raw - is_water_transport - MAX - - - has_possibility_reg_residence_raw - is_possibility_reg_residence - MAX - - - N - - - recruit_id - - - N - grp - - - 688 - 352 + 1744 + 496 Identify last row in a stream DetectLastRow - Y + N 1 @@ -383,8 +310,8 @@ DO UPDATE SET last_row - 880 - 352 + 768 + 496 @@ -487,7 +414,7 @@ DO UPDATE SET N - 688 + 1024 496 @@ -513,7 +440,6 @@ base as( FROM ervu_dashboard.recruits_info ri join ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at ), prop AS ( SELECT @@ -594,7 +520,7 @@ all_data as( to_date(property_data->>'dataRegPrav', 'YYYY-MM-DD') AS start_date, to_date(property_data->>'dataPrekPrav', 'YYYY-MM-DD') AS close_date, 'realty' AS type_property, - 1 AS is_possibility_reg_residence + true AS has_possibility_reg_residence FROM base JOIN prop ON base.recruit_id = prop.recruit_id WHERE prop.property_type = 'realty' @@ -616,7 +542,7 @@ all_data as( to_date(property_data->>'dataRegPr', 'YYYY-MM-DD') AS start_date, to_date(property_data->>'dataPrekPr', 'YYYY-MM-DD') AS close_date, 'ground_transportation' AS type_property, - 0 AS is_possibility_reg_residence + false AS has_possibility_reg_residence FROM base JOIN prop ON base.recruit_id = prop.recruit_id WHERE prop.property_type = 'ground_transportation' @@ -638,7 +564,7 @@ all_data as( to_date(property_data->>'dataRegPr', 'YYYY-MM-DD') AS start_date, to_date(property_data->>'dataPrekPr', 'YYYY-MM-DD') AS close_date, 'air_vehicles' AS type_property, - 0 AS is_possibility_reg_residence + false AS has_possibility_reg_residence FROM base JOIN prop ON base.recruit_id = prop.recruit_id WHERE prop.property_type = 'air_vehicles' @@ -660,7 +586,7 @@ all_data as( to_date(property_data->>'dataRegPr', 'YYYY-MM-DD') AS start_date, to_date(property_data->>'dataPrekPr', 'YYYY-MM-DD') AS close_date, 'water_vehicles' AS type_property, - 0 AS is_possibility_reg_residence + false AS has_possibility_reg_residence FROM base JOIN prop ON base.recruit_id = prop.recruit_id WHERE prop.property_type = 'water_vehicles' @@ -679,23 +605,23 @@ select start_date, close_date, type_property, - is_possibility_reg_residence, + has_possibility_reg_residence, CASE - WHEN type_property = 'realty' THEN 1 - ELSE 0 - END AS is_property, -- недвиж//has_property + WHEN type_property = 'realty' THEN true + ELSE false + END AS has_property, -- недвиж//has_property CASE - WHEN type_property = 'ground_transportation'THEN 1 - ELSE 0 - END AS is_ground_transport, -- наземный транспорт//has_ground_transport + WHEN type_property = 'ground_transportation' THEN true + ELSE false + END AS has_ground_transport, -- наземный транспорт//has_ground_transport CASE - WHEN type_property = 'air_vehicles' THEN 1 - ELSE 0 - END AS is_air_transport, -- воздушный транспорт//has_air_transport + WHEN type_property = 'air_vehicles' THEN true + ELSE false + END AS has_air_transport, -- воздушный транспорт//has_air_transport CASE - WHEN type_property = 'water_vehicles' THEN 1 - ELSE 0 - END AS is_water_transport -- водный транспорт//has_water_transport + WHEN type_property = 'water_vehicles' THEN true + ELSE false + END AS has_water_transport -- водный транспорт//has_water_transport from all_data Y @@ -724,13 +650,24 @@ from all_data recruit_id recruit_id - - <> - has_possibility_reg_residence - has_possibility_reg_residence - ervu_dashboard citizen
+ + has_property + has_property + + + has_ground_transport + has_ground_transport + + + has_air_transport + has_air_transport + + + has_water_transport + has_water_transport + has_possibility_reg_residence has_possibility_reg_residence @@ -740,13 +677,13 @@ from all_data Y - 1648 - 352 + 992 + 688 - Update 2 - Update + sort_by_recruit_id + SortRows Y @@ -755,210 +692,25 @@ from all_data none - 10000 - ervu-dashboard - Y - - - = - recruit_id + N + ${java.io.tmpdir} + + + Y + N + N + 0 recruit_id - - - <> - has_water_transport - has_water_transport - - ervu_dashboard - citizen
- - has_water_transport - has_water_transport - - - N - Y + N + + + srt + 1000000 + Y - 1552 - 352 - - - - Update 3 - Update - - Y - - 1 - - none - - - 10000 - ervu-dashboard - Y - - - = - recruit_id - recruit_id - - - <> - has_air_transport - has_air_transport - - ervu_dashboard - citizen
- - has_air_transport - has_air_transport - - - N - Y - - - 1456 - 352 - - - - Update 4 - Update - - Y - - 1 - - none - - - 10000 - ervu-dashboard - Y - - - = - recruit_id - recruit_id - - - <> - has_ground_transport - has_ground_transport - - ervu_dashboard - citizen
- - has_ground_transport - has_ground_transport - - - N - Y - - - 1360 - 352 - - - - Update 5 - Update - - Y - - 1 - - none - - - 10000 - ervu-dashboard - Y - - - = - recruit_id - recruit_id - - - <> - has_property - has_property - - ervu_dashboard - citizen
- - has_property - has_property - - - N - Y - - - 1264 - 352 - - - - User defined Java expression - Janino - - Y - - 1 - - none - - - - has_property - has_property_raw == 1 - Boolean - -1 - -1 - - - - has_ground_transport - has_ground_transport_raw == 1 - Boolean - -1 - -1 - - - - has_air_transport - has_air_transport_raw == 1 - Boolean - -1 - -1 - - - - has_water_transport - has_water_transport_raw == 1 - Boolean - -1 - -1 - - - - has_possibility_reg_residence - has_possibility_reg_residence_raw == 1 - Boolean - -1 - -1 - - - - - 1104 - 352 + 768 + 688 @@ -967,7 +719,7 @@ from all_data Change job status on error Y - + error_description @@ -979,55 +731,7 @@ from all_data Change job status on error Y - - - - - - - - - Update 2 - Change job status on error - Y - - - - - - - - - - Update 3 - Change job status on error - Y - - - - - - - - - - Update 4 - Change job status on error - Y - - - - - - - - - - Update 5 - Change job status on error - Y - - + error_description diff --git a/mappings/info_recruits/citizen_tables/property/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/property/recruitment_five_flow_delta.hpl index c9c2eb5..89d88a7 100644 --- a/mappings/info_recruits/citizen_tables/property/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/property/recruitment_five_flow_delta.hpl @@ -65,22 +65,27 @@ ervu-dashboard N 0 - WITH mud AS ( - SELECT - recruitment_id, - MAX(execution_datetime) AS max_upd_date - FROM etl.job_execution - WHERE job_name = '${JOB_NAME}' - AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING') - GROUP BY recruitment_id -) -SELECT - r.idm_id + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r JOIN mud ON mud.recruitment_id = r.idm_id JOIN recruits_info ri ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id - AND ri.updated_at > mud.max_upd_date; + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; Y diff --git a/mappings/info_recruits/citizen_tables/property/recruitment_five_flow_on_error.hpl b/mappings/info_recruits/citizen_tables/property/recruitment_five_flow_on_error.hpl index dfc534b..c87e9ad 100644 --- a/mappings/info_recruits/citizen_tables/property/recruitment_five_flow_on_error.hpl +++ b/mappings/info_recruits/citizen_tables/property/recruitment_five_flow_on_error.hpl @@ -65,9 +65,9 @@ FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - and job_name = 'property_job' + and job_name = '${JOB_NAME}' where je.status is null or je.status in('ERROR', 'PROCESSING'); - N + Y 272 diff --git a/mappings/info_recruits/citizen_tables/property/recruitments_five_flow.hpl b/mappings/info_recruits/citizen_tables/property/recruitments_five_flow.hpl index 4bf6bb0..1b2d941 100644 --- a/mappings/info_recruits/citizen_tables/property/recruitments_five_flow.hpl +++ b/mappings/info_recruits/citizen_tables/property/recruitments_five_flow.hpl @@ -63,24 +63,8 @@ SELECT idm_id AS recruitment FROM ervu_dashboard.recruitment -${COMMENT}/* -where idm_id in ('110686b2-4a01-4976-885b-d26a2980f53a', -'cff66da8-2be0-4839-bd8e-332b1a150031', -'34c8685e-f08a-4877-afb1-a0589800a21c', -'0e10678c-f272-41c3-94dd-c15830321f9a', -'9703f108-761a-467a-aad3-89617e5539ed', -'55eae374-9bb5-4295-9e3a-5fbcdaba870d', -'dad6c101-34be-45c3-8173-57190cb69dfb', -'fc10523f-af31-4f6b-b0e2-00799758f634', -'61807cc2-9913-4060-8802-0672877c28c8', -'eba7fd7d-e3da-49a7-8e65-8fc85e47190e', -'f59afe6d-c166-4196-a489-6d17f02fc05e', -'22c64b79-f8e4-41e7-a2f3-1aa9df424fd9', -'0fe2bdf0-afbb-402d-93da-67e511799afa', -'6139de3b-2378-496f-9fca-7df7dab90832', -'50ba73dd-1726-48f1-8b7c-7a77ff965299') -${COMMENT}*/ - Y + + N 432 diff --git a/mappings/info_recruits/citizen_tables/recruit_archive/job_last_update_date_recruit_archive.hwf b/mappings/info_recruits/citizen_tables/recruit_archive/job_last_update_date_recruit_archive.hwf deleted file mode 100644 index 76566d3..0000000 --- a/mappings/info_recruits/citizen_tables/recruit_archive/job_last_update_date_recruit_archive.hwf +++ /dev/null @@ -1,126 +0,0 @@ - - - job_last_update_date_recruit_archive - Y - - - - - - 2025/06/02 14:48:05.666 - - - 2025/06/02 14:48:05.666 - - - - - Start - - SPECIAL - - 1 - 12 - 60 - 0 - 0 - N - 0 - 1 - N - 400 - 352 - - - - recruit_archive_update_date_ervu_dashboard.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/recruit_archive/recruit_archive_update_date_ervu_dashboard.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 736 - 352 - - - - row_last_recruit_archive_update_date.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/recruit_archive/row_last_recruit_archive_update_date.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 1152 - 352 - - - - Success - - SUCCESS - - N - 1504 - 352 - - - - - - Start - recruit_archive_update_date_ervu_dashboard.hpl - Y - Y - Y - - - recruit_archive_update_date_ervu_dashboard.hpl - row_last_recruit_archive_update_date.hpl - Y - Y - N - - - row_last_recruit_archive_update_date.hpl - Success - Y - Y - N - - - - - - diff --git a/mappings/info_recruits/citizen_tables/recruit_archive/parallel/recruit_archive_flow.hpl b/mappings/info_recruits/citizen_tables/recruit_archive/parallel/recruit_archive_flow.hpl index ddcd15f..f37e40e 100644 --- a/mappings/info_recruits/citizen_tables/recruit_archive/parallel/recruit_archive_flow.hpl +++ b/mappings/info_recruits/citizen_tables/recruit_archive/parallel/recruit_archive_flow.hpl @@ -41,48 +41,48 @@ Y - Table input - Table output + Table output + Change job status on error Y - Group by + Update + Change job status on error + Y + + + Table input Identify last row in a stream Y Identify last row in a stream - User defined Java expression + Table output Y - User defined Java expression - Update - Y - - - Table output - Change job status on error - Y - - - Update - Filter rows - Y - - - Update + Identify last row in a stream Detect empty stream Y - Update - Change job status on error + Table output + Block until transforms finish Y - Table output - Group by + Block until transforms finish + Filter rows + Y + + + Identify last row in a stream + sort_by_recruit_id + Y + + + sort_by_recruit_id + Update Y @@ -103,7 +103,29 @@ 1344 - 464 + 480 + + + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + Update + + + + + 1168 + 320 @@ -123,11 +145,8 @@ ervu-dashboard - Y - N - Y Y N @@ -139,11 +158,10 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - - 1344 - 320 + 1136 + 480 @@ -160,11 +178,8 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard - Y - N - Y N N @@ -175,10 +190,9 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - - 1792 + 1360 176 @@ -224,7 +238,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 1584 + 720 176 @@ -260,50 +274,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1584 - 304 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - has_card_in_archive_raw - is_card_in_archive - MAX - - - N - - - recruit_id - - - N - grp - - - 704 - 176 + 1360 + 320 Identify last row in a stream DetectLastRow - Y + N 1 @@ -313,8 +292,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); last_row - 896 - 176 + 720 + 320 @@ -330,6 +309,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu_person_archive N + SELECT id, recruit_id, @@ -339,13 +319,12 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); hidden, recruit_extract_enable, CASE - WHEN recruit_extract_enable IS true THEN 1 - ELSE 0 - END AS is_card_in_archive + WHEN recruit_extract_enable IS true THEN true + ELSE false + END AS card_in_archive FROM public.recruit_archive WHERE '${IDM_ID}' != '' -- Проверка на пустую строку AND vk_id = '${IDM_ID}' - AND '${M_R_CR_DATE}'::timestamp >= created_at Y @@ -412,7 +391,7 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку Y - 704 + 912 320 @@ -447,13 +426,13 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку Y - 1344 - 176 + 912 + 480 - User defined Java expression - Janino + sort_by_recruit_id + SortRows Y @@ -462,18 +441,25 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку none - - card_in_archive - has_card_in_archive_raw == 1 - Boolean - -1 - -1 - - + N + ${java.io.tmpdir} + + + Y + N + N + 0 + recruit_id + N + + + srt + 1000000 + Y - 1120 - 176 + 720 + 480 @@ -482,7 +468,7 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку Change job status on error Y - + error_description @@ -494,7 +480,7 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/recruit_archive/parallel/recruit_archive_flow_delta.hpl b/mappings/info_recruits/citizen_tables/recruit_archive/parallel/recruit_archive_flow_delta.hpl index 7d7ae20..af151f0 100644 --- a/mappings/info_recruits/citizen_tables/recruit_archive/parallel/recruit_archive_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/recruit_archive/parallel/recruit_archive_flow_delta.hpl @@ -41,38 +41,43 @@ Y - Table input - Insert / update + Update + Change job status on error Y - Group by + Table input Identify last row in a stream Y Identify last row in a stream - User defined Java expression + Insert / update Y - User defined Java expression - Update - Y - - - Update + Identify last row in a stream Detect empty stream Y - Update + Insert / update + Block until transforms finish + Y + + + Block until transforms finish Filter rows Y - Update - Change job status on error + Identify last row in a stream + sort_by_recruit_id + Y + + + sort_by_recruit_id + Update Y @@ -80,11 +85,6 @@ Change job status on error Y - - Insert / update - Group by - Y - Abort @@ -102,8 +102,30 @@ 0 - 1344 - 480 + 1392 + 496 + + + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + Update + + + + + 1200 + 352 @@ -138,8 +160,8 @@ and recruitment_id = '${IDM_ID}'; - 1344 - 352 + 1152 + 496 @@ -170,7 +192,7 @@ and recruitment_id = '${IDM_ID}'; - 1760 + 1376 224 @@ -219,7 +241,7 @@ and recruitment_id = '${IDM_ID}'; - 1568 + 704 224 @@ -255,50 +277,15 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1568 - 336 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - has_card_in_archive_raw - is_card_in_archive - MAX - - - N - - - recruit_id - - - N - grp - - - 704 - 224 + 1376 + 352 Identify last row in a stream DetectLastRow - Y + N 1 @@ -308,8 +295,8 @@ and recruitment_id = '${IDM_ID}'; last_row - 896 - 224 + 704 + 352 @@ -372,7 +359,7 @@ and recruitment_id = '${IDM_ID}'; N - 704 + 944 352 @@ -389,6 +376,7 @@ and recruitment_id = '${IDM_ID}'; ervu_person_archive N + SELECT id, recruit_id, @@ -398,8 +386,8 @@ and recruitment_id = '${IDM_ID}'; hidden, recruit_extract_enable, CASE - WHEN recruit_extract_enable IS true THEN 1 - ELSE 0 + WHEN recruit_extract_enable IS true THEN true + ELSE false END AS card_in_archive FROM public.recruit_archive WHERE '${IDM_ID}' != '' -- Проверка на пустую строку @@ -448,13 +436,13 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку Y - 1344 - 224 + 944 + 496 - User defined Java expression - Janino + sort_by_recruit_id + SortRows Y @@ -463,18 +451,25 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку none - - card_in_archive - has_card_in_archive_raw == 1 - Boolean - -1 - -1 - - + N + ${java.io.tmpdir} + + + Y + N + N + 0 + recruit_id + N + + + srt + 1000000 + Y - 1120 - 224 + 704 + 496 @@ -483,7 +478,7 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку Change job status on error Y - + error_description @@ -495,7 +490,7 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/recruit_archive/parallel/recruit_archive_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/recruit_archive/parallel/recruit_archive_flow_repeat.hpl index 60f0257..fb38c85 100644 --- a/mappings/info_recruits/citizen_tables/recruit_archive/parallel/recruit_archive_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/recruit_archive/parallel/recruit_archive_flow_repeat.hpl @@ -41,48 +41,48 @@ Y - Group by + Insert / update + Change job status on error + Y + + + Update + Change job status on error + Y + + + Table input Identify last row in a stream Y Identify last row in a stream - User defined Java expression - Y - - - User defined Java expression - Update - Y - - - Update - Detect empty stream - Y - - - Update - Filter rows - Y - - - Table input Insert / update Y - Insert / update - Group by + Identify last row in a stream + Detect empty stream Y Insert / update - Change job status on error + Block until transforms finish Y - Update - Change job status on error + Block until transforms finish + Filter rows + Y + + + Identify last row in a stream + sort_by_recruit_id + Y + + + sort_by_recruit_id + Update Y @@ -103,7 +103,29 @@ 1120 - 336 + 352 + + + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + Update + + + + + 976 + 208 @@ -123,11 +145,8 @@ ervu-dashboard - Y - N - Y Y N @@ -139,11 +158,10 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - - 1120 - 208 + 928 + 352 @@ -160,11 +178,8 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard - Y - N - Y N N @@ -175,10 +190,9 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - - 1552 + 1152 80 @@ -242,7 +256,7 @@ DO UPDATE SET - 1328 + 528 80 @@ -278,50 +292,15 @@ DO UPDATE SET Change job status on success - 1328 - 176 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - has_card_in_archive_raw - is_card_in_archive - MAX - - - N - - - recruit_id - - - N - grp - - - 480 - 80 + 1152 + 208 Identify last row in a stream DetectLastRow - Y + N 1 @@ -331,8 +310,8 @@ DO UPDATE SET last_row - 672 - 80 + 528 + 208 @@ -395,7 +374,7 @@ DO UPDATE SET N - 480 + 720 208 @@ -412,6 +391,7 @@ DO UPDATE SET ervu_person_archive N + SELECT id, recruit_id, @@ -421,13 +401,12 @@ DO UPDATE SET hidden, recruit_extract_enable, CASE - WHEN recruit_extract_enable IS true THEN 1 - ELSE 0 + WHEN recruit_extract_enable IS true THEN true + ELSE false END AS card_in_archive FROM public.recruit_archive WHERE '${IDM_ID}' != '' -- Проверка на пустую строку AND vk_id = '${IDM_ID}' - AND '${M_R_CR_DATE}'::timestamp >= created_at Y @@ -472,13 +451,13 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку Y - 1120 - 80 + 720 + 352 - User defined Java expression - Janino + sort_by_recruit_id + SortRows Y @@ -487,18 +466,25 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку none - - card_in_archive - has_card_in_archive_raw == 1 - Boolean - -1 - -1 - - + N + ${java.io.tmpdir} + + + Y + N + N + 0 + recruit_id + N + + + srt + 1000000 + Y - 896 - 80 + 528 + 352 @@ -507,7 +493,7 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку Change job status on error Y - + error_description @@ -519,7 +505,7 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/recruit_archive/recruit_archive_update_date_ervu_dashboard.hpl b/mappings/info_recruits/citizen_tables/recruit_archive/recruit_archive_update_date_ervu_dashboard.hpl deleted file mode 100644 index 5eac034..0000000 --- a/mappings/info_recruits/citizen_tables/recruit_archive/recruit_archive_update_date_ervu_dashboard.hpl +++ /dev/null @@ -1,93 +0,0 @@ - - - - recruit_archive_update_date_ervu_dashboard - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/06/02 14:50:31.833 - - - 2025/06/02 14:50:31.833 - - - - - - Table input - Table output - Y - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT - COALESCE(MAX(created_at), '0001-01-01 00:00:00'::timestamp) AS system_create_date, - COALESCE(MAX(updated_at), '0001-01-01 00:00:00'::timestamp) AS system_update_date, - current_timestamp AS record_created, - 'recruit_archive_table' AS workflow -FROM ervu_dashboard.recruit_archive; - N - - - 896 - 304 - - - - Table output - TableOutput - - Y - - 1 - - none - - - 1000 - ervu-dashboard - - - N - N - N - N - Y - N - public - N - recruit_create
- N - Y - N - Y - - - 1216 - 304 - - - - - - diff --git a/mappings/info_recruits/citizen_tables/recruit_archive/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/recruit_archive/recruitment_five_flow_delta.hpl index d68ba20..2741a20 100644 --- a/mappings/info_recruits/citizen_tables/recruit_archive/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/recruit_archive/recruitment_five_flow_delta.hpl @@ -60,14 +60,28 @@ ervu-dashboard N 0 - SELECT - r.idm_id as recruitment_id + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'recruit_archive_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -95,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -146,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -197,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -248,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -299,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -325,7 +339,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 640 + 656 448 diff --git a/mappings/info_recruits/citizen_tables/recruit_archive/recruitment_five_flow_on_error.hpl b/mappings/info_recruits/citizen_tables/recruit_archive/recruitment_five_flow_on_error.hpl index 6296ccc..265d00b 100644 --- a/mappings/info_recruits/citizen_tables/recruit_archive/recruitment_five_flow_on_error.hpl +++ b/mappings/info_recruits/citizen_tables/recruit_archive/recruitment_five_flow_on_error.hpl @@ -65,9 +65,9 @@ FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - and job_name = 'recruit_archive_job' + and job_name = '${JOB_NAME}' where je.status is null or je.status in('ERROR', 'PROCESSING'); - N + Y 272 diff --git a/mappings/info_recruits/citizen_tables/recruit_archive/recruitments_five_flow.hpl b/mappings/info_recruits/citizen_tables/recruit_archive/recruitments_five_flow.hpl index b71eb9c..556ef23 100644 --- a/mappings/info_recruits/citizen_tables/recruit_archive/recruitments_five_flow.hpl +++ b/mappings/info_recruits/citizen_tables/recruit_archive/recruitments_five_flow.hpl @@ -63,24 +63,8 @@ SELECT idm_id AS recruitment FROM ervu_dashboard.recruitment -${COMMENT}/* -where idm_id in ('110686b2-4a01-4976-885b-d26a2980f53a', -'cff66da8-2be0-4839-bd8e-332b1a150031', -'34c8685e-f08a-4877-afb1-a0589800a21c', -'0e10678c-f272-41c3-94dd-c15830321f9a', -'9703f108-761a-467a-aad3-89617e5539ed', -'55eae374-9bb5-4295-9e3a-5fbcdaba870d', -'dad6c101-34be-45c3-8173-57190cb69dfb', -'fc10523f-af31-4f6b-b0e2-00799758f634', -'61807cc2-9913-4060-8802-0672877c28c8', -'eba7fd7d-e3da-49a7-8e65-8fc85e47190e', -'f59afe6d-c166-4196-a489-6d17f02fc05e', -'22c64b79-f8e4-41e7-a2f3-1aa9df424fd9', -'0fe2bdf0-afbb-402d-93da-67e511799afa', -'6139de3b-2378-496f-9fca-7df7dab90832', -'50ba73dd-1726-48f1-8b7c-7a77ff965299') -${COMMENT}*/ - Y + + N 432 diff --git a/mappings/info_recruits/citizen_tables/recruit_archive/row_last_recruit_archive_update_date.hpl b/mappings/info_recruits/citizen_tables/recruit_archive/row_last_recruit_archive_update_date.hpl index 894136a..034a924 100644 --- a/mappings/info_recruits/citizen_tables/recruit_archive/row_last_recruit_archive_update_date.hpl +++ b/mappings/info_recruits/citizen_tables/recruit_archive/row_last_recruit_archive_update_date.hpl @@ -71,7 +71,6 @@ - MAX_RECRUIT_ARCHIVE_UPDATE_DATE M_RECARCH_UP_DATE GP_WORKFLOW @@ -98,13 +97,17 @@ ervu-dashboard N 0 - SELECT - system_update_date AS MAX_RECRUIT_ARCHIVE_UPDATE_DATE--, ---record_created -FROM public.recruit_create -WHERE system_update_date = (select max(system_update_date) from public.recruit_create where workflow = 'recruit_archive_table') -order by record_created desc -limit 1 + with +base as ( +select max(updated_at) AS MAX_RECRUIT_ARCHIVE_UPDATE_DATE +from ervu_dashboard.recruit_archive +) +select + CASE + WHEN MAX_RECRUIT_ARCHIVE_UPDATE_DATE is null THEN '0001-01-01 00:00:00'::timestamp + ELSE MAX_RECRUIT_ARCHIVE_UPDATE_DATE + END AS MAX_RECRUIT_ARCHIVE_UPDATE_DATE +from base N diff --git a/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow.hpl b/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow.hpl index 26761d2..d0d7926 100644 --- a/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow.hpl +++ b/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow.hpl @@ -267,7 +267,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - WITH base as ( SELECT @@ -364,7 +363,7 @@ WHERE LENGTH(REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'sn Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_delta.hpl b/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_delta.hpl index 2da9b4d..2db0854 100644 --- a/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_delta.hpl @@ -324,7 +324,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH base AS ( SELECT ri.recruit_id, @@ -350,7 +349,6 @@ and recruitment_id = '${IDM_ID}'; AND LENGTH(REGEXP_REPLACE(ri.info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'snils', '[^0-9]', '', 'g')) = 11 ) ) -limit 10 ), snils_data AS ( SELECT @@ -401,7 +399,7 @@ FROM snils_data; Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_repeat.hpl index 3a80e34..6d2d740 100644 --- a/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_repeat.hpl @@ -339,7 +339,6 @@ DO UPDATE SET ervu-dashboard N - WITH base as ( SELECT @@ -348,7 +347,6 @@ base as ( FROM ervu_dashboard.recruits_info ri join ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at ) SELECT recruit_id, @@ -382,7 +380,7 @@ WHERE LENGTH(REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'sn Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/snils/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/snils/recruitment_five_flow_delta.hpl index 13b0cd1..95d7d0a 100644 --- a/mappings/info_recruits/citizen_tables/snils/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/snils/recruitment_five_flow_delta.hpl @@ -65,22 +65,27 @@ ervu-dashboard N 0 - WITH mud AS ( - SELECT - recruitment_id, - MAX(execution_datetime) AS max_upd_date - FROM etl.job_execution - WHERE job_name = '${JOB_NAME}' - AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING') - GROUP BY recruitment_id -) -SELECT - r.idm_id + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r JOIN mud ON mud.recruitment_id = r.idm_id JOIN recruits_info ri ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id - AND ri.updated_at > mud.max_upd_date; + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; Y diff --git a/mappings/info_recruits/citizen_tables/snils/recruitment_five_flow_on_error.hpl b/mappings/info_recruits/citizen_tables/snils/recruitment_five_flow_on_error.hpl index 65f8604..0db9bfd 100644 --- a/mappings/info_recruits/citizen_tables/snils/recruitment_five_flow_on_error.hpl +++ b/mappings/info_recruits/citizen_tables/snils/recruitment_five_flow_on_error.hpl @@ -65,9 +65,9 @@ FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - and job_name = 'snils_inn_job' + and job_name = '${JOB_NAME}' where je.status is null or je.status in('ERROR', 'PROCESSING'); - N + Y 272 diff --git a/mappings/info_recruits/citizen_tables/snils/recruitments_five_flow.hpl b/mappings/info_recruits/citizen_tables/snils/recruitments_five_flow.hpl index 3762f21..2757f86 100644 --- a/mappings/info_recruits/citizen_tables/snils/recruitments_five_flow.hpl +++ b/mappings/info_recruits/citizen_tables/snils/recruitments_five_flow.hpl @@ -63,24 +63,8 @@ SELECT idm_id AS recruitment FROM ervu_dashboard.recruitment -${COMMENT}/* -where idm_id in ('110686b2-4a01-4976-885b-d26a2980f53a', -'cff66da8-2be0-4839-bd8e-332b1a150031', -'34c8685e-f08a-4877-afb1-a0589800a21c', -'0e10678c-f272-41c3-94dd-c15830321f9a', -'9703f108-761a-467a-aad3-89617e5539ed', -'55eae374-9bb5-4295-9e3a-5fbcdaba870d', -'dad6c101-34be-45c3-8173-57190cb69dfb', -'fc10523f-af31-4f6b-b0e2-00799758f634', -'61807cc2-9913-4060-8802-0672877c28c8', -'eba7fd7d-e3da-49a7-8e65-8fc85e47190e', -'f59afe6d-c166-4196-a489-6d17f02fc05e', -'22c64b79-f8e4-41e7-a2f3-1aa9df424fd9', -'0fe2bdf0-afbb-402d-93da-67e511799afa', -'6139de3b-2378-496f-9fca-7df7dab90832', -'50ba73dd-1726-48f1-8b7c-7a77ff965299') -${COMMENT}*/ - Y + + N 432 diff --git a/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow.hpl b/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow.hpl index c622019..09438c0 100644 --- a/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow.hpl +++ b/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow.hpl @@ -45,43 +45,43 @@ Change job status on error Y - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - User defined Java expression - Y - - - User defined Java expression - Update - Y - - - Table input - Table output - Y - - - Table output - Group by - Y - Update Change job status on error Y - Update + Table input + Identify last row in a stream + Y + + + Identify last row in a stream + Table output + Y + + + Identify last row in a stream Detect empty stream Y - Update + Identify last row in a stream + sort_by_recruit_id + Y + + + sort_by_recruit_id + Update + Y + + + Table output + Block until transforms finish + Y + + + Block until transforms finish Filter rows Y @@ -102,8 +102,30 @@ 0 - 1504 - 496 + 1424 + 512 + + + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + Update + + + + + 1184 + 320 @@ -123,11 +145,8 @@ ervu-dashboard - Y - N - Y Y N @@ -139,11 +158,10 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - - 1504 - 320 + 1200 + 512 @@ -160,11 +178,8 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard - Y - N - Y N N @@ -175,10 +190,9 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - - 2032 + 1408 176 @@ -224,7 +238,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 1712 + 704 176 @@ -260,65 +274,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1712 - 304 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - has_sent_summons_raw - is_sent_summons - MAX - - - has_served_summons_raw - is_served_summons - MAX - - - has_appeared_to_recruitment_center_raw - is_appeared_to_recruitment_center - MAX - - - has_not_appeared_by_valid_excuse_raw - is_not_appeared_by_valid_excuse - MAX - - - N - - - recruit_id - - - N - grp - - - 688 - 16 + 1408 + 320 Identify last row in a stream DetectLastRow - Y + N 1 @@ -328,8 +292,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); last_row - 880 - 16 + 704 + 320 @@ -345,24 +309,25 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); postgres.subpoena N + SELECT s.id, --id повестки CASE - WHEN s.send_date is not null THEN 1 - ELSE 0 - END AS is_sent_summons, --факт направления повестки///Признак наличия направленной повестки + WHEN s.send_date is not null THEN true + ELSE false + END AS has_sent_summons, --факт направления повестки///Признак наличия направленной повестки CASE - WHEN s.system_stages->'subpoenaAwarded' != 'true' THEN 1 - ELSE 0 - END AS is_served_summons, --has_served_summons//Признак наличия врученной повестки + WHEN s.system_stages->'subpoenaAwarded' != 'true' THEN true + ELSE false + END AS has_served_summons, --has_served_summons//Признак наличия врученной повестки CASE - WHEN s.system_stages->'recruitAttendanceConfirm' != 'true' THEN 1 - ELSE 0 - END AS is_appeared_to_recruitment_center, -- appeared_to_recruitment_center//Признак явки в военный комиссариат + WHEN s.system_stages->'recruitAttendanceConfirm' != 'true' THEN true + ELSE false + END AS appeared_to_recruitment_center, -- appeared_to_recruitment_center//Признак явки в военный комиссариат CASE - WHEN ss.code = '5.1' THEN 1 - ELSE 0 - END AS is_not_appeared_by_valid_excuse, -- not_appeared_by_valid_excuse//Признак неявки в военкомат по уважительной причине + WHEN ss.code = '5.1' THEN true + ELSE false + END AS not_appeared_by_valid_excuse, -- not_appeared_by_valid_excuse//Признак неявки в военкомат по уважительной причине s.recruit_id, s.send_date, --дата направления повестки s.series, --серия повестки @@ -389,13 +354,7 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку - AND s.department_id = '${IDM_ID}' - - AND '${M_R_CR_DATE}' >= s.create_date - - - -${LIMIT_FW} + AND s.department_id = '${IDM_ID}' Y @@ -493,7 +452,7 @@ ${LIMIT_FW} Y - 688 + 912 320 @@ -540,13 +499,13 @@ ${LIMIT_FW} Y - 1504 - 176 + 912 + 512 - User defined Java expression - Janino + sort_by_recruit_id + SortRows Y @@ -555,42 +514,25 @@ ${LIMIT_FW} none - - has_sent_summons - has_sent_summons_raw == 1 - Boolean - -1 - -1 - - - - has_served_summons - has_served_summons_raw == 1 - Boolean - -1 - -1 - - - - appeared_to_recruitment_center - has_appeared_to_recruitment_center_raw == 1 - Boolean - -1 - -1 - - - - not_appeared_by_valid_excuse - has_not_appeared_by_valid_excuse_raw == 1 - Boolean - -1 - -1 - - + N + ${java.io.tmpdir} + + + Y + N + N + 0 + recruit_id + N + + + srt + 1000000 + Y - 1104 - 16 + 704 + 512 @@ -601,7 +543,7 @@ ${LIMIT_FW} error_description - error_code + @@ -611,7 +553,7 @@ ${LIMIT_FW} Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow_delta.hpl b/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow_delta.hpl index 7d7a58e..81b0267 100644 --- a/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow_delta.hpl @@ -40,48 +40,48 @@ Abort Y - - Table input - Insert / update - Y - Insert / update Change job status on error Y - Group by + Update + Change job status on error + Y + + + Table input Identify last row in a stream Y Identify last row in a stream - User defined Java expression + Insert / update Y - User defined Java expression + Identify last row in a stream + Detect empty stream + Y + + + Identify last row in a stream + sort_by_recruit_id + Y + + + sort_by_recruit_id Update Y Insert / update - Group by + Block until transforms finish Y - Update - Change job status on error - Y - - - Update - Detect empty stream - Y - - - Update + Block until transforms finish Filter rows Y @@ -102,10 +102,32 @@ 0 - 1440 + 1488 880 + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + Update + + + + + 1312 + 704 + + Change job status on error ExecSql @@ -123,11 +145,8 @@ ervu-dashboard - Y - N - Y Y N @@ -139,11 +158,10 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - - 1440 - 704 + 1312 + 880 @@ -160,11 +178,8 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard - Y - N - Y N N @@ -175,11 +190,10 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - - 1920 - 384 + 1488 + 560 @@ -227,8 +241,8 @@ and recruitment_id = '${IDM_ID}'; - 1680 - 384 + 848 + 560 @@ -263,65 +277,15 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1680 - 496 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - has_sent_summons_raw - is_sent_summons - MAX - - - has_served_summons_raw - is_served_summons - MAX - - - has_appeared_to_recruitment_center_raw - is_appeared_to_recruitment_center - MAX - - - has_not_appeared_by_valid_excuse_raw - is_not_appeared_by_valid_excuse - MAX - - - N - - - recruit_id - - - N - grp - - - 800 - 384 + 1488 + 704 Identify last row in a stream DetectLastRow - Y + N 1 @@ -331,8 +295,8 @@ and recruitment_id = '${IDM_ID}'; last_row - 992 - 384 + 848 + 704 @@ -435,7 +399,7 @@ and recruitment_id = '${IDM_ID}'; N - 800 + 1056 704 @@ -452,24 +416,25 @@ and recruitment_id = '${IDM_ID}'; postgres.subpoena N + SELECT s.id, --id повестки CASE - WHEN s.send_date is not null THEN 1 - ELSE 0 - END AS is_sent_summons, --факт направления повестки///Признак наличия направленной повестки + WHEN s.send_date is not null THEN true + ELSE false + END AS has_sent_summons, --факт направления повестки///Признак наличия направленной повестки CASE - WHEN s.system_stages->'subpoenaAwarded' != 'true' THEN 1 - ELSE 0 - END AS is_served_summons, --has_served_summons//Признак наличия врученной повестки + WHEN s.system_stages->'subpoenaAwarded' != 'true' THEN true + ELSE false + END AS has_served_summons, --has_served_summons//Признак наличия врученной повестки CASE - WHEN s.system_stages->'recruitAttendanceConfirm' != 'true' THEN 1 - ELSE 0 - END AS is_appeared_to_recruitment_center, -- appeared_to_recruitment_center//Признак явки в военный комиссариат + WHEN s.system_stages->'recruitAttendanceConfirm' != 'true' THEN true + ELSE false + END AS appeared_to_recruitment_center, -- appeared_to_recruitment_center//Признак явки в военный комиссариат CASE - WHEN ss.code = '5.1' THEN 1 - ELSE 0 - END AS is_not_appeared_by_valid_excuse, -- not_appeared_by_valid_excuse//Признак неявки в военкомат по уважительной причине + WHEN ss.code = '5.1' THEN true + ELSE false + END AS not_appeared_by_valid_excuse, -- not_appeared_by_valid_excuse//Признак неявки в военкомат по уважительной причине s.recruit_id, s.send_date, --дата направления повестки s.series, --серия повестки @@ -494,9 +459,7 @@ join public.subpoena_reason sr on sr.id = s.reason_id WHERE '${IDM_ID}' != '' -- Проверка на пустую строку AND s.department_id = '${IDM_ID}' - AND s.status_change_date >= '${M_S_UP_DATE}'::timestamp - -${LIMIT_FW} + AND s.status_change_date >= '${M_S_UP_DATE}'::timestamp Y @@ -547,13 +510,13 @@ ${LIMIT_FW} Y - 1440 - 384 + 1056 + 880 - User defined Java expression - Janino + sort_by_recruit_id + SortRows Y @@ -562,42 +525,25 @@ ${LIMIT_FW} none - - has_sent_summons - has_sent_summons_raw == 1 - Boolean - -1 - -1 - - - - has_served_summons - has_served_summons_raw == 1 - Boolean - -1 - -1 - - - - appeared_to_recruitment_center - has_appeared_to_recruitment_center_raw == 1 - Boolean - -1 - -1 - - - - not_appeared_by_valid_excuse - has_not_appeared_by_valid_excuse_raw == 1 - Boolean - -1 - -1 - - + N + ${java.io.tmpdir} + + + Y + N + N + 0 + recruit_id + N + + + srt + 1000000 + Y - 1216 - 384 + 848 + 880 @@ -606,7 +552,7 @@ ${LIMIT_FW} Change job status on error Y - + error_description @@ -618,7 +564,7 @@ ${LIMIT_FW} Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow_repeat.hpl index 4ade27d..9692d38 100644 --- a/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow_repeat.hpl @@ -45,43 +45,43 @@ Change job status on error Y - - Table input - Insert / update - Y - - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - User defined Java expression - Y - - - User defined Java expression - Update - Y - - - Insert / update - Group by - Y - Update Change job status on error Y - Update + Table input + Identify last row in a stream + Y + + + Identify last row in a stream + Insert / update + Y + + + Identify last row in a stream Detect empty stream Y - Update + Identify last row in a stream + sort_by_recruit_id + Y + + + sort_by_recruit_id + Update + Y + + + Insert / update + Block until transforms finish + Y + + + Block until transforms finish Filter rows Y @@ -106,6 +106,28 @@ 752 + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + Update + + + + + 1296 + 560 + + Change job status on error ExecSql @@ -123,11 +145,8 @@ ervu-dashboard - Y - N - Y Y N @@ -139,11 +158,10 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - - 1472 - 560 + 1280 + 752 @@ -160,11 +178,8 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard - Y - N - Y N N @@ -175,11 +190,10 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - - 1936 - 352 + 1472 + 400 @@ -242,8 +256,8 @@ DO UPDATE SET - 1728 - 352 + 848 + 400 @@ -278,65 +292,15 @@ DO UPDATE SET Change job status on success - 1728 - 464 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - has_sent_summons_raw - is_sent_summons - MAX - - - has_served_summons_raw - is_served_summons - MAX - - - has_appeared_to_recruitment_center_raw - is_appeared_to_recruitment_center - MAX - - - has_not_appeared_by_valid_excuse_raw - is_not_appeared_by_valid_excuse - MAX - - - N - - - recruit_id - - - N - grp - - - 832 - 352 + 1472 + 560 Identify last row in a stream DetectLastRow - Y + N 1 @@ -346,8 +310,8 @@ DO UPDATE SET last_row - 1024 - 352 + 848 + 560 @@ -450,7 +414,7 @@ DO UPDATE SET N - 832 + 1056 560 @@ -467,24 +431,25 @@ DO UPDATE SET postgres.subpoena N + SELECT s.id, --id повестки CASE - WHEN s.send_date is not null THEN 1 - ELSE 0 - END AS is_sent_summons, --факт направления повестки///Признак наличия направленной повестки + WHEN s.send_date is not null THEN true + ELSE false + END AS has_sent_summons, --факт направления повестки///Признак наличия направленной повестки CASE - WHEN s.system_stages->'subpoenaAwarded' != 'true' THEN 1 - ELSE 0 - END AS is_served_summons, --has_served_summons//Признак наличия врученной повестки + WHEN s.system_stages->'subpoenaAwarded' != 'true' THEN true + ELSE false + END AS has_served_summons, --has_served_summons//Признак наличия врученной повестки CASE - WHEN s.system_stages->'recruitAttendanceConfirm' != 'true' THEN 1 - ELSE 0 - END AS is_appeared_to_recruitment_center, -- appeared_to_recruitment_center//Признак явки в военный комиссариат + WHEN s.system_stages->'recruitAttendanceConfirm' != 'true' THEN true + ELSE false + END AS appeared_to_recruitment_center, -- appeared_to_recruitment_center//Признак явки в военный комиссариат CASE - WHEN ss.code = '5.1' THEN 1 - ELSE 0 - END AS is_not_appeared_by_valid_excuse, -- not_appeared_by_valid_excuse//Признак неявки в военкомат по уважительной причине + WHEN ss.code = '5.1' THEN true + ELSE false + END AS not_appeared_by_valid_excuse, -- not_appeared_by_valid_excuse//Признак неявки в военкомат по уважительной причине s.recruit_id, s.send_date, --дата направления повестки s.series, --серия повестки @@ -509,9 +474,7 @@ join public.subpoena_reason sr on sr.id = s.reason_id WHERE '${IDM_ID}' != '' -- Проверка на пустую строку AND s.department_id = '${IDM_ID}' - AND '${M_R_CR_DATE}' >= s.create_date - -${LIMIT_FW} + Y @@ -562,13 +525,13 @@ ${LIMIT_FW} Y - 1472 - 352 + 1056 + 752 - User defined Java expression - Janino + sort_by_recruit_id + SortRows Y @@ -577,42 +540,25 @@ ${LIMIT_FW} none - - has_sent_summons - has_sent_summons_raw == 1 - Boolean - -1 - -1 - - - - has_served_summons - has_served_summons_raw == 1 - Boolean - -1 - -1 - - - - appeared_to_recruitment_center - has_appeared_to_recruitment_center_raw == 1 - Boolean - -1 - -1 - - - - not_appeared_by_valid_excuse - has_not_appeared_by_valid_excuse_raw == 1 - Boolean - -1 - -1 - - + N + ${java.io.tmpdir} + + + Y + N + N + 0 + recruit_id + N + + + srt + 1000000 + Y - 1248 - 352 + 848 + 752 @@ -621,7 +567,7 @@ ${LIMIT_FW} Change job status on error Y - + error_description @@ -633,7 +579,7 @@ ${LIMIT_FW} Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/subpoena/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/subpoena/recruitment_five_flow_delta.hpl index fc3874f..ff6c74c 100644 --- a/mappings/info_recruits/citizen_tables/subpoena/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/subpoena/recruitment_five_flow_delta.hpl @@ -60,14 +60,28 @@ ervu-dashboard N 0 - SELECT - r.idm_id as recruitment_id + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'subpoena_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -95,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -146,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -197,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -248,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -299,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y diff --git a/mappings/info_recruits/citizen_tables/subpoena/recruitment_five_flow_on_error.hpl b/mappings/info_recruits/citizen_tables/subpoena/recruitment_five_flow_on_error.hpl index b34a53e..2451223 100644 --- a/mappings/info_recruits/citizen_tables/subpoena/recruitment_five_flow_on_error.hpl +++ b/mappings/info_recruits/citizen_tables/subpoena/recruitment_five_flow_on_error.hpl @@ -65,9 +65,9 @@ FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - and job_name = 'subpoena_job' + and job_name = '${JOB_NAME}' where je.status is null or je.status in('ERROR', 'PROCESSING'); - N + Y 272 diff --git a/mappings/info_recruits/citizen_tables/subpoena/recruitments_five_flow.hpl b/mappings/info_recruits/citizen_tables/subpoena/recruitments_five_flow.hpl index 4ae9e09..c434964 100644 --- a/mappings/info_recruits/citizen_tables/subpoena/recruitments_five_flow.hpl +++ b/mappings/info_recruits/citizen_tables/subpoena/recruitments_five_flow.hpl @@ -63,24 +63,8 @@ SELECT idm_id AS recruitment FROM ervu_dashboard.recruitment -${COMMENT}/* -where idm_id in ('110686b2-4a01-4976-885b-d26a2980f53a', -'cff66da8-2be0-4839-bd8e-332b1a150031', -'34c8685e-f08a-4877-afb1-a0589800a21c', -'0e10678c-f272-41c3-94dd-c15830321f9a', -'9703f108-761a-467a-aad3-89617e5539ed', -'55eae374-9bb5-4295-9e3a-5fbcdaba870d', -'dad6c101-34be-45c3-8173-57190cb69dfb', -'fc10523f-af31-4f6b-b0e2-00799758f634', -'61807cc2-9913-4060-8802-0672877c28c8', -'eba7fd7d-e3da-49a7-8e65-8fc85e47190e', -'f59afe6d-c166-4196-a489-6d17f02fc05e', -'22c64b79-f8e4-41e7-a2f3-1aa9df424fd9', -'0fe2bdf0-afbb-402d-93da-67e511799afa', -'6139de3b-2378-496f-9fca-7df7dab90832', -'50ba73dd-1726-48f1-8b7c-7a77ff965299') -${COMMENT}*/ - Y + + N 432 diff --git a/mappings/info_recruits/job_parallel_subpoena_decision_archive_person.hwf b/mappings/info_recruits/job_parallel_subpoena_decision_archive_person.hwf index 68d9007..0975e46 100644 --- a/mappings/info_recruits/job_parallel_subpoena_decision_archive_person.hwf +++ b/mappings/info_recruits/job_parallel_subpoena_decision_archive_person.hwf @@ -440,30 +440,6 @@ 912 - - job_last_update_date_infringement.hwf - - WORKFLOW - - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_liability_fz53/job_last_update_date_infringement.hwf - Nothing - - Y - - N - local - N - N - Y - Y - 944 - 976 - - job_last_update_date_deferment_decision.hwf @@ -488,30 +464,6 @@ 1104 - - job_last_update_date_recruit_archive.hwf - - WORKFLOW - - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/recruit_archive/job_last_update_date_recruit_archive.hwf - Nothing - - Y - - N - local - N - N - Y - Y - 944 - 1232 - - job_last_update_date_restriction_document.hwf @@ -584,6 +536,58 @@ 1424 + + row_last_infringement_update_date.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_liability_fz53/row_last_infringement_update_date.hpl + Basic + + Y + + N + local + N + N + Y + Y + 944 + 976 + + + + row_last_recruit_archive_update_date.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/recruit_archive/row_last_recruit_archive_update_date.hpl + Basic + + Y + + N + local + N + N + Y + N + 944 + 1232 + + @@ -698,27 +702,6 @@ Y N - - Start - job_last_update_date_infringement.hwf - Y - Y - Y - - - job_last_update_date_infringement.hwf - citizen_criminal_liability_fz53_job.hwf - Y - Y - N - - - job_last_update_date_infringement.hwf - citizen_liability_fz53_job.hwf - Y - Y - N - job_last_update_date_deferment_decision.hwf deferment_liberation_job.hwf @@ -733,20 +716,6 @@ Y Y - - Start - job_last_update_date_recruit_archive.hwf - N - Y - Y - - - job_last_update_date_recruit_archive.hwf - recruit_archive_job.hwf - Y - Y - N - Start job_last_update_date_restriction_document.hwf @@ -799,10 +768,45 @@ Start citizen_appealing_violations_fz53_job.hwf + N + Y + Y + + + Start + row_last_infringement_update_date.hpl Y Y Y + + row_last_infringement_update_date.hpl + citizen_criminal_liability_fz53_job.hwf + Y + Y + N + + + row_last_infringement_update_date.hpl + citizen_liability_fz53_job.hwf + Y + Y + N + + + Start + row_last_recruit_archive_update_date.hpl + Y + Y + Y + + + row_last_recruit_archive_update_date.hpl + recruit_archive_job.hwf + Y + Y + N + diff --git a/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow.hpl b/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow.hpl index 6fc55ff..eaa3249 100644 --- a/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow.hpl +++ b/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow.hpl @@ -271,8 +271,7 @@ FROM public.restriction_document WHERE '${IDM_ID}' != '' -- Проверка на пустую строку AND vk_id = '${IDM_ID}' - AND '${M_R_CR_DATE}' >= created_at -${LIMIT_FW} + Y @@ -355,14 +354,11 @@ ${LIMIT_FW} N N N - Y - N ervu_dashboard Y restriction_document
- N Y N @@ -381,7 +377,7 @@ ${LIMIT_FW} error_description - error_code + diff --git a/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_delta.hpl b/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_delta.hpl index 8464675..592136d 100644 --- a/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_delta.hpl +++ b/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_delta.hpl @@ -380,8 +380,7 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку AND vk_id = '${IDM_ID}' AND updated_at >= '${M_RESTRDOC_UP_DATE}'::timestamp - -${LIMIT_FW} + Y @@ -395,7 +394,7 @@ ${LIMIT_FW} Change job status on error Y - + error_description diff --git a/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_repeat.hpl b/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_repeat.hpl index 3db2497..b74cd46 100644 --- a/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_repeat.hpl +++ b/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_repeat.hpl @@ -288,13 +288,11 @@ DO UPDATE SET = id id - = created_at created_at - ervu_dashboard restriction_document
@@ -395,8 +393,7 @@ FROM public.restriction_document WHERE '${IDM_ID}' != '' -- Проверка на пустую строку AND vk_id = '${IDM_ID}' - AND '${M_R_CR_DATE}' >= created_at -${LIMIT_FW} + Y @@ -410,7 +407,7 @@ ${LIMIT_FW} Change job status on error Y - + error_description diff --git a/mappings/info_recruits/raw_data/restriction_document/recruitment_five_flow_delta.hpl b/mappings/info_recruits/raw_data/restriction_document/recruitment_five_flow_delta.hpl index d6812fc..bdbf431 100644 --- a/mappings/info_recruits/raw_data/restriction_document/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/raw_data/restriction_document/recruitment_five_flow_delta.hpl @@ -60,14 +60,28 @@ ervu-dashboard N 0 - SELECT - r.idm_id as recruitment_id + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'restriction_document_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -95,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -146,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -197,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -248,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y @@ -299,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id Y diff --git a/mappings/info_recruits/raw_data/restriction_document/recruitment_five_flow_on_error.hpl b/mappings/info_recruits/raw_data/restriction_document/recruitment_five_flow_on_error.hpl index 33c0426..bcb439a 100644 --- a/mappings/info_recruits/raw_data/restriction_document/recruitment_five_flow_on_error.hpl +++ b/mappings/info_recruits/raw_data/restriction_document/recruitment_five_flow_on_error.hpl @@ -65,9 +65,9 @@ FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - and job_name = 'restriction_document_job' + and job_name = '${JOB_NAME}' where je.status is null or je.status in('ERROR', 'PROCESSING'); - N + Y 272 diff --git a/mappings/info_recruits/raw_data/restriction_document/recruitments_five_flow.hpl b/mappings/info_recruits/raw_data/restriction_document/recruitments_five_flow.hpl index 7371f01..bca46ae 100644 --- a/mappings/info_recruits/raw_data/restriction_document/recruitments_five_flow.hpl +++ b/mappings/info_recruits/raw_data/restriction_document/recruitments_five_flow.hpl @@ -63,24 +63,8 @@ SELECT idm_id AS recruitment FROM ervu_dashboard.recruitment -${COMMENT}/* -where idm_id in ('110686b2-4a01-4976-885b-d26a2980f53a', -'cff66da8-2be0-4839-bd8e-332b1a150031', -'34c8685e-f08a-4877-afb1-a0589800a21c', -'0e10678c-f272-41c3-94dd-c15830321f9a', -'9703f108-761a-467a-aad3-89617e5539ed', -'55eae374-9bb5-4295-9e3a-5fbcdaba870d', -'dad6c101-34be-45c3-8173-57190cb69dfb', -'fc10523f-af31-4f6b-b0e2-00799758f634', -'61807cc2-9913-4060-8802-0672877c28c8', -'eba7fd7d-e3da-49a7-8e65-8fc85e47190e', -'f59afe6d-c166-4196-a489-6d17f02fc05e', -'22c64b79-f8e4-41e7-a2f3-1aa9df424fd9', -'0fe2bdf0-afbb-402d-93da-67e511799afa', -'6139de3b-2378-496f-9fca-7df7dab90832', -'50ba73dd-1726-48f1-8b7c-7a77ff965299') -${COMMENT}*/ - Y + + N 432 diff --git a/mappings/info_recruits/raw_data/restriction_document_item/parallel/restriction_document_item_flow.hpl b/mappings/info_recruits/raw_data/restriction_document_item/parallel/restriction_document_item_flow.hpl index f70a1e4..737c73c 100644 --- a/mappings/info_recruits/raw_data/restriction_document_item/parallel/restriction_document_item_flow.hpl +++ b/mappings/info_recruits/raw_data/restriction_document_item/parallel/restriction_document_item_flow.hpl @@ -45,43 +45,43 @@ Change job status on error Y - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - User defined Java expression - Y - - - User defined Java expression - Update - Y - - - Table input - Table output - Y - - - Table output - Group by - Y - Update Change job status on error Y - Update + Table input + Identify last row in a stream + Y + + + Identify last row in a stream + Table output + Y + + + Identify last row in a stream Detect empty stream Y - Update + Identify last row in a stream + sort_by_recruit_id + Y + + + sort_by_recruit_id + Update + Y + + + Table output + Block until transforms finish + Y + + + Block until transforms finish Filter rows Y @@ -102,8 +102,30 @@ 0 - 1312 - 464 + 1360 + 496 + + + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + Update + + + + + 1152 + 320 @@ -137,8 +159,8 @@ and recruitment_id = '${IDM_ID}'; - 1312 - 320 + 1168 + 496 @@ -168,8 +190,8 @@ and recruitment_id = '${IDM_ID}'; - 1808 - 192 + 1344 + 176 @@ -214,8 +236,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 1552 - 192 + 704 + 176 @@ -250,52 +272,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1552 - 288 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - has_active_temporary_measure_raw - is_active_temporary_measure - MAX - - - - N - - - recruit_id - - - N - - grp - - - 672 - 192 + 1344 + 320 Identify last row in a stream DetectLastRow - Y + N 1 @@ -305,8 +290,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); last_row - 864 - 192 + 704 + 320 @@ -322,12 +307,13 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); postgres.subpoena N + select s.recruit_id, CASE - WHEN rdi.applied_fact is true THEN 1 - ELSE 0 -END AS is_active_temporary_measure, -- has_active_temporary_measure//Признак наличия примененных временных мер + WHEN rdi.applied_fact is true THEN true + ELSE false +END AS has_active_temporary_measure, -- has_active_temporary_measure//Признак наличия примененных временных мер rdi.* from public.restriction_document_item rdi join public.restriction_document rd on @@ -337,9 +323,7 @@ s.id = rd.subpoena_id WHERE '${IDM_ID}' != '' -- Проверка на пустую строку AND rd.vk_id = '${IDM_ID}' - AND '${M_R_CR_DATE}' >= rdi.created_at - -${LIMIT_FW} + Y @@ -437,7 +421,7 @@ ${LIMIT_FW} Y - 672 + 928 320 @@ -472,13 +456,13 @@ ${LIMIT_FW} Y - 1312 - 192 + 928 + 496 - User defined Java expression - Janino + sort_by_recruit_id + SortRows Y @@ -487,18 +471,25 @@ ${LIMIT_FW} none - - has_active_temporary_measure - has_active_temporary_measure_raw == 1 - Boolean - -1 - -1 - - + N + ${java.io.tmpdir} + + + Y + N + N + 0 + recruit_id + N + + + srt + 1000000 + Y - 1088 - 192 + 704 + 496 @@ -509,7 +500,7 @@ ${LIMIT_FW} error_description - error_code + @@ -519,7 +510,7 @@ ${LIMIT_FW} Change job status on error Y - + error_description diff --git a/mappings/info_recruits/raw_data/restriction_document_item/parallel/restriction_document_item_flow_delta.hpl b/mappings/info_recruits/raw_data/restriction_document_item/parallel/restriction_document_item_flow_delta.hpl index 57d0c84..f8cdb0f 100644 --- a/mappings/info_recruits/raw_data/restriction_document_item/parallel/restriction_document_item_flow_delta.hpl +++ b/mappings/info_recruits/raw_data/restriction_document_item/parallel/restriction_document_item_flow_delta.hpl @@ -45,46 +45,46 @@ Change job status on error Y - - Table input - Insert / update - Y - - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - User defined Java expression - Y - - - User defined Java expression - Update - Y - - - Insert / update - Group by - Y - Update Change job status on error Y - Update + Table input + Identify last row in a stream + Y + + + Identify last row in a stream + Insert / update + Y + + + Identify last row in a stream Detect empty stream Y - Update + Insert / update + Block until transforms finish + Y + + + Block until transforms finish Filter rows Y + + Identify last row in a stream + sort_by_recruit_id + Y + + + sort_by_recruit_id + Update + Y + Abort @@ -102,10 +102,32 @@ 0 - 1376 + 1408 368 + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + Update + + + + + 1264 + 224 + + Change job status on error ExecSql @@ -137,8 +159,8 @@ and recruitment_id = '${IDM_ID}'; - 1376 - 224 + 1232 + 368 @@ -168,8 +190,8 @@ and recruitment_id = '${IDM_ID}'; - 1920 - 96 + 1472 + 80 @@ -217,8 +239,8 @@ and recruitment_id = '${IDM_ID}'; - 1648 - 96 + 768 + 80 @@ -253,50 +275,15 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1648 - 192 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - has_active_temporary_measure - is_active_temporary_measure - MAX - - - N - - - recruit_id - - - N - grp - - - 736 - 96 + 1472 + 224 Identify last row in a stream DetectLastRow - Y + N 1 @@ -306,8 +293,8 @@ and recruitment_id = '${IDM_ID}'; last_row - 928 - 96 + 768 + 224 @@ -415,7 +402,7 @@ and recruitment_id = '${IDM_ID}'; N - 736 + 992 224 @@ -432,13 +419,12 @@ and recruitment_id = '${IDM_ID}'; ervu_person_registry N - select s.recruit_id, CASE - WHEN rdi.applied_fact is true THEN 1 - ELSE 0 -END AS is_active_temporary_measure, -- has_active_temporary_measure//Признак наличия примененных временных мер + WHEN rdi.applied_fact is true THEN true + ELSE false +END AS has_active_temporary_measure, -- has_active_temporary_measure//Признак наличия примененных временных мер rdi.* from public.restriction_document_item rdi join public.restriction_document rd on @@ -448,9 +434,7 @@ s.id = rd.subpoena_id WHERE '${IDM_ID}' != '' -- Проверка на пустую строку AND rd.vk_id = '${IDM_ID}' - AND rdi.updated_at >= '${M_RDI_UP_DATE}'::timestamp - -${LIMIT_FW} + AND rdi.updated_at >= '${M_RDI_UP_DATE}'::timestamp Y @@ -494,13 +478,13 @@ ${LIMIT_FW} Y - 1376 - 96 + 992 + 368 - User defined Java expression - Janino + sort_by_recruit_id + SortRows Y @@ -509,18 +493,25 @@ ${LIMIT_FW} none - - has_active_temporary_measure - has_active_temporary_measure_raw == 1 - Boolean - -1 - -1 - - + N + ${java.io.tmpdir} + + + Y + N + N + 0 + recruit_id + N + + + srt + 1000000 + Y - 1152 - 96 + 768 + 368 @@ -529,7 +520,7 @@ ${LIMIT_FW} Change job status on error Y - + error_description @@ -541,7 +532,7 @@ ${LIMIT_FW} Change job status on error Y - + error_description diff --git a/mappings/info_recruits/raw_data/restriction_document_item/parallel/restriction_document_item_flow_repeat.hpl b/mappings/info_recruits/raw_data/restriction_document_item/parallel/restriction_document_item_flow_repeat.hpl index 1dec4b9..efddae3 100644 --- a/mappings/info_recruits/raw_data/restriction_document_item/parallel/restriction_document_item_flow_repeat.hpl +++ b/mappings/info_recruits/raw_data/restriction_document_item/parallel/restriction_document_item_flow_repeat.hpl @@ -45,43 +45,43 @@ Change job status on error Y - - Table input - Insert / update - Y - - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - User defined Java expression - Y - - - User defined Java expression - Update - Y - - - Insert / update - Group by - Y - Update Change job status on error Y - Update + Table input + Identify last row in a stream + Y + + + Identify last row in a stream + Insert / update + Y + + + Insert / update + Block until transforms finish + Y + + + Identify last row in a stream + sort_by_recruit_id + Y + + + sort_by_recruit_id + Update + Y + + + Identify last row in a stream Detect empty stream Y - Update + Block until transforms finish Filter rows Y @@ -102,8 +102,30 @@ 0 - 1280 - 400 + 1312 + 368 + + + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + Update + + + + + 1184 + 208 @@ -137,8 +159,8 @@ and recruitment_id = '${IDM_ID}'; - 1280 - 208 + 1136 + 368 @@ -168,8 +190,8 @@ and recruitment_id = '${IDM_ID}'; - 1744 - 80 + 1440 + 64 @@ -232,8 +254,8 @@ DO UPDATE SET - 1488 - 80 + 672 + 64 @@ -268,50 +290,15 @@ DO UPDATE SET Change job status on success - 1504 - 192 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - has_active_temporary_measure - is_active_temporary_measure - MAX - - - N - - - recruit_id - - - N - grp - - - 640 - 80 + 1440 + 208 Identify last row in a stream DetectLastRow - Y + N 1 @@ -321,8 +308,8 @@ DO UPDATE SET last_row - 832 - 80 + 672 + 208 @@ -343,13 +330,11 @@ DO UPDATE SET = id id - = created_at created_at - ervu_dashboard restriction_document_item
@@ -432,7 +417,7 @@ DO UPDATE SET N - 640 + 912 208 @@ -449,12 +434,13 @@ DO UPDATE SET postgres.subpoena N + select s.recruit_id, CASE - WHEN rdi.applied_fact is true THEN 1 - ELSE 0 -END AS is_active_temporary_measure, -- has_active_temporary_measure//Признак наличия примененных временных мер + WHEN rdi.applied_fact is true THEN true + ELSE false +END AS has_active_temporary_measure, -- has_active_temporary_measure//Признак наличия примененных временных мер rdi.* from public.restriction_document_item rdi join public.restriction_document rd on @@ -464,9 +450,7 @@ s.id = rd.subpoena_id WHERE '${IDM_ID}' != '' -- Проверка на пустую строку AND rd.vk_id = '${IDM_ID}' - AND '${M_R_CR_DATE}' >= rdi.created_at - -${LIMIT_FW} + Y @@ -488,19 +472,16 @@ ${LIMIT_FW} 10000 ervu-dashboard Y - = recruit_id recruit_id - <> has_active_temporary_measure has_active_temporary_measure - ervu_dashboard citizen
@@ -513,13 +494,13 @@ ${LIMIT_FW} Y - 1280 - 80 + 912 + 368 - User defined Java expression - Janino + sort_by_recruit_id + SortRows Y @@ -528,18 +509,25 @@ ${LIMIT_FW} none - - has_active_temporary_measure - has_active_temporary_measure_raw == 1 - Boolean - -1 - -1 - - + N + ${java.io.tmpdir} + + + Y + N + N + 0 + recruit_id + N + + + srt + 1000000 + Y - 1056 - 80 + 672 + 368 @@ -548,7 +536,7 @@ ${LIMIT_FW} Change job status on error Y - + error_description @@ -560,7 +548,7 @@ ${LIMIT_FW} Change job status on error Y - + error_description diff --git a/mappings/info_recruits/raw_data/restriction_document_item/recruitment_five_flow_delta.hpl b/mappings/info_recruits/raw_data/restriction_document_item/recruitment_five_flow_delta.hpl index 00d0826..93012ae 100644 --- a/mappings/info_recruits/raw_data/restriction_document_item/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/raw_data/restriction_document_item/recruitment_five_flow_delta.hpl @@ -60,14 +60,28 @@ ervu-dashboard N 0 - SELECT - r.idm_id as recruitment_id + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'restriction_document_item_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 diff --git a/mappings/info_recruits/raw_data/restriction_document_item/recruitment_five_flow_on_error.hpl b/mappings/info_recruits/raw_data/restriction_document_item/recruitment_five_flow_on_error.hpl index 4480180..e559118 100644 --- a/mappings/info_recruits/raw_data/restriction_document_item/recruitment_five_flow_on_error.hpl +++ b/mappings/info_recruits/raw_data/restriction_document_item/recruitment_five_flow_on_error.hpl @@ -65,9 +65,9 @@ FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - and job_name = 'restriction_document_item_job' + and job_name = '${JOB_NAME}' where je.status is null or je.status in('ERROR', 'PROCESSING'); - N + Y 272 diff --git a/mappings/info_recruits/raw_data/restriction_document_item/recruitments_five_flow.hpl b/mappings/info_recruits/raw_data/restriction_document_item/recruitments_five_flow.hpl index 5b99064..1636897 100644 --- a/mappings/info_recruits/raw_data/restriction_document_item/recruitments_five_flow.hpl +++ b/mappings/info_recruits/raw_data/restriction_document_item/recruitments_five_flow.hpl @@ -63,24 +63,8 @@ SELECT idm_id AS recruitment FROM ervu_dashboard.recruitment -${COMMENT}/* -where idm_id in ('110686b2-4a01-4976-885b-d26a2980f53a', -'cff66da8-2be0-4839-bd8e-332b1a150031', -'34c8685e-f08a-4877-afb1-a0589800a21c', -'0e10678c-f272-41c3-94dd-c15830321f9a', -'9703f108-761a-467a-aad3-89617e5539ed', -'55eae374-9bb5-4295-9e3a-5fbcdaba870d', -'dad6c101-34be-45c3-8173-57190cb69dfb', -'fc10523f-af31-4f6b-b0e2-00799758f634', -'61807cc2-9913-4060-8802-0672877c28c8', -'eba7fd7d-e3da-49a7-8e65-8fc85e47190e', -'f59afe6d-c166-4196-a489-6d17f02fc05e', -'22c64b79-f8e4-41e7-a2f3-1aa9df424fd9', -'0fe2bdf0-afbb-402d-93da-67e511799afa', -'6139de3b-2378-496f-9fca-7df7dab90832', -'50ba73dd-1726-48f1-8b7c-7a77ff965299') -${COMMENT}*/ - Y + + N 432 diff --git a/mappings/info_recruits/supportive_jobs/job2_info_from_json.hwf b/mappings/info_recruits/supportive_jobs/job2_info_from_json.hwf index f8e19f1..44b13dc 100644 --- a/mappings/info_recruits/supportive_jobs/job2_info_from_json.hwf +++ b/mappings/info_recruits/supportive_jobs/job2_info_from_json.hwf @@ -376,7 +376,7 @@ row_last_recruit_update_date.hpl drivers_licence_job.hwf - N + Y Y N @@ -390,7 +390,7 @@ row_last_recruit_update_date.hpl property_job.hwf - N + Y Y N diff --git a/mappings/info_recruits/supportive_jobs/job3_info_from_json.hwf b/mappings/info_recruits/supportive_jobs/job3_info_from_json.hwf index ae61e85..79e32f7 100644 --- a/mappings/info_recruits/supportive_jobs/job3_info_from_json.hwf +++ b/mappings/info_recruits/supportive_jobs/job3_info_from_json.hwf @@ -411,7 +411,7 @@ row_last_recruit_update_date.hpl snils_job.hwf - N + Y Y N @@ -453,7 +453,7 @@ row_last_recruit_update_date.hpl inn_job.hwf - N + Y Y N