From 81a2d44fb08b5ae684cb9f13d2f3d1be8f43774c Mon Sep 17 00:00:00 2001 From: Fusionshh Date: Tue, 23 Sep 2025 15:59:34 +0300 Subject: [PATCH] Revert "ERVU-517 : election_candidate flow" This reverts commit f5242a17 --- .../parallel/border_crossing_flow.hpl} | 350 ++++++++------- .../parallel/border_crossing_flow_repeat.hpl} | 406 +++++++++--------- .../job_guardianship.hwf} | 142 +++--- .../recruitment_five_flow.hpl | 330 -------------- .../recruitment_five_flow_delta.hpl | 399 ----------------- .../recruitment_five_flow_on_error.hpl} | 95 ++-- .../parallel/recruit_archive_flow_delta.hpl} | 405 +++++++++-------- 7 files changed, 735 insertions(+), 1392 deletions(-) rename mappings/info_recruits/citizen_tables/{election_candidate/parallel/election_candidate_flow.hpl => border_crossing/parallel/border_crossing_flow.hpl} (69%) rename mappings/info_recruits/citizen_tables/{election_candidate/parallel/election_candidate_repeat.hpl => border_crossing/parallel/border_crossing_flow_repeat.hpl} (61%) rename mappings/info_recruits/citizen_tables/{election_candidate/election_candidate_job.hwf => citizen_guardianship/job_guardianship.hwf} (71%) delete mode 100644 mappings/info_recruits/citizen_tables/election_candidate/recruitment_five_flow.hpl delete mode 100644 mappings/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_delta.hpl rename mappings/info_recruits/citizen_tables/{election_candidate/recruitment_five_flow_repeat.hpl => property/recruitment_five_flow_on_error.hpl} (83%) rename mappings/info_recruits/citizen_tables/{election_candidate/parallel/election_candidate_delta.hpl => recruit_archive/parallel/recruit_archive_flow_delta.hpl} (65%) diff --git a/mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_flow.hpl b/mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow.hpl similarity index 69% rename from mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_flow.hpl rename to mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow.hpl index 56a02ef..fd811a6 100644 --- a/mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_flow.hpl +++ b/mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow.hpl @@ -1,7 +1,7 @@ - election_candidate_flow + border_crossing_flow Y @@ -13,31 +13,16 @@ 1000 100 - - 2025/09/08 19:24:20.455 + 2025/08/05 14:31:22.799 - - 2025/09/08 19:24:20.455 + 2025/08/05 14:31:22.799 - - Change job status on error - Abort - Y - - - Block until transforms finish - election_candidate_flag_update - Y - Create job execution record - election_candidate_input - Y - - - Identify last row in a stream - Block until transforms finish + Table input Y @@ -45,41 +30,61 @@ Change job status on success Y - - election_candidate_flag_update - Change job status on error - Y - - - Identify last row in a stream - Detect empty stream - Y - - - election_candidate_flag_update - Filter rows - Y - - - Identify last row in a stream - election_candidate_output - Y - Detect empty stream Change job status on success Y - election_candidate_output + Change job status on error + Abort + Y + + + Table output Change job status on error Y - election_candidate_input + Group by Identify last row in a stream Y + + Identify last row in a stream + User defined Java expression + Y + + + User defined Java expression + Update + Y + + + Table input + Table output + Y + + + Table output + Group by + Y + + + Update + Change job status on error + Y + + + Update + Detect empty stream + Y + + + Update + Filter rows + Y + Abort @@ -92,35 +97,13 @@ none - ABORT + ABORT_WITH_ERROR Y 0 - 1504 - 592 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - deputy_output - - - - - 976 - 416 + 1328 + 496 @@ -154,8 +137,8 @@ and recruitment_id = '${IDM_ID}'; - 1216 - 592 + 1328 + 320 @@ -185,8 +168,8 @@ and recruitment_id = '${IDM_ID}'; - 1504 - 240 + 1856 + 176 @@ -215,7 +198,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 288 - 416 + 320 @@ -231,8 +214,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 704 - 240 + 1536 + 176 @@ -267,15 +250,50 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1504 - 416 + 1536 + 304 + + + + Group by + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + border_crossed_raw + is_border_crossed + MAX + + + N + + + recruit_id + + + N + grp + + + 688 + 176 Identify last row in a stream DetectLastRow - N + Y 1 @@ -285,47 +303,12 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); last_row - 704 - 416 + 880 + 176 - election_candidate_flag_update - Update - - Y - - 1 - - none - - - 10000 - ervu-dashboard - N - - - = - recruit_id - recruit_id - - ervu_dashboard - citizen
- - has_election_candidate_info - has_election_candidate_info - -
- N - Y - - - 1216 - 416 - -
- - election_candidate_input + Table input TableInput Y @@ -338,31 +321,28 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem - FROM recruits_info ri - WHERE ri.info -> 'svedFL' -> 'svedKandidat' ->> 'kandidat' <> 'null' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' -) -SELECT fd.recruit_id, - deputy_elem ->> 'id' AS source_id, - to_date(deputy_elem ->> 'dataSved', 'YYYY-MM-DD') AS source_update_date, - deputy_elem -> 'extend' ->> 'unikIdentGrazhd' AS citizen_unique_id, - deputy_elem ->> 'priznakAktualNaim' AS actual_status_name, - to_date(deputy_elem ->> 'dataReg', 'YYYY-MM-DD') AS registration_date, - deputy_elem ->> 'prichinaSnyatNaim' AS termination_reason, - to_date(deputy_elem ->> 'dataSnyatKandidat', 'YYYY-MM-DD') AS termination_date, - true as has_election_candidate_info -FROM filteredData fd; + SELECT + recruit_id, + (string_agg(elem->>'dataVyezd', ', '))::date as date_crossing, + (string_agg(elem->>'dataVozvrashh', ', '))::date as return_date, + 1 as is_border_crossed +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND '${M_R_CR_DATE}'::timestamp >= ri.created_at, +LATERAL jsonb_array_elements(info->'svedFL'->'extend'->'svedPeresechGran') as elem +WHERE jsonb_typeof(info->'svedFL'->'extend'->'svedPeresechGran') = 'array' + AND (elem->>'dataVyezd' IS NOT NULL OR elem->>'dataVozvrashh' IS NOT NULL) +GROUP BY recruit_id; Y - 512 - 416 + 496 + 320 - election_candidate_output + Table output TableOutput Y @@ -372,7 +352,7 @@ FROM filteredData fd; none - 10000 + 1000 ervu-dashboard @@ -380,32 +360,12 @@ FROM filteredData fd; recruit_id - source_id - source_id + date_crossing + date_crossing - source_update_date - source_update_date - - - citizen_unique_id - citizen_unique_id - - - actual_status_name - actual_status_name - - - registration_date - registration_date - - - termination_reason - termination_reason - - - termination_date - termination_date + return_date + return_date N @@ -416,36 +376,96 @@ FROM filteredData fd; N ervu_dashboard Y - election_candidate
+ border_crossing
N Y N Y - 976 - 592 + 688 + 320 + +
+ + Update + Update + + N + + 1 + + none + + + 100 + ervu-dashboard + Y + + + = + recruit_id + recruit_id + + ervu_dashboard + citizen
+ + border_crossed + border_crossed + +
+ N + Y + + + 1328 + 176 + +
+ + User defined Java expression + Janino + + Y + + 1 + + none + + + + border_crossed + border_crossed_raw == 1 + Boolean + -1 + -1 + + + + + 1104 + 176 - election_candidate_flag_update + Table output Change job status on error Y error_description - + error_code - election_candidate_output + Update Change job status on error Y - error_description + diff --git a/mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_repeat.hpl b/mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_repeat.hpl similarity index 61% rename from mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_repeat.hpl rename to mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_repeat.hpl index a655d09..fc86c6a 100644 --- a/mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_repeat.hpl @@ -1,7 +1,7 @@ - election_candidate_repeat + border_crossing_flow_repeat Y @@ -13,26 +13,16 @@ 1000 100 - - 2025/09/14 15:17:48.470 + 2025/08/08 13:25:52.319 - - 2025/09/14 15:17:48.470 + 2025/08/08 13:25:52.319 - Block until transforms finish - election_candidate_flag_update - Y - - - Change job status on error - Abort - Y - - - Detect empty stream - Change job status on success + Create job execution record + Table input Y @@ -41,45 +31,60 @@ Y - Identify last row in a stream - Block until transforms finish + Detect empty stream + Change job status on success + Y + + + Change job status on error + Abort + Y + + + Insert / update + Change job status on error + Y + + + Table input + Insert / update + Y + + + Group by + Identify last row in a stream Y Identify last row in a stream + User defined Java expression + Y + + + User defined Java expression + Update + Y + + + Insert / update + Group by + Y + + + Update Detect empty stream Y - election_candidate_flag_update - Change job status on error - Y - - - election_candidate_flag_update + Update Filter rows Y - Identify last row in a stream - election_candidate_upsert - Y - - - election_candidate_upsert + Update Change job status on error Y - - Create job execution record - election_candidate_input - Y - - - election_candidate_input - Identify last row in a stream - Y - Abort @@ -92,35 +97,13 @@ none - ABORT + ABORT_WITH_ERROR Y 0 - 1584 - 784 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - deputy_upsert - - - - - 1056 - 512 + 1312 + 400 @@ -154,8 +137,8 @@ and recruitment_id = '${IDM_ID}'; - 1296 - 784 + 1312 + 208 @@ -185,8 +168,8 @@ and recruitment_id = '${IDM_ID}'; - 1584 - 336 + 1776 + 64 @@ -208,17 +191,32 @@ and recruitment_id = '${IDM_ID}'; Y N N - UPDATE etl.job_execution -SET + INSERT INTO etl.job_execution ( + id, + job_name, + status, + execution_datetime, + error_description, + recruitment_id +) +VALUES ( + DEFAULT, + '${JOB_NAME}', + 'PROCESSING', + DEFAULT, + NULL, + '${IDM_ID}' +) +ON CONFLICT (job_name, recruitment_id) +DO UPDATE SET status = 'PROCESSING', - execution_datetime = current_timestamp, + execution_datetime = DEFAULT, error_description = NULL -where job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; + - 368 - 512 + 272 + 208 @@ -234,8 +232,8 @@ and recruitment_id = '${IDM_ID}'; - 784 - 336 + 1568 + 64 @@ -270,15 +268,50 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1584 - 512 + 1568 + 176 + + + + Group by + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + border_crossed_raw + is_border_crossed + MAX + + + N + + + recruit_id + + + N + grp + + + 672 + 64 Identify last row in a stream DetectLastRow - N + Y 1 @@ -288,49 +321,12 @@ and recruitment_id = '${IDM_ID}'; last_row - 784 - 512 + 864 + 64 - election_candidate_flag_update - Update - - Y - - 1 - - none - - - 10000 - ervu-dashboard - N - - - - = - recruit_id - recruit_id - - - ervu_dashboard - citizen
- - has_election_candidate_info - has_election_candidate_info - -
- N - Y - - - 1296 - 512 - -
- - election_candidate_upsert + Insert / update InsertUpdate Y @@ -340,67 +336,41 @@ and recruitment_id = '${IDM_ID}'; none - 10000 + 100 ervu-dashboard = - source_id - source_id - + recruit_id + recruit_id ervu_dashboard - election_candidate
+ border_crossing
recruit_id recruit_id N - source_id - source_id - N - - - source_update_date - source_update_date + date_crossing + date_crossing Y - citizen_unique_id - citizen_unique_id - Y - - - actual_status_name - actual_status_name - Y - - - registration_date - registration_date - Y - - - termination_reason - termination_reason - Y - - - termination_date - termination_date + return_date + return_date Y
N - 1056 - 784 + 672 + 208
- election_candidate_input + Table input TableInput Y @@ -413,57 +383,93 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N -<<<<<<< HEAD:mappings/info_recruits/raw_data/restriction_document_item/parallel/restriction_document_item_flow_delta.hpl - select -s.recruit_id, -CASE - WHEN rdi.applied_fact is true THEN 1 - ELSE 0 -END AS is_active_temporary_measure, -- has_active_temporary_measure//Признак наличия примененных временных мер -rdi.* -from public.restriction_document_item rdi -join public.restriction_document rd on -rd.id = coalesce(rdi.restriction_document_create_id,rdi.restriction_document_cancel_id) -join public.subpoena s on -s.id = rd.subpoena_id -WHERE - '${IDM_ID}' != '' -- Проверка на пустую строку - AND rd.vk_id = '${IDM_ID}' - AND rdi.updated_at >= '${M_RDI_UP_DATE}'::timestamp - -${LIMIT_FW} -======= - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem - FROM recruits_info ri - WHERE ri.info -> 'svedFL' -> 'svedKandidat' ->> 'kandidat' <> 'null' - --and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' -) -SELECT fd.recruit_id, - deputy_elem ->> 'id' AS source_id, - to_date(deputy_elem ->> 'dataSved', 'YYYY-MM-DD') AS source_update_date, - deputy_elem -> 'extend' ->> 'unikIdentGrazhd' AS citizen_unique_id, - deputy_elem ->> 'priznakAktualNaim' AS actual_status_name, - to_date(deputy_elem ->> 'dataReg', 'YYYY-MM-DD') AS registration_date, - deputy_elem ->> 'prichinaSnyatNaim' AS termination_reason, - to_date(deputy_elem ->> 'dataSnyatKandidat', 'YYYY-MM-DD') AS termination_date, - true as has_election_candidate_info -FROM filteredData fd; ->>>>>>> 10bcf8dc200fb1b4ce7b6f5b82d5b8cd20b5b935:mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_repeat.hpl + SELECT + recruit_id, + (string_agg(elem->>'dataVyezd', ', '))::date as date_crossing, + (string_agg(elem->>'dataVozvrashh', ', '))::date as return_date, + 1 as is_border_crossed +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND '${M_R_CR_DATE}'::timestamp >= ri.created_at, +LATERAL jsonb_array_elements(info->'svedFL'->'extend'->'svedPeresechGran') as elem +WHERE jsonb_typeof(info->'svedFL'->'extend'->'svedPeresechGran') = 'array' + AND (elem->>'dataVyezd' IS NOT NULL OR elem->>'dataVozvrashh' IS NOT NULL) +GROUP BY recruit_id; Y - 576 - 512 + 480 + 208 + + + + Update + Update + + N + + 1 + + none + + + 100 + ervu-dashboard + Y + + + = + recruit_id + recruit_id + + ervu_dashboard + citizen
+ + border_crossed + border_crossed + +
+ N + Y + + + 1312 + 64 + +
+ + User defined Java expression + Janino + + Y + + 1 + + none + + + + border_crossed + border_crossed_raw == 1 + Boolean + -1 + -1 + + + + + 1088 + 64 - election_candidate_flag_update + Insert / update Change job status on error Y - error_description + @@ -471,11 +477,11 @@ FROM filteredData fd; - election_candidate_upsert + Update Change job status on error Y - error_description + diff --git a/mappings/info_recruits/citizen_tables/election_candidate/election_candidate_job.hwf b/mappings/info_recruits/citizen_tables/citizen_guardianship/job_guardianship.hwf similarity index 71% rename from mappings/info_recruits/citizen_tables/election_candidate/election_candidate_job.hwf rename to mappings/info_recruits/citizen_tables/citizen_guardianship/job_guardianship.hwf index 394679b..a09271a 100644 --- a/mappings/info_recruits/citizen_tables/election_candidate/election_candidate_job.hwf +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/job_guardianship.hwf @@ -1,16 +1,27 @@ - election_candidate_job + job_guardianship Y + 0 - - 2025/09/08 16:38:17.982 + 2025/06/05 14:27:15.055 - - 2025/09/08 16:38:17.982 + 2025/06/05 14:27:15.055 - + + M_R_CR_DATE + 3001-01-01 00:00:00 + + + + M_R_UP_DATE + + + + Start @@ -26,12 +37,12 @@ 0 1 N - 688 - 176 + 416 + 208 - check_if_job_execution_exists.hpl + check_if_citizen_guardianship_job_exists.hpl PIPELINE @@ -41,7 +52,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_job_execution_exists.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_guardianship/checkpoints/check_if_citizen_guardianship_job_exists.hpl Basic Y @@ -52,16 +63,21 @@ N Y N - 1056 - 176 + 608 + 208 - employer_job_execution_exists_check + citizen_guardianship_job_exists_check SIMPLE_EVAL + + boolean + + + false equal equal @@ -69,12 +85,12 @@ variable JOB_EXECUTED_FLAG N - 1312 - 176 + 880 + 208 - recruitment_five_flow.hpl + recruitments_five_flow.hpl PIPELINE @@ -84,7 +100,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/recruitment_five_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_guardianship/recruitments_five_flow.hpl Basic Y @@ -95,12 +111,12 @@ N Y Y - 1584 - 176 + 1248 + 208 - recruitment_five_flow_repeat.hpl + recruitment_five_flow_on_error.hpl PIPELINE @@ -110,7 +126,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_guardianship/recruitment_five_flow_on_error.hpl Basic Y @@ -121,12 +137,12 @@ N Y N - 1584 - 480 + 1120 + 512 - check_if_need_to_repeat.hpl + check_if_need_to_repeat_job.hpl PIPELINE @@ -136,7 +152,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_need_to_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_guardianship/check_if_need_to_repeat_job.hpl Basic Y @@ -147,8 +163,8 @@ N Y N - 1312 - 320 + 880 + 368 @@ -164,8 +180,8 @@ variable NEED_TO_REPEAT_JOB N - 1312 - 480 + 880 + 512 @@ -179,7 +195,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_guardianship/recruitment_five_flow_delta.hpl Basic Y @@ -190,69 +206,50 @@ N Y N - 1312 - 640 - - - - init_job_name - - SET_VARIABLES - - - - JOB_NAME - CURRENT_WORKFLOW - election_candidate_job - - - CURRENT_WORKFLOW - N - N - 864 - 176 + 880 + 704 - check_if_job_execution_exists.hpl - employer_job_execution_exists_check + Start + check_if_citizen_guardianship_job_exists.hpl + Y + Y + Y + + + check_if_citizen_guardianship_job_exists.hpl + citizen_guardianship_job_exists_check Y Y N - employer_job_execution_exists_check - check_if_need_to_repeat.hpl + citizen_guardianship_job_exists_check + recruitments_five_flow.hpl + Y + Y + N + + + citizen_guardianship_job_exists_check + check_if_need_to_repeat_job.hpl Y N N - check_if_need_to_repeat.hpl + check_if_need_to_repeat_job.hpl Simple evaluation Y Y N - - init_job_name - check_if_job_execution_exists.hpl - Y - Y - N - - - employer_job_execution_exists_check - recruitment_five_flow.hpl - Y - Y - N - Simple evaluation - recruitment_five_flow_repeat.hpl + recruitment_five_flow_on_error.hpl Y Y N @@ -264,13 +261,6 @@ N N - - Start - init_job_name - Y - Y - Y - diff --git a/mappings/info_recruits/citizen_tables/election_candidate/recruitment_five_flow.hpl b/mappings/info_recruits/citizen_tables/election_candidate/recruitment_five_flow.hpl deleted file mode 100644 index 74c6478..0000000 --- a/mappings/info_recruits/citizen_tables/election_candidate/recruitment_five_flow.hpl +++ /dev/null @@ -1,330 +0,0 @@ - - - - recruitment_five_flow - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/08 19:24:07.006 - - - 2025/09/08 19:24:07.006 - - - - - - Get all recruitments - election_candidate_flow.hpl - Y - - - Get all recruitments - election_candidate_flow.hpl 2 - Y - - - Get all recruitments - election_candidate_flow.hpl 3 - Y - - - Get all recruitments - election_candidate_flow.hpl 4 - Y - - - Get all recruitments - election_candidate_flow.hpl 5 - Y - - - - Get all recruitments - TableInput - - Y - - 1 - - none - - - ervu-dashboard-test - N - SELECT -idm_id -FROM ervu_dashboard.recruitment; - N - - - 560 - 448 - - - - election_candidate_flow.hpl - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 832 - 288 - - - - election_candidate_flow.hpl 2 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 832 - 368 - - - - election_candidate_flow.hpl 3 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 832 - 448 - - - - election_candidate_flow.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 832 - 528 - - - - election_candidate_flow.hpl 5 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 832 - 608 - - - - - - diff --git a/mappings/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_delta.hpl deleted file mode 100644 index b106c8c..0000000 --- a/mappings/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_delta.hpl +++ /dev/null @@ -1,399 +0,0 @@ - - - - recruitment_five_flow_delta - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/08 20:38:40.987 - - - 2025/09/08 20:38:40.987 - - - - - - Table input - election_candidate_delta.hpl - Y - - - get_max_source_update_date - Table input - Y - - - Table input - election_candidate_delta.hpl 2 - Y - - - Table input - election_candidate_delta.hpl 3 - Y - - - Table input - election_candidate_delta.hpl 4 - Y - - - Table input - election_candidate_delta.hpl 5 - Y - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard-test - N - 0 - get_max_source_update_date - WITH mud AS ( - SELECT - recruitment_id, - MAX(execution_datetime) AS max_upd_date - FROM etl.job_execution - WHERE job_name = '${JOB_NAME}' - AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING') - GROUP BY recruitment_id -) -SELECT - r.idm_id, - ? max_source_update_date -FROM ervu_dashboard.recruitment r - JOIN mud ON mud.recruitment_id = r.idm_id - JOIN recruits_info ri - ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id - AND ri.updated_at > mud.max_upd_date; - Y - - - 704 - 496 - - - - election_candidate_delta.hpl - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 336 - - - - get_max_source_update_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT max(source_update_date) AS max_source_update_date -FROM election_candidate; - N - - - 528 - 496 - - - - election_candidate_delta.hpl 2 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 416 - - - - election_candidate_delta.hpl 3 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 496 - - - - election_candidate_delta.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 576 - - - - election_candidate_delta.hpl 5 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 656 - - - - - - diff --git a/mappings/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/property/recruitment_five_flow_on_error.hpl similarity index 83% rename from mappings/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_repeat.hpl rename to mappings/info_recruits/citizen_tables/property/recruitment_five_flow_on_error.hpl index 58e64a8..b7c2cde 100644 --- a/mappings/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/property/recruitment_five_flow_on_error.hpl @@ -1,7 +1,7 @@ - recruitment_five_flow_repeat + recruitment_five_flow_on_error Y @@ -13,36 +13,36 @@ 1000 100 - - 2025/09/08 20:31:52.575 + 2025/08/05 12:54:50.126 - - 2025/09/08 20:31:52.575 + 2025/08/05 12:54:50.126 Table input - recruitment_five_flow_repeat.hpl + property_flow_repeat.hpl Y Table input - recruitment_five_flow_repeat.hpl 2 + property_flow_repeat.hpl 2 Y Table input - recruitment_five_flow_repeat.hpl 3 + property_flow_repeat.hpl 3 Y Table input - recruitment_five_flow_repeat.hpl 4 + property_flow_repeat.hpl 4 Y Table input - recruitment_five_flow_repeat.hpl 5 + property_flow_repeat.hpl 5 Y @@ -57,25 +57,25 @@ none - ervu-dashboard-test + ervu-dashboard N 0 - SELECT r.idm_id AS recruitment_id + SELECT + r.idm_id as recruitment_id FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - AND je.job_name = 'disability_job' -WHERE je.id IS NULL - OR je.status IN ('ERROR', 'PROCESSING'); + and job_name = 'property_job' +where je.status is null or je.status in('ERROR', 'PROCESSING'); N - 656 - 464 + 272 + 368 - recruitment_five_flow_repeat.hpl + property_flow_repeat.hpl PipelineExecutor Y @@ -86,7 +86,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/property/parallel/property_flow_repeat.hpl N 1 @@ -98,6 +98,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + property_job + Y @@ -121,12 +126,12 @@ WHERE je.id IS NULL - 864 - 304 + 544 + 208 - recruitment_five_flow_repeat.hpl 2 + property_flow_repeat.hpl 2 PipelineExecutor Y @@ -137,7 +142,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/property/parallel/property_flow_repeat.hpl N 1 @@ -149,6 +154,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + property_job + Y @@ -172,12 +182,12 @@ WHERE je.id IS NULL - 864 - 384 + 544 + 288 - recruitment_five_flow_repeat.hpl 3 + property_flow_repeat.hpl 3 PipelineExecutor Y @@ -188,7 +198,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/property/parallel/property_flow_repeat.hpl N 1 @@ -200,6 +210,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + property_job + Y @@ -223,12 +238,12 @@ WHERE je.id IS NULL - 864 - 464 + 544 + 368 - recruitment_five_flow_repeat.hpl 4 + property_flow_repeat.hpl 4 PipelineExecutor Y @@ -239,7 +254,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/property/parallel/property_flow_repeat.hpl N 1 @@ -251,6 +266,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + property_job + Y @@ -274,12 +294,12 @@ WHERE je.id IS NULL - 864 - 544 + 544 + 448 - recruitment_five_flow_repeat.hpl 5 + property_flow_repeat.hpl 5 PipelineExecutor Y @@ -290,7 +310,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/property/parallel/property_flow_repeat.hpl N 1 @@ -302,6 +322,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + property_job + Y @@ -325,8 +350,8 @@ WHERE je.id IS NULL - 864 - 624 + 544 + 528 diff --git a/mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_delta.hpl b/mappings/info_recruits/citizen_tables/recruit_archive/parallel/recruit_archive_flow_delta.hpl similarity index 65% rename from mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_delta.hpl rename to mappings/info_recruits/citizen_tables/recruit_archive/parallel/recruit_archive_flow_delta.hpl index f96b7b0..df3019c 100644 --- a/mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_delta.hpl +++ b/mappings/info_recruits/citizen_tables/recruit_archive/parallel/recruit_archive_flow_delta.hpl @@ -1,7 +1,7 @@ - election_candidate_delta + recruit_archive_flow_delta Y @@ -13,26 +13,16 @@ 1000 100 - - 2025/09/14 15:17:52.106 + 2025/08/11 13:22:56.048 - - 2025/09/14 15:17:52.106 + 2025/08/11 13:22:56.048 - - Block until transforms finish - election_candidate_flag_update - Y - - - Change job status on error - Abort - Y - Create job execution record - election_candidate_input + Table input Y @@ -46,38 +36,53 @@ Y - Identify last row in a stream - Block until transforms finish + Change job status on error + Abort Y - Identify last row in a stream - Detect empty stream + Table input + Insert / update Y - election_candidate_flag_update - Change job status on error - Y - - - election_candidate_input + Group by Identify last row in a stream Y Identify last row in a stream - election_candidate_upsert + User defined Java expression Y - election_candidate_upsert + User defined Java expression + Update + Y + + + Update + Detect empty stream + Y + + + Update + Filter rows + Y + + + Update Change job status on error Y - election_candidate_flag_update - Filter rows + Insert / update + Change job status on error + Y + + + Insert / update + Group by Y @@ -92,35 +97,13 @@ none - ABORT + ABORT_WITH_ERROR Y 0 - 1552 - 688 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - deputy_upsert - - - - - 896 - 416 + 1344 + 480 @@ -146,7 +129,7 @@ Y N UPDATE etl.job_execution -SET status = 'ERROR', +SET status = 'DELTA_ERROR', error_description = ? WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; @@ -154,8 +137,8 @@ and recruitment_id = '${IDM_ID}'; - 1136 - 688 + 1344 + 352 @@ -178,15 +161,15 @@ and recruitment_id = '${IDM_ID}'; N N UPDATE etl.job_execution -SET status = 'SUCCESS' +SET status = 'DELTA_SUCCESS' WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - 1552 - 240 + 1760 + 224 @@ -208,14 +191,17 @@ and recruitment_id = '${IDM_ID}'; Y N N - INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id) -VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - - + UPDATE etl.job_execution +SET + status = 'DELTA_PROCESSING', + execution_datetime = DEFAULT, + error_description = NULL +where job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; - 208 - 416 + 352 + 352 @@ -231,8 +217,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 624 - 240 + 1568 + 224 @@ -267,15 +253,50 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1552 - 416 + 1568 + 336 + + + + Group by + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + has_card_in_archive_raw + is_card_in_archive + MAX + + + N + + + recruit_id + + + N + grp + + + 704 + 224 Identify last row in a stream DetectLastRow - N + Y 1 @@ -285,93 +306,12 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); last_row - 624 - 416 + 896 + 224 - election_candidate_flag_update - Update - - Y - - 1 - - none - - - 10000 - ervu-dashboard - N - - - - = - recruit_id - recruit_id - - - - <> - has_election_candidate_info - has_election_candidate_info - - - ervu_dashboard - citizen
- - has_election_candidate_info - has_election_candidate_info - -
- N - Y - - - 1136 - 416 - -
- - election_candidate_input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem - FROM recruits_info ri - WHERE ri.info -> 'svedFL' -> 'svedKandidat' ->> 'kandidat' <> 'null' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - and to_date(ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' -) -SELECT fd.recruit_id, - deputy_elem ->> 'id' AS source_id, - to_date(deputy_elem ->> 'dataSved', 'YYYY-MM-DD') AS source_update_date, - deputy_elem -> 'extend' ->> 'unikIdentGrazhd' AS citizen_unique_id, - deputy_elem ->> 'priznakAktualNaim' AS actual_status_name, - to_date(deputy_elem ->> 'dataReg', 'YYYY-MM-DD') AS registration_date, - deputy_elem ->> 'prichinaSnyatNaim' AS termination_reason, - to_date(deputy_elem ->> 'dataSnyatKandidat', 'YYYY-MM-DD') AS termination_date, - true as has_election_candidate_info -FROM filteredData fd; - Y - - - 432 - 416 - - - - election_candidate_upsert + Insert / update InsertUpdate Y @@ -381,72 +321,163 @@ FROM filteredData fd; none - 10000 + 100 ervu-dashboard = - source_id - source_id - + id + id ervu_dashboard - election_candidate
+ recruit_archive
+ + id + id + N + recruit_id recruit_id + N + + + archivation_reason_id + archivation_reason_id Y - source_id - source_id + created_at + created_at Y - source_update_date - source_update_date + updated_at + updated_at Y - citizen_unique_id - citizen_unique_id + hidden + hidden Y - actual_status_name - actual_status_name - Y - - - registration_date - registration_date - Y - - - termination_reason - termination_reason - Y - - - termination_date - termination_date + recruit_extract_enable + recruit_extract_enable Y
N - 896 - 688 + 704 + 352 + +
+ + Table input + TableInput + + Y + + 1 + + none + + + ervu_person_archive + N + SELECT + id, + recruit_id, + archivation_reason_id, + created_at, + updated_at, + hidden, + recruit_extract_enable, + CASE + WHEN recruit_extract_enable IS true THEN 1 + ELSE 0 + END AS card_in_archive +FROM public.recruit_archive +WHERE '${IDM_ID}' != '' -- Проверка на пустую строку + AND vk_id = '${IDM_ID}' + --AND '${M_R_CR_DATE}'::timestamp >= ri.created_at + AND updated_at >= '${M_R_UP_DATE}'::timestamp + Y + + + 544 + 352 + + + + Update + Update + + N + + 1 + + none + + + 100 + ervu-dashboard + Y + + + = + recruit_id + recruit_id + + ervu_dashboard + citizen
+ + card_in_archive + card_in_archive + +
+ N + Y + + + 1344 + 224 + +
+ + User defined Java expression + Janino + + Y + + 1 + + none + + + + card_in_archive + has_card_in_archive_raw == 1 + Boolean + -1 + -1 + + + + + 1120 + 224 - election_candidate_flag_update + Insert / update Change job status on error Y - error_description + @@ -454,7 +485,7 @@ FROM filteredData fd; - election_candidate_upsert + Update Change job status on error Y