From f3b188dc79060177afc329ad1c25cb8e74785b0d Mon Sep 17 00:00:00 2001 From: Fusionshh Date: Tue, 23 Sep 2025 15:59:25 +0300 Subject: [PATCH] Revert "ERVU-514 : medicine flow" This reverts commit 31b1f6f2 --- .../parallel/border_crossing_flow_delta.hpl} | 366 ++++++++------- .../job_citizen_address.hwf} | 142 +++--- .../recruitment_five_flow_delta.hpl | 360 +++++++++++++++ .../job_citizen_information_search.hwf} | 164 +++---- .../citizen_information_search_flow.hpl} | 231 +++++----- ...itizen_information_search_flow_repeat.hpl} | 400 +++++++++-------- .../job_spouse.hwf} | 137 +++--- .../citizen_spouse/recruitments_five_flow.hpl | 356 +++++++++++++++ .../recruitment_five_flow_delta.hpl} | 282 ++++++------ ...nt_decision_update_date_ervu_dashboard.hpl | 98 ++++ .../recruitment_five_flow_delta.hpl} | 375 ++++++++-------- .../recruitment_five_flow_delta.hpl | 360 +++++++++++++++ .../disability/recruitment_five_flow.hpl | 330 -------------- .../recruitment_five_flow_delta.hpl | 399 ---------------- .../constants/health_state_constants_flow.hpl | 165 ------- .../health_state/health_state_job.hwf | 414 ----------------- .../parallel/health_state_flow_delta.hpl | 398 ---------------- .../health_state/recruitment_five_flow.hpl | 330 -------------- .../recruitment_five_flow_delta.hpl | 398 ---------------- .../parallel/incapacity_flow_delta.hpl | 418 ----------------- .../recruitment_five_flow_delta.hpl | 424 ------------------ .../medicine/vich/recruitment_five_flow.hpl | 330 -------------- .../vich/recruitment_five_flow_delta.hpl | 399 ---------------- .../parallel/passport_flow.hpl} | 189 ++++---- .../parallel/passport_flow_delta.hpl} | 295 ++++++------ .../recruitment_five_flow_on_error.hpl} | 97 ++-- .../recruitments_five_flow.hpl} | 97 ++-- .../recruitment_five_flow_delta.hpl} | 95 ++-- .../recruitment_five_flow_delta.hpl | 360 +++++++++++++++ .../recruitment_five_flow_on_error.hpl | 360 +++++++++++++++ .../employer/recruitment_five_flow_repeat.hpl | 25 -- .../support/check_if_need_to_repeat_job.hpl} | 29 +- .../parallel/restriction_document_flow.hpl} | 242 +++++----- .../restriction_document_flow_delta.hpl | 408 +++++++++++++++++ .../parallel/subpoena_appearance_flow.hpl} | 158 +++---- .../subpoena_appearance_flow_delta.hpl} | 202 +++++---- .../subpoena_appearance_flow_repeat.hpl} | 210 +++++---- .../subpoena_send_info_flow_delta.hpl} | 205 ++++----- .../recruitment_five_flow_delta.hpl | 360 +++++++++++++++ .../recruit_archivation_reason.hpl} | 145 +++--- 40 files changed, 4768 insertions(+), 5985 deletions(-) rename mappings/info_recruits/citizen_tables/{medicine/disability/parallel/disability_flow_delta.hpl => border_crossing/parallel/border_crossing_flow_delta.hpl} (67%) rename mappings/info_recruits/citizen_tables/{medicine/incapacity/incapacity_job.hwf => citizen_address/job_citizen_address.hwf} (72%) create mode 100644 mappings/info_recruits/citizen_tables/citizen_guardianship/recruitment_five_flow_delta.hpl rename mappings/info_recruits/citizen_tables/{medicine/disability/disability_job.hwf => citizen_information_search/job_citizen_information_search.hwf} (68%) rename mappings/info_recruits/citizen_tables/{medicine/disability/parallel/disability_flow.hpl => citizen_information_search/parallel/citizen_information_search_flow.hpl} (69%) rename mappings/info_recruits/citizen_tables/{medicine/disability/parallel/disability_flow_repeat.hpl => citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl} (66%) rename mappings/info_recruits/citizen_tables/{medicine/vich/vich_job.hwf => citizen_spouse/job_spouse.hwf} (72%) create mode 100644 mappings/info_recruits/citizen_tables/citizen_spouse/recruitments_five_flow.hpl rename mappings/info_recruits/citizen_tables/{medicine/incapacity/recruitment_five_flow_repeat.hpl => citizenship/recruitment_five_flow_delta.hpl} (86%) create mode 100644 mappings/info_recruits/citizen_tables/deferment_liberation/deferment_decision_update_date_ervu_dashboard.hpl rename mappings/info_recruits/citizen_tables/{medicine/vich/recruitment_five_flow_repeat.hpl => deferment_liberation/recruitment_five_flow_delta.hpl} (81%) create mode 100644 mappings/info_recruits/citizen_tables/drivers_licence/recruitment_five_flow_delta.hpl delete mode 100644 mappings/info_recruits/citizen_tables/medicine/disability/recruitment_five_flow.hpl delete mode 100644 mappings/info_recruits/citizen_tables/medicine/disability/recruitment_five_flow_delta.hpl delete mode 100644 mappings/info_recruits/citizen_tables/medicine/health_state/constants/health_state_constants_flow.hpl delete mode 100644 mappings/info_recruits/citizen_tables/medicine/health_state/health_state_job.hwf delete mode 100644 mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_delta.hpl delete mode 100644 mappings/info_recruits/citizen_tables/medicine/health_state/recruitment_five_flow.hpl delete mode 100644 mappings/info_recruits/citizen_tables/medicine/health_state/recruitment_five_flow_delta.hpl delete mode 100644 mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_delta.hpl delete mode 100644 mappings/info_recruits/citizen_tables/medicine/incapacity/recruitment_five_flow_delta.hpl delete mode 100644 mappings/info_recruits/citizen_tables/medicine/vich/recruitment_five_flow.hpl delete mode 100644 mappings/info_recruits/citizen_tables/medicine/vich/recruitment_five_flow_delta.hpl rename mappings/info_recruits/citizen_tables/{medicine/incapacity/parallel/incapacity_flow.hpl => passport/parallel/passport_flow.hpl} (74%) rename mappings/info_recruits/citizen_tables/{medicine/vich/parallel/vich_flow_repeat.hpl => passport/parallel/passport_flow_delta.hpl} (72%) rename mappings/info_recruits/citizen_tables/{medicine/disability/recruitment_five_flow_repeat.hpl => passport/recruitment_five_flow_on_error.hpl} (83%) rename mappings/info_recruits/citizen_tables/{medicine/incapacity/recruitment_five_flow.hpl => passport/recruitments_five_flow.hpl} (84%) rename mappings/info_recruits/citizen_tables/{medicine/health_state/recruitment_five_flow_repeat.hpl => property/recruitment_five_flow_delta.hpl} (83%) create mode 100644 mappings/info_recruits/citizen_tables/punishment/recruitment_five_flow_delta.hpl create mode 100644 mappings/info_recruits/citizen_tables/punishment/recruitment_five_flow_on_error.hpl rename mappings/info_recruits/citizen_tables/{medicine/health_state/constants/support/check_if_constants_exists.hpl => work/work_activity/support/check_if_need_to_repeat_job.hpl} (70%) rename mappings/info_recruits/{citizen_tables/medicine/health_state/parallel/health_state_flow.hpl => raw_data/restriction_document/parallel/restriction_document_flow.hpl} (65%) create mode 100644 mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_delta.hpl rename mappings/info_recruits/{citizen_tables/medicine/vich/parallel/vich_flow.hpl => raw_data/subpoena_appearance/parallel/subpoena_appearance_flow.hpl} (74%) rename mappings/info_recruits/{citizen_tables/medicine/vich/parallel/vich_flow_delta.hpl => raw_data/subpoena_appearance/parallel/subpoena_appearance_flow_delta.hpl} (72%) rename mappings/info_recruits/{citizen_tables/medicine/incapacity/parallel/incapacity_flow_repeat.hpl => raw_data/subpoena_appearance/parallel/subpoena_appearance_flow_repeat.hpl} (71%) rename mappings/info_recruits/{citizen_tables/medicine/health_state/parallel/health_state_flow_repeat.hpl => raw_data/subpoena_send_info/parallel/subpoena_send_info_flow_delta.hpl} (68%) create mode 100644 mappings/info_recruits/raw_data/subpoena_send_info/recruitment_five_flow_delta.hpl rename mappings/info_recruits/{citizen_tables/medicine/health_state/constants/health_state_constants_flow_repeat.hpl => reference_data/recruit_archivation_reason.hpl} (55%) diff --git a/mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_delta.hpl b/mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl similarity index 67% rename from mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_delta.hpl rename to mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl index 611a7d9..b3924d1 100644 --- a/mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl @@ -1,7 +1,7 @@ - disability_flow_delta + border_crossing_flow_delta Y @@ -13,46 +13,16 @@ 1000 100 - - 2025/09/08 20:36:38.686 + 2025/08/11 13:22:56.048 - - 2025/09/08 20:36:38.686 + 2025/08/11 13:22:56.048 - - Change job status on error - Abort - Y - Create job execution record - disability_input - Y - - - Filter rows - Change job status on success - Y - - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - disability_upsert - Y - - - disability_input - Identify last row in a stream - Y - - - disability_upsert - Change job status on error + Table input Y @@ -61,17 +31,57 @@ Y - disability_upsert - has_disability_update + Filter rows + Change job status on success Y - has_disability_update + Change job status on error + Abort + Y + + + Table input + Insert / update + Y + + + Group by + Identify last row in a stream + Y + + + Identify last row in a stream + User defined Java expression + Y + + + User defined Java expression + Update + Y + + + Insert / update + Group by + Y + + + Update + Detect empty stream + Y + + + Update Filter rows Y - has_disability_update + Update + Change job status on error + Y + + + Insert / update Change job status on error Y @@ -87,13 +97,13 @@ none - ABORT + ABORT_WITH_ERROR Y 0 - 1936 - 560 + 1392 + 528 @@ -127,8 +137,8 @@ and recruitment_id = '${IDM_ID}'; - 1712 - 560 + 1392 + 352 @@ -158,7 +168,7 @@ and recruitment_id = '${IDM_ID}'; - 1936 + 1872 208 @@ -184,14 +194,14 @@ and recruitment_id = '${IDM_ID}'; UPDATE etl.job_execution SET status = 'DELTA_PROCESSING', - execution_datetime = current_timestamp, + execution_datetime = DEFAULT, error_description = NULL where job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - 816 - 384 + 352 + 352 @@ -207,7 +217,7 @@ and recruitment_id = '${IDM_ID}'; - 1232 + 1632 208 @@ -243,15 +253,50 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1936 - 384 + 1632 + 320 + + + + Group by + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + border_crossed_raw + is_border_crossed + MAX + + + N + + + recruit_id + + + N + grp + + + 752 + 208 Identify last row in a stream DetectLastRow - N + Y 1 @@ -261,12 +306,56 @@ and recruitment_id = '${IDM_ID}'; last_row - 1232 - 384 + 944 + 208 - disability_input + Insert / update + InsertUpdate + + Y + + 1 + + none + + + 100 + ervu-dashboard + + + = + recruit_id + recruit_id + + ervu_dashboard + border_crossing
+ + recruit_id + recruit_id + N + + + date_crossing + date_crossing + Y + + + return_date + return_date + Y + +
+ N + + + 752 + 352 + +
+ + Table input TableInput Y @@ -278,159 +367,94 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' AS invalid_elem - FROM recruits_info ri - WHERE ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' <> 'null' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - and to_date(ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' -) -SELECT fd.recruit_id, - invalid_elem ->> 'id' AS source_id, - to_date(invalid_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, - invalid_elem ->> 'gruppa' as disability_group, - invalid_elem ->> 'ustanOrg' as disability_establish_org_name, - to_date(invalid_elem -> 'ustanInvalid' ->> 'dataUstan', 'YYYY-MM-DD') as register_date, - to_date(invalid_elem -> 'ustanInvalid' ->> 'dataPodtverzhdPlan', 'YYYY-MM-DD') as confirmation_date, - to_date(invalid_elem ->> 'dataSnyat', 'YYYY-MM-DD') as deregistration_date, - coalesce(invalid_elem -> 'ustanInvalid' ->> 'prBessrInvl' = '1', false) as permanent -FROM filteredData fd; + + SELECT + recruit_id, + (string_agg(elem->>'dataVyezd', ', '))::date as date_crossing, + (string_agg(elem->>'dataVozvrashh', ', '))::date as return_date, + 1 as border_crossed +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp, +LATERAL jsonb_array_elements(info->'svedFL'->'extend'->'svedPeresechGran') as elem +WHERE jsonb_typeof(info->'svedFL'->'extend'->'svedPeresechGran') = 'array' + AND (elem->>'dataVyezd' IS NOT NULL OR elem->>'dataVozvrashh' IS NOT NULL) +GROUP BY recruit_id; Y - 1040 - 384 + 560 + 352 - disability_upsert - InsertUpdate - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - = - source_id - source_id - - ervu_dashboard - disability
- - recruit_id - recruit_id - N - - - source_id - source_id - N - - - source_update_date - source_update_date - Y - - - disability_group - disability_group - Y - - - disability_establish_org_name - disability_establish_org_name - Y - - - register_date - register_date - Y - - - confirmation_date - confirmation_date - Y - - - deregistration_date - deregistration_date - Y - - - permanent - permanent - Y - - - last_row - last_row - Y - -
- N - - - 1520 - 384 - -
- - has_disability_update + Update Update - Y + N 1 none - 10000 + 100 ervu-dashboard - N - + Y = recruit_id recruit_id - - - - <> - has_disability - has_disability - ervu_dashboard citizen
- has_disability - has_disability + border_crossed + border_crossed
N Y - 1712 - 384 + 1392 + 208 + +
+ + User defined Java expression + Janino + + Y + + 1 + + none + + + + border_crossed + border_crossed_raw == 1 + Boolean + -1 + -1 + + + + + 1168 + 208 - disability_upsert + Insert / update Change job status on error Y - error_description + @@ -438,7 +462,7 @@ FROM filteredData fd; - has_disability_update + Update Change job status on error Y diff --git a/mappings/info_recruits/citizen_tables/medicine/incapacity/incapacity_job.hwf b/mappings/info_recruits/citizen_tables/citizen_address/job_citizen_address.hwf similarity index 72% rename from mappings/info_recruits/citizen_tables/medicine/incapacity/incapacity_job.hwf rename to mappings/info_recruits/citizen_tables/citizen_address/job_citizen_address.hwf index 0f4eaf4..495a2c2 100644 --- a/mappings/info_recruits/citizen_tables/medicine/incapacity/incapacity_job.hwf +++ b/mappings/info_recruits/citizen_tables/citizen_address/job_citizen_address.hwf @@ -1,16 +1,27 @@ - incapacity_job + job_citizen_address Y + 0 - - 2025/09/08 16:38:17.982 + 2025/06/05 14:27:15.055 - - 2025/09/08 16:38:17.982 + 2025/06/05 14:27:15.055 - + + M_R_CR_DATE + 3001-01-01 00:00:00 + + + + M_R_UP_DATE + + + + Start @@ -26,12 +37,12 @@ 0 1 N - 96 - 160 + 416 + 208 - check_if_job_execution_exists.hpl + check_if_citizen_address_job_exists.hpl PIPELINE @@ -41,7 +52,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_job_execution_exists.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_address/checkpoints/check_if_citizen_address_job_exists.hpl Basic Y @@ -52,12 +63,12 @@ N Y N - 480 - 160 + 608 + 208 - employer_job_execution_exists_check + citizen_address_job_exists_check SIMPLE_EVAL @@ -69,12 +80,12 @@ variable JOB_EXECUTED_FLAG N - 784 - 160 + 880 + 208 - recruitment_five_flow.hpl + recruitments_five_flow.hpl PIPELINE @@ -84,7 +95,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/incapacity/recruitment_five_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_address/recruitments_five_flow.hpl Basic Y @@ -95,12 +106,12 @@ N Y Y - 1104 - 160 + 1120 + 208 - recruitment_five_flow_repeat.hpl + recruitment_five_flow_on_error.hpl PIPELINE @@ -110,7 +121,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/incapacity/recruitment_five_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_address/recruitment_five_flow_on_error.hpl Basic Y @@ -121,12 +132,12 @@ N Y N - 1040 - 464 + 1120 + 512 - check_if_need_to_repeat.hpl + check_if_need_to_repeat_job.hpl PIPELINE @@ -136,9 +147,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_need_to_repeat.hpl - - + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_address/check_if_need_to_repeat_job.hpl Basic Y @@ -149,8 +158,8 @@ N Y N - 784 - 304 + 880 + 368 @@ -158,7 +167,12 @@ SIMPLE_EVAL + + boolean + + + true equal equal @@ -166,8 +180,8 @@ variable NEED_TO_REPEAT_JOB N - 784 - 464 + 880 + 512 @@ -181,7 +195,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/incapacity/recruitment_five_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_address/recruitment_five_flow_delta.hpl Basic Y @@ -192,62 +206,50 @@ N Y N - 784 + 880 704 - - init_job_name - - SET_VARIABLES - - - - JOB_NAME - CURRENT_WORKFLOW - incapacity_job - - - CURRENT_WORKFLOW - N - N - 256 - 160 - - - check_if_job_execution_exists.hpl - employer_job_execution_exists_check + Start + check_if_citizen_address_job_exists.hpl + Y + Y + Y + + + check_if_citizen_address_job_exists.hpl + citizen_address_job_exists_check Y Y N - employer_job_execution_exists_check - check_if_need_to_repeat.hpl + citizen_address_job_exists_check + recruitments_five_flow.hpl + Y + Y + N + + + citizen_address_job_exists_check + check_if_need_to_repeat_job.hpl Y N N - check_if_need_to_repeat.hpl + check_if_need_to_repeat_job.hpl Simple evaluation Y Y N - - employer_job_execution_exists_check - recruitment_five_flow.hpl - Y - Y - N - Simple evaluation - recruitment_five_flow_repeat.hpl + recruitment_five_flow_on_error.hpl Y Y N @@ -259,20 +261,6 @@ N N - - Start - init_job_name - Y - Y - Y - - - init_job_name - check_if_job_execution_exists.hpl - Y - Y - N - diff --git a/mappings/info_recruits/citizen_tables/citizen_guardianship/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_guardianship/recruitment_five_flow_delta.hpl new file mode 100644 index 0000000..034d7b5 --- /dev/null +++ b/mappings/info_recruits/citizen_tables/citizen_guardianship/recruitment_five_flow_delta.hpl @@ -0,0 +1,360 @@ + + + + recruitment_five_flow_delta + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/08/11 13:16:11.204 + - + 2025/08/11 13:16:11.204 + + + + + + Table input + citizen_guardianship_flow_delta.hpl + Y + + + Table input + citizen_guardianship_flow_delta.hpl 2 + Y + + + Table input + citizen_guardianship_flow_delta.hpl 3 + Y + + + Table input + citizen_guardianship_flow_delta.hpl 4 + Y + + + Table input + citizen_guardianship_flow_delta.hpl 5 + Y + + + + Table input + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT + r.idm_id as recruitment_id +FROM ervu_dashboard.recruitment r + LEFT JOIN etl.job_execution je + ON r.idm_id = je.recruitment_id + and job_name = 'citizen_guardianship_job' +where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); + N + + + 352 + 288 + + + + citizen_guardianship_flow_delta.hpl + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + citizen_guardianship_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 544 + 128 + + + + citizen_guardianship_flow_delta.hpl 2 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + citizen_guardianship_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 544 + 208 + + + + citizen_guardianship_flow_delta.hpl 3 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + citizen_guardianship_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 544 + 288 + + + + citizen_guardianship_flow_delta.hpl 4 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + citizen_guardianship_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 544 + 368 + + + + citizen_guardianship_flow_delta.hpl 5 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + citizen_guardianship_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 544 + 448 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/medicine/disability/disability_job.hwf b/mappings/info_recruits/citizen_tables/citizen_information_search/job_citizen_information_search.hwf similarity index 68% rename from mappings/info_recruits/citizen_tables/medicine/disability/disability_job.hwf rename to mappings/info_recruits/citizen_tables/citizen_information_search/job_citizen_information_search.hwf index 704af69..79f22b2 100644 --- a/mappings/info_recruits/citizen_tables/medicine/disability/disability_job.hwf +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/job_citizen_information_search.hwf @@ -1,16 +1,27 @@ - disability_job + job_citizen_information_search Y + 0 - - 2025/09/08 16:38:17.982 + 2025/06/05 14:27:15.055 - - 2025/09/08 16:38:17.982 + 2025/06/05 14:27:15.055 - + + M_R_CR_DATE + 3001-01-01 00:00:00 + + + + M_R_UP_DATE + + + + Start @@ -26,12 +37,12 @@ 0 1 N - 112 - 160 + 416 + 208 - check_if_job_execution_exists.hpl + check_if_citizen_information_search_job_exists.hpl PIPELINE @@ -41,7 +52,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/support/check_if_job_execution_exists.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_information_search/checkpoints/check_if_citizen_information_search_job_exists.hpl Basic Y @@ -52,12 +63,12 @@ N Y N - 480 - 160 + 608 + 208 - employer_job_execution_exists_check + citizen_information_search_job_exists_check SIMPLE_EVAL @@ -69,12 +80,12 @@ variable JOB_EXECUTED_FLAG N - 784 - 160 + 880 + 208 - recruitment_five_flow.hpl + recruitments_five_flow.hpl PIPELINE @@ -84,7 +95,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/recruitment_five_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_information_search/recruitments_five_flow.hpl Basic Y @@ -95,12 +106,12 @@ N Y Y - 1104 - 160 + 1120 + 208 - recruitment_five_flow_repeat.hpl + recruitment_five_flow_on_error.hpl PIPELINE @@ -110,7 +121,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/recruitment_five_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_information_search/recruitment_five_flow_on_error.hpl Basic Y @@ -121,12 +132,12 @@ N Y N - 1040 - 464 + 1120 + 512 - check_if_need_to_repeat.hpl + check_if_need_to_repeat_job.hpl PIPELINE @@ -136,7 +147,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/support/check_if_need_to_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_information_search/check_if_need_to_repeat_job.hpl Basic Y @@ -147,8 +158,8 @@ N Y N - 784 - 304 + 880 + 368 @@ -164,8 +175,8 @@ variable NEED_TO_REPEAT_JOB N - 784 - 464 + 880 + 512 @@ -179,7 +190,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/recruitment_five_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_information_search/recruitment_five_flow_delta.hpl Basic Y @@ -190,63 +201,50 @@ N Y N - 784 + 880 704 - - init_job_name - - SET_VARIABLES - - - - JOB_NAME - CURRENT_WORKFLOW - disability_job - - - CURRENT_WORKFLOW - - N - N - 272 - 160 - - - check_if_job_execution_exists.hpl - employer_job_execution_exists_check + Start + check_if_citizen_information_search_job_exists.hpl + Y + Y + Y + + + check_if_citizen_information_search_job_exists.hpl + citizen_information_search_job_exists_check Y Y N - employer_job_execution_exists_check - check_if_need_to_repeat.hpl + citizen_information_search_job_exists_check + recruitments_five_flow.hpl + Y + Y + N + + + citizen_information_search_job_exists_check + check_if_need_to_repeat_job.hpl Y N N - check_if_need_to_repeat.hpl + check_if_need_to_repeat_job.hpl Simple evaluation Y Y N - - employer_job_execution_exists_check - recruitment_five_flow.hpl - Y - Y - N - Simple evaluation - recruitment_five_flow_repeat.hpl + recruitment_five_flow_on_error.hpl Y Y N @@ -258,22 +256,36 @@ N N - - Start - init_job_name - Y - Y - Y - - - init_job_name - check_if_job_execution_exists.hpl - Y - Y - N - + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 149 + 976 + 0 + + +M_R_UP_DATE + +M_R_CR_DATE + +0001-01-01 00:00:00 + +3001-01-01 00:00:00 + 114 + diff --git a/mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl similarity index 69% rename from mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow.hpl rename to mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl index c0ed99b..a511083 100644 --- a/mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl @@ -1,7 +1,7 @@ - disability_flow + citizen_information_search_flow Y @@ -13,21 +13,26 @@ 1000 100 - - 2025/09/08 19:24:20.455 + 2025/08/05 14:31:22.799 - - 2025/09/08 19:24:20.455 + 2025/08/05 14:31:22.799 - Change job status on error - Abort + Create job execution record + Table input Y - Create job execution record - disability_input + Table output + Change job status on error + Y + + + Change job status on error + Abort Y @@ -36,45 +41,45 @@ Y - Identify last row in a stream + Table output Detect empty stream Y - - Identify last row in a stream - disability_output - Y - - - disability_output - Change job status on error - Y - - - disability_input - Identify last row in a stream - Y - Detect empty stream Change job status on success Y - disability_output - has_disability_update + Table input + Table output Y - has_disability_update + Identify last row in a stream + Add constants + Y + + + Add constants + Update + Y + + + Update Filter rows Y - has_disability_update + Update Change job status on error Y + + Table output + Identify last row in a stream + Y + Abort @@ -87,13 +92,40 @@ none - ABORT + ABORT_WITH_ERROR Y 0 - 1504 - 576 + 1168 + 448 + + + + Add constants + Constant + + Y + + 1 + + none + + + + + -1 + is_wanted + true + -1 + N + Boolean + + + + + 880 + 624 @@ -127,8 +159,8 @@ and recruitment_id = '${IDM_ID}'; - 1216 - 576 + 1008 + 448 @@ -153,13 +185,14 @@ and recruitment_id = '${IDM_ID}'; UPDATE etl.job_execution SET status = 'SUCCESS' WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; +and recruitment_id = '${IDM_ID}' +and status = 'PROCESSING'; - 1504 - 240 + 1488 + 624 @@ -187,8 +220,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 288 - 416 + 80 + 320 @@ -204,8 +237,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 704 - 240 + 1488 + 320 @@ -240,15 +273,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1504 - 416 + 1200 + 624 Identify last row in a stream DetectLastRow - N + Y 1 @@ -258,12 +291,12 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); last_row - 704 - 416 + 720 + 624 - disability_input + Table input TableInput Y @@ -275,43 +308,41 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' AS invalid_elem - FROM recruits_info ri - WHERE ri.info -> 'svedFL' -> 'svedInvalid' ->> 'invalid' <> 'null' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' -) -SELECT fd.recruit_id, - invalid_elem ->> 'id' AS source_id, - to_date(invalid_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, - invalid_elem ->> 'gruppa' as disability_group, - invalid_elem ->> 'ustanOrg' as disability_establish_org_name, - to_date(invalid_elem -> 'ustanInvalid' ->> 'dataUstan', 'YYYY-MM-DD') as register_date, - to_date(invalid_elem -> 'ustanInvalid' ->> 'dataPodtverzhdPlan', 'YYYY-MM-DD') as confirmation_date, - to_date(invalid_elem ->> 'dataSnyat', 'YYYY-MM-DD') as deregistration_date, - coalesce(invalid_elem -> 'ustanInvalid' ->> 'prBessrInvl' = '1', false) as permanent, - true as has_disability -FROM filteredData fd; + +SELECT + ri.recruit_id, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code, + TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, + TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search +FROM recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id +WHERE'${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND '${M_R_CR_DATE}'::timestamp >= ri.created_at + AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' + Y - 512 - 416 + 336 + 320 - disability_output + Table output TableOutput - Y + N 1 none - 10000 + 1000 ervu-dashboard @@ -319,36 +350,28 @@ FROM filteredData fd; recruit_id - source_id - source_id + name_organ_start_search + name_organ_start_search - source_update_date - source_update_date + article_criminal_code + article_criminal_code - disability_group - disability_group + wanted_start_date + wanted_start_date - disability_establish_org_name - disability_establish_org_name + wanted_end_date + wanted_end_date - register_date - register_date + grounds_stopping_search + grounds_stopping_search - confirmation_date - confirmation_date - - - deregistration_date - deregistration_date - - - permanent - permanent + name_organ_end_search + name_organ_end_search N @@ -359,29 +382,29 @@ FROM filteredData fd; N ervu_dashboard Y - disability
+ citizen_information_search
N Y N Y - 992 - 416 + 720 + 320
- has_disability_update + Update Update - Y + N 1 none - 10000 + 1000 ervu-dashboard N @@ -395,39 +418,39 @@ FROM filteredData fd; ervu_dashboard citizen
- has_disability - has_disability + is_wanted + is_wanted N - Y + N - 1216 - 416 + 1008 + 624
- disability_output + Table output Change job status on error Y error_description - + error_code - has_disability_update + Update Change job status on error Y - error_description + error_decription - + error_code diff --git a/mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl similarity index 66% rename from mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_repeat.hpl rename to mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl index 92e4e86..92f0032 100644 --- a/mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl @@ -1,7 +1,7 @@ - disability_flow_repeat + citizen_information_search_flow_repeat Y @@ -13,68 +13,73 @@ 1000 100 - - 2025/09/08 20:32:10.265 + 2025/08/08 13:25:52.319 - - 2025/09/08 20:32:10.265 + 2025/08/08 13:25:52.319 + + Create job execution record + Table input + Y + + + Add constants + Update + Y + Change job status on error Abort Y - - Create job execution record - disability_input - Y - - - Filter rows - Change job status on success - Y - - - Identify last row in a stream - Detect empty stream - Y - - - disability_input - Identify last row in a stream - Y - - - Identify last row in a stream - disability_upsert - Y - - - disability_upsert - Change job status on error - Y - Detect empty stream Change job status on success Y - disability_upsert - has_disability_update + Filter rows + Change job status on success Y - has_disability_update + Identify last row in a stream 2 + Add constants + Y + + + Update + Change job status on error + Y + + + Update Filter rows Y - has_disability_update + Insert / update + Detect empty stream + Y + + + Insert / update Change job status on error Y + + Table input + Insert / update + Y + + + Insert / update + Identify last row in a stream 2 + Y + Abort @@ -87,13 +92,40 @@ none - ABORT + ABORT_WITH_ERROR Y 0 - 1504 - 592 + 1152 + 432 + + + + Add constants + Constant + + Y + + 1 + + none + + + + + -1 + is_wanted + true + -1 + N + Boolean + + + + + 880 + 640 @@ -127,8 +159,8 @@ and recruitment_id = '${IDM_ID}'; - 1120 - 592 + 992 + 432 @@ -153,13 +185,14 @@ and recruitment_id = '${IDM_ID}'; UPDATE etl.job_execution SET status = 'SUCCESS' WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; +and recruitment_id = '${IDM_ID}' +and status = 'PROCESSING'; - 1504 - 256 + 1312 + 640 @@ -181,17 +214,31 @@ and recruitment_id = '${IDM_ID}'; Y N N - UPDATE etl.job_execution -SET + INSERT INTO etl.job_execution ( + id, + job_name, + status, + execution_datetime, + error_description, + recruitment_id +) +VALUES ( + DEFAULT, + '${JOB_NAME}', + 'PROCESSING', + DEFAULT, + NULL, + '${IDM_ID}' +) +ON CONFLICT (job_name, recruitment_id) +DO UPDATE SET status = 'PROCESSING', - execution_datetime = current_timestamp, - error_description = NULL -where job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; + execution_datetime = DEFAULT, + error_description = NULL; - 416 - 432 + 176 + 272 @@ -207,8 +254,8 @@ and recruitment_id = '${IDM_ID}'; - 832 - 256 + 1312 + 272 @@ -243,15 +290,15 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1504 - 432 + 1136 + 640 - Identify last row in a stream + Identify last row in a stream 2 DetectLastRow - N + Y 1 @@ -261,12 +308,76 @@ and recruitment_id = '${IDM_ID}'; last_row - 832 - 432 + 688 + 640 - disability_input + Insert / update + InsertUpdate + + N + + 1 + + none + + + 1000 + ervu-dashboard + + + = + recruit_id + recruit_id + + ervu_dashboard + citizen_information_search
+ + recruit_id + recruit_id + N + + + name_organ_start_search + name_organ_start_search + Y + + + article_criminal_code + article_criminal_code + Y + + + wanted_start_date + wanted_start_date + Y + + + wanted_end_date + wanted_end_date + Y + + + grounds_stopping_search + grounds_stopping_search + Y + + + name_organ_end_search + name_organ_end_search + Y + +
+ N + + + 688 + 272 + +
+ + Table input TableInput Y @@ -278,154 +389,79 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' AS invalid_elem - FROM recruits_info ri - WHERE ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' <> 'null' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' -) -SELECT fd.recruit_id, - invalid_elem ->> 'id' AS source_id, - to_date(invalid_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, - invalid_elem ->> 'gruppa' as disability_group, - invalid_elem ->> 'ustanOrg' as disability_establish_org_name, - to_date(invalid_elem -> 'ustanInvalid' ->> 'dataUstan', 'YYYY-MM-DD') as register_date, - to_date(invalid_elem -> 'ustanInvalid' ->> 'dataPodtverzhdPlan', 'YYYY-MM-DD') as confirmation_date, - to_date(invalid_elem ->> 'dataSnyat', 'YYYY-MM-DD') as deregistration_date, - coalesce(invalid_elem -> 'ustanInvalid' ->> 'prBessrInvl' = '1', false) as permanent -FROM filteredData fd; + +SELECT + ri.recruit_id, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code, + TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date, + TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search, + ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search +FROM recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id +WHERE'${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND '${M_R_CR_DATE}'::timestamp >= ri.created_at + AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object' + Y - 640 - 432 + 416 + 272 - disability_upsert - InsertUpdate - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - = - source_id - source_id - - ervu_dashboard - disability
- - recruit_id - recruit_id - N - - - source_id - source_id - N - - - source_update_date - source_update_date - Y - - - disability_group - disability_group - Y - - - disability_establish_org_name - disability_establish_org_name - Y - - - register_date - register_date - Y - - - confirmation_date - confirmation_date - Y - - - deregistration_date - deregistration_date - Y - - - permanent - permanent - Y - - - last_row - last_row - Y - -
- N - - - 1120 - 432 - -
- - has_disability_update + Update Update - Y + N 1 none - 10000 + 1000 ervu-dashboard N - = recruit_id recruit_id - - - - <> - has_disability - has_disability - ervu_dashboard citizen
- has_disability - has_disability + is_wanted + is_wanted
N - Y + N - 1312 - 432 + 992 + 640
- disability_upsert + Insert / update + Change job status on error + Y + + error_description + + error_code + + + + + + Update Change job status on error Y @@ -436,18 +472,6 @@ FROM filteredData fd; - - has_disability_update - Change job status on error - Y - - - - - - - -
diff --git a/mappings/info_recruits/citizen_tables/medicine/vich/vich_job.hwf b/mappings/info_recruits/citizen_tables/citizen_spouse/job_spouse.hwf similarity index 72% rename from mappings/info_recruits/citizen_tables/medicine/vich/vich_job.hwf rename to mappings/info_recruits/citizen_tables/citizen_spouse/job_spouse.hwf index b784620..52a8cd1 100644 --- a/mappings/info_recruits/citizen_tables/medicine/vich/vich_job.hwf +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/job_spouse.hwf @@ -1,16 +1,27 @@ - vich_job + job_citizen_spouse Y + 0 - - 2025/09/08 16:38:17.982 + 2025/06/05 14:27:15.055 - - 2025/09/08 16:38:17.982 + 2025/06/05 14:27:15.055 - + + M_R_CR_DATE + 3001-01-01 00:00:00 + + + + M_R_UP_DATE + + + + Start @@ -26,12 +37,12 @@ 0 1 N - 128 - 160 + 416 + 208 - check_if_job_execution_exists.hpl + check_if_citizen_spouse_job_exists.hpl PIPELINE @@ -41,7 +52,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_job_execution_exists.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_spouse/checkpoints/check_if_citizen_spouse_job_exists.hpl Basic Y @@ -52,12 +63,12 @@ N Y N - 480 - 160 + 608 + 208 - employer_job_execution_exists_check + citizen_spouse_job_exists_check SIMPLE_EVAL @@ -69,12 +80,12 @@ variable JOB_EXECUTED_FLAG N - 784 - 160 + 880 + 208 - recruitment_five_flow.hpl + recruitments_five_flow.hpl PIPELINE @@ -84,7 +95,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/vich/recruitment_five_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_spouse/recruitments_five_flow.hpl Basic @@ -97,12 +108,12 @@ N Y Y - 1104 - 160 + 1120 + 208 - recruitment_five_flow_repeat.hpl + recruitment_five_flow_on_error.hpl PIPELINE @@ -112,7 +123,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/vich/recruitment_five_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_spouse/recruitment_five_flow_on_error.hpl Basic @@ -125,12 +136,12 @@ N Y N - 1040 - 464 + 1120 + 512 - check_if_need_to_repeat.hpl + check_if_need_to_repeat_job.hpl PIPELINE @@ -140,7 +151,9 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_need_to_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_spouse/check_if_need_to_repeat_job.hpl + + Basic Y @@ -151,8 +164,8 @@ N Y N - 784 - 304 + 880 + 368 @@ -168,8 +181,8 @@ variable NEED_TO_REPEAT_JOB N - 784 - 464 + 880 + 512 @@ -183,7 +196,7 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/vich/recruitment_five_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_spouse/recruitment_five_flow_delta.hpl Basic @@ -196,62 +209,50 @@ N Y N - 784 + 880 704 - - init_job_name - - SET_VARIABLES - - - - JOB_NAME - CURRENT_WORKFLOW - vich_job - - - CURRENT_WORKFLOW - N - N - 272 - 160 - - - check_if_job_execution_exists.hpl - employer_job_execution_exists_check + Start + check_if_citizen_spouse_job_exists.hpl + Y + Y + Y + + + check_if_citizen_spouse_job_exists.hpl + citizen_spouse_job_exists_check Y Y N - employer_job_execution_exists_check - check_if_need_to_repeat.hpl + citizen_spouse_job_exists_check + recruitments_five_flow.hpl + Y + Y + N + + + citizen_spouse_job_exists_check + check_if_need_to_repeat_job.hpl Y N N - check_if_need_to_repeat.hpl + check_if_need_to_repeat_job.hpl Simple evaluation Y Y N - - employer_job_execution_exists_check - recruitment_five_flow.hpl - Y - Y - N - Simple evaluation - recruitment_five_flow_repeat.hpl + recruitment_five_flow_on_error.hpl Y Y N @@ -263,20 +264,6 @@ N N - - Start - init_job_name - Y - Y - Y - - - init_job_name - check_if_job_execution_exists.hpl - Y - Y - N - diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/recruitments_five_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/recruitments_five_flow.hpl new file mode 100644 index 0000000..6dccdb6 --- /dev/null +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/recruitments_five_flow.hpl @@ -0,0 +1,356 @@ + + + + recruitments_five_flow + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/04/18 09:48:01.970 + - + 2025/04/18 09:48:01.970 + + + + + + Get all recruitments ordered by created_date + citizen_spouse_flow.hpl + Y + + + Get all recruitments ordered by created_date + citizen_spouse_flow.hpl 2 + Y + + + Get all recruitments ordered by created_date + citizen_spouse_flow.hpl 3 + Y + + + Get all recruitments ordered by created_date + citizen_spouse_flow.hpl 4 + Y + + + Get all recruitments ordered by created_date + citizen_spouse_flow.hpl 5 + Y + + + + Get all recruitments ordered by created_date + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + + SELECT +idm_id AS recruitment +FROM ervu_dashboard.recruitment; + N + + + 432 + 304 + + + + citizen_spouse_flow.hpl + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl + N + + 1 + + + + + IDM_ID + recruitment + + + + JOB_NAME + + citizen_spouse_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 704 + 144 + + + + citizen_spouse_flow.hpl 2 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl + N + + 1 + + + + + IDM_ID + recruitment + + + + JOB_NAME + + citizen_spouse_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 704 + 224 + + + + citizen_spouse_flow.hpl 3 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl + N + + 1 + + + + + IDM_ID + recruitment + + + + JOB_NAME + + citizen_spouse_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 704 + 304 + + + + citizen_spouse_flow.hpl 4 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl + N + + 1 + + + + + IDM_ID + recruitment + + + + JOB_NAME + + citizen_spouse_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 704 + 384 + + + + citizen_spouse_flow.hpl 5 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl + N + + 1 + + + + + IDM_ID + recruitment + + + + JOB_NAME + + citizen_spouse_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 704 + 464 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/medicine/incapacity/recruitment_five_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizenship/recruitment_five_flow_delta.hpl similarity index 86% rename from mappings/info_recruits/citizen_tables/medicine/incapacity/recruitment_five_flow_repeat.hpl rename to mappings/info_recruits/citizen_tables/citizenship/recruitment_five_flow_delta.hpl index 0234694..be774d3 100644 --- a/mappings/info_recruits/citizen_tables/medicine/incapacity/recruitment_five_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship/recruitment_five_flow_delta.hpl @@ -1,7 +1,7 @@ - recruitment_five_flow_repeat + recruitment_five_flow_delta Y @@ -13,36 +13,36 @@ 1000 100 - - 2025/09/08 20:31:52.575 + 2025/08/11 13:16:11.204 - - 2025/09/08 20:31:52.575 + 2025/08/11 13:16:11.204 Table input - incapacity_flow_repeat.hpl + citizenship_flow_delta.hpl Y Table input - incapacity_flow_repeat.hpl 2 + citizenship_flow_delta.hpl 2 Y Table input - incapacity_flow_repeat.hpl 3 + citizenship_flow_delta.hpl 3 Y Table input - incapacity_flow_repeat.hpl 4 + citizenship_flow_delta.hpl 4 Y Table input - incapacity_flow_repeat.hpl 5 + citizenship_flow_delta.hpl 5 Y @@ -57,25 +57,137 @@ none - ervu-dashboard-test + ervu-dashboard N 0 - SELECT r.idm_id AS recruitment_id + SELECT + r.idm_id as recruitment_id FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - AND je.job_name = 'incapacity_job' -WHERE je.id IS NULL - OR je.status IN ('ERROR', 'PROCESSING'); + and job_name = 'citizenship_job' +where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); N + + 352 + 288 + + + + citizenship_flow_delta.hpl + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + citizenship_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 640 + 128 + + + + citizenship_flow_delta.hpl 2 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + citizenship_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + 656 - 464 + 208 - incapacity_flow_repeat.hpl + citizenship_flow_delta.hpl 3 PipelineExecutor Y @@ -86,7 +198,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow_delta.hpl N 1 @@ -101,7 +213,7 @@ WHERE je.id IS NULL JOB_NAME - incapacity_job + citizenship_job Y @@ -126,12 +238,12 @@ WHERE je.id IS NULL - 864 - 304 + 656 + 288 - incapacity_flow_repeat.hpl 2 + citizenship_flow_delta.hpl 4 PipelineExecutor Y @@ -142,7 +254,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow_delta.hpl N 1 @@ -157,7 +269,7 @@ WHERE je.id IS NULL JOB_NAME - incapacity_job + citizenship_job Y @@ -182,12 +294,12 @@ WHERE je.id IS NULL - 864 - 384 + 656 + 368 - incapacity_flow_repeat.hpl 3 + citizenship_flow_delta.hpl 5 PipelineExecutor Y @@ -198,7 +310,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow_delta.hpl N 1 @@ -213,7 +325,7 @@ WHERE je.id IS NULL JOB_NAME - incapacity_job + citizenship_job Y @@ -238,120 +350,8 @@ WHERE je.id IS NULL - 864 - 464 - - - - incapacity_flow_repeat.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_repeat.hpl - N - - 1 - - - - - IDM_ID - recruitment_id - - - - JOB_NAME - - incapacity_job - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 864 - 544 - - - - incapacity_flow_repeat.hpl 5 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_repeat.hpl - N - - 1 - - - - - IDM_ID - recruitment_id - - - - JOB_NAME - - incapacity_job - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 864 - 624 + 656 + 448 diff --git a/mappings/info_recruits/citizen_tables/deferment_liberation/deferment_decision_update_date_ervu_dashboard.hpl b/mappings/info_recruits/citizen_tables/deferment_liberation/deferment_decision_update_date_ervu_dashboard.hpl new file mode 100644 index 0000000..339243a --- /dev/null +++ b/mappings/info_recruits/citizen_tables/deferment_liberation/deferment_decision_update_date_ervu_dashboard.hpl @@ -0,0 +1,98 @@ + + + + deferment_decision_update_date_ervu_dashboard + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/06/02 14:50:31.833 + - + 2025/06/02 14:50:31.833 + + + + + + Table input + Table output + Y + + + + Table input + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT + created_at AS system_create_date, + CASE + WHEN updated_at is null THEN '0001-01-01 00:00:00' + ELSE updated_at + END as system_update_date, + current_timestamp AS record_created, + 'deferment_decision_table' AS workflow +FROM ervu_dashboard.deferment_liberation +WHERE updated_at = (SELECT MAX(updated_at) FROM ervu_dashboard.deferment_liberation) +limit 1 + N + + + 896 + 304 + + + + Table output + TableOutput + + Y + + 1 + + none + + + 1000 + ervu-dashboard + + + N + N + N + N + Y + N + public + N + recruit_create
+ N + Y + N + Y + + + 1216 + 304 + +
+ + + +
diff --git a/mappings/info_recruits/citizen_tables/medicine/vich/recruitment_five_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/deferment_liberation/recruitment_five_flow_delta.hpl similarity index 81% rename from mappings/info_recruits/citizen_tables/medicine/vich/recruitment_five_flow_repeat.hpl rename to mappings/info_recruits/citizen_tables/deferment_liberation/recruitment_five_flow_delta.hpl index 254dc34..dd1d7f6 100644 --- a/mappings/info_recruits/citizen_tables/medicine/vich/recruitment_five_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/deferment_liberation/recruitment_five_flow_delta.hpl @@ -1,7 +1,7 @@ - recruitment_five_flow_repeat + recruitment_five_flow_delta Y @@ -13,36 +13,36 @@ 1000 100 - - 2025/09/08 20:31:52.575 + 2025/08/11 13:16:11.204 - - 2025/09/08 20:31:52.575 + 2025/08/11 13:16:11.204 Table input - vich_flow_repeat.hpl + deferment_liberation_flow_delta.hpl Y Table input - vich_flow_repeat.hpl 2 + deferment_liberation_flow_delta.hpl 2 Y Table input - vich_flow_repeat.hpl 3 + deferment_liberation_flow_delta.hpl 3 Y Table input - vich_flow_repeat.hpl 4 + deferment_liberation_flow_delta.hpl 4 Y Table input - vich_flow_repeat.hpl 5 + deferment_liberation_flow_delta.hpl 5 Y @@ -57,25 +57,193 @@ none - ervu-dashboard-test + ervu-dashboard N 0 - SELECT r.idm_id AS recruitment_id + SELECT + r.idm_id as recruitment_id FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - AND je.job_name = 'incapacity_job' -WHERE je.id IS NULL - OR je.status IN ('ERROR', 'PROCESSING'); + and job_name = 'deferment_liberation_job' +where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); N + + 352 + 288 + + + + deferment_liberation_flow_delta.hpl + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation/parallel/deferment_liberation_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + deferment_liberation_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 640 + 128 + + + + deferment_liberation_flow_delta.hpl 2 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation/parallel/deferment_liberation_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + deferment_liberation_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 640 + 208 + + + + deferment_liberation_flow_delta.hpl 3 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation/parallel/deferment_liberation_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + deferment_liberation_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + 656 - 464 + 288 - vich_flow_repeat.hpl + deferment_liberation_flow_delta.hpl 4 PipelineExecutor Y @@ -86,7 +254,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation/parallel/deferment_liberation_flow_delta.hpl N 1 @@ -98,6 +266,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + deferment_liberation_job + Y @@ -121,12 +294,12 @@ WHERE je.id IS NULL - 864 - 304 + 656 + 368 - vich_flow_repeat.hpl 2 + deferment_liberation_flow_delta.hpl 5 PipelineExecutor Y @@ -137,7 +310,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation/parallel/deferment_liberation_flow_delta.hpl N 1 @@ -149,6 +322,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + deferment_liberation_job + Y @@ -172,161 +350,8 @@ WHERE je.id IS NULL - 864 - 384 - - - - vich_flow_repeat.hpl 3 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_repeat.hpl - N - - 1 - - - - - IDM_ID - recruitment_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 864 - 464 - - - - vich_flow_repeat.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_repeat.hpl - N - - 1 - - - - - IDM_ID - recruitment_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 864 - 544 - - - - vich_flow_repeat.hpl 5 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_repeat.hpl - N - - 1 - - - - - IDM_ID - recruitment_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 864 - 624 + 656 + 448 diff --git a/mappings/info_recruits/citizen_tables/drivers_licence/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/drivers_licence/recruitment_five_flow_delta.hpl new file mode 100644 index 0000000..d72c66c --- /dev/null +++ b/mappings/info_recruits/citizen_tables/drivers_licence/recruitment_five_flow_delta.hpl @@ -0,0 +1,360 @@ + + + + recruitment_five_flow_delta + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/08/11 13:16:11.204 + - + 2025/08/11 13:16:11.204 + + + + + + Table input + drivers_licence_flow_delta.hpl + Y + + + Table input + drivers_licence_flow_delta.hpl 2 + Y + + + Table input + drivers_licence_flow_delta.hpl 3 + Y + + + Table input + drivers_licence_flow_delta.hpl 4 + Y + + + Table input + drivers_licence_flow_delta.hpl 5 + Y + + + + Table input + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT + r.idm_id as recruitment_id +FROM ervu_dashboard.recruitment r + LEFT JOIN etl.job_execution je + ON r.idm_id = je.recruitment_id + and job_name = 'drivers_licence_job' +where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); + N + + + 352 + 288 + + + + drivers_licence_flow_delta.hpl + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/drivers_licence/parallel/drivers_licence_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + drivers_licence_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 640 + 128 + + + + drivers_licence_flow_delta.hpl 2 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/drivers_licence/parallel/drivers_licence_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + drivers_licence_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 640 + 208 + + + + drivers_licence_flow_delta.hpl 3 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/drivers_licence/parallel/drivers_licence_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + drivers_licence_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 640 + 288 + + + + drivers_licence_flow_delta.hpl 4 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/drivers_licence/parallel/drivers_licence_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + drivers_licence_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 640 + 368 + + + + drivers_licence_flow_delta.hpl 5 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/drivers_licence/parallel/drivers_licence_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + drivers_licence_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 640 + 464 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/medicine/disability/recruitment_five_flow.hpl b/mappings/info_recruits/citizen_tables/medicine/disability/recruitment_five_flow.hpl deleted file mode 100644 index f0ab5ab..0000000 --- a/mappings/info_recruits/citizen_tables/medicine/disability/recruitment_five_flow.hpl +++ /dev/null @@ -1,330 +0,0 @@ - - - - recruitment_five_flow - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/08 19:24:07.006 - - - 2025/09/08 19:24:07.006 - - - - - - Get all recruitments - disability_flow.hpl - Y - - - Get all recruitments - disability_flow.hpl 2 - Y - - - Get all recruitments - disability_flow.hpl 3 - Y - - - Get all recruitments - disability_flow.hpl 4 - Y - - - Get all recruitments - disability_flow.hpl 5 - Y - - - - Get all recruitments - TableInput - - Y - - 1 - - none - - - ervu-dashboard-test - N - SELECT -idm_id -FROM ervu_dashboard.recruitment; - N - - - 560 - 448 - - - - disability_flow.hpl - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 832 - 288 - - - - disability_flow.hpl 2 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 832 - 368 - - - - disability_flow.hpl 3 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 832 - 448 - - - - disability_flow.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 832 - 528 - - - - disability_flow.hpl 5 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 832 - 608 - - - - - - diff --git a/mappings/info_recruits/citizen_tables/medicine/disability/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/medicine/disability/recruitment_five_flow_delta.hpl deleted file mode 100644 index aeb507e..0000000 --- a/mappings/info_recruits/citizen_tables/medicine/disability/recruitment_five_flow_delta.hpl +++ /dev/null @@ -1,399 +0,0 @@ - - - - recruitment_five_flow_delta - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/08 20:38:40.987 - - - 2025/09/08 20:38:40.987 - - - - - - Table input - disability_flow_delta.hpl - Y - - - get_max_source_update_date - Table input - Y - - - Table input - disability_flow_delta.hpl 2 - Y - - - Table input - disability_flow_delta.hpl 3 - Y - - - Table input - disability_flow_delta.hpl 4 - Y - - - Table input - disability_flow_delta.hpl 5 - Y - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard-test - N - 0 - get_max_source_update_date - WITH mud AS ( - SELECT - recruitment_id, - MAX(execution_datetime) AS max_upd_date - FROM etl.job_execution - WHERE job_name = '${JOB_NAME}' - AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING') - GROUP BY recruitment_id -) -SELECT - r.idm_id, - ? max_source_update_date -FROM ervu_dashboard.recruitment r - JOIN mud ON mud.recruitment_id = r.idm_id - JOIN recruits_info ri - ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id - AND ri.updated_at > mud.max_upd_date; - Y - - - 704 - 496 - - - - disability_flow_delta.hpl - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_delta.hpl - N - - 1 - - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 336 - - - - get_max_source_update_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - select max(source_update_date) -from disability; - N - - - 528 - 496 - - - - disability_flow_delta.hpl 2 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_delta.hpl - N - - 1 - - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 416 - - - - disability_flow_delta.hpl 3 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_delta.hpl - N - - 1 - - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 496 - - - - disability_flow_delta.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_delta.hpl - N - - 1 - - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 576 - - - - disability_flow_delta.hpl 5 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_delta.hpl - N - - 1 - - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 656 - - - - - - diff --git a/mappings/info_recruits/citizen_tables/medicine/health_state/constants/health_state_constants_flow.hpl b/mappings/info_recruits/citizen_tables/medicine/health_state/constants/health_state_constants_flow.hpl deleted file mode 100644 index 502451b..0000000 --- a/mappings/info_recruits/citizen_tables/medicine/health_state/constants/health_state_constants_flow.hpl +++ /dev/null @@ -1,165 +0,0 @@ - - - - health_state_constants_flow - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/08 16:43:33.050 - - - 2025/09/08 16:43:33.050 - - - - - - disease_input - Table output - Y - - - -<<<<<<< HEAD:mappings/info_recruits/citizen_tables/deferment_liberation/deferment_decision_update_date_ervu_dashboard.hpl - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT * FROM ( - SELECT - created_at AS system_create_date, - CASE - WHEN updated_at IS NULL THEN '0001-01-01 00:00:00'::timestamp - ELSE updated_at - END AS system_update_date, - current_timestamp AS record_created, - 'deferment_decision_table' AS workflow - FROM ervu_dashboard.deferment_liberation - WHERE permission_fact = 'fact' - AND updated_at = (SELECT MAX(updated_at) FROM ervu_dashboard.deferment_liberation WHERE permission_fact = 'fact') - LIMIT 1 -) AS first_query - -UNION ALL - -SELECT * FROM ( - SELECT - created_at AS system_create_date, - CASE - WHEN updated_at IS NULL THEN '0001-01-01 00:00:00'::timestamp - ELSE updated_at - END AS system_update_date, - current_timestamp AS record_created, - 'recruits_deferment_table' AS workflow - FROM ervu_dashboard.deferment_liberation - WHERE permission_fact = 'permission' - AND updated_at = (SELECT MAX(updated_at) FROM ervu_dashboard.deferment_liberation WHERE permission_fact = 'permission') - LIMIT 1 -) AS second_query - N - - - 896 - 304 - - - -======= ->>>>>>> 10bcf8dc200fb1b4ce7b6f5b82d5b8cd20b5b935:mappings/info_recruits/citizen_tables/medicine/health_state/constants/health_state_constants_flow.hpl - Table output - TableOutput - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - code - code - - - value - value - - - actual - actual - - - update_date - update_date - - - N - N - N - N - Y - N - ervu_dashboard - Y - disease
- N - Y - N - Y - - - 608 - 288 - -
- - disease_input - TableInput - - Y - - 1 - - none - - - nsi - N - 0 - SELECT - key as code, - value ->> 'value' as value, - NOT hidden as actual, - updated_at as update_date -FROM classifier_records -WHERE code = 'diseaseCode'; - Y - - - 400 - 288 - - - - - -
diff --git a/mappings/info_recruits/citizen_tables/medicine/health_state/health_state_job.hwf b/mappings/info_recruits/citizen_tables/medicine/health_state/health_state_job.hwf deleted file mode 100644 index c6360c7..0000000 --- a/mappings/info_recruits/citizen_tables/medicine/health_state/health_state_job.hwf +++ /dev/null @@ -1,414 +0,0 @@ - - - health_state_job - Y - - - - - - 2025/09/08 20:45:23.262 - - - 2025/09/08 20:45:23.262 - - - - - Start - - SPECIAL - - 1 - 12 - 60 - 0 - 0 - N - 0 - 1 - N - 256 - 256 - - - - check_if_job_execution_exists.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_job_execution_exists.hpl - Basic - - Y - - N - local - N - N - Y - N - 1136 - 272 - - - - employer_job_execution_exists_check - - SIMPLE_EVAL - - boolean - false - equal - equal - N - variable - JOB_EXECUTED_FLAG - N - 1440 - 272 - - - - recruitment_five_flow.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/recruitment_five_flow.hpl - Basic - - Y - - N - local - N - N - Y - Y - 1744 - 272 - - - - recruitment_five_flow_repeat.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/recruitment_five_flow_repeat.hpl - Basic - - Y - - N - local - N - N - Y - N - 1744 - 576 - - - - check_if_need_to_repeat.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_need_to_repeat.hpl - Basic - - Y - - N - local - N - N - Y - N - 1440 - 416 - - - - Simple evaluation - - SIMPLE_EVAL - - boolean - true - equal - equal - N - variable - NEED_TO_REPEAT_JOB - N - 1440 - 576 - - - - recruitment_five_flow_delta.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/recruitment_five_flow_delta.hpl - Basic - - Y - - N - local - N - N - Y - N - 1440 - 768 - - - - init_job_name - - SET_VARIABLES - - - - JOB_NAME - CURRENT_WORKFLOW - health_state_job - - - CURRENT_WORKFLOW - N - N - 944 - 272 - - - - check_if_constants_exists.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/constants/support/check_if_constants_exists.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 384 - 256 - - - - constants_exists - - SIMPLE_EVAL - - boolean - true - equal - equal - N - variable - CONSTANTS_EXISTS - N - 560 - 256 - - - - health_state_constants_flow.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/constants/health_state_constants_flow.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 752 - 176 - - - - health_state_constants_flow_repeat.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/constants/health_state_constants_flow_repeat.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 752 - 352 - - - - - - check_if_job_execution_exists.hpl - employer_job_execution_exists_check - Y - Y - N - - - employer_job_execution_exists_check - check_if_need_to_repeat.hpl - Y - N - N - - - check_if_need_to_repeat.hpl - Simple evaluation - Y - Y - N - - - init_job_name - check_if_job_execution_exists.hpl - Y - Y - N - - - check_if_constants_exists.hpl - constants_exists - Y - Y - N - - - constants_exists - health_state_constants_flow.hpl - Y - N - N - - - constants_exists - health_state_constants_flow_repeat.hpl - Y - Y - N - - - Start - check_if_constants_exists.hpl - Y - Y - Y - - - health_state_constants_flow.hpl - init_job_name - Y - Y - N - - - health_state_constants_flow_repeat.hpl - init_job_name - Y - Y - N - - - Simple evaluation - recruitment_five_flow_repeat.hpl - Y - Y - N - - - employer_job_execution_exists_check - recruitment_five_flow.hpl - Y - Y - N - - - Simple evaluation - recruitment_five_flow_delta.hpl - Y - N - N - - - - - - diff --git a/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_delta.hpl b/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_delta.hpl deleted file mode 100644 index 0c59209..0000000 --- a/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_delta.hpl +++ /dev/null @@ -1,398 +0,0 @@ - - - - health_state_flow_delta - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/08 22:10:33.736 - - - 2025/09/08 22:10:33.736 - - - - - - Change job status on error - Abort - Y - - - Create job execution record - health_state_input - Y - - - Filter rows - Change job status on success - Y - - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - health_state_upsert - Y - - - health_state_input - Identify last row in a stream - Y - - - health_state_upsert - Change job status on error - Y - - - health_state_upsert - Filter rows - Y - - - Detect empty stream - Change job status on success - Y - - - - Abort - Abort - - Y - - 1 - - none - - - ABORT - Y - 0 - - - 1584 - 704 - - - - Change job status on error - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'DELTA_ERROR', - error_description = ? -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 1360 - 704 - - - - Change job status on success - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'DELTA_SUCCESS' -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 1584 - 336 - - - - Create job execution record - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - N - N - Y - N - N - UPDATE etl.job_execution -SET - status = 'DELTA_PROCESSING', - execution_datetime = current_timestamp, - error_description = NULL -where job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - 656 - 512 - - - - Detect empty stream - DetectEmptyStream - - Y - - 1 - - none - - - - - 1072 - 336 - - - - Filter rows - FilterRows - - Y - - 1 - - none - - - - - - - = - last_row - N - - - - N - -1 - constant - -1 - Y - Boolean - - - - Change job status on success - - - 1584 - 512 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 1072 - 512 - - - - health_state_input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved' as source_update_date, - ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov' as zdorov_arr - FROM recruits_info ri - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov') = - 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - and to_date(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved', - 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}') -SELECT fd.recruit_id, - zdorov_elem ->> 'id' AS source_id, - source_update_date, - zdorov_elem ->> 'kodZabol' AS disease_code, - to_date(zdorov_elem ->> 'dataDiagn', 'YYYY-MM-DD') AS diagnosis_date -FROM filteredData fd - CROSS JOIN LATERAL jsonb_array_elements(zdorov_arr) AS zdorov_elem; - Y - - - 880 - 512 - - - - health_state_upsert - InsertUpdate - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - = - source_id - source_id - - ervu_dashboard - health_state
- - recruit_id - recruit_id - Y - - - source_id - source_id - Y - - - source_update_date - source_update_date - Y - - - disease_code - disease_code - Y - - - diagnosis_date - diagnosis_date - Y - -
- N - - -<<<<<<< HEAD:mappings/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow_delta.hpl - 944 - 224 - -
- - Table input - TableInput - - N - - 1 - - none - - - postgres.subpoena - N - - select ssi.* -from public.subpoena_send_info ssi -join public.subpoena s on s.id = ssi.subpoena_id -WHERE - '${IDM_ID}' != '' -- Проверка на пустую строку - AND s.department_id = '${IDM_ID}' - AND s.status_change_date >= '${M_S_UP_DATE}'::timestamp -${LIMIT_FW} - Y - - - 544 - 224 -======= - 1360 - 512 ->>>>>>> 10bcf8dc200fb1b4ce7b6f5b82d5b8cd20b5b935:mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_delta.hpl - - - - - health_state_upsert - Change job status on error - Y - - error_description - - - - - - - - -
diff --git a/mappings/info_recruits/citizen_tables/medicine/health_state/recruitment_five_flow.hpl b/mappings/info_recruits/citizen_tables/medicine/health_state/recruitment_five_flow.hpl deleted file mode 100644 index ff54894..0000000 --- a/mappings/info_recruits/citizen_tables/medicine/health_state/recruitment_five_flow.hpl +++ /dev/null @@ -1,330 +0,0 @@ - - - - recruitment_five_flow - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/08 20:59:52.647 - - - 2025/09/08 20:59:52.647 - - - - - - Get all recruitments - health_state_flow.hpl - Y - - - Get all recruitments - health_state_flow.hpl 2 - Y - - - Get all recruitments - health_state_flow.hpl 3 - Y - - - Get all recruitments - health_state_flow.hpl 4 - Y - - - Get all recruitments - health_state_flow.hpl 5 - Y - - - - Get all recruitments - TableInput - - Y - - 1 - - none - - - ervu-dashboard-test - N - SELECT -idm_id -FROM ervu_dashboard.recruitment; - N - - - 480 - 432 - - - - health_state_flow.hpl - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 752 - 272 - - - - health_state_flow.hpl 2 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 752 - 352 - - - - health_state_flow.hpl 3 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 752 - 432 - - - - health_state_flow.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 752 - 512 - - - - health_state_flow.hpl 5 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 752 - 592 - - - - - - diff --git a/mappings/info_recruits/citizen_tables/medicine/health_state/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/medicine/health_state/recruitment_five_flow_delta.hpl deleted file mode 100644 index 806d981..0000000 --- a/mappings/info_recruits/citizen_tables/medicine/health_state/recruitment_five_flow_delta.hpl +++ /dev/null @@ -1,398 +0,0 @@ - - - - recruitment_five_flow_delta - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/08 21:58:26.774 - - - 2025/09/08 21:58:26.774 - - - - - - Table input - health_state_flow_delta.hpl - Y - - - get_max_source_update_date - Table input - Y - - - Table input - health_state_flow_delta.hpl 2 - Y - - - Table input - health_state_flow_delta.hpl 3 - Y - - - Table input - health_state_flow_delta.hpl 4 - Y - - - Table input - health_state_flow_delta.hpl 5 - Y - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard-test - N - 0 - get_max_source_update_date - SELECT - r.idm_id, - ? max_source_update_date -FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'health_state_job' - JOIN recruits_info ri - ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id - AND ri.updated_at > ( - SELECT MAX(execution_datetime) - FROM etl.job_execution - WHERE job_name = 'health_state_job' - AND recruitment_id = r.idm_id - ) -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N - - - 816 - 432 - - - - get_max_source_update_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - select max(source_update_date) -from health_state; - N - - - 640 - 432 - - - - health_state_flow_delta.hpl - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 1008 - 272 - - - - health_state_flow_delta.hpl 2 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 1008 - 352 - - - - health_state_flow_delta.hpl 3 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 1008 - 432 - - - - health_state_flow_delta.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 1008 - 512 - - - - health_state_flow_delta.hpl 5 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 1008 - 592 - - - - - - diff --git a/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_delta.hpl b/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_delta.hpl deleted file mode 100644 index 6972dd6..0000000 --- a/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_delta.hpl +++ /dev/null @@ -1,418 +0,0 @@ - - - - incapacity_flow_delta - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/08 20:36:38.686 - - - 2025/09/08 20:36:38.686 - - - - - - Change job status on error - Abort - Y - - - Create job execution record - incapacity_input - Y - - - Filter rows - Change job status on success - Y - - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - incapacity_upsert - Y - - - incapacity_input - Identify last row in a stream - Y - - - incapacity_upsert - Change job status on error - Y - - - incapacity_upsert - Filter rows - Y - - - Detect empty stream - Change job status on success - Y - - - - Abort - Abort - - Y - - 1 - - none - - - ABORT - Y - 0 - - - 1744 - 560 - - - - Change job status on error - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'DELTA_ERROR', - error_description = ? -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 1520 - 560 - - - - Change job status on success - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'DELTA_SUCCESS' -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 1744 - 208 - - - - Create job execution record - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - N - N - Y - N - N - UPDATE etl.job_execution -SET - status = 'DELTA_PROCESSING', - execution_datetime = current_timestamp, - error_description = NULL -where job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - 816 - 384 - - - - Detect empty stream - DetectEmptyStream - - Y - - 1 - - none - - - - - 1232 - 208 - - - - Filter rows - FilterRows - - Y - - 1 - - none - - - - - - - = - last_row - N - - - - N - -1 - constant - -1 - Y - Boolean - - - - Change job status on success - - - 1744 - 384 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 1232 - 384 - - - - incapacity_input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob' as nedeesposob_arr - FROM recruits_info ri - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob') = 'array' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}') -SELECT fd.recruit_id, - nedeesposob_elem ->> 'id' AS source_id, - to_date(nedeesposob_elem ->> 'dataSved', 'YYYY-MM-DD') AS source_update_date, - nedeesposob_elem ->> 'naimStatus' AS capacity_status, - nedeesposob_elem ->> 'naimOrg' AS court_name, - to_date(nedeesposob_elem ->> 'dataPrisv', 'YYYY-MM-DD') AS register_date, - to_date(nedeesposob_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS deregistration_date -FROM filteredData fd - CROSS JOIN LATERAL jsonb_array_elements(fd.nedeesposob_arr -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob') AS nedeesposob_elem -WHERE to_date(nedeesposob_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'; - Y - - - 1040 - 384 - - - - incapacity_upsert - InsertUpdate - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - = -<<<<<<< HEAD:mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_delta.hpl - id - id - - - = - created_at - created_at -======= - source_id - source_id ->>>>>>> 10bcf8dc200fb1b4ce7b6f5b82d5b8cd20b5b935:mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_delta.hpl - - ervu_dashboard - disability
- - recruit_id - recruit_id - N - - - source_id - source_id - N - - - source_update_date - source_update_date - Y - - - capacity_status - capacity_status - Y - - - court_name - court_name - Y - - - register_date - register_date - Y - - - deregistration_date - deregistration_date - Y - -
- N - - -<<<<<<< HEAD:mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_delta.hpl - 944 - 224 - -
- - Table input - TableInput - - N - - 1 - - none - - - postgres.subpoena - N - - SELECT -* -FROM public.restriction_document -WHERE - '${IDM_ID}' != '' -- Проверка на пустую строку - AND vk_id = '${IDM_ID}' - AND updated_at >= '${M_RESTRDOC_UP_DATE}'::timestamp - -${LIMIT_FW} - Y - - - 544 - 224 -======= - 1520 - 384 ->>>>>>> 10bcf8dc200fb1b4ce7b6f5b82d5b8cd20b5b935:mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_delta.hpl - - - - - incapacity_upsert - Change job status on error - Y - - error_description - - - - - - - - -
diff --git a/mappings/info_recruits/citizen_tables/medicine/incapacity/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/medicine/incapacity/recruitment_five_flow_delta.hpl deleted file mode 100644 index bdae4c3..0000000 --- a/mappings/info_recruits/citizen_tables/medicine/incapacity/recruitment_five_flow_delta.hpl +++ /dev/null @@ -1,424 +0,0 @@ - - - - recruitment_five_flow_delta - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/08 20:38:40.987 - - - 2025/09/08 20:38:40.987 - - - - - - Table input - incapacity_flow_delta.hpl - Y - - - get_max_source_update_date - Table input - Y - - - Table input - incapacity_flow_delta.hpl 2 - Y - - - Table input - incapacity_flow_delta.hpl 3 - Y - - - Table input - incapacity_flow_delta.hpl 4 - Y - - - Table input - incapacity_flow_delta.hpl 5 - Y - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard-test - N - 0 - get_max_source_update_date - WITH mud AS ( - SELECT - recruitment_id, - MAX(execution_datetime) AS max_upd_date - FROM etl.job_execution - WHERE job_name = '${JOB_NAME}' - AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING') - GROUP BY recruitment_id -) -SELECT - r.idm_id, - ? max_source_update_date -FROM ervu_dashboard.recruitment r - JOIN mud ON mud.recruitment_id = r.idm_id - JOIN recruits_info ri - ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id - AND ri.updated_at > mud.max_upd_date; - N - - - 704 - 496 - - - - get_max_source_update_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - select max(source_update_date) -from incapacity; - N - - - 528 - 496 - - - - incapacity_flow_delta.hpl - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - JOB_NAME - - incapacity_job - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 336 - - - - incapacity_flow_delta.hpl 2 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - JOB_NAME - - incapacity_job - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 416 - - - - incapacity_flow_delta.hpl 3 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - JOB_NAME - - incapacity_job - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 496 - - - - incapacity_flow_delta.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - JOB_NAME - - incapacity_job - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 576 - - - - incapacity_flow_delta.hpl 5 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - JOB_NAME - - incapacity_job - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 656 - - - - - - diff --git a/mappings/info_recruits/citizen_tables/medicine/vich/recruitment_five_flow.hpl b/mappings/info_recruits/citizen_tables/medicine/vich/recruitment_five_flow.hpl deleted file mode 100644 index 95f4a25..0000000 --- a/mappings/info_recruits/citizen_tables/medicine/vich/recruitment_five_flow.hpl +++ /dev/null @@ -1,330 +0,0 @@ - - - - recruitment_five_flow - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/08 19:24:07.006 - - - 2025/09/08 19:24:07.006 - - - - - - Get all recruitments - vich_flow.hpl - Y - - - Get all recruitments - vich_flow.hpl 2 - Y - - - Get all recruitments - vich_flow.hpl 3 - Y - - - Get all recruitments - vich_flow.hpl 4 - Y - - - Get all recruitments - vich_flow.hpl 5 - Y - - - - Get all recruitments - TableInput - - Y - - 1 - - none - - - ervu-dashboard-test - N - SELECT -idm_id -FROM ervu_dashboard.recruitment; - N - - - 560 - 448 - - - - vich_flow.hpl - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 832 - 288 - - - - vich_flow.hpl 2 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 832 - 368 - - - - vich_flow.hpl 3 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 832 - 448 - - - - vich_flow.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 832 - 528 - - - - vich_flow.hpl 5 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 832 - 608 - - - - - - diff --git a/mappings/info_recruits/citizen_tables/medicine/vich/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/medicine/vich/recruitment_five_flow_delta.hpl deleted file mode 100644 index 014da7d..0000000 --- a/mappings/info_recruits/citizen_tables/medicine/vich/recruitment_five_flow_delta.hpl +++ /dev/null @@ -1,399 +0,0 @@ - - - - recruitment_five_flow_delta - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/08 20:38:40.987 - - - 2025/09/08 20:38:40.987 - - - - - - Table input - vich_flow_delta.hpl - Y - - - get_max_source_update_date - Table input - Y - - - Table input - vich_flow_delta.hpl 2 - Y - - - Table input - vich_flow_delta.hpl 3 - Y - - - Table input - vich_flow_delta.hpl 4 - Y - - - Table input - vich_flow_delta.hpl 5 - Y - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard-test - N - 0 - get_max_source_update_date - WITH mud AS ( - SELECT - recruitment_id, - MAX(execution_datetime) AS max_upd_date - FROM etl.job_execution - WHERE job_name = '${JOB_NAME}' - AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING') - GROUP BY recruitment_id -) -SELECT - r.idm_id, - ? max_source_update_date -FROM ervu_dashboard.recruitment r - JOIN mud ON mud.recruitment_id = r.idm_id - JOIN recruits_info ri - ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id - AND ri.updated_at > mud.max_upd_date; - N - - - 704 - 496 - - - - get_max_source_update_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - select max(source_update_date) -from vich_registered; - N - - - 528 - 496 - - - - vich_flow_delta.hpl - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 336 - - - - vich_flow_delta.hpl 2 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 416 - - - - vich_flow_delta.hpl 3 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 496 - - - - vich_flow_delta.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 576 - - - - vich_flow_delta.hpl 5 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 896 - 656 - - - - - - diff --git a/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl similarity index 74% rename from mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow.hpl rename to mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl index 02cd627..cb2fe0c 100644 --- a/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl @@ -1,7 +1,7 @@ - incapacity_flow + passport_flow Y @@ -13,21 +13,46 @@ 1000 100 - - 2025/09/08 19:24:20.455 + 2025/08/05 14:31:22.799 - - 2025/09/08 19:24:20.455 + 2025/08/05 14:31:22.799 - Change job status on error - Abort + Create job execution record + Table input Y - Create job execution record - incapacity_input + Table output + Change job status on error + Y + + + Table output + Detect empty stream + Y + + + Detect empty stream + Change job status on success + Y + + + Table input + Identify last row in a stream + Y + + + Identify last row in a stream + Table output + Y + + + Table output + Filter rows Y @@ -36,33 +61,8 @@ Y - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - incapacity_output - Y - - - incapacity_output - Change job status on error - Y - - - incapacity_output - Filter rows - Y - - - incapacity_input - Identify last row in a stream - Y - - - Detect empty stream - Change job status on success + Change job status on error + Abort Y @@ -77,13 +77,13 @@ none - ABORT + ABORT_WITH_ERROR Y 0 - 1216 - 576 + 1264 + 608 @@ -117,8 +117,8 @@ and recruitment_id = '${IDM_ID}'; - 992 - 576 + 896 + 608 @@ -143,13 +143,14 @@ and recruitment_id = '${IDM_ID}'; UPDATE etl.job_execution SET status = 'SUCCESS' WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; +and recruitment_id = '${IDM_ID} +and status = 'PROCESSING'; - 1216 - 240 + 1264 + 176 @@ -178,7 +179,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 288 - 416 + 320 @@ -194,8 +195,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 704 - 240 + 896 + 176 @@ -230,15 +231,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1216 - 416 + 1264 + 320 Identify last row in a stream DetectLastRow - N + Y 1 @@ -248,12 +249,12 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); last_row - 704 - 416 + 640 + 320 - incapacity_input + Table input TableInput Y @@ -266,41 +267,51 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - WITH filteredData AS ( - SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob' as nedeesposob_arr - FROM recruits_info ri - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob') = 'array' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' -) -SELECT fd.recruit_id, - nedeesposob_elem ->> 'id' AS source_id, - to_date(nedeesposob_elem ->> 'dataSved', 'YYYY-MM-DD') AS source_update_date, - nedeesposob_elem ->> 'naimStatus' AS capacity_status, - nedeesposob_elem ->> 'naimOrg' AS court_name, - to_date(nedeesposob_elem ->> 'dataPrisv', 'YYYY-MM-DD') AS register_date, - to_date(nedeesposob_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS deregistration_date -FROM filteredData fd - CROSS JOIN LATERAL jsonb_array_elements(fd.nedeesposob_arr -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob') AS nedeesposob_elem; + SELECT + ri.recruit_id, + NULLIF(p.passport_data->>'nomDok', '') AS passport_number, + NULLIF(p.passport_data->>'serDok', '') AS passport_series, + NULLIF(p.passport_data->>'vydDok', '') AS organization_name, + NULLIF(p.passport_data->>'kodVydDok', '') AS unit_code, + TO_DATE(NULLIF(p.passport_data->>'dataDok', ''), 'YYYY-MM-DD') AS issue_date, + CASE + WHEN NULLIF(p.passport_data->>'kodStatus', '') = '1' THEN TRUE + ELSE FALSE + END AS actual +FROM recruits_info ri +JOIN ervu_dashboard.citizen r + ON r.recruit_id = ri.recruit_id + AND '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date +JOIN LATERAL ( + SELECT passport_data + FROM UNNEST(ARRAY[ + ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF', + ri.info->'svedFL'->'svedBS'->'pasportRF'->'predPasportRF' + ]) AS passport_data + WHERE passport_data IS NOT NULL AND passport_data::text <> 'null' +) AS p ON TRUE + Y - 512 - 416 + 496 + 320 - incapacity_output + Table output TableOutput - Y + N 1 none - 10000 + 1000 ervu-dashboard @@ -308,28 +319,28 @@ FROM filteredData fd recruit_id - source_id - source_id + passport_number + passport_number - source_update_date - source_update_date + passport_series + passport_series - capacity_status - capacity_status + organization_name + organization_name - court_name - court_name + unit_code + unit_code - register_date - register_date + issue_date + issue_date - deregistration_date - deregistration_date + actual + actual N @@ -340,26 +351,26 @@ FROM filteredData fd N ervu_dashboard Y - incapacity
+ passport
N Y N Y - 992 - 416 + 896 + 320
- incapacity_output + Table output Change job status on error Y error_description - + error_code diff --git a/mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl similarity index 72% rename from mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_repeat.hpl rename to mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl index 7c11f4f..b7c0e1c 100644 --- a/mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl @@ -1,7 +1,7 @@ - vich_flow_repeat + passport_flow_delta Y @@ -13,21 +13,46 @@ 1000 100 - - 2025/09/08 20:32:10.265 + 2025/08/11 13:22:56.048 - - 2025/09/08 20:32:10.265 + 2025/08/11 13:22:56.048 - Change job status on error - Abort + Create job execution record + Table input Y - Create job execution record - incapacity_input + Insert / update + Change job status on error + Y + + + Insert / update + Detect empty stream + Y + + + Detect empty stream + Change job status on success + Y + + + Table input + Identify last row in a stream + Y + + + Identify last row in a stream + Insert / update + Y + + + Insert / update + Filter rows Y @@ -36,33 +61,8 @@ Y - Identify last row in a stream - Detect empty stream - Y - - - incapacity_input - Identify last row in a stream - Y - - - Identify last row in a stream - incapacity_upsert - Y - - - incapacity_upsert - Filter rows - Y - - - incapacity_upsert - Change job status on error - Y - - - Detect empty stream - Change job status on success + Change job status on error + Abort Y @@ -77,13 +77,13 @@ none - ABORT + ABORT_WITH_ERROR Y 0 - 1344 - 608 + 1008 + 464 @@ -109,7 +109,7 @@ Y N UPDATE etl.job_execution -SET status = 'ERROR', +SET status = 'DELTA_ERROR', error_description = ? WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; @@ -117,8 +117,8 @@ and recruitment_id = '${IDM_ID}'; - 1120 - 608 + 752 + 464 @@ -141,15 +141,16 @@ and recruitment_id = '${IDM_ID}'; N N UPDATE etl.job_execution -SET status = 'SUCCESS' +SET status = 'DELTA_SUCCESS' WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; +and recruitment_id = '${IDM_ID}' +and status = 'DELTA_PROCESSING'; - 1344 - 256 + 1120 + 80 @@ -173,15 +174,15 @@ and recruitment_id = '${IDM_ID}'; N UPDATE etl.job_execution SET - status = 'PROCESSING', - execution_datetime = current_timestamp, + status = 'DELTA_PROCESSING', + execution_datetime = DEFAULT, error_description = NULL where job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - 416 - 432 + 128 + 224 @@ -197,8 +198,8 @@ and recruitment_id = '${IDM_ID}'; - 832 - 256 + 752 + 80 @@ -233,15 +234,15 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1344 - 432 + 1120 + 224 Identify last row in a stream DetectLastRow - N + Y 1 @@ -251,12 +252,81 @@ and recruitment_id = '${IDM_ID}'; last_row - 832 - 432 + 560 + 224 - incapacity_input + Insert / update + InsertUpdate + + N + + 1 + + none + + + 100 + ervu-dashboard + + + = + recruit_id + recruit_id + + + = + actual + actual + + ervu_dashboard + passport
+ + recruit_id + recruit_id + N + + + passport_number + passport_number + Y + + + passport_series + passport_series + Y + + + organization_name + organization_name + Y + + + unit_code + unit_code + Y + + + issue_date + issue_date + Y + + + actual + actual + N + +
+ N + + + 752 + 224 + +
+ + Table input TableInput Y @@ -269,100 +339,49 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, - to_date(ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' ->> 'dataSved', - 'YYYY-MM-DD') as source_update_date, - ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' -> 'svedUchet' as vich_arr - FROM recruits_info ri - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' -> 'svedUchet') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}') -SELECT fd.recruit_id, - vich_elem ->> 'id' AS source_id, - fd.source_update_date, - to_date(vich_elem ->> 'dataSnyat', 'YYYY-MM-DD') AS register_date, - to_date(vich_elem ->> 'dataPostUchet', 'YYYY-MM-DD') AS deregistration_date -FROM filteredData fd - CROSS JOIN LATERAL jsonb_array_elements(vich_arr) AS vich_elem; + SELECT + ri.recruit_id, + NULLIF(p.passport_data->>'nomDok', '') AS passport_number, + NULLIF(p.passport_data->>'serDok', '') AS passport_series, + NULLIF(p.passport_data->>'vydDok', '') AS organization_name, + NULLIF(p.passport_data->>'kodVydDok', '') AS unit_code, + TO_DATE(NULLIF(p.passport_data->>'dataDok', ''), 'YYYY-MM-DD') AS issue_date, + CASE + WHEN NULLIF(p.passport_data->>'kodStatus', '') = '1' THEN TRUE + ELSE FALSE + END AS actual +FROM recruits_info ri +JOIN ervu_dashboard.citizen r + ON r.recruit_id = ri.recruit_id + AND '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date + AND r.update_date >= '${M_R_UP_DATE}'::timestamp +JOIN LATERAL ( + SELECT passport_data + FROM UNNEST(ARRAY[ + ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF', + ri.info->'svedFL'->'svedBS'->'pasportRF'->'predPasportRF' + ]) AS passport_data + WHERE passport_data IS NOT NULL AND passport_data::text <> 'null' +) AS p ON TRUE + Y - 640 - 432 - - - - incapacity_upsert - InsertUpdate - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - = - source_id - source_id - - ervu_dashboard - disability
- - recruit_id - recruit_id - N - - - source_id - source_id - N - - - source_update_date - source_update_date - Y - - - capacity_status - capacity_status - Y - - - court_name - court_name - Y - - - register_date - register_date - Y - - - deregistration_date - deregistration_date - Y - -
- N - - - 1120 - 432 + 400 + 224
- incapacity_upsert + Insert / update Change job status on error Y error_description - + error_code diff --git a/mappings/info_recruits/citizen_tables/medicine/disability/recruitment_five_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/passport/recruitment_five_flow_on_error.hpl similarity index 83% rename from mappings/info_recruits/citizen_tables/medicine/disability/recruitment_five_flow_repeat.hpl rename to mappings/info_recruits/citizen_tables/passport/recruitment_five_flow_on_error.hpl index 65d9cfc..56c602d 100644 --- a/mappings/info_recruits/citizen_tables/medicine/disability/recruitment_five_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/passport/recruitment_five_flow_on_error.hpl @@ -1,7 +1,7 @@ - recruitment_five_flow_repeat + recruitment_five_flow_on_error Y @@ -13,36 +13,36 @@ 1000 100 - - 2025/09/08 20:31:52.575 + 2025/08/05 12:54:50.126 - - 2025/09/08 20:31:52.575 + 2025/08/05 12:54:50.126 Table input - disability_flow_repeat.hpl + passport_flow_repeat.hpl Y Table input - disability_flow_repeat.hpl 2 + passport_flow_repeat.hpl 3 Y Table input - disability_flow_repeat.hpl 3 + passport_flow_repeat.hpl 2 Y Table input - disability_flow_repeat.hpl 4 + passport_flow_repeat.hpl 4 Y Table input - disability_flow_repeat.hpl 5 + passport_flow_repeat.hpl 5 Y @@ -57,25 +57,25 @@ none - ervu-dashboard-test + ervu-dashboard N - 0 - SELECT r.idm_id AS recruitment_id + + SELECT + r.idm_id as recruitment_id FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - AND je.job_name = 'disability_job' -WHERE je.id IS NULL - OR je.status IN ('ERROR', 'PROCESSING'); + and job_name = 'passport_job' +where je.status is null or je.status in('ERROR', 'PROCESSING'); N - 656 - 464 + 272 + 368 - disability_flow_repeat.hpl + passport_flow_repeat.hpl PipelineExecutor Y @@ -86,7 +86,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl N 1 @@ -98,6 +98,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + passport_job + Y @@ -121,12 +126,12 @@ WHERE je.id IS NULL - 864 - 304 + 464 + 208 - disability_flow_repeat.hpl 2 + passport_flow_repeat.hpl 2 PipelineExecutor Y @@ -137,7 +142,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl N 1 @@ -149,6 +154,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + passport_job + Y @@ -172,12 +182,12 @@ WHERE je.id IS NULL - 864 - 384 + 464 + 288 - disability_flow_repeat.hpl 3 + passport_flow_repeat.hpl 3 PipelineExecutor Y @@ -188,7 +198,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl N 1 @@ -200,6 +210,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + passport_job + Y @@ -223,12 +238,12 @@ WHERE je.id IS NULL - 864 - 464 + 464 + 368 - disability_flow_repeat.hpl 4 + passport_flow_repeat.hpl 4 PipelineExecutor Y @@ -239,7 +254,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl N 1 @@ -251,6 +266,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + passport_job + Y @@ -274,12 +294,12 @@ WHERE je.id IS NULL - 864 - 544 + 464 + 448 - disability_flow_repeat.hpl 5 + passport_flow_repeat.hpl 5 PipelineExecutor Y @@ -290,7 +310,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl N 1 @@ -302,6 +322,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + passport_job + Y @@ -325,8 +350,8 @@ WHERE je.id IS NULL - 864 - 624 + 464 + 528 diff --git a/mappings/info_recruits/citizen_tables/medicine/incapacity/recruitment_five_flow.hpl b/mappings/info_recruits/citizen_tables/passport/recruitments_five_flow.hpl similarity index 84% rename from mappings/info_recruits/citizen_tables/medicine/incapacity/recruitment_five_flow.hpl rename to mappings/info_recruits/citizen_tables/passport/recruitments_five_flow.hpl index 3d15cae..18397ed 100644 --- a/mappings/info_recruits/citizen_tables/medicine/incapacity/recruitment_five_flow.hpl +++ b/mappings/info_recruits/citizen_tables/passport/recruitments_five_flow.hpl @@ -1,7 +1,7 @@ - recruitment_five_flow + recruitments_five_flow Y @@ -13,41 +13,41 @@ 1000 100 - - 2025/09/08 19:24:07.006 + 2025/04/18 09:48:01.970 - - 2025/09/08 19:24:07.006 + 2025/04/18 09:48:01.970 - Get all recruitments - incapacity_flow.hpl + Get all recruitments ordered by created_date + passport_flow.hpl Y - Get all recruitments - incapacity_flow.hpl 2 + Get all recruitments ordered by created_date + passport_flow.hpl 2 Y - Get all recruitments - incapacity_flow.hpl 3 + Get all recruitments ordered by created_date + passport_flow.hpl 3 Y - Get all recruitments - incapacity_flow.hpl 4 + Get all recruitments ordered by created_date + passport_flow.hpl 4 Y - Get all recruitments - incapacity_flow.hpl 5 + Get all recruitments ordered by created_date + passport_flow.hpl 5 Y - Get all recruitments + Get all recruitments ordered by created_date TableInput Y @@ -57,20 +57,21 @@ none - ervu-dashboard-test + ervu-dashboard N + SELECT -idm_id +idm_id AS recruitment FROM ervu_dashboard.recruitment; N - 560 - 448 + 432 + 304 - incapacity_flow.hpl + passport_flow.hpl PipelineExecutor Y @@ -81,7 +82,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl N 1 @@ -90,13 +91,13 @@ FROM ervu_dashboard.recruitment; IDM_ID - idm_id + recruitment JOB_NAME - incapacity_job + passport_job Y @@ -121,12 +122,12 @@ FROM ervu_dashboard.recruitment; - 832 - 288 + 704 + 144 - incapacity_flow.hpl 2 + passport_flow.hpl 2 PipelineExecutor Y @@ -137,7 +138,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl N 1 @@ -146,13 +147,13 @@ FROM ervu_dashboard.recruitment; IDM_ID - idm_id + recruitment JOB_NAME - incapacity_job + passport_job Y @@ -177,12 +178,12 @@ FROM ervu_dashboard.recruitment; - 832 - 368 + 704 + 224 - incapacity_flow.hpl 3 + passport_flow.hpl 3 PipelineExecutor Y @@ -193,7 +194,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl N 1 @@ -202,13 +203,13 @@ FROM ervu_dashboard.recruitment; IDM_ID - idm_id + recruitment JOB_NAME - incapacity_job + passport_job Y @@ -233,12 +234,12 @@ FROM ervu_dashboard.recruitment; - 832 - 448 + 704 + 304 - incapacity_flow.hpl 4 + passport_flow.hpl 4 PipelineExecutor Y @@ -249,7 +250,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl N 1 @@ -258,13 +259,13 @@ FROM ervu_dashboard.recruitment; IDM_ID - idm_id + recruitment JOB_NAME - incapacity_job + passport_job Y @@ -289,12 +290,12 @@ FROM ervu_dashboard.recruitment; - 832 - 528 + 704 + 384 - incapacity_flow.hpl 5 + passport_flow.hpl 5 PipelineExecutor Y @@ -305,7 +306,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl N 1 @@ -314,13 +315,13 @@ FROM ervu_dashboard.recruitment; IDM_ID - idm_id + recruitment JOB_NAME - incapacity_job + passport_job Y @@ -345,8 +346,8 @@ FROM ervu_dashboard.recruitment; - 832 - 608 + 704 + 464 diff --git a/mappings/info_recruits/citizen_tables/medicine/health_state/recruitment_five_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/property/recruitment_five_flow_delta.hpl similarity index 83% rename from mappings/info_recruits/citizen_tables/medicine/health_state/recruitment_five_flow_repeat.hpl rename to mappings/info_recruits/citizen_tables/property/recruitment_five_flow_delta.hpl index 3971440..0ac96e5 100644 --- a/mappings/info_recruits/citizen_tables/medicine/health_state/recruitment_five_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/property/recruitment_five_flow_delta.hpl @@ -1,7 +1,7 @@ - recruitment_five_flow_repeat + recruitment_five_flow_delta Y @@ -13,36 +13,36 @@ 1000 100 - - 2025/09/08 21:44:04.866 + 2025/08/11 13:16:11.204 - - 2025/09/08 21:44:04.866 + 2025/08/11 13:16:11.204 Table input - health_state_flow_repeat.hpl + property_flow_delta.hpl Y Table input - health_state_flow_repeat.hpl 2 + property_flow_delta.hpl 2 Y Table input - health_state_flow_repeat.hpl 3 + property_flow_delta.hpl 3 Y Table input - health_state_flow_repeat.hpl 4 + property_flow_delta.hpl 4 Y Table input - health_state_flow_repeat.hpl 5 + property_flow_delta.hpl 5 Y @@ -57,25 +57,25 @@ none - ervu-dashboard-test + ervu-dashboard N 0 - SELECT r.idm_id AS recruitment_id + SELECT + r.idm_id as recruitment_id FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - AND je.job_name = 'health_state_job' -WHERE je.id IS NULL - OR je.status IN ('ERROR', 'PROCESSING'); + and job_name = 'property_job' +where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); N - 400 - 464 + 352 + 288 - health_state_flow_repeat.hpl + property_flow_delta.hpl PipelineExecutor Y @@ -86,7 +86,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/property/parallel/property_flow_delta.hpl N 1 @@ -98,6 +98,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + property_job + Y @@ -121,12 +126,12 @@ WHERE je.id IS NULL - 608 - 304 + 640 + 128 - health_state_flow_repeat.hpl 2 + property_flow_delta.hpl 2 PipelineExecutor Y @@ -137,7 +142,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/property/parallel/property_flow_delta.hpl N 1 @@ -149,6 +154,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + property_job + Y @@ -172,12 +182,12 @@ WHERE je.id IS NULL - 608 - 384 + 640 + 208 - health_state_flow_repeat.hpl 3 + property_flow_delta.hpl 3 PipelineExecutor Y @@ -188,7 +198,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/property/parallel/property_flow_delta.hpl N 1 @@ -200,6 +210,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + property_job + Y @@ -223,12 +238,12 @@ WHERE je.id IS NULL - 608 - 464 + 640 + 288 - health_state_flow_repeat.hpl 4 + property_flow_delta.hpl 4 PipelineExecutor Y @@ -239,7 +254,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/property/parallel/property_flow_delta.hpl N 1 @@ -251,6 +266,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + property_job + Y @@ -274,12 +294,12 @@ WHERE je.id IS NULL - 608 - 544 + 640 + 368 - health_state_flow_repeat.hpl 5 + property_flow_delta.hpl 5 PipelineExecutor Y @@ -290,7 +310,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/property/parallel/property_flow_delta.hpl N 1 @@ -302,6 +322,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + property_job + Y @@ -325,8 +350,8 @@ WHERE je.id IS NULL - 608 - 624 + 640 + 448 diff --git a/mappings/info_recruits/citizen_tables/punishment/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/punishment/recruitment_five_flow_delta.hpl new file mode 100644 index 0000000..45065a2 --- /dev/null +++ b/mappings/info_recruits/citizen_tables/punishment/recruitment_five_flow_delta.hpl @@ -0,0 +1,360 @@ + + + + recruitment_five_flow_delta + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/08/11 13:16:11.204 + - + 2025/08/11 13:16:11.204 + + + + + + Table input + punishment_flow_delta.hpl + Y + + + Table input + punishment_flow_delta.hpl 2 + Y + + + Table input + punishment_flow_delta.hpl 3 + Y + + + Table input + punishment_flow_delta.hpl 4 + Y + + + Table input + punishment_flow_delta.hpl 5 + Y + + + + Table input + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + + SELECT + r.idm_id as recruitment_id +FROM ervu_dashboard.recruitment r + LEFT JOIN etl.job_execution je + ON r.idm_id = je.recruitment_id + and job_name = 'punishment_job' +where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); + N + + + 352 + 288 + + + + punishment_flow_delta.hpl + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + punishment_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 544 + 128 + + + + punishment_flow_delta.hpl 2 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + punishment_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 544 + 208 + + + + punishment_flow_delta.hpl 3 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + punishment_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 544 + 288 + + + + punishment_flow_delta.hpl 4 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + punishment_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 544 + 368 + + + + punishment_flow_delta.hpl 5 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + punishment_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 544 + 448 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/punishment/recruitment_five_flow_on_error.hpl b/mappings/info_recruits/citizen_tables/punishment/recruitment_five_flow_on_error.hpl new file mode 100644 index 0000000..da6ae9d --- /dev/null +++ b/mappings/info_recruits/citizen_tables/punishment/recruitment_five_flow_on_error.hpl @@ -0,0 +1,360 @@ + + + + recruitment_five_flow_on_error + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/08/05 12:54:50.126 + - + 2025/08/05 12:54:50.126 + + + + + + Table input + punishment_flow_repeat.hpl + Y + + + Table input + punishment_flow_repeat.hpl 3 + Y + + + Table input + punishment_flow_repeat.hpl 2 + Y + + + Table input + punishment_flow_repeat.hpl 4 + Y + + + Table input + punishment_flow_repeat.hpl 5 + Y + + + + Table input + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + + SELECT + r.idm_id as recruitment_id +FROM ervu_dashboard.recruitment r + LEFT JOIN etl.job_execution je + ON r.idm_id = je.recruitment_id + and job_name = 'punishment_job' +where je.status is null or je.status in('ERROR', 'PROCESSING'); + N + + + 272 + 368 + + + + punishment_flow_repeat.hpl + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + punishment_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 464 + 208 + + + + punishment_flow_repeat.hpl 2 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + punishment_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 464 + 288 + + + + punishment_flow_repeat.hpl 3 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + punishment_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 464 + 368 + + + + punishment_flow_repeat.hpl 4 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + punishment_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 464 + 448 + + + + punishment_flow_repeat.hpl 5 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + punishment_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 464 + 528 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/work/employer/recruitment_five_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/work/employer/recruitment_five_flow_repeat.hpl index fd60da1..ed96db3 100644 --- a/mappings/info_recruits/citizen_tables/work/employer/recruitment_five_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/work/employer/recruitment_five_flow_repeat.hpl @@ -98,11 +98,6 @@ WHERE je.id IS NULL recruitment_id - - JOB_NAME - - employer_job - Y @@ -154,11 +149,6 @@ WHERE je.id IS NULL recruitment_id - - JOB_NAME - - employer_job - Y @@ -210,11 +200,6 @@ WHERE je.id IS NULL recruitment_id - - JOB_NAME - - employer_job - Y @@ -266,11 +251,6 @@ WHERE je.id IS NULL recruitment_id - - JOB_NAME - - employer_job - Y @@ -322,11 +302,6 @@ WHERE je.id IS NULL recruitment_id - - JOB_NAME - - employer_job - Y diff --git a/mappings/info_recruits/citizen_tables/medicine/health_state/constants/support/check_if_constants_exists.hpl b/mappings/info_recruits/citizen_tables/work/work_activity/support/check_if_need_to_repeat_job.hpl similarity index 70% rename from mappings/info_recruits/citizen_tables/medicine/health_state/constants/support/check_if_constants_exists.hpl rename to mappings/info_recruits/citizen_tables/work/work_activity/support/check_if_need_to_repeat_job.hpl index e1e87ed..5ca8d4b 100644 --- a/mappings/info_recruits/citizen_tables/medicine/health_state/constants/support/check_if_constants_exists.hpl +++ b/mappings/info_recruits/citizen_tables/work/work_activity/support/check_if_need_to_repeat_job.hpl @@ -1,7 +1,7 @@ - check_if_constants_exists + check_if_need_to_repeat_job Y @@ -13,9 +13,9 @@ 1000 100 - - 2025/09/22 17:28:33.897 + 2025/08/08 12:27:51.630 - - 2025/09/22 17:28:33.897 + 2025/08/08 12:27:51.630 @@ -39,17 +39,16 @@ - - constants_exists - CONSTANTS_EXISTS - PARENT_WORKFLOW + need_to_repeat_job + NEED_TO_REPEAT_JOB + ROOT_WORKFLOW Y 624 - 304 + 288 @@ -66,14 +65,18 @@ ervu-dashboard N 0 - SELECT EXISTS ( - SELECT 1 FROM disease -) AS constants_exists - Y + SELECT EXISTS (SELECT 1 +FROM ervu_dashboard.recruitment r + LEFT JOIN etl.job_execution je + ON r.idm_id = je.recruitment_id + and job_name = 'work_activity_job' +where je.status is null or je.status in ('ERROR', 'PROCESSING') +) as need_to_repeat_job; + N 336 - 304 + 288 diff --git a/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow.hpl b/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow.hpl similarity index 65% rename from mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow.hpl rename to mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow.hpl index bc512a9..4486fd5 100644 --- a/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow.hpl +++ b/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow.hpl @@ -1,7 +1,7 @@ - health_state_flow + restriction_document_flow Y @@ -13,21 +13,16 @@ 1000 100 - - 2025/09/08 21:46:18.176 + 2025/08/05 14:31:22.799 - - 2025/09/08 21:46:18.176 + 2025/08/05 14:31:22.799 - - Change job status on error - Abort - Y - Create job execution record - health_state_input + Table input Y @@ -35,46 +30,41 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - health_state_output - Y - - - health_state_input - Identify last row in a stream - Y - - - health_state_output - Change job status on error - Y - Detect empty stream Change job status on success Y - health_state_output - has_health_state_update - Y - - - has_health_state_update + Change job status on error Abort Y - has_health_state_update + Table input + Identify last row in a stream + Y + + + Identify last row in a stream + Table output + Y + + + Table output Filter rows Y + + Table output + Change job status on error + Y + + + Table output + Detect empty stream + Y + Abort @@ -87,13 +77,13 @@ none - ABORT + ABORT_WITH_ERROR Y 0 - 1264 - 672 + 1376 + 512 @@ -127,8 +117,8 @@ and recruitment_id = '${IDM_ID}'; - 1040 - 672 + 1024 + 512 @@ -158,8 +148,8 @@ and recruitment_id = '${IDM_ID}'; - 1504 - 288 + 1632 + 160 @@ -187,8 +177,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 336 - 464 + 288 + 320 @@ -204,8 +194,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 752 - 288 + 1632 + 320 @@ -240,15 +230,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1504 - 464 + 1024 + 160 Identify last row in a stream DetectLastRow - N + Y 1 @@ -259,111 +249,107 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 752 - 464 + 320 - has_health_state_update - Update - - Y - - 1 - - none - - - 10000 - ervu-dashboard - N - - - = - recruit_id - recruit_id - - ervu_dashboard - citizen
- - has_health_state_info - has_health_state_info - -
- N - Y - - - 1264 - 464 - -
- - health_state_input + Table input TableInput - Y + N 1 none - ervu-dashboard + postgres.subpoena N - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved' as source_update_date, - ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov' as zdorov_arr - FROM recruits_info ri - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}') -SELECT fd.recruit_id, - zdorov_elem ->> 'id' AS source_id, - source_update_date, - zdorov_elem ->> 'kodZabol' AS disease_code, - to_date(zdorov_elem ->> 'dataDiagn', 'YYYY-MM-DD') AS diagnosis_date -FROM filteredData fd - CROSS JOIN LATERAL jsonb_array_elements(zdorov_arr) AS zdorov_elem; + SELECT +* +FROM public.restriction_document +WHERE + '${IDM_ID}' != '' -- Проверка на пустую строку + AND vk_id = '${IDM_ID}' + AND '${M_R_CR_DATE}' >= created_at +${LIMIT_FW} Y - 560 - 464 + 496 + 320 - health_state_output + Table output TableOutput - Y + N 1 none - 10000 + 1000 ervu-dashboard - recruit_id - recruit_id + id + id - source_id - source_id + subpoena_id + subpoena_id - source_update_date - source_update_date + created_at + created_at - disease_code - disease_code + updated_at + updated_at - diagnosis_date - diagnosis_date + vk_id + vk_id + + + user_id + user_id + + + user_name + user_name + + + status + status + + + decision_number + decision_number + + + decision_date + decision_date + + + decision_reason + decision_reason + + + extra_info + extra_info + + + type + type + + + recruitment_name + recruitment_name N @@ -374,38 +360,26 @@ FROM filteredData fd N ervu_dashboard Y - health_state
+ restriction_document
N Y N Y - 1040 - 464 + 1024 + 320
- has_health_state_update - Abort - Y - - error_description - - - - - - - - health_state_output + Table output Change job status on error Y error_description - + error_code diff --git a/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_delta.hpl b/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_delta.hpl new file mode 100644 index 0000000..0c9ab5d --- /dev/null +++ b/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_delta.hpl @@ -0,0 +1,408 @@ + + + + restriction_document_flow_delta + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/08/11 13:22:56.048 + - + 2025/08/11 13:22:56.048 + + + + + + Create job execution record + Table input + Y + + + Table input + Identify last row in a stream + Y + + + Detect empty stream + Change job status on success + Y + + + Filter rows + Change job status on success + Y + + + Change job status on error + Abort + Y + + + Identify last row in a stream + Insert / update + Y + + + Insert / update + Filter rows + Y + + + Insert / update + Detect empty stream + Y + + + Insert / update + Change job status on error + Y + + + + Abort + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 1168 + 400 + + + + Change job status on error + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'DELTA_ERROR', + error_description = ? +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 944 + 400 + + + + Change job status on success + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'DELTA_SUCCESS' +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1248 + 64 + + + + Create job execution record + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + N + N + Y + N + N + UPDATE etl.job_execution +SET + status = 'DELTA_PROCESSING', + execution_datetime = DEFAULT, + error_description = NULL +where job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + 336 + 224 + + + + Detect empty stream + DetectEmptyStream + + Y + + 1 + + none + + + + + 1248 + 224 + + + + Filter rows + FilterRows + + Y + + 1 + + none + + + + + + + = + last_row + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + Change job status on success + + + 944 + 64 + + + + Identify last row in a stream + DetectLastRow + + Y + + 1 + + none + + + last_row + + + 736 + 224 + + + + Insert / update + InsertUpdate + + N + + 1 + + none + + + 100 + ervu-dashboard + + + = + id + id + + + + = + created_at + created_at + + + ervu_dashboard + restriction_document
+ + id + id + N + + + subpoena_id + subpoena_id + Y + + + created_at + created_at + N + + + updated_at + updated_at + Y + + + vk_id + vk_id + Y + + + user_id + user_id + Y + + + user_name + user_name + Y + + + status + status + Y + + + decision_number + decision_number + Y + + + decision_date + decision_date + Y + + + decision_reason + decision_reason + Y + + + extra_info + extra_info + Y + + + type + type + Y + + + recruitment_name + recruitment_name + Y + +
+ N + + + 944 + 224 + +
+ + Table input + TableInput + + N + + 1 + + none + + + postgres.subpoena + N + SELECT +* +FROM public.restriction_document +WHERE + '${IDM_ID}' != '' -- Проверка на пустую строку + AND vk_id = '${IDM_ID}' + AND updated_at >= '${M_R_UP_DATE}'::timestamp + +${LIMIT_FW} + Y + + + 544 + 224 + + + + + Insert / update + Change job status on error + Y + + + + + + + + + + +
diff --git a/mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow.hpl b/mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow.hpl similarity index 74% rename from mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow.hpl rename to mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow.hpl index b1fa401..cddb2f7 100644 --- a/mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow.hpl +++ b/mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow.hpl @@ -1,7 +1,7 @@ - vich_flow + subpoena_appearance_flow Y @@ -13,21 +13,16 @@ 1000 100 - - 2025/09/08 19:24:20.455 + 2025/08/05 14:31:22.799 - - 2025/09/08 19:24:20.455 + 2025/08/05 14:31:22.799 - - Change job status on error - Abort - Y - Create job execution record - vich_input + Table input Y @@ -36,33 +31,38 @@ Y - Identify last row in a stream - Detect empty stream + Detect empty stream + Change job status on success Y - Identify last row in a stream - vich_output + Change job status on error + Abort Y - vich_output - Change job status on error - Y - - - vich_output - Filter rows - Y - - - vich_input + Table input Identify last row in a stream Y - Detect empty stream - Change job status on success + Identify last row in a stream + Table output + Y + + + Table output + Filter rows + Y + + + Table output + Change job status on error + Y + + + Table output + Detect empty stream Y @@ -77,13 +77,13 @@ none - ABORT + ABORT_WITH_ERROR Y 0 - 1216 - 592 + 1376 + 512 @@ -117,8 +117,8 @@ and recruitment_id = '${IDM_ID}'; - 992 - 592 + 1024 + 512 @@ -148,8 +148,8 @@ and recruitment_id = '${IDM_ID}'; - 1216 - 240 + 1632 + 160 @@ -178,7 +178,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 288 - 416 + 320 @@ -194,8 +194,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 704 - 240 + 1632 + 320 @@ -230,15 +230,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1216 - 416 + 1024 + 160 Identify last row in a stream DetectLastRow - N + Y 1 @@ -248,78 +248,80 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); last_row - 704 - 416 + 752 + 320 - vich_input + Table input TableInput - Y + N 1 none - ervu-dashboard + postgres.subpoena N - WITH filteredData AS (SELECT ri.recruit_id, - to_date(ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' ->> 'dataSved', - 'YYYY-MM-DD') as source_update_date, - ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' -> 'svedUchet' as vich_arr - FROM recruits_info ri - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' -> 'svedUchet') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}') -SELECT fd.recruit_id, - vich_elem ->> 'id' AS source_id, - fd.source_update_date, - to_date(vich_elem ->> 'dataSnyat', 'YYYY-MM-DD') AS register_date, - to_date(vich_elem ->> 'dataPostUchet', 'YYYY-MM-DD') AS deregistration_date -FROM filteredData fd - CROSS JOIN LATERAL jsonb_array_elements(vich_arr) AS vich_elem; + select sa.* +from public.subpoena_appearance sa +join public.subpoena s on s.id = sa.subpoena_id +WHERE + '${IDM_ID}' != '' -- Проверка на пустую строку + AND s.department_id = '${IDM_ID}' + AND '${M_R_CR_DATE}' >= sa.created_date_time +${LIMIT_FW} Y - 512 - 416 + 496 + 320 - vich_output + Table output TableOutput - Y + N 1 none - 10000 + 1000 ervu-dashboard - recruit_id - recruit_id + id + id - source_id - source_id + subpoena_id + subpoena_id - source_update_date - source_update_date + fact_appearance + fact_appearance - register_date - register_date + date_appearance + date_appearance - deregistration_date - deregistration_date + nonappearance_info + nonappearance_info + + + created_date_time + created_date_time + + + change_date_time + change_date_time N @@ -330,26 +332,26 @@ FROM filteredData fd N ervu_dashboard Y - incapacity
+ subpoena_appearance
N Y N Y - 992 - 416 + 1024 + 320
- vich_output + Table output Change job status on error Y error_description - + error_code diff --git a/mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_delta.hpl b/mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow_delta.hpl similarity index 72% rename from mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_delta.hpl rename to mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow_delta.hpl index 7decaf1..d772d1a 100644 --- a/mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_delta.hpl +++ b/mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow_delta.hpl @@ -1,7 +1,7 @@ - vich_flow_delta + subpoena_appearance_flow_delta Y @@ -13,21 +13,26 @@ 1000 100 - - 2025/09/08 20:36:38.686 + 2025/08/11 13:22:56.048 - - 2025/09/08 20:36:38.686 + 2025/08/11 13:22:56.048 - Change job status on error - Abort + Create job execution record + Table input Y - Create job execution record - vich_input + Table input + Identify last row in a stream + Y + + + Detect empty stream + Change job status on success Y @@ -36,33 +41,28 @@ Y - Identify last row in a stream - Detect empty stream + Change job status on error + Abort Y Identify last row in a stream - vich_upsert + Insert / update Y - vich_input - Identify last row in a stream - Y - - - vich_upsert - Change job status on error - Y - - - vich_upsert + Insert / update Filter rows Y - Detect empty stream - Change job status on success + Insert / update + Detect empty stream + Y + + + Insert / update + Change job status on error Y @@ -77,13 +77,13 @@ none - ABORT + ABORT_WITH_ERROR Y 0 - 1744 - 560 + 1168 + 400 @@ -117,8 +117,8 @@ and recruitment_id = '${IDM_ID}'; - 1520 - 560 + 944 + 400 @@ -148,8 +148,8 @@ and recruitment_id = '${IDM_ID}'; - 1744 - 208 + 1248 + 64 @@ -174,14 +174,14 @@ and recruitment_id = '${IDM_ID}'; UPDATE etl.job_execution SET status = 'DELTA_PROCESSING', - execution_datetime = current_timestamp, + execution_datetime = DEFAULT, error_description = NULL where job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - 816 - 384 + 336 + 224 @@ -197,8 +197,8 @@ and recruitment_id = '${IDM_ID}'; - 1232 - 208 + 1248 + 224 @@ -233,15 +233,15 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1744 - 384 + 944 + 64 Identify last row in a stream DetectLastRow - N + Y 1 @@ -251,108 +251,116 @@ and recruitment_id = '${IDM_ID}'; last_row - 1232 - 384 + 736 + 224 - vich_input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - - WITH filteredData AS (SELECT ri.recruit_id, - to_date(ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' ->> 'dataSved', - 'YYYY-MM-DD') as source_update_date, - ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' -> 'svedUchet' as vich_arr - FROM recruits_info ri - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' -> 'svedUchet') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - and to_date(ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' - ) -SELECT fd.recruit_id, - vich_elem ->> 'id' AS source_id, - fd.source_update_date, - to_date(vich_elem ->> 'dataSnyat', 'YYYY-MM-DD') AS register_date, - to_date(vich_elem ->> 'dataPostUchet', 'YYYY-MM-DD') AS deregistration_date -FROM filteredData fd - CROSS JOIN LATERAL jsonb_array_elements(vich_arr) AS vich_elem; - Y - - - 1040 - 384 - - - - vich_upsert + Insert / update InsertUpdate - Y + N 1 none - 10000 + 100 ervu-dashboard = - source_id - source_id + id + id + + + = + created_date_time + created_date_time ervu_dashboard - disability
+ subpoena_appearance
- recruit_id - recruit_id + id + id N - source_id - source_id + created_date_time + created_date_time N - source_update_date - source_update_date + subpoena_id + subpoena_id Y - register_date - register_date + fact_appearance + fact_appearance Y - deregistration_date - deregistration_date + date_appearance + date_appearance + Y + + + nonappearance_info + nonappearance_info + Y + + + change_date_time + change_date_time Y
N - 1520 - 384 + 944 + 224 + +
+ + Table input + TableInput + + N + + 1 + + none + + + postgres.subpoena + N + + select sa.* +from public.subpoena_appearance sa +join public.subpoena s on s.id = sa.subpoena_id +WHERE + '${IDM_ID}' != '' -- Проверка на пустую строку + AND s.department_id = '${IDM_ID}' + AND sa.change_date_time >= '${M_R_UP_DATE}'::timestamp + +${LIMIT_FW} + Y + + + 544 + 224 - vich_upsert + Insert / update Change job status on error Y - error_description + diff --git a/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_repeat.hpl b/mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow_repeat.hpl similarity index 71% rename from mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_repeat.hpl rename to mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow_repeat.hpl index 3d10789..2e0ee69 100644 --- a/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_repeat.hpl +++ b/mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow_repeat.hpl @@ -1,7 +1,7 @@ - incapacity_flow_repeat + subpoena_appearance_flow_repeat Y @@ -13,21 +13,21 @@ 1000 100 - - 2025/09/08 20:32:10.265 + 2025/08/08 13:25:52.319 - - 2025/09/08 20:32:10.265 + 2025/08/08 13:25:52.319 - Change job status on error - Abort + Create job execution record + Table input Y - Create job execution record - incapacity_input + Table input + Identify last row in a stream Y @@ -36,33 +36,33 @@ Y - Identify last row in a stream - Detect empty stream + Detect empty stream + Change job status on success Y - incapacity_input - Identify last row in a stream + Change job status on error + Abort Y Identify last row in a stream - incapacity_upsert + Insert / update Y - incapacity_upsert + Insert / update Filter rows Y - incapacity_upsert - Change job status on error + Insert / update + Detect empty stream Y - Detect empty stream - Change job status on success + Insert / update + Change job status on error Y @@ -77,13 +77,13 @@ none - ABORT + ABORT_WITH_ERROR Y 0 - 1344 - 640 + 1248 + 400 @@ -117,8 +117,8 @@ and recruitment_id = '${IDM_ID}'; - 1120 - 640 + 960 + 400 @@ -148,8 +148,8 @@ and recruitment_id = '${IDM_ID}'; - 1344 - 256 + 1440 + 48 @@ -171,17 +171,32 @@ and recruitment_id = '${IDM_ID}'; Y N N - UPDATE etl.job_execution -SET + INSERT INTO etl.job_execution ( + id, + job_name, + status, + execution_datetime, + error_description, + recruitment_id +) +VALUES ( + DEFAULT, + '${JOB_NAME}', + 'PROCESSING', + DEFAULT, + NULL, + '${IDM_ID}' +) +ON CONFLICT (job_name, recruitment_id) +DO UPDATE SET status = 'PROCESSING', - execution_datetime = current_timestamp, + execution_datetime = DEFAULT, error_description = NULL -where job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; + - 416 - 432 + 272 + 208 @@ -197,8 +212,8 @@ and recruitment_id = '${IDM_ID}'; - 832 - 256 + 1440 + 208 @@ -233,15 +248,15 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1344 - 432 + 960 + 48 Identify last row in a stream DetectLastRow - N + Y 1 @@ -251,118 +266,115 @@ and recruitment_id = '${IDM_ID}'; last_row - 832 - 432 + 720 + 208 - incapacity_input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - - WITH filteredData AS ( - SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob' as nedeesposob_arr - FROM recruits_info ri - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob') = 'array' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' -) -SELECT fd.recruit_id, - nedeesposob_elem ->> 'id' AS source_id, - to_date(nedeesposob_elem ->> 'dataSved', 'YYYY-MM-DD') AS source_update_date, - nedeesposob_elem ->> 'naimStatus' AS capacity_status, - nedeesposob_elem ->> 'naimOrg' AS court_name, - to_date(nedeesposob_elem ->> 'dataPrisv', 'YYYY-MM-DD') AS register_date, - to_date(nedeesposob_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS deregistration_date -FROM filteredData fd - CROSS JOIN LATERAL jsonb_array_elements(fd.nedeesposob_arr -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob') AS nedeesposob_elem; - Y - - - 640 - 432 - - - - incapacity_upsert + Insert / update InsertUpdate - Y + N 1 none - 10000 + 100 ervu-dashboard = - source_id - source_id + id + id + + + = + created_date_time + created_date_time ervu_dashboard - disability
+ subpoena_appearance
- recruit_id - recruit_id + id + id N - source_id - source_id + created_date_time + created_date_time N - source_update_date - source_update_date + subpoena_id + subpoena_id Y - capacity_status - capacity_status + fact_appearance + fact_appearance Y - court_name - court_name + date_appearance + date_appearance Y - register_date - register_date + nonappearance_info + nonappearance_info Y - deregistration_date - deregistration_date + change_date_time + change_date_time Y
N - 1120 - 432 + 960 + 208 + +
+ + Table input + TableInput + + N + + 1 + + none + + + postgres.subpoena + N + + select sa.* +from public.subpoena_appearance sa +join public.subpoena s on s.id = sa.subpoena_id +WHERE + '${IDM_ID}' != '' -- Проверка на пустую строку + AND s.department_id = '${IDM_ID}' + AND '${M_R_CR_DATE}' >= sa.created_date_time +${LIMIT_FW} + Y + + + 480 + 208 - incapacity_upsert + Insert / update Change job status on error Y - error_description + diff --git a/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_repeat.hpl b/mappings/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow_delta.hpl similarity index 68% rename from mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_repeat.hpl rename to mappings/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow_delta.hpl index 73d920b..2bc3a65 100644 --- a/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_repeat.hpl +++ b/mappings/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow_delta.hpl @@ -1,7 +1,7 @@ - health_state_flow_repeat + subpoena_send_info_flow_delta Y @@ -13,21 +13,26 @@ 1000 100 - - 2025/09/08 21:48:46.051 + 2025/08/11 13:22:56.048 - - 2025/09/08 21:48:46.051 + 2025/08/11 13:22:56.048 - Change job status on error - Abort + Create job execution record + Table input Y - Create job execution record - health_state_input + Table input + Identify last row in a stream + Y + + + Detect empty stream + Change job status on success Y @@ -36,33 +41,28 @@ Y - Identify last row in a stream - Detect empty stream - Y - - - health_state_input - Identify last row in a stream + Change job status on error + Abort Y Identify last row in a stream - health_state_upsert + Insert / update Y - health_state_upsert + Insert / update Filter rows Y - health_state_upsert - Change job status on error + Insert / update + Detect empty stream Y - Detect empty stream - Change job status on success + Insert / update + Change job status on error Y @@ -77,13 +77,13 @@ none - ABORT + ABORT_WITH_ERROR Y 0 1168 - 640 + 400 @@ -109,7 +109,7 @@ Y N UPDATE etl.job_execution -SET status = 'ERROR', +SET status = 'DELTA_ERROR', error_description = ? WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; @@ -118,7 +118,7 @@ and recruitment_id = '${IDM_ID}'; 944 - 640 + 400 @@ -141,15 +141,15 @@ and recruitment_id = '${IDM_ID}'; N N UPDATE etl.job_execution -SET status = 'SUCCESS' +SET status = 'DELTA_SUCCESS' WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - 1168 - 272 + 1248 + 64 @@ -173,15 +173,15 @@ and recruitment_id = '${IDM_ID}'; N UPDATE etl.job_execution SET - status = 'PROCESSING', - execution_datetime = current_timestamp, + status = 'DELTA_PROCESSING', + execution_datetime = DEFAULT, error_description = NULL where job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - 240 - 448 + 336 + 224 @@ -197,8 +197,8 @@ and recruitment_id = '${IDM_ID}'; - 656 - 272 + 1248 + 224 @@ -233,15 +233,15 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1168 - 448 + 944 + 64 Identify last row in a stream DetectLastRow - N + Y 1 @@ -251,89 +251,99 @@ and recruitment_id = '${IDM_ID}'; last_row - 656 - 448 + 736 + 224 - health_state_input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved' as source_update_date, - ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov' as zdorov_arr - FROM recruits_info ri - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov') = - 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}') -SELECT fd.recruit_id, - zdorov_elem ->> 'id' AS source_id, - source_update_date, - zdorov_elem ->> 'kodZabol' AS disease_code, - to_date(zdorov_elem ->> 'dataDiagn', 'YYYY-MM-DD') AS diagnosis_date -FROM filteredData fd - CROSS JOIN LATERAL jsonb_array_elements(zdorov_arr) AS zdorov_elem; - Y - - - 464 - 448 - - - - health_state_upsert + Insert / update InsertUpdate - Y + N 1 none - 10000 + 100 ervu-dashboard = - source_id - source_id + id + id ervu_dashboard - health_state
+ subpoena_send_info
- recruit_id - recruit_id + id + id + N + + + subpoena_id + subpoena_id Y - source_id - source_id + send_code + send_code Y - source_update_date - source_update_date + send_address + send_address Y - disease_code - disease_code + send_address_type + send_address_type Y - diagnosis_date - diagnosis_date + send_date + send_date + Y + + + track_number + track_number + Y + + + is_delivered + is_delivered + Y + + + delivery_code + delivery_code + Y + + + delivery_date + delivery_date + Y + + + act_number + act_number + Y + + + act_date + act_date + Y + + + delivery_fio + delivery_fio + Y + + + auto_delivery + auto_delivery Y
@@ -341,7 +351,6 @@ FROM filteredData fd 944 -<<<<<<< HEAD:mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow_delta.hpl 224
@@ -359,32 +368,28 @@ FROM filteredData fd postgres.subpoena N - select sa.* -from public.subpoena_appearance sa -join public.subpoena s on s.id = sa.subpoena_id + select ssi.* +from public.subpoena_send_info ssi +join public.subpoena s on s.id = ssi.subpoena_id WHERE '${IDM_ID}' != '' -- Проверка на пустую строку AND s.department_id = '${IDM_ID}' - AND sa.change_date_time >= '${M_SA_UP_DATE}'::timestamp - + AND s.status_change_date >= '${M_R_UP_DATE}'::timestamp ${LIMIT_FW} Y 544 224 -======= - 448 ->>>>>>> 10bcf8dc200fb1b4ce7b6f5b82d5b8cd20b5b935:mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_repeat.hpl - health_state_upsert + Insert / update Change job status on error Y - error_description + diff --git a/mappings/info_recruits/raw_data/subpoena_send_info/recruitment_five_flow_delta.hpl b/mappings/info_recruits/raw_data/subpoena_send_info/recruitment_five_flow_delta.hpl new file mode 100644 index 0000000..befe783 --- /dev/null +++ b/mappings/info_recruits/raw_data/subpoena_send_info/recruitment_five_flow_delta.hpl @@ -0,0 +1,360 @@ + + + + recruitment_five_flow_delta + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/08/11 13:16:11.204 + - + 2025/08/11 13:16:11.204 + + + + + + Table input + subpoena_send_info_flow_delta.hpl + Y + + + Table input + subpoena_send_info_flow_delta.hpl 2 + Y + + + Table input + subpoena_send_info_flow_delta.hpl 3 + Y + + + Table input + subpoena_send_info_flow_delta.hpl 4 + Y + + + Table input + subpoena_send_info_flow_delta.hpl 5 + Y + + + + Table input + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT + r.idm_id as recruitment_id +FROM ervu_dashboard.recruitment r + LEFT JOIN etl.job_execution je + ON r.idm_id = je.recruitment_id + and job_name = 'subpoena_send_info_job' +where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); + N + + + 352 + 288 + + + + subpoena_send_info_flow_delta.hpl + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + subpoena_send_info_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 624 + 128 + + + + subpoena_send_info_flow_delta.hpl 2 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + subpoena_send_info_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 624 + 208 + + + + subpoena_send_info_flow_delta.hpl 3 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + subpoena_send_info_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 624 + 288 + + + + subpoena_send_info_flow_delta.hpl 4 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + subpoena_send_info_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 624 + 368 + + + + subpoena_send_info_flow_delta.hpl 5 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + subpoena_send_info_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 624 + 448 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/medicine/health_state/constants/health_state_constants_flow_repeat.hpl b/mappings/info_recruits/reference_data/recruit_archivation_reason.hpl similarity index 55% rename from mappings/info_recruits/citizen_tables/medicine/health_state/constants/health_state_constants_flow_repeat.hpl rename to mappings/info_recruits/reference_data/recruit_archivation_reason.hpl index 0eb4d87..2bc6733 100644 --- a/mappings/info_recruits/citizen_tables/medicine/health_state/constants/health_state_constants_flow_repeat.hpl +++ b/mappings/info_recruits/reference_data/recruit_archivation_reason.hpl @@ -1,7 +1,7 @@ - health_state_constants_flow_repeat + recruit_archivation_reason Y @@ -13,81 +13,21 @@ 1000 100 - - 2025/09/22 17:34:05.067 + 2025/09/09 16:35:56.339 - - 2025/09/22 17:34:05.067 + 2025/09/09 16:35:56.339 - get_max_disease_update_date - disease_input - Y - - - disease_input - minobr_disease_upsert + Table input + Insert / update Y - disease_input - TableInput - - Y - - 1 - - none - - - nsi - N - 0 - get_max_disease_update_date - SELECT - key as code, - value ->> 'value' as value, - NOT hidden as actual, - updated_at as update_date -FROM classifier_records -WHERE code = 'diseaseCode' -AND updated_at > ?; - Y - - - 448 - 256 - - - - get_max_disease_update_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT - max(update_date) as max_update_date -FROM disease; - - N - - - 224 - 256 - - - - minobr_disease_upsert + Insert / update InsertUpdate Y @@ -97,43 +37,86 @@ FROM disease; none - 10000 + 100 ervu-dashboard = - code - code + id + id ervu_dashboard - disease
- - code - code - N - + recruit_archivation_reason
value value Y - actual - actual + code + code Y - update_date - update_date + created_at + created_at Y + + updated_at + updated_at + Y + + + description + description + Y + + + extra_info + extra_info + Y + + + hidden + hidden + Y + + + id + id + N +
N - 640 - 256 + 848 + 288 + +
+ + Table input + TableInput + + Y + + 1 + + none + + + ervu_person_archive + N + 0 + SELECT * +from public.recruit_archivation_reason + N + + + 480 + 288