From 4997754b03aeaf7e5447735cb6572cc5e4505d9d Mon Sep 17 00:00:00 2001 From: "r.gaztdinov" Date: Fri, 5 Sep 2025 13:29:39 +0300 Subject: [PATCH] border_crossing --- .../border_crossing/border_crossing_job.hwf | 360 +++++++++++++ .../check_if_need_to_repeat_job.hpl | 86 +++ .../check_if_border_crossing_job_exists.hpl | 113 ++++ .../parallel/border_crossing_flow.hpl | 478 +++++++++++++++++ .../parallel/border_crossing_flow_delta.hpl | 477 +++++++++++++++++ .../parallel/border_crossing_flow_repeat.hpl | 492 ++++++++++++++++++ .../recruitment_five_flow_delta.hpl | 360 +++++++++++++ .../recruitment_five_flow_on_error.hpl | 360 +++++++++++++ .../recruitments_five_flow.hpl | 373 +++++++++++++ .../property/parallel/property_flow_delta.hpl | 8 +- .../parallel/property_flow_repeat.hpl | 8 +- 11 files changed, 3109 insertions(+), 6 deletions(-) create mode 100644 mappings/info_recruits/citizen_tables/border_crossing/border_crossing_job.hwf create mode 100644 mappings/info_recruits/citizen_tables/border_crossing/check_if_need_to_repeat_job.hpl create mode 100644 mappings/info_recruits/citizen_tables/border_crossing/checkpoints/check_if_border_crossing_job_exists.hpl create mode 100644 mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow.hpl create mode 100644 mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl create mode 100644 mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_repeat.hpl create mode 100644 mappings/info_recruits/citizen_tables/border_crossing/recruitment_five_flow_delta.hpl create mode 100644 mappings/info_recruits/citizen_tables/border_crossing/recruitment_five_flow_on_error.hpl create mode 100644 mappings/info_recruits/citizen_tables/border_crossing/recruitments_five_flow.hpl diff --git a/mappings/info_recruits/citizen_tables/border_crossing/border_crossing_job.hwf b/mappings/info_recruits/citizen_tables/border_crossing/border_crossing_job.hwf new file mode 100644 index 0000000..110e498 --- /dev/null +++ b/mappings/info_recruits/citizen_tables/border_crossing/border_crossing_job.hwf @@ -0,0 +1,360 @@ + + + border_crossing_job + Y + + + + 0 + - + 2025/08/05 12:45:28.561 + - + 2025/08/05 12:45:28.561 + + + COMMENT + + + + + LIMIT_FW + ; + + + + M_R_CR_DATE + 3001-01-01 00:00:00 + + + + M_R_UP_DATE + 0001-01-01 00:00:00 + + + + + + Start + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 224 + 304 + + + + check_if_border_crossing_job_exists.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/checkpoints/check_if_border_crossing_job_exists.hpl + Basic + + Y + + N + local + N + N + Y + N + 416 + 304 + + + + border_crossing_job_exists_check + + SIMPLE_EVAL + + boolean + false + equal + equal + N + variable + JOB_EXECUTED_FLAG + N + 688 + 304 + + + + recruitments_five_flow.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/recruitments_five_flow.hpl + Basic + + Y + + N + local + N + N + Y + Y + 928 + 304 + + + + recruitment_five_flow_on_error.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/recruitment_five_flow_on_error.hpl + Basic + + Y + + N + local + N + N + Y + N + 928 + 608 + + + + check_if_need_to_repeat_job.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/check_if_need_to_repeat_job.hpl + Basic + + Y + + N + local + N + N + Y + N + 688 + 464 + + + + Simple evaluation + + SIMPLE_EVAL + + boolean + true + equal + equal + N + variable + NEED_TO_REPEAT_JOB + N + 688 + 608 + + + + recruitment_five_flow_delta.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/recruitment_five_flow_delta.hpl + + + Basic + + Y + + N + local + N + N + Y + N + 688 + 800 + + + + + + Start + check_if_border_crossing_job_exists.hpl + Y + Y + Y + + + check_if_border_crossing_job_exists.hpl + border_crossing_job_exists_check + Y + Y + N + + + border_crossing_job_exists_check + recruitments_five_flow.hpl + Y + Y + N + + + border_crossing_job_exists_check + check_if_need_to_repeat_job.hpl + Y + N + N + + + check_if_need_to_repeat_job.hpl + Simple evaluation + Y + Y + N + + + Simple evaluation + recruitment_five_flow_on_error.hpl + Y + Y + N + + + Simple evaluation + recruitment_five_flow_delta.hpl + Y + N + N + + + + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + -1 + 26 + 800 + 256 + Первичка + 65 + + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + -1 + 26 + 800 + 560 + Повторка + 64 + + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + -1 + 26 + 608 + 720 + Дельта + 49 + + + 251 + 232 + 201 + 90 + 58 + 14 + N + 90 + 58 + 14 + N + Segoe UI + 9 + 150 + 1200 + 80 + border_crossing_job + +M_R_UP_DATE + +M_R_CR_DATE + +0001-01-01 00:00:00 + +3001-01-01 00:00:00 + 117 + + + + diff --git a/mappings/info_recruits/citizen_tables/border_crossing/check_if_need_to_repeat_job.hpl b/mappings/info_recruits/citizen_tables/border_crossing/check_if_need_to_repeat_job.hpl new file mode 100644 index 0000000..612b37a --- /dev/null +++ b/mappings/info_recruits/citizen_tables/border_crossing/check_if_need_to_repeat_job.hpl @@ -0,0 +1,86 @@ + + + + check_if_need_to_repeat_job + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/08/08 12:27:51.630 + - + 2025/08/08 12:27:51.630 + + + + + + Table input + Set variables + Y + + + + Set variables + SetVariable + + Y + + 1 + + none + + + + + need_to_repeat_job + NEED_TO_REPEAT_JOB + ROOT_WORKFLOW + + + Y + + + 624 + 288 + + + + Table input + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT EXISTS (SELECT 1 +FROM ervu_dashboard.recruitment r + LEFT JOIN etl.job_execution je + ON r.idm_id = je.recruitment_id + and job_name = 'border_crossing_job' +where je.status is null or je.status in ('ERROR', 'PROCESSING') +limit 1 +) as need_to_repeat_job; + N + + + 336 + 288 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/border_crossing/checkpoints/check_if_border_crossing_job_exists.hpl b/mappings/info_recruits/citizen_tables/border_crossing/checkpoints/check_if_border_crossing_job_exists.hpl new file mode 100644 index 0000000..99219bf --- /dev/null +++ b/mappings/info_recruits/citizen_tables/border_crossing/checkpoints/check_if_border_crossing_job_exists.hpl @@ -0,0 +1,113 @@ + + + + check_if_border_crossing_job_exists + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/08/05 13:28:46.661 + - + 2025/08/05 13:28:46.661 + + + + + + Set variables + Write to log + N + + + Table input + Set variables + Y + + + + Set variables + SetVariable + + Y + + 1 + + none + + + + + job_executed_flag + JOB_EXECUTED_FLAG + PARENT_WORKFLOW + + + Y + + + 672 + 336 + + + + Table input + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT EXISTS ( + SELECT 1 + FROM etl.job_execution + WHERE job_name = 'border_crossing_job' + limit 1 + ) AS job_executed_flag; + N + + + 384 + 336 + + + + Write to log + WriteToLog + + Y + + 1 + + none + + + Y + + + N + 0 + Basic + ${STATUS} + + + 944 + 224 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow.hpl b/mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow.hpl new file mode 100644 index 0000000..be02496 --- /dev/null +++ b/mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow.hpl @@ -0,0 +1,478 @@ + + + + border_crossing_flow + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/08/05 14:31:22.799 + - + 2025/08/05 14:31:22.799 + + + + + + Create job execution record + Table input + Y + + + Filter rows + Change job status on success + Y + + + Detect empty stream + Change job status on success + Y + + + Change job status on error + Abort + Y + + + Table output + Change job status on error + Y + + + Group by + Identify last row in a stream + Y + + + Identify last row in a stream + User defined Java expression + Y + + + User defined Java expression + Update + Y + + + Table input + Table output + Y + + + Table output + Group by + Y + + + Update + Change job status on error + Y + + + Update + Detect empty stream + Y + + + Update + Filter rows + Y + + + + Abort + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 1328 + 496 + + + + Change job status on error + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ? +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1328 + 320 + + + + Change job status on success + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS' +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1856 + 176 + + + + Create job execution record + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + N + N + Y + N + N + INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id) +VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); + + + + + 288 + 320 + + + + Detect empty stream + DetectEmptyStream + + Y + + 1 + + none + + + + + 1536 + 176 + + + + Filter rows + FilterRows + + Y + + 1 + + none + + + + + + + = + last_row + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + Change job status on success + + + 1536 + 304 + + + + Group by + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + border_crossed_raw + is_border_crossed + MAX + + + N + + + recruit_id + + + N + grp + + + 688 + 176 + + + + Identify last row in a stream + DetectLastRow + + Y + + 1 + + none + + + last_row + + + 880 + 176 + + + + Table input + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + SELECT + recruit_id, + (string_agg(elem->>'dataVyezd', ', '))::date as date_crossing, + (string_agg(elem->>'dataVozvrashh', ', '))::date as return_date, + 1 as is_border_crossed +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND '${M_R_CR_DATE}'::timestamp >= ri.created_at, +LATERAL jsonb_array_elements(info->'extend'->'svedPeresechGran') as elem +WHERE jsonb_typeof(info->'extend'->'svedPeresechGran') = 'array' + AND (elem->>'dataVyezd' IS NOT NULL OR elem->>'dataVozvrashh' IS NOT NULL) +GROUP BY recruit_id; + Y + + + 496 + 320 + + + + Table output + TableOutput + + Y + + 1 + + none + + + 1000 + ervu-dashboard + + + recruit_id + recruit_id + + + date_crossing + date_crossing + + + return_date + return_date + + + N + N + N + N + Y + N + ervu_dashboard + Y + border_crossing
+ N + Y + N + Y + + + 688 + 320 + + + + Update + Update + + N + + 1 + + none + + + 100 + ervu-dashboard + Y + + + + = + recruit_id + recruit_id + + + ervu_dashboard + citizen
+ + border_crossed + border_crossed + + + N + Y + + + 1328 + 176 + + + + User defined Java expression + Janino + + Y + + 1 + + none + + + + border_crossed + border_crossed_raw == 1 + Boolean + -1 + -1 + + + + + 1104 + 176 + + + + + Table output + Change job status on error + Y + + error_description + + error_code + + + + + + Update + Change job status on error + Y + + + + + + + + + + + diff --git a/mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl b/mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl new file mode 100644 index 0000000..5c50844 --- /dev/null +++ b/mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl @@ -0,0 +1,477 @@ + + + + border_crossing_flow_delta + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/08/11 13:22:56.048 + - + 2025/08/11 13:22:56.048 + + + + + + Create job execution record + Table input + Y + + + Detect empty stream + Change job status on success + Y + + + Filter rows + Change job status on success + Y + + + Change job status on error + Abort + Y + + + Table input + Insert / update + Y + + + Group by + Identify last row in a stream + Y + + + Identify last row in a stream + User defined Java expression + Y + + + User defined Java expression + Update + Y + + + Insert / update + Group by + Y + + + Update + Detect empty stream + Y + + + Update + Filter rows + Y + + + Update + Change job status on error + Y + + + Insert / update + Change job status on error + Y + + + + Abort + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 1392 + 528 + + + + Change job status on error + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'DELTA_ERROR', + error_description = ? +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1392 + 352 + + + + Change job status on success + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'DELTA_SUCCESS' +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1872 + 208 + + + + Create job execution record + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + N + N + Y + N + N + UPDATE etl.job_execution +SET + status = 'DELTA_PROCESSING', + execution_datetime = DEFAULT, + error_description = NULL +where job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + 352 + 352 + + + + Detect empty stream + DetectEmptyStream + + Y + + 1 + + none + + + + + 1632 + 208 + + + + Filter rows + FilterRows + + Y + + 1 + + none + + + + + + + = + last_row + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + Change job status on success + + + 1632 + 320 + + + + Group by + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + border_crossed_raw + is_border_crossed + MAX + + + N + + + recruit_id + + + N + grp + + + 752 + 208 + + + + Identify last row in a stream + DetectLastRow + + Y + + 1 + + none + + + last_row + + + 944 + 208 + + + + Insert / update + InsertUpdate + + Y + + 1 + + none + + + 100 + ervu-dashboard + + + = + recruit_id + recruit_id + + ervu_dashboard + border_crossing
+ + recruit_id + recruit_id + N + + + date_crossing + date_crossing + Y + + + return_date + return_date + Y + + + N + + + 752 + 352 + + + + Table input + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + SELECT + recruit_id, + (string_agg(elem->>'dataVyezd', ', '))::date as date_crossing, + (string_agg(elem->>'dataVozvrashh', ', '))::date as return_date, + 1 as border_crossed +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp, +LATERAL jsonb_array_elements(info->'extend'->'svedPeresechGran') as elem +WHERE jsonb_typeof(info->'extend'->'svedPeresechGran') = 'array' + AND (elem->>'dataVyezd' IS NOT NULL OR elem->>'dataVozvrashh' IS NOT NULL) +GROUP BY recruit_id; + Y + + + 560 + 352 + + + + Update + Update + + N + + 1 + + none + + + 100 + ervu-dashboard + Y + + + = + recruit_id + recruit_id + + ervu_dashboard + citizen
+ + border_crossed + border_crossed + + + N + Y + + + 1392 + 208 + + + + User defined Java expression + Janino + + Y + + 1 + + none + + + + border_crossed + border_crossed_raw == 1 + Boolean + -1 + -1 + + + + + 1168 + 208 + + + + + Insert / update + Change job status on error + Y + + + + + + + + + + Update + Change job status on error + Y + + + + + + + + + + + diff --git a/mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_repeat.hpl new file mode 100644 index 0000000..2c67d23 --- /dev/null +++ b/mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_repeat.hpl @@ -0,0 +1,492 @@ + + + + border_crossing_flow_repeat + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/08/08 13:25:52.319 + - + 2025/08/08 13:25:52.319 + + + + + + Create job execution record + Table input + Y + + + Filter rows + Change job status on success + Y + + + Detect empty stream + Change job status on success + Y + + + Change job status on error + Abort + Y + + + Insert / update + Change job status on error + Y + + + Table input + Insert / update + Y + + + Group by + Identify last row in a stream + Y + + + Identify last row in a stream + User defined Java expression + Y + + + User defined Java expression + Update + Y + + + Insert / update + Group by + Y + + + Update + Detect empty stream + Y + + + Update + Filter rows + Y + + + Update + Change job status on error + Y + + + + Abort + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 1312 + 400 + + + + Change job status on error + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ? +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1312 + 208 + + + + Change job status on success + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS' +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1776 + 64 + + + + Create job execution record + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + N + N + Y + N + N + INSERT INTO etl.job_execution ( + id, + job_name, + status, + execution_datetime, + error_description, + recruitment_id +) +VALUES ( + DEFAULT, + '${JOB_NAME}', + 'PROCESSING', + DEFAULT, + NULL, + '${IDM_ID}' +) +ON CONFLICT (job_name, recruitment_id) +DO UPDATE SET + status = 'PROCESSING', + execution_datetime = DEFAULT, + error_description = NULL + + + + 272 + 208 + + + + Detect empty stream + DetectEmptyStream + + Y + + 1 + + none + + + + + 1568 + 64 + + + + Filter rows + FilterRows + + Y + + 1 + + none + + + + + + + = + last_row + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + Change job status on success + + + 1568 + 176 + + + + Insert / update + InsertUpdate + + Y + + 1 + + none + + + 100 + ervu-dashboard + + + = + recruit_id + recruit_id + + ervu_dashboard + border_crossing
+ + recruit_id + recruit_id + N + + + date_crossing + date_crossing + Y + + + return_date + return_date + Y + + + N + + + 672 + 208 + + + + Table input + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + SELECT + recruit_id, + (string_agg(elem->>'dataVyezd', ', '))::date as date_crossing, + (string_agg(elem->>'dataVozvrashh', ', '))::date as return_date, + 1 as is_border_crossed +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND '${M_R_CR_DATE}'::timestamp >= ri.created_at, +LATERAL jsonb_array_elements(info->'extend'->'svedPeresechGran') as elem +WHERE jsonb_typeof(info->'extend'->'svedPeresechGran') = 'array' + AND (elem->>'dataVyezd' IS NOT NULL OR elem->>'dataVozvrashh' IS NOT NULL) +GROUP BY recruit_id; + Y + + + 480 + 208 + + + + Group by + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + border_crossed_raw + is_border_crossed + MAX + + + N + + + recruit_id + + + N + grp + + + 672 + 64 + + + + Identify last row in a stream + DetectLastRow + + Y + + 1 + + none + + + last_row + + + 864 + 64 + + + + Update + Update + + N + + 1 + + none + + + 100 + ervu-dashboard + Y + + + = + recruit_id + recruit_id + + ervu_dashboard + citizen
+ + border_crossed + border_crossed + + + N + Y + + + 1312 + 64 + + + + User defined Java expression + Janino + + Y + + 1 + + none + + + + border_crossed + border_crossed_raw == 1 + Boolean + -1 + -1 + + + + + 1088 + 64 + + + + + Insert / update + Change job status on error + Y + + + + + + + + + + Update + Change job status on error + Y + + + + + + + + + + + diff --git a/mappings/info_recruits/citizen_tables/border_crossing/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/border_crossing/recruitment_five_flow_delta.hpl new file mode 100644 index 0000000..2fbab78 --- /dev/null +++ b/mappings/info_recruits/citizen_tables/border_crossing/recruitment_five_flow_delta.hpl @@ -0,0 +1,360 @@ + + + + recruitment_five_flow_delta + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/08/11 13:16:11.204 + - + 2025/08/11 13:16:11.204 + + + + + + Table input + border_crossing_flow_delta.hpl + Y + + + Table input + border_crossing_flow_delta.hpl 2 + Y + + + Table input + border_crossing_flow_delta.hpl 3 + Y + + + Table input + border_crossing_flow_delta.hpl 4 + Y + + + Table input + border_crossing_flow_delta.hpl 5 + Y + + + + Table input + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT + r.idm_id as recruitment_id +FROM ervu_dashboard.recruitment r + LEFT JOIN etl.job_execution je + ON r.idm_id = je.recruitment_id + and job_name = 'property_job' +where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); + N + + + 352 + 288 + + + + border_crossing_flow_delta.hpl + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + border_crossing_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 640 + 128 + + + + border_crossing_flow_delta.hpl 2 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + border_crossing_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 656 + 224 + + + + border_crossing_flow_delta.hpl 3 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + border_crossing_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 656 + 304 + + + + border_crossing_flow_delta.hpl 4 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + border_crossing_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 656 + 384 + + + + border_crossing_flow_delta.hpl 5 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + border_crossing_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 656 + 480 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/border_crossing/recruitment_five_flow_on_error.hpl b/mappings/info_recruits/citizen_tables/border_crossing/recruitment_five_flow_on_error.hpl new file mode 100644 index 0000000..3a232e2 --- /dev/null +++ b/mappings/info_recruits/citizen_tables/border_crossing/recruitment_five_flow_on_error.hpl @@ -0,0 +1,360 @@ + + + + recruitment_five_flow_on_error + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/08/05 12:54:50.126 + - + 2025/08/05 12:54:50.126 + + + + + + Table input + border_crossing_flow_repeat.hpl + Y + + + Table input + border_crossing_flow_repeat.hpl 2 + Y + + + Table input + border_crossing_flow_repeat.hpl 3 + Y + + + Table input + border_crossing_flow_repeat.hpl 4 + Y + + + Table input + border_crossing_flow_repeat.hpl 5 + Y + + + + Table input + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT + r.idm_id as recruitment_id +FROM ervu_dashboard.recruitment r + LEFT JOIN etl.job_execution je + ON r.idm_id = je.recruitment_id + and job_name = 'border_crossing_job' +where je.status is null or je.status in('ERROR', 'PROCESSING'); + N + + + 272 + 368 + + + + border_crossing_flow_repeat.hpl + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_repeat.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + border_crossing_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 544 + 208 + + + + border_crossing_flow_repeat.hpl 2 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_repeat.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + border_crossing_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 544 + 288 + + + + border_crossing_flow_repeat.hpl 3 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_repeat.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + border_crossing_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 544 + 368 + + + + border_crossing_flow_repeat.hpl 4 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_repeat.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + border_crossing_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 544 + 464 + + + + border_crossing_flow_repeat.hpl 5 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_repeat.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + border_crossing_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 560 + 544 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/border_crossing/recruitments_five_flow.hpl b/mappings/info_recruits/citizen_tables/border_crossing/recruitments_five_flow.hpl new file mode 100644 index 0000000..97f1461 --- /dev/null +++ b/mappings/info_recruits/citizen_tables/border_crossing/recruitments_five_flow.hpl @@ -0,0 +1,373 @@ + + + + recruitments_five_flow + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/04/18 09:48:01.970 + - + 2025/04/18 09:48:01.970 + + + + + + Get all recruitments ordered by created_date + border_crossing_flow.hpl + Y + + + Get all recruitments ordered by created_date + border_crossing_flow.hpl 2 + Y + + + Get all recruitments ordered by created_date + border_crossing_flow.hpl 3 + Y + + + Get all recruitments ordered by created_date + border_crossing_flow.hpl 4 + Y + + + Get all recruitments ordered by created_date + border_crossing_flow.hpl 5 + Y + + + + Get all recruitments ordered by created_date + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT +idm_id AS recruitment +FROM ervu_dashboard.recruitment +${COMMENT}/* +where idm_id in ('110686b2-4a01-4976-885b-d26a2980f53a', +'cff66da8-2be0-4839-bd8e-332b1a150031', +'34c8685e-f08a-4877-afb1-a0589800a21c', +'0e10678c-f272-41c3-94dd-c15830321f9a', +'9703f108-761a-467a-aad3-89617e5539ed', +'55eae374-9bb5-4295-9e3a-5fbcdaba870d', +'dad6c101-34be-45c3-8173-57190cb69dfb', +'fc10523f-af31-4f6b-b0e2-00799758f634', +'61807cc2-9913-4060-8802-0672877c28c8', +'eba7fd7d-e3da-49a7-8e65-8fc85e47190e', +'f59afe6d-c166-4196-a489-6d17f02fc05e', +'22c64b79-f8e4-41e7-a2f3-1aa9df424fd9', +'0fe2bdf0-afbb-402d-93da-67e511799afa', +'6139de3b-2378-496f-9fca-7df7dab90832', +'50ba73dd-1726-48f1-8b7c-7a77ff965299') +${COMMENT}*/ + Y + + + 432 + 304 + + + + border_crossing_flow.hpl + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow.hpl + N + + 1 + + + + + IDM_ID + recruitment + + + + JOB_NAME + + border_crossing_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 736 + 144 + + + + border_crossing_flow.hpl 2 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow.hpl + N + + 1 + + + + + IDM_ID + recruitment + + + + JOB_NAME + + border_crossing_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 736 + 240 + + + + border_crossing_flow.hpl 3 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow.hpl + N + + 1 + + + + + IDM_ID + recruitment + + + + JOB_NAME + + border_crossing_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 752 + 336 + + + + border_crossing_flow.hpl 4 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow.hpl + N + + 1 + + + + + IDM_ID + recruitment + + + + JOB_NAME + + border_crossing_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 768 + 416 + + + + border_crossing_flow.hpl 5 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow.hpl + N + + 1 + + + + + IDM_ID + recruitment + + + + JOB_NAME + + border_crossing_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 752 + 512 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/property/parallel/property_flow_delta.hpl b/mappings/info_recruits/citizen_tables/property/parallel/property_flow_delta.hpl index 2499f87..5e45d5b 100644 --- a/mappings/info_recruits/citizen_tables/property/parallel/property_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/property/parallel/property_flow_delta.hpl @@ -341,18 +341,21 @@ and recruitment_id = '${IDM_ID}'; = - id - id + recruit_id + recruit_id + = start_date start_date + = type_property type_property + ervu_dashboard property
@@ -437,7 +440,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH base as( SELECT diff --git a/mappings/info_recruits/citizen_tables/property/parallel/property_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/property/parallel/property_flow_repeat.hpl index 2aa2f33..46ef46c 100644 --- a/mappings/info_recruits/citizen_tables/property/parallel/property_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/property/parallel/property_flow_repeat.hpl @@ -356,18 +356,21 @@ DO UPDATE SET = - id - id + recruit_id + recruit_id + = start_date start_date + = type_property type_property + ervu_dashboard property
@@ -452,7 +455,6 @@ DO UPDATE SET ervu-dashboard N - WITH base as( SELECT