diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index b3e41e5..0000000 --- a/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -ARG RUNTIME_IMAGE=repo.micord.ru/tools/etl-base:0.0.1-sha1e61336f - -FROM $RUNTIME_IMAGE - -COPY mappings /opt/etl/config/projects/mappings diff --git a/db.env b/db.env deleted file mode 100644 index b0c9cb4..0000000 --- a/db.env +++ /dev/null @@ -1,41 +0,0 @@ -DB_DASHBOARD_HOST=10.10.31.119 -DB_DASHBOARD_PORT=5432 -DB_DASHBOARD_NAME=ervu-dashboard-recr -DB_DASHBOARD_USERNAME=ervu-dashboard -DB_DASHBOARD_PASSWORD=ervu-dashboard - -DB_APPEAL_HOST=10.10.31.118 -DB_APPEAL_PORT=5432 -DB_APPEAL_NAME=appeal-document-service -DB_APPEAL_USERNAME=ervu -DB_APPEAL_PASSWORD=ervu - -DB_DECISION_HOST=10.10.31.118 -DB_DECISION_PORT=5432 -DB_DECISION_NAME=ervu-decision-document-service -DB_DECISION_USERNAME=ervu -DB_DECISION_PASSWORD=ervu - -DB_PERSON_HOST=10.10.31.118 -DB_PERSON_PORT=5432 -DB_PERSON_NAME=ervu-person-registry -DB_PERSON_USERNAME=ervu -DB_PERSON_PASSWORD=ervu - -DB_SUBPOENA_HOST=10.10.31.118 -DB_SUBPOENA_PORT=5432 -DB_SUBPOENA_NAME=ervu-subpoena-registry -DB_SUBPOENA_USERNAME=ervu -DB_SUBPOENA_PASSWORD=ervu - -DB_JOURNAL_HOST=10.10.31.118 -DB_JOURNAL_PORT=5432 -DB_JOURNAL_NAME=journal_dev -DB_JOURNAL_USERNAME=ervu -DB_JOURNAL_PASSWORD=ervu - -DB_METRICS_HOST=10.10.31.119 -DB_METRICS_PORT=5432 -DB_METRICS_NAME=ervu-dashboard-copy -DB_METRICS_USERNAME=ervu_business_metrics -DB_METRICS_PASSWORD=ervu_business_metrics diff --git a/entrypoint.sh b/entrypoint.sh deleted file mode 100755 index ad5d0d8..0000000 --- a/entrypoint.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash - -# https://gist.github.com/athlan/b6f09977e2f5cf20840ef61ca3cda932 -printenv | sed 's/^\(.*\)$/export \1/g' > /root/.env.sh -chmod +x /root/.env.sh - -export TZ=Europe/Moscow - -#. /etc/profile -#. ~/.bash_profile -#. ~/.bashrc -cd /opt/pentaho-di/mappings_old - -for F in $(find . -type f); do - sed -i "s/1111/$DB_DASHBOARD_PORT/g" $F - sed -i "s/dashboard-dbhost/$DB_DASHBOARD_HOST/g" $F - sed -i "s/dashboard-dbname/$DB_DASHBOARD_NAME/g" $F - sed -i "s/dashboard-dbuser/$DB_DASHBOARD_USERNAME/g" $F - sed -i "s/Encrypted 2daf9dca008c89396af54aa72ce93bcc9/$DB_DASHBOARD_PASSWORD/g" $F - - sed -i "s/2222/$DB_APPEAL_PORT/g" $F - sed -i "s/appeal-dbhost/$DB_APPEAL_HOST/g" $F - sed -i "s/appeal-dbname/$DB_APPEAL_NAME/g" $F - sed -i "s/appeal-dbuser/$DB_APPEAL_USERNAME/g" $F - sed -i "s/Encrypted 2be98afa91ad79785a754aa72ce93bcc9/$DB_APPEAL_PASSWORD/g" $F - - sed -i "s/3333/$DB_DECISION_PORT/g" $F - sed -i "s/decision-dbhost/$DB_DECISION_HOST/g" $F - sed -i "s/decision-dbname/$DB_DECISION_NAME/g" $F - sed -i "s/decision-dbuser/$DB_DECISION_USERNAME/g" $F - sed -i "s/Encrypted 2befccaab03d49b8ba554aa72ce93bcc9/$DB_DECISION_PASSWORD/g" $F - - sed -i "s/4444/$DB_PERSON_PORT/g" $F - sed -i "s/person-dbhost/$DB_PERSON_HOST/g" $F - sed -i "s/person-dbname/$DB_PERSON_NAME/g" $F - sed -i "s/person-dbuser/$DB_PERSON_USERNAME/g" $F - sed -i "s/Encrypted 2be98afb80fd5818ba554aa72ce93bcc9/$DB_PERSON_PASSWORD/g" $F - - sed -i "s/5555/$DB_SUBPOENA_PORT/g" $F - sed -i "s/subpoena-dbhost/$DB_SUBPOENA_HOST/g" $F - sed -i "s/subpoena-dbname/$DB_SUBPOENA_NAME/g" $F - sed -i "s/subpoena-dbuser/$DB_SUBPOENA_USERNAME/g" $F - sed -i "s/Encrypted 2beebdaaa1ac8978aaa54aa72ce93bcc9/$DB_SUBPOENA_PASSWORD/g" $F - - sed -i "s/6666/$DB_JOURNAL_PORT/g" $F - sed -i "s/journal-dbhost/$DB_JOURNAL_HOST/g" $F - sed -i "s/journal-dbname/$DB_JOURNAL_NAME/g" $F - sed -i "s/journal-dbuser/$DB_JOURNAL_USERNAME/g" $F - sed -i "s/Encrypted 2be98afc86aa7f2e4cb79ce10db80b9cf/$DB_JOURNAL_PASSWORD/g" $F - - sed -i "s/7777/$DB_METRICS_PORT/g" $F - sed -i "s/metrics-dbhost/$DB_METRICS_HOST/g" $F - sed -i "s/metrics-dbname/$DB_METRICS_NAME/g" $F - sed -i "s/metrics-dbuser/$DB_METRICS_USERNAME/g" $F - sed -i "s/Encrypted 657276755ddceddca104c281979414ab64cc9bacc9/$DB_METRICS_PASSWORD/g" $F -done; - - #Запускаем job один раз при старте контейнера - echo "---- Run info_recruits/job_info_recruits.kjb ----" - bash -c ". /root/.env.sh; /opt/pentaho-di/data-integration/kitchen.sh /file:////opt/pentaho-di/mappings_old/info_recruits/job_info_recruits.kjb > /proc/1/fd/1 2>&1" - -if [ $RUNONCE == true ]; then - echo "---- Run job_general.kjb ----" - bash -c ". /root/.env.sh; /opt/pentaho-di/data-integration/kitchen.sh /file:////opt/pentaho-di/mappings_old/job_general.kjb > /proc/1/fd/1 2>&1" - echo "---- Run job_metr.kjb ----" - bash -c ". /root/.env.sh; /opt/pentaho-di/data-integration/kitchen.sh /file:////opt/pentaho-di/mappings_old/metrics/job_metr.kjb > /proc/1/fd/1 2>&1" - exit $? -else - cat /root/crontab | crontab - - cron -f -fi - diff --git a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/job_last_update_date_appeal_document.hwf b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/job_last_update_date_appeal_document.hwf deleted file mode 100644 index 3b23c49..0000000 --- a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/job_last_update_date_appeal_document.hwf +++ /dev/null @@ -1,142 +0,0 @@ - - - job_last_update_date_appeal_document - Y - - - - - - 2025/06/02 14:48:05.666 - - - 2025/06/02 14:48:05.666 - - - - - Start - - SPECIAL - - 1 - 12 - 60 - 0 - 0 - N - 0 - 1 - N - 592 - 352 - - - - recruit_update_date_ervu_dashboard.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/raw_data/recruits_info/recruit_update_date_ervu_dashboard.hpl - Basic - - Y - - N - local - N - N - Y - N - 784 - 352 - - - - row_last_recruit_update_date.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/raw_data/recruits_info/row_last_recruit_update_date.hpl - Basic - - Y - - N - local - N - N - Y - N - 1056 - 352 - - - - Success - - SUCCESS - - N - 1232 - 352 - - - - - - Start - recruit_update_date_ervu_dashboard.hpl - Y - Y - Y - - - recruit_update_date_ervu_dashboard.hpl - row_last_recruit_update_date.hpl - Y - Y - N - - - row_last_recruit_update_date.hpl - Success - Y - Y - N - - - - - 251 - 232 - 201 - 90 - 58 - 14 - N - 90 - 58 - 14 - N - Segoe UI - 9 - 26 - 581 - 546 - А где испольуезтся джоба? - 162 - - - - diff --git a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow.hpl index 0a4cd5f..2ca38f2 100644 --- a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow.hpl @@ -18,86 +18,76 @@ 2025/08/05 14:31:22.799 - - 251 - 232 - 201 - 90 - 58 - 14 - N - 90 - 58 - 14 - N - Segoe UI - 9 - 26 - 352 - 480 - в таблице нет апдейт даты - 154 - - Create job execution record - Table input + сreate_job_execution_record_CAV + table_input_CAV Y - Filter rows - Change job status on success + filter_last_row_CAV + change_job_status_on_success_CAV Y - Detect empty stream - Change job status on success + detect_empty_stream_CAV + change_job_status_on_success_CAV Y - Change job status on error - Abort + change_job_status_on_error_CAV + abort_CAV Y - Update - Change job status on error + update_CAV + change_job_status_on_error_CAV Y - Table output - Change job status on error + table_output_CAV + change_job_status_on_error_CAV Y - sort_by_recruit_id - Update + table_output_CAV + filter_last_row_CAV Y - Table output - Filter rows + table_input_CAV + identify_last_row_CAV Y - Table input - Detect empty stream + identify_last_row_CAV + detect_empty_stream_CAV Y - Table input - Table output + identify_last_row_CAV + table_output_CAV Y - Table input - sort_by_recruit_id + identify_last_row_CAV + filter_true_appeal_violations + Y + + + filter_true_appeal_violations + sort_rows_CAV + Y + + + sort_rows_CAV + update_CAV Y - Abort + abort_CAV Abort Y @@ -112,12 +102,12 @@ 0 - 1264 + 1488 832 - Change job status on error + change_job_status_on_error_CAV ExecSql Y @@ -148,12 +138,12 @@ and recruitment_id = '${IDM_ID}'; - 1088 + 1296 832 - Change job status on success + change_job_status_on_success_CAV ExecSql Y @@ -180,41 +170,12 @@ and recruitment_id = '${IDM_ID}'; - 1088 + 1296 480 - Create job execution record - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - N - N - Y - N - N - INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id) -VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - - - - - 432 - 656 - - - - Detect empty stream + detect_empty_stream_CAV DetectEmptyStream Y @@ -226,12 +187,12 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 640 + 736 480 - Filter rows + filter_last_row_CAV FilterRows Y @@ -259,16 +220,52 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Change job status on success + change_job_status_on_success_CAV - 1088 + 1296 656 - Table input - TableInput + filter_true_appeal_violations + FilterRows + + Y + + 1 + + none + + + + + + + = + has_appealing_violations_fz53 + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + sort_rows_CAV + + + 736 + 832 + + + + identify_last_row_CAV + DetectLastRow N @@ -277,43 +274,86 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); none - postgres.decision-document-service - N - - with filtered_data as ( SELECT - ad.recruit_id, - ad.create_date, -- дата создания обжалования - ad.appeal_number AS number_appealed, -- номер обжалуемого решения - ad.appeal_date AS date_appealed, -- дата обжалуемого решения - ad.result_number AS case_number, -- номер принятого решения - ad.result_date AS decision_date, -- дата принятия решения - CASE - WHEN ad.result = '1' THEN 'Удовлетворена' - WHEN ad.result = '2' THEN 'Частично удовлетворена' - WHEN ad.result = '3' THEN 'Отказано в удовлетворении' - END AS result_appeal, -- код рассмотрения - ad.commission_name AS name_commission, -- наименование комиссии, принявшей решение - ad.reason_for_add_entry AS grounds, -- основание для внесения записи - ad.hidden, - NOT ad.hidden AS has_appealing_violations_fz53, - ctid -FROM public.appeal_document_dto ad - JOIN public.recruit r ON r.id = ad.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку - AND (r.current_recruitment = '${IDM_ID}' or (r.current_recruitment is null and r.target_recruitment = '${IDM_ID}'))) -select - fd.*, - fd.ctid = (select max(ctid) from filtered_data) as last_row - from filtered_data fd; - - Y + last_row - 640 + 736 656 - Table output + sort_rows_CAV + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 912 + 832 + + + + table_input_CAV + TableInput + + Y + + 1 + + none + + + appeal_test + N + SELECT ad.recruit_id, + ad.updated_at, -- дата создания обжалования + ad.appeal_number AS appeal_number, -- номер обжалуемого решения + ad.appeal_date AS appeal_date, -- дата обжалуемого решения + ad.extra_info ->> 'resultNumber' AS decision_number, -- номер принятого решения + to_date(ad.extra_info ->> 'resultDate', 'YYYY-MM-DD') AS decision_date, -- дата принятия решения + CASE ad.extra_info ->> 'result' + WHEN '1' THEN 'Удовлетворена' + WHEN '2' THEN 'Частично удовлетворена' + WHEN '3' THEN 'Отказано в удовлетворении' + END AS decision_result, -- код рассмотрения + ad.extra_info ->> 'commissionName' AS commision_name, -- наименование комиссии, принявшей решение + ad.extra_info ->> 'reasonForAddEntry' AS reason, -- основание для внесения записи + ad.hidden, + NOT ad.hidden AS has_appealing_violations_fz53 +FROM public.appeal_document ad +--where ad.vk_id = '${IDM_ID}' + Y + + + 560 + 656 + + + + table_output_CAV TableOutput Y @@ -324,43 +364,43 @@ select 10000 - ervu-dashboard + ervu_dashboard_test recruit_id recruit_id - create_date - create_date + updated_at + updated_at - number_appealed - number_appealed + appeal_number + appeal_number - date_appealed - date_appealed + appeal_date + appeal_date - case_number - case_number + decision_number + decision_number decision_date decision_date - result_appeal - result_appeal + decision_result + decision_result - name_commission - name_commission + commision_name + commision_name - grounds - grounds + reason + reason hidden @@ -371,23 +411,26 @@ select N N N + Y + N ervu_dashboard Y citizen_appealing_violations_fz53
+ N Y N Y - 848 + 1056 656 - Update + update_CAV Update N @@ -417,13 +460,13 @@ select Y - 848 + 1056 832 - sort_by_recruit_id - SortRows + сreate_job_execution_record_CAV + ExecSql Y @@ -432,33 +475,28 @@ select none - ${java.io.tmpdir} - srt - 100000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - + + + ervu-dashboard + N + N + Y + N + N + INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id) +VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); + + - 640 - 832 + 352 + 656 - Table output - Change job status on error + table_output_CAV + change_job_status_on_error_CAV Y error_description @@ -469,8 +507,8 @@ select - Update - Change job status on error + update_CAV + change_job_status_on_error_CAV Y error_description diff --git a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow_delta.hpl index 9415dc6..719cdb7 100644 --- a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow_delta.hpl @@ -21,68 +21,73 @@ - Create job execution record - Table input + update_job_execution_record_CAV + table_input_CAV Y - Detect empty stream - Change job status on success + detect_empty_stream_CAV + change_job_status_on_success_CAV Y - Filter rows - Change job status on success + filter_last_row_CAV + change_job_status_on_success_CAV Y - Change job status on error - Abort + change_job_status_on_error_CAV + abort_CAV Y - Insert / update - Change job status on error + upsert_CAV + change_job_status_on_error_CAV Y - Update - Change job status on error + update_CAV + change_job_status_on_error_CAV Y - sort_by_recruit_id - Update + table_input_CAV + identify_last_row_CAV Y - Table input - Identify last row in a stream + identify_last_row_CAV + upsert_CAV Y - Identify last row in a stream - Insert / update + identify_last_row_CAV + detect_empty_stream_CAV Y - Identify last row in a stream - sort_by_recruit_id + upsert_CAV + filter_last_row_CAV Y - Identify last row in a stream - Detect empty stream + filter_true_appeal_violations + sort_rows_CAV Y - Insert / update - Filter rows + identify_last_row_CAV + filter_true_appeal_violations + Y + + + sort_rows_CAV + update_CAV Y - Abort + abort_CAV Abort Y @@ -98,11 +103,11 @@ 1472 - 400 + 384 - Change job status on error + change_job_status_on_error_CAV ExecSql Y @@ -133,12 +138,12 @@ and recruitment_id = '${IDM_ID}'; - 1200 - 400 + 1312 + 384 - Change job status on success + change_job_status_on_success_CAV ExecSql Y @@ -170,7 +175,224 @@ and recruitment_id = '${IDM_ID}'; - Create job execution record + detect_empty_stream_CAV + DetectEmptyStream + + Y + + 1 + + none + + + + + 736 + 80 + + + + filter_last_row_CAV + FilterRows + + Y + + 1 + + none + + + + + + + = + last_row + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + change_job_status_on_success_CAV + + + 1472 + 224 + + + + filter_true_appeal_violations + FilterRows + + Y + + 1 + + none + + + + + + + = + has_appealing_violations_fz53 + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + sort_rows_CAV + + + 736 + 384 + + + + identify_last_row_CAV + DetectLastRow + + N + + 1 + + none + + + last_row + + + 736 + 224 + + + + sort_rows_CAV + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 928 + 384 + + + + table_input_CAV + TableInput + + Y + + 1 + + none + + + appeal_test + N + SELECT ad.recruit_id, + ad.updated_at, -- дата создания обжалования + ad.appeal_number AS appeal_number, -- номер обжалуемого решения + ad.appeal_date AS appeal_date, -- дата обжалуемого решения + ad.extra_info ->> 'resultNumber' AS decision_number, -- номер принятого решения + to_date(ad.extra_info ->> 'resultDate', 'YYYY-MM-DD') AS decision_date, -- дата принятия решения + CASE ad.extra_info ->> 'result' + WHEN '1' THEN 'Удовлетворена' + WHEN '2' THEN 'Частично удовлетворена' + WHEN '3' THEN 'Отказано в удовлетворении' + END AS decision_result, -- код рассмотрения + ad.extra_info ->> 'commissionName' AS commission_name, -- наименование комиссии, принявшей решение + ad.extra_info ->> 'reasonForAddEntry' AS reason, -- основание для внесения записи + ad.hidden, + NOT ad.hidden AS has_appealing_violations_fz53 +FROM public.appeal_document ad +where ad.vk_id = '${IDM_ID}' +and case when '${MAX_UPDATE_DATE}' <> '' + then to_date(ri.info -> 'svedFL' -> 'svedDeputat' -> 'deputat' ->> 'dataSved', 'YYYY-MM-DD') > + to_date('${MAX_UPDATE_DATE}', 'YYYY-MM-DD') + else true end; + Y + + + 544 + 224 + + + + update_CAV + Update + + N + + 1 + + none + + + 10000 + ervu-dashboard + Y + + + = + recruit_id + recruit_id + + ervu_dashboard + citizen
+ + has_appealing_violations_fz53 + has_appealing_violations_fz53 + + + N + Y + + + 1088 + 384 + + + + update_job_execution_record_CAV ExecSql Y @@ -202,78 +424,7 @@ and recruitment_id = '${IDM_ID}'; - Detect empty stream - DetectEmptyStream - - Y - - 1 - - none - - - - - 736 - 80 - - - - Filter rows - FilterRows - - Y - - 1 - - none - - - - - - - = - last_row - N - - - - N - -1 - constant - -1 - Y - Boolean - - - - Change job status on success - - - 1472 - 224 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 736 - 224 - - - - Insert / update + upsert_CAV InsertUpdate Y @@ -284,7 +435,7 @@ and recruitment_id = '${IDM_ID}'; 10000 - ervu-dashboard + ervu_dashboard_test = @@ -293,8 +444,8 @@ and recruitment_id = '${IDM_ID}'; = - create_date - create_date + appeal_number + appeal_number ervu_dashboard citizen_appealing_violations_fz53
@@ -304,23 +455,23 @@ and recruitment_id = '${IDM_ID}'; N - create_date - create_date + updated_at + updated_at + Y + + + appeal_number + appeal_number N - number_appealed - number_appealed + appeal_date + appeal_date Y - date_appealed - date_appealed - Y - - - case_number - case_number + decision_number + decision_number Y @@ -329,18 +480,18 @@ and recruitment_id = '${IDM_ID}'; Y - result_appeal - result_appeal + decision_result + decision_result Y - name_commission - name_commission + commission_name + commission_name Y - grounds - grounds + reason + reason Y @@ -352,127 +503,14 @@ and recruitment_id = '${IDM_ID}'; N - 928 + 1088 224 - - Table input - TableInput - - Y - - 1 - - none - - - postgres.decision-document-service - N - SELECT - ad.recruit_id, - ad.create_date, -- дата создания обжалования - ad.appeal_number AS number_appealed, -- номер обжалуемого решения - ad.appeal_date AS date_appealed, -- дата обжалуемого решения - ad.result_number AS case_number, -- номер принятого решения - ad.result_date AS decision_date, -- дата принятия решения - CASE - WHEN ad.result = '1' THEN 'Удовлетворена' - WHEN ad.result = '2' THEN 'Частично удовлетворена' - WHEN ad.result = '3' THEN 'Отказано в удовлетворении' - ELSE NULL - END AS result_appeal, -- код рассмотрения - ad.commission_name AS name_commission, -- наименование комиссии, принявшей решение - ad.reason_for_add_entry AS grounds, -- основание для внесения записи - ad.hidden, - NOT ad.hidden AS has_appealing_violations_fz53 -FROM public.appeal_document_dto ad -JOIN public.recruit r ON r.id = ad.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку - AND (r.current_recruitment = '${IDM_ID}' or (r.current_recruitment is null and r.target_recruitment = '${IDM_ID}')) - AND ad.create_date >= '${MAX_CREATE_DATE}'::timestamp - - Y - - - 544 - 224 - - - - Update - Update - - N - - 1 - - none - - - 10000 - ervu-dashboard - Y - - - - = - recruit_id - recruit_id - - - ervu_dashboard - citizen
- - has_appealing_violations_fz53 - has_appealing_violations_fz53 - - - N - Y - - - 928 - 400 - - - - sort_by_recruit_id - SortRows - - Y - - 1 - - none - - - ${java.io.tmpdir} - srt - 100000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - - - - 736 - 400 - - - Insert / update - Change job status on error + update_CAV + change_job_status_on_error_CAV Y error_description @@ -483,8 +521,8 @@ JOIN public.recruit r ON r.id = ad.recruit_id AND '${IDM_ID}' != '' -- Пров - Update - Change job status on error + upsert_CAV + change_job_status_on_error_CAV Y error_description diff --git a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow_repeat.hpl index d46fd0e..5b27f4e 100644 --- a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow_repeat.hpl @@ -21,63 +21,73 @@ - Create job execution record - Table input + upsert_job_execution_record_CAV + table_input_CAV Y - Filter rows - Change job status on success + filter_last_row_CAV + change_job_status_on_success_CAV Y - Detect empty stream - Change job status on success + detect_empty_stream_CAV + change_job_status_on_success_CAV Y - Change job status on error - Abort + change_job_status_on_error_CAV + abort_CAV Y - Insert / update - Change job status on error + upsert_CAV + change_job_status_on_error_CAV Y - Update - Change job status on error + update_CAV + change_job_status_on_error_CAV Y - sort_by_recruit_id - Update + upsert_CAV + filter_last_row_CAV Y - Insert / update - Filter rows + table_input_CAV + identify_last_row_CAV Y - Table input - Detect empty stream + filter_true_appeal_violations + sort_rows_CAV Y - Table input - Insert / update + identify_last_row_CAV + filter_true_appeal_violations Y - Table input - sort_by_recruit_id + identify_last_row_CAV + detect_empty_stream_CAV + Y + + + identify_last_row_CAV + upsert_CAV + Y + + + sort_rows_CAV + update_CAV Y - Abort + abort_CAV Abort Y @@ -92,12 +102,12 @@ 0 - 1120 - 368 + 1440 + 352 - Change job status on error + change_job_status_on_error_CAV ExecSql Y @@ -128,12 +138,12 @@ and recruitment_id = '${IDM_ID}'; - 944 - 368 + 1248 + 352 - Change job status on success + change_job_status_on_success_CAV ExecSql Y @@ -160,59 +170,12 @@ and recruitment_id = '${IDM_ID}'; - 944 + 1248 64 - Create job execution record - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - N - N - Y - N - N - INSERT INTO etl.job_execution ( - id, - job_name, - status, - execution_datetime, - error_description, - recruitment_id -) -VALUES ( - DEFAULT, - '${JOB_NAME}', - 'PROCESSING', - DEFAULT, - NULL, - '${IDM_ID}' -) -ON CONFLICT (job_name, recruitment_id) -DO UPDATE SET - status = 'PROCESSING', - execution_datetime = DEFAULT, - error_description = NULL - - - - 272 - 208 - - - - Detect empty stream + detect_empty_stream_CAV DetectEmptyStream Y @@ -224,12 +187,12 @@ DO UPDATE SET - 480 + 688 64 - Filter rows + filter_last_row_CAV FilterRows Y @@ -257,16 +220,16 @@ DO UPDATE SET - Change job status on success + change_job_status_on_success_CAV - 944 + 1248 208 - Insert / update - InsertUpdate + filter_true_appeal_violations + FilterRows Y @@ -275,82 +238,34 @@ DO UPDATE SET none - 10000 - ervu-dashboard - - - = - recruit_id - recruit_id - - - = - create_date - create_date - - ervu_dashboard - citizen_appealing_violations_fz53
- - recruit_id - recruit_id - N - - - create_date - create_date - N - - - number_appealed - number_appealed - Y - - - date_appealed - date_appealed - Y - - - case_number - case_number - Y - - - decision_date - decision_date - Y - - - result_appeal - result_appeal - Y - - - name_commission - name_commission - Y - - - grounds - grounds - Y - - - hidden - hidden - Y - - - Y + + + + + = + has_appealing_violations_fz53 + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + sort_rows_CAV - 704 - 208 + 688 + 352 - Table input - TableInput + identify_last_row_CAV + DetectLastRow N @@ -359,34 +274,77 @@ DO UPDATE SET none - postgres.decision-document-service + last_row + + + 688 + 208 + + + + sort_rows_CAV + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 880 + 352 + + + + table_input_CAV + TableInput + + Y + + 1 + + none + + + appeal_test N - - with filtered_data as ( SELECT - ad.recruit_id, - ad.create_date, -- дата создания обжалования - ad.appeal_number AS number_appealed, -- номер обжалуемого решения - ad.appeal_date AS date_appealed, -- дата обжалуемого решения - ad.result_number AS case_number, -- номер принятого решения - ad.result_date AS decision_date, -- дата принятия решения - CASE - WHEN ad.result = '1' THEN 'Удовлетворена' - WHEN ad.result = '2' THEN 'Частично удовлетворена' - WHEN ad.result = '3' THEN 'Отказано в удовлетворении' - END AS result_appeal, -- код рассмотрения - ad.commission_name AS name_commission, -- наименование комиссии, принявшей решение - ad.reason_for_add_entry AS grounds, -- основание для внесения записи - ad.hidden, - NOT ad.hidden AS has_appealing_violations_fz53, - ctid -FROM public.appeal_document_dto ad - JOIN public.recruit r ON r.id = ad.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку - AND (r.current_recruitment = '${IDM_ID}' or (r.current_recruitment is null and r.target_recruitment = '${IDM_ID}'))) -select - fd.*, - fd.ctid = (select max(ctid) from filtered_data) as last_row - from filtered_data fd - + SELECT ad.recruit_id, + ad.updated_at, -- дата создания обжалования + ad.appeal_number AS appeal_number, -- номер обжалуемого решения + ad.appeal_date AS appeal_date, -- дата обжалуемого решения + ad.extra_info ->> 'resultNumber' AS decision_number, -- номер принятого решения + to_date(ad.extra_info ->> 'resultDate', 'YYYY-MM-DD') AS decision_date, -- дата принятия решения + CASE ad.extra_info ->> 'result' + WHEN '1' THEN 'Удовлетворена' + WHEN '2' THEN 'Частично удовлетворена' + WHEN '3' THEN 'Отказано в удовлетворении' + END AS decision_result, -- код рассмотрения + ad.extra_info ->> 'commissionName' AS commission_name, -- наименование комиссии, принявшей решение + ad.extra_info ->> 'reasonForAddEntry' AS reason, -- основание для внесения записи + ad.hidden, + NOT ad.hidden AS has_appealing_violations_fz53 +FROM public.appeal_document ad +--where ad.vk_id = '${IDM_ID}' Y @@ -395,7 +353,7 @@ select - Update + update_CAV Update N @@ -430,13 +388,13 @@ select Y - 704 - 368 + 1056 + 352 - sort_by_recruit_id - SortRows + upsert_CAV + InsertUpdate Y @@ -445,33 +403,134 @@ select none - ${java.io.tmpdir} - srt - 100000 - - N - - Y - - + 10000 + ervu-dashboard + + + = + recruit_id recruit_id - Y - N - N - 0 - N - - + + + = + appeal_number + appeal_number + + ervu_dashboard + citizen_appealing_violations_fz53
+ + recruit_id + recruit_id + N + + + updated_at + updated_at + Y + + + appeal_number + appeal_number + N + + + appeal_date + appeal_date + Y + + + decision_number + decision_number + Y + + + decision_date + decision_date + Y + + + decision_result + decision_result + Y + + + commission_name + commission_name + Y + + + reason + reason + Y + + + hidden + hidden + Y + + + Y - 480 - 368 + 1056 + 208 + + + + upsert_job_execution_record_CAV + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + + N + + N + + Y + N + N + INSERT INTO etl.job_execution ( + id, + job_name, + status, + execution_datetime, + error_description, + recruitment_id +) +VALUES ( + DEFAULT, + '${JOB_NAME}', + 'PROCESSING', + DEFAULT, + NULL, + '${IDM_ID}' +) +ON CONFLICT (job_name, recruitment_id) +DO UPDATE SET + status = 'PROCESSING', + execution_datetime = DEFAULT, + error_description = NULL + + + + + 272 + 208 - Insert / update - Change job status on error + update_CAV + change_job_status_on_error_CAV Y error_description @@ -482,8 +541,8 @@ select - Update - Change job status on error + upsert_CAV + change_job_status_on_error_CAV Y error_description diff --git a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/recruitment_five_flow_delta.hpl index bedc64f..f8c373b 100644 --- a/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_appealing_violations_fz53/recruitment_five_flow_delta.hpl @@ -18,62 +18,417 @@ 2025/08/11 13:16:11.204 - - 251 - 232 - 201 - 90 - 58 - 14 - N - 90 - 58 - 14 - N - Segoe UI - 9 - 42 - 48 - 112 - тащить только те военкоматы по которым появились новые записи в бд dds? -не факт что мы можешь считать обновление recruits_info как обновление данных в dds? - 500 - - Table input - citizen_appealing_violations_fz53_flow_delta.hpl + Append streams + Sort rows Y - get_max_source_update_date - Table input + failed_appeal_document_idm_ids_on_delta + Append streams Y - Table input - citizen_appealing_violations_fz53_flow_delta.hpl 2 + updated_idm_ids_from_appeal_document + Append streams Y - Table input - citizen_appealing_violations_fz53_flow_delta.hpl 3 + get_max_source_update_date_CAV + updated_idm_ids_from_appeal_document Y - Table input - citizen_appealing_violations_fz53_flow_delta.hpl 4 + get_max_source_update_date_CAV + failed_appeal_document_idm_ids_on_delta Y - Table input - citizen_appealing_violations_fz53_flow_delta.hpl 5 + Sort rows + CAV_flow_delta.hpl + Y + + + Sort rows + CAV_flow_delta.hpl 2 + Y + + + Sort rows + CAV_flow_delta.hpl 3 + Y + + + Sort rows + CAV_flow_delta.hpl 4 + Y + + + Sort rows + CAV_flow_delta.hpl 5 Y - Table input + CAV_flow_delta.hpl + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow_delta.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + + MAX_UPDATE_DATE + max_update_date + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 928 + 128 + + + + CAV_flow_delta.hpl 2 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow_delta.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + + MAX_UPDATE_DATE + max_update_date + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 928 + 208 + + + + CAV_flow_delta.hpl 3 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow_delta.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + + MAX_UPDATE_DATE + max_update_date + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 928 + 288 + + + + CAV_flow_delta.hpl 4 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow_delta.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + + MAX_UPDATE_DATE + max_update_date + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 928 + 368 + + + + CAV_flow_delta.hpl 5 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow_delta.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + + MAX_UPDATE_DATE + max_update_date + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 928 + 448 + + + + get_max_source_update_date_CAV + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT max(updated_at) AS max_update_date +FROM ervu_dashboard.citizen_appealing_violations_fz53; + N + + + 80 + 288 + + + + Append streams + Append + + Y + + 1 + + none + + + failed_appeal_document_idm_ids_on_delta + updated_idm_ids_from_appeal_document + + + 512 + 288 + + + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruitment_id + Y + N + N + 0 + N + + + + + 688 + 288 + + + + failed_appeal_document_idm_ids_on_delta TableInput Y @@ -86,316 +441,23 @@ ervu-dashboard N 0 - WITH mud AS (SELECT recruitment_id, - MAX(execution_datetime) AS max_upd_date - FROM etl.job_execution - WHERE job_name = '${JOB_NAME}' - AND status IN ('SUCCESS', 'DELTA_SUCCESS') - GROUP BY recruitment_id) -SELECT r.idm_id -FROM ervu_dashboard.recruitment r - JOIN mud ON mud.recruitment_id = r.idm_id - JOIN recruits_info ri - ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id - AND ri.updated_at > mud.max_upd_date - -union - -select r2.idm_id -from ervu_dashboard.recruitment r2 + get_max_source_update_date_CAV + select r.idm_id as recruitment_id, + ? as max_update_date +from ervu_dashboard.recruitment r join etl.job_execution je - on r2.idm_id = je.recruitment_id + on r.idm_id = je.recruitment_id where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') and je.job_name = '${JOB_NAME}'; Y - 352 - 304 - - - - citizen_appealing_violations_fz53_flow_delta.hpl - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_CREATE_DATE - max_create_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 592 - 144 - - - - citizen_appealing_violations_fz53_flow_delta.hpl 2 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_CREATE_DATE - max_create_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 592 - 224 - - - - citizen_appealing_violations_fz53_flow_delta.hpl 3 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_CREATE_DATE - max_create_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 592 - 304 - - - - citizen_appealing_violations_fz53_flow_delta.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_CREATE_DATE - max_create_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 592 + 304 384 - citizen_appealing_violations_fz53_flow_delta.hpl 5 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_appealing_violations_fz53/parallel/citizen_appealing_violations_fz53_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_CREATE_DATE - max_create_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 592 - 464 - - - - get_max_source_update_date + updated_idm_ids_from_appeal_document TableInput Y @@ -405,16 +467,21 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') none - ervu-dashboard + ervu_person_registry N 0 - SELECT max(create_date) AS max_create_date -FROM ervu_dashboard.citizen_appealing_violations_fz53; + get_max_source_update_date_CAV + WITH max_update_date(val) AS (SELECT CAST(? AS timestamp)) +SELECT DISTINCT vk_id AS recruitment_id, + mud.val as max_update_date +FROM appeal_document ad + JOIN max_update_date mud ON TRUE +WHERE (mud.val IS NULL OR ad.updated_at > mud.val); N - 192 - 304 + 304 + 192 diff --git a/mappings/info_recruits/citizen_tables/citizen_criminal_liability_fz53/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_criminal_liability_fz53/recruitment_five_flow_delta.hpl index 36d016d..70d414c 100644 --- a/mappings/info_recruits/citizen_tables/citizen_criminal_liability_fz53/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_criminal_liability_fz53/recruitment_five_flow_delta.hpl @@ -21,39 +21,59 @@ - Table input + Append streams + Sort rows + Y + + + failed_deferment_liberation_permission_idm_ids_on_delta + Append streams + Y + + + max_update_date_from_deferment_liberation + failed_deferment_liberation_permission_idm_ids_on_delta + Y + + + max_update_date_from_deferment_liberation + updated_idm_ids_fromrecruits_deferment + Y + + + updated_idm_ids_fromrecruits_deferment + Append streams + Y + + + Sort rows citizen_criminal_liability_fz53_flow_delta.hpl Y - get_max_source_update_date - Table input - Y - - - Table input + Sort rows citizen_criminal_liability_fz53_flow_delta.hpl 2 Y - Table input + Sort rows citizen_criminal_liability_fz53_flow_delta.hpl 3 Y - Table input + Sort rows citizen_criminal_liability_fz53_flow_delta.hpl 4 Y - Table input + Sort rows citizen_criminal_liability_fz53_flow_delta.hpl 5 Y - Table input - TableInput + Append streams + Append Y @@ -62,35 +82,46 @@ none - ervu-dashboard - N - 0 - WITH mud AS (SELECT recruitment_id, - MAX(execution_datetime) AS max_upd_date - FROM etl.job_execution - WHERE job_name = '${JOB_NAME}' - AND status IN ('SUCCESS', 'DELTA_SUCCESS') - GROUP BY recruitment_id) -SELECT r.idm_id -FROM ervu_dashboard.recruitment r - JOIN mud ON mud.recruitment_id = r.idm_id - JOIN ervu_dashboard.recruits_info ri - ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id - AND ri.updated_at > mud.max_upd_date - -union - -select r2.idm_id -from ervu_dashboard.recruitment r2 - join etl.job_execution je - on r2.idm_id = je.recruitment_id -where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - and je.job_name = '${JOB_NAME}'; - Y + failed_deferment_liberation_permission_idm_ids_on_delta + updated_idm_ids_fromrecruits_deferment - 336 - 288 + 768 + 272 + + + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruitment_id + Y + N + N + 0 + N + + + + + 944 + 272 @@ -145,33 +176,10 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - 624 + 1264 128 - - get_max_source_update_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT max(updated_at) AS max_update_date -FROM ervu_dashboard.citizen_criminal_liability_fz53; - N - - - 176 - 288 - - citizen_criminal_liability_fz53_flow_delta.hpl 2 PipelineExecutor @@ -224,7 +232,7 @@ FROM ervu_dashboard.citizen_criminal_liability_fz53; - 624 + 1264 208 @@ -280,7 +288,7 @@ FROM ervu_dashboard.citizen_criminal_liability_fz53; - 624 + 1264 288 @@ -336,7 +344,7 @@ FROM ervu_dashboard.citizen_criminal_liability_fz53; - 624 + 1264 368 @@ -392,10 +400,91 @@ FROM ervu_dashboard.citizen_criminal_liability_fz53; - 624 + 1264 448 + + failed_deferment_liberation_permission_idm_ids_on_delta + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + max_update_date_from_deferment_liberation + select r.idm_id as recruitment_id, + ? as max_update_date +from ervu_dashboard.recruitment r + join etl.job_execution je + on r.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y + + + 560 + 368 + + + + max_update_date_from_deferment_liberation + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + 0 + select max(updated_at) as max_update_date + from citizen_criminal_liability_fz53; + N + + + 368 + 272 + + + + updated_idm_ids_fromrecruits_deferment + TableInput + + Y + + 1 + + none + + + ervu_person_registry + N + 0 + max_update_date_from_deferment_liberation + WITH max_update_date(val) AS (SELECT CAST(? AS timestamp)) +SELECT DISTINCT COALESCE(r.current_recruitment, r.target_recruitment) AS recruitment_id, + mud.val as max_update_date +FROM recruit r + JOIN infringement i ON i.recruit_id = r.id + JOIN max_update_date mud ON TRUE +WHERE (mud.val IS NULL OR i.updated_at > mud.val); + N + + + 560 + 176 + + diff --git a/mappings/info_recruits/citizen_tables/citizen_liability_fz53/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_liability_fz53/recruitment_five_flow_delta.hpl index b5ea8f8..5f2e67f 100644 --- a/mappings/info_recruits/citizen_tables/citizen_liability_fz53/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_liability_fz53/recruitment_five_flow_delta.hpl @@ -82,12 +82,48 @@ none + failed_deferment_liberation_permission_idm_ids_on_delta + updated_idm_ids_fromrecruits_deferment 864 288 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruitment_id + Y + N + N + 0 + N + + + + + 1040 + 288 + + citizen_liability_fz53_flow_delta.hpl PipelineExecutor @@ -144,121 +180,6 @@ 128 - - failed_deferment_liberation_permission_idm_ids_on_delta - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - max_update_date_from_deferment_liberation - select r.idm_id as recruitment_id, - ? as max_update_date -from ervu_dashboard.recruitment r - join etl.job_execution je - on r.idm_id = je.recruitment_id -where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - and je.job_name = '${JOB_NAME}'; - Y - - - 656 - 384 - - - - max_update_date_from_deferment_liberation - TableInput - - N - - 1 - - none - - - ervu-dashboard - N - 0 - select max(updated_at) as max_update_date - from citizen_liability_fz53; - N - - - 464 - 288 - - - - updated_idm_ids_fromrecruits_deferment - TableInput - - Y - - 1 - - none - - - ervu_person_registry - N - 0 - max_update_date_from_deferment_liberation - WITH max_update_date(val) AS (SELECT CAST(? AS timestamp)) -SELECT DISTINCT COALESCE(r.current_recruitment, r.target_recruitment) AS recruitment_id, - ? as max_update_date -FROM recruit r - JOIN infringement i ON i.recruit_id = r.id - JOIN max_update_date mud ON TRUE -WHERE (mud.val IS NULL OR i.updated_at > mud.val); - N - - - 656 - 192 - - - - Sort rows - SortRows - - Y - - 1 - - none - - - ${java.io.tmpdir} - out - 100000 - - N - - Y - - - recruitment_id - Y - N - N - 0 - N - - - - - 1040 - 288 - - citizen_liability_fz53_flow_delta.hpl 2 PipelineExecutor @@ -483,6 +404,87 @@ WHERE (mud.val IS NULL OR i.updated_at > mud.val); 448 + + failed_deferment_liberation_permission_idm_ids_on_delta + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + max_update_date_from_deferment_liberation + select r.idm_id as recruitment_id, + ? as max_update_date +from ervu_dashboard.recruitment r + join etl.job_execution je + on r.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y + + + 656 + 384 + + + + max_update_date_from_deferment_liberation + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + 0 + select max(updated_at) as max_update_date + from citizen_liability_fz53; + N + + + 464 + 288 + + + + updated_idm_ids_fromrecruits_deferment + TableInput + + Y + + 1 + + none + + + ervu_person_registry + N + 0 + max_update_date_from_deferment_liberation + WITH max_update_date(val) AS (SELECT CAST(? AS timestamp)) +SELECT DISTINCT COALESCE(r.current_recruitment, r.target_recruitment) AS recruitment_id, + mud.val as max_update_date +FROM recruit r + JOIN infringement i ON i.recruit_id = r.id + JOIN max_update_date mud ON TRUE +WHERE (mud.val IS NULL OR i.updated_at > mud.val); + N + + + 656 + 192 + + diff --git a/mappings/info_recruits/citizen_tables/citizen_liability_fz53/row_last_infringement_update_date.hpl b/mappings/info_recruits/citizen_tables/citizen_liability_fz53/row_last_infringement_update_date.hpl deleted file mode 100644 index 17a0bcc..0000000 --- a/mappings/info_recruits/citizen_tables/citizen_liability_fz53/row_last_infringement_update_date.hpl +++ /dev/null @@ -1,159 +0,0 @@ - - - - row_last_infringement_update_date - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/06/02 14:51:59.611 - - - 2025/06/02 14:51:59.611 - - - - - - Copy rows to result - Set variables - N - - - Table input - Copy rows to result - N - - - Copy rows to result - Write to log - N - - - Table input - Set variables - Y - - - Set variables - Write to log - N - - - - Copy rows to result - RowsToResult - - Y - - 1 - - none - - - - - 944 - 192 - - - - Set variables - SetVariable - - Y - - 1 - - none - - - - - max_infringement_update_date - M_INFR_UP_DATE - GP_WORKFLOW - - - Y - - - 1232 - 320 - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - with -base as ( -select max(updated_at) AS max_infringement_update_date -from ervu_dashboard.citizen_criminal_liability_fz53 - -union all - -select max(updated_at) AS max_infringement_update_date -from ervu_dashboard.citizen_liability_fz53 -), -b AS ( -select max(max_infringement_update_date) AS max_infringement_update_date -from base -) -select - CASE - WHEN max_infringement_update_date is null THEN '0001-01-01 00:00:00'::timestamp - ELSE max_infringement_update_date - END AS max_infringement_update_date -from b - N - - - 688 - 320 - - - - Write to log - WriteToLog - - Y - - 1 - - none - - - Y - - - N - 0 - Basic - ${M_INFR_UP_DATE} - - - 1376 - 192 - - - - - - diff --git a/mappings/info_recruits/citizen_tables/citizenship_foreign/citizenship_foreign_job.hwf b/mappings/info_recruits/citizen_tables/citizenship_foreign/citizenship_foreign_job.hwf index 4972faa..46e9864 100644 --- a/mappings/info_recruits/citizen_tables/citizenship_foreign/citizenship_foreign_job.hwf +++ b/mappings/info_recruits/citizen_tables/citizenship_foreign/citizenship_foreign_job.hwf @@ -34,7 +34,7 @@ - Start + start_job_CF SPECIAL @@ -228,7 +228,6 @@ CURRENT_WORKFLOW - N N 240 @@ -266,7 +265,7 @@ N - Start + start_job_CF init_job_name Y Y diff --git a/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow.hpl b/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow.hpl index 163e9c1..10ff2df 100644 --- a/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow.hpl @@ -21,68 +21,78 @@ - Create job execution record - Table input + create_job_execution_record_CF + table_input_CF Y - Filter rows - Change job status on success + filter_last_row_CF + change_job_status_on_success_CF Y - Detect empty stream - Change job status on success + detect_empty_stream_CF + change_job_status_on_success_CF Y - Change job status on error - Abort + change_job_status_on_error_CF + abort_CF Y - Table output - Change job status on error + table_output_CF + change_job_status_on_error_CF Y - Update - Change job status on error + table_input_CF + identify_last_row_CF Y - Table input - Identify last row in a stream + identify_last_row_CF + table_output_CF Y - Identify last row in a stream - Table output + identify_last_row_CF + detect_empty_stream_CF Y - Identify last row in a stream - Detect empty stream + table_output_CF + filter_last_row_CF Y - Identify last row in a stream - sort_by_recruit_id + filter_true_citizenship + update_citizen_has_foreign_citizenship Y - sort_by_recruit_id - Update + filter_true_citizenship + update_citizen_has_foreign_residence Y - Table output - Filter rows + update_citizen_has_foreign_citizenship + change_job_status_on_error_CF + Y + + + update_citizen_has_foreign_residence + change_job_status_on_error_CF + Y + + + identify_last_row_CF + filter_true_citizenship Y - Abort + abort_CF Abort Y @@ -94,15 +104,16 @@ ABORT_WITH_ERROR Y + 0 - 1344 - 480 + 1456 + 496 - Change job status on error + change_job_status_on_error_CF ExecSql Y @@ -133,12 +144,12 @@ and recruitment_id = '${IDM_ID}'; - 1104 - 480 + 1200 + 496 - Change job status on success + change_job_status_on_success_CF ExecSql Y @@ -165,12 +176,12 @@ and recruitment_id = '${IDM_ID}'; - 1328 + 1456 176 - Create job execution record + create_job_execution_record_CF ExecSql Y @@ -199,7 +210,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Detect empty stream + detect_empty_stream_CF DetectEmptyStream Y @@ -216,7 +227,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Filter rows + filter_last_row_CF FilterRows Y @@ -244,15 +255,52 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Change job status on success + change_job_status_on_success_CF - 1328 + 1456 320 - Identify last row in a stream + filter_true_citizenship + FilterRows + + Y + + 1 + + none + + + + + + + = + has_foreign_citizenship + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + update_citizen_has_foreign_residence + update_citizen_has_foreign_citizenship + + + 688 + 496 + + + + identify_last_row_CF DetectLastRow N @@ -270,7 +318,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Table input + table_input_CF TableInput Y @@ -282,58 +330,36 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - WITH base AS ( - SELECT - ri.recruit_id, - ri.info - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку - AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - WHERE jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedIG') = 'array' - OR jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedPP') = 'array' -), -citizen AS ( - SELECT - ri.recruit_id, - jsonb_array_elements(ri.info->'svedFL'->'svedIGPP'->'svedIG') AS data, + WITH filtered_data AS ( + SELECT + ri.recruit_id, + jsonb_array_elements(ri.info -> 'svedFL' -> 'svedIGPP' -> 'svedIG') AS data, 'citizenship' AS type - FROM base ri - WHERE jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedIG') = 'array' + FROM ervu_dashboard.recruits_info ri + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedIGPP' -> 'svedIG') = 'array' + AND coalesce(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' UNION ALL - SELECT - ri.recruit_id, - jsonb_array_elements(ri.info->'svedFL'->'svedIGPP'->'svedPP') AS data, - 'residense' AS type - FROM base ri WHERE jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedPP') = 'array' + SELECT + ri.recruit_id, + jsonb_array_elements(ri.info -> 'svedFL' -> 'svedIGPP' -> 'svedPP') AS data, + 'residence' AS type + FROM ervu_dashboard.recruits_info ri + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedIGPP' -> 'svedPP') = 'array' + AND coalesce(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' ) SELECT recruit_id, - to_date(data->>'dataSved', 'YYYY-MM-DD') AS source_update_date, -- дата обновления сведений - CASE - WHEN type='residense' THEN data->>'osnovPriobrPP' - END::text AS residense_right_basis, - CASE - WHEN type='residense' THEN to_date(NULLIF(data->>'dataPriobrPP',''), 'YYYY-MM-DD') - END::date AS residense_right_date, - data->'oksm'->>'naimOKSM' AS country_name, - CASE - WHEN type='citizenship' THEN data->>'osnovPriobrIG' - END::text AS citizenship_basis, - CASE - WHEN type='citizenship' THEN to_date(NULLIF(data->>'dataPriobrIG',''), 'YYYY-MM-DD') - END::date AS citizenship_date, + coalesce(to_date(NULLIF(data ->> 'dataSved', ''), 'YYYY-MM-DD'), current_date) AS source_update_date, -- дата обновления сведений + data ->> CASE WHEN type = 'residence' THEN 'osnovPriobrPP' ELSE 'osnovPriobrIG' END AS basis, + to_date(data ->> CASE WHEN type = 'residence' THEN 'dataPriobrPP' ELSE 'dataPriobrIG' END, 'YYYY-MM-DD') AS acquisition_date, + data -> 'oksm' ->> 'kodOKSM' AS oksm_code, type, - CASE - WHEN type = 'residense' THEN true - ELSE false - END AS has_foreign_residence, - CASE - WHEN type = 'citizenship' THEN true - ELSE false - END AS has_foreign_citizenship -FROM citizen + (type = 'residence') AS has_foreign_residence, + (type = 'citizenship') AS has_foreign_citizenship +FROM filtered_data; + Y @@ -342,7 +368,7 @@ FROM citizen - Table output + table_output_CF TableOutput Y @@ -360,32 +386,28 @@ FROM citizen recruit_id - residense_right_basis - residense_right_basis + source_update_date + source_update_date - residense_right_date - residense_right_date + basis + basis - country_name - country_name + acquisition_date + acquisition_date - citizenship_basis - citizenship_basis - - - citizenship_date - citizenship_date + oksm_code + oksm_code type type - source_update_date - source_update_date + source_id + source_id N @@ -408,7 +430,7 @@ FROM citizen - Update + update_citizen_has_foreign_citizenship Update N @@ -421,18 +443,16 @@ FROM citizen 10000 ervu-dashboard Y + = recruit_id recruit_id + ervu_dashboard citizen
- - has_foreign_residence - has_foreign_residence - has_foreign_citizenship has_foreign_citizenship @@ -443,47 +463,50 @@ FROM citizen 896 - 480 + 496 - sort_by_recruit_id - SortRows + update_citizen_has_foreign_residence + Update - Y + N 1 none - ${java.io.tmpdir} - srt - 100000 - - N - - Y - - + 10000 + ervu-dashboard + Y + + + + = + recruit_id recruit_id - Y - N - N - 0 - N - - + + + ervu_dashboard + citizen
+ + has_foreign_residence + has_foreign_residence + + + N + Y - 688 - 480 + 896 + 640 - Table output - Change job status on error + table_output_CF + change_job_status_on_error_CF Y error_description @@ -494,8 +517,20 @@ FROM citizen - Update - Change job status on error + update_citizen_has_foreign_citizenship + change_job_status_on_error_CF + Y + + error_description + + + + + + + + update_citizen_has_foreign_residence + change_job_status_on_error_CF Y error_description diff --git a/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl index d718186..f5ad91d 100644 --- a/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl @@ -21,68 +21,78 @@ - Create job execution record - Table input + update_job_execution_record_CF + table_input_CF Y - Detect empty stream - Change job status on success + detect_empty_stream_CF + change_job_status_on_success_CF Y - Filter rows - Change job status on success + filter_last_row_CF + change_job_status_on_success_CF Y - Change job status on error - Abort + change_job_status_on_error_CF + abort_CF Y - Update - Change job status on error + upsert_CF + change_job_status_on_error_CF Y - Insert / update - Change job status on error + table_input_CF + identify_last_row_CF Y - Table input - Identify last row in a stream + identify_last_row_CF + upsert_CF Y - Identify last row in a stream - Insert / update + identify_last_row_CF + detect_empty_stream_CF Y - Identify last row in a stream - sort_by_recruit_id + upsert_CF + filter_last_row_CF Y - Identify last row in a stream - Detect empty stream + filter_true_citizenship + update_citizen_has_foreign_citizenship Y - sort_by_recruit_id - Update + filter_true_citizenship + update_citizen_has_foreign_residence Y - Insert / update - Filter rows + identify_last_row_CF + filter_true_citizenship + Y + + + update_citizen_has_foreign_residence + change_job_status_on_error_CF + Y + + + update_citizen_has_foreign_citizenship + change_job_status_on_error_CF Y - Abort + abort_CF Abort Y @@ -94,15 +104,16 @@ ABORT_WITH_ERROR Y + 0 - 1392 - 528 + 1408 + 512 - Change job status on error + change_job_status_on_error_CF ExecSql Y @@ -118,8 +129,11 @@ ervu-dashboard + Y + N + Y Y N @@ -131,14 +145,15 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + - 1168 - 528 + 1248 + 512 - Change job status on success + change_job_status_on_success_CF ExecSql Y @@ -151,8 +166,11 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard + Y + N + Y N N @@ -163,6 +181,7 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + 1408 @@ -170,7 +189,319 @@ and recruitment_id = '${IDM_ID}'; - Create job execution record + detect_empty_stream_CF + DetectEmptyStream + + Y + + 1 + + none + + + + + 784 + 192 + + + + filter_last_row_CF + FilterRows + + Y + + 1 + + none + + + + + + + = + last_row + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + change_job_status_on_success_CF + + + 1408 + 352 + + + + upsert_CF + InsertUpdate + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + = + recruit_id + recruit_id + + + + = + source_id + source_id + + + ervu_dashboard + citizenship_foreign
+ + recruit_id + recruit_id + N + + + source_update_date + source_update_date + Y + + + basis + basis + Y + + + acquisition_date + acquisition_date + Y + + + oksm_code + oksm_code + Y + + + type + type + Y + + + source_id + source_id + N + + + N + + + 976 + 352 + + + + filter_true_citizenship + FilterRows + + Y + + 1 + + none + + + + + + + = + has_foreign_citizenship + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + update_citizen_has_foreign_residence + update_citizen_has_foreign_citizenship + + + 784 + 576 + + + + identify_last_row_CF + DetectLastRow + + N + + 1 + + none + + + last_row + + + 784 + 352 + + + + table_input_CF + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + WITH filtered_data AS ( + SELECT + ri.recruit_id, + jsonb_array_elements(ri.info -> 'svedFL' -> 'svedIGPP' -> 'svedIG') AS data, + 'citizenship' AS type + FROM ervu_dashboard.recruits_info ri + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedIGPP' -> 'svedIG') = 'array' + AND coalesce(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + and case when '${MAX_SOURCE_UPDATE_DATE}' <> '' + then to_date(ri.info -> 'svedFL' -> 'svedDeputat' -> 'deputat' ->> 'dataSved', 'YYYY-MM-DD') > + to_date('${MAX_SOURCE_UPDATE_DATE}', 'YYYY-MM-DD') + else true end + + UNION ALL + + SELECT + ri.recruit_id, + jsonb_array_elements(ri.info -> 'svedFL' -> 'svedIGPP' -> 'svedPP') AS data, + 'residence' AS type + FROM ervu_dashboard.recruits_info ri + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedIGPP' -> 'svedPP') = 'array' + AND coalesce(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + and case when '${MAX_SOURCE_UPDATE_DATE}' <> '' + then to_date(ri.info -> 'svedFL' -> 'svedDeputat' -> 'deputat' ->> 'dataSved', 'YYYY-MM-DD') > + to_date('${MAX_SOURCE_UPDATE_DATE}', 'YYYY-MM-DD') + else true end +) +SELECT + recruit_id, + coalesce(to_date(NULLIF(data ->> 'dataSved', ''), 'YYYY-MM-DD'), current_date) AS source_update_date, -- дата обновления сведений + data ->> CASE WHEN type = 'residence' THEN 'osnovPriobrPP' ELSE 'osnovPriobrIG' END AS basis, + to_date(data ->> CASE WHEN type = 'residence' THEN 'dataPriobrPP' ELSE 'dataPriobrIG' END, 'YYYY-MM-DD') AS acquisition_date, + data -> 'oksm' ->> 'kodOKSM' AS oksm_code, + type, + (type = 'residence') AS has_foreign_residence, + (type = 'citizenship') AS has_foreign_citizenship +FROM filtered_data; + + Y + + + 560 + 352 + + + + update_citizen_has_foreign_citizenship + Update + + N + + 1 + + none + + + 10000 + ervu-dashboard + Y + + + = + recruit_id + recruit_id + + + <> + has_foreign_citizenship + has_foreign_citizenship + + ervu_dashboard + citizen
+ + has_foreign_citizenship + has_foreign_citizenship + + + N + Y + + + 976 + 512 + + + + update_citizen_has_foreign_residence + Update + + N + + 1 + + none + + + 10000 + ervu-dashboard + Y + + + = + recruit_id + recruit_id + + ervu_dashboard + citizen
+ + has_foreign_residence + has_foreign_residence + + + N + Y + + + 976 + 656 + + + + update_job_execution_record_CF ExecSql Y @@ -201,308 +532,10 @@ and recruitment_id = '${IDM_ID}'; 352 - - Detect empty stream - DetectEmptyStream - - Y - - 1 - - none - - - - - 784 - 192 - - - - Filter rows - FilterRows - - Y - - 1 - - none - - - - - - - = - last_row - N - - - - N - -1 - constant - -1 - Y - Boolean - - - - Change job status on success - - - 1408 - 352 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 784 - 352 - - - - Insert / update - InsertUpdate - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - = - recruit_id - recruit_id - - - = - type - type - - ervu_dashboard - citizenship_foreign
- - recruit_id - recruit_id - N - - - residense_right_basis - residense_right_basis - Y - - - residense_right_date - residense_right_date - Y - - - country_name - country_name - Y - - - citizenship_basis - citizenship_basis - Y - - - citizenship_date - citizenship_date - Y - - - type - type - Y - - - source_update_date - source_update_date - Y - - - N - - - 976 - 352 - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - WITH base AS ( - SELECT - ri.recruit_id, - ri.info - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - WHERE '${IDM_ID}' != '' - AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND ( - jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedIG') = 'array' - OR jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedPP') = 'array' - ) -), -filtered_data AS ( - SELECT - recruit_id, - elem AS data, - 'citizenship' AS type - FROM base ri, - LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedIGPP'->'svedIG') AS elem - WHERE jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedIG') = 'array' - AND to_date(elem->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date - - UNION ALL - - SELECT - recruit_id, - elem AS data, - 'residense' AS type - FROM base ri, - LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedIGPP'->'svedPP') AS elem - WHERE jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedPP') = 'array' - AND to_date(elem->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date -) -SELECT - recruit_id, - to_date(data->>'dataSved', 'YYYY-MM-DD') AS source_update_date, -- дата обновления сведений - CASE - WHEN type='residense' THEN data->>'osnovPriobrPP' - END::text AS residense_right_basis, - CASE - WHEN type='residense' THEN to_date(NULLIF(data->>'dataPriobrPP',''), 'YYYY-MM-DD') - END::date AS residense_right_date, - data->'oksm'->>'naimOKSM' AS country_name, - CASE - WHEN type='citizenship' THEN data->>'osnovPriobrIG' - END::text AS citizenship_basis, - CASE - WHEN type='citizenship' THEN to_date(NULLIF(data->>'dataPriobrIG',''), 'YYYY-MM-DD') - END::date AS citizenship_date, - type, - CASE - WHEN type = 'residense' THEN true - ELSE false - END AS has_foreign_residence, - CASE - WHEN type = 'citizenship' THEN true - ELSE false - END AS has_foreign_citizenship -FROM citizen - Y - - - 560 - 352 - - - - Update - Update - - N - - 1 - - none - - - 10000 - ervu-dashboard - Y - - - = - recruit_id - recruit_id - - ervu_dashboard - citizen
- - has_foreign_residence - has_foreign_residence - - - has_foreign_citizenship - has_foreign_citizenship - - - N - Y - - - 992 - 528 - - - - sort_by_recruit_id - SortRows - - Y - - 1 - - none - - - ${java.io.tmpdir} - srt - 100000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - - - - 784 - 528 - - - Insert / update - Change job status on error + upsert_CF + change_job_status_on_error_CF Y error_description @@ -513,8 +546,20 @@ FROM citizen - Update - Change job status on error + update_citizen_has_foreign_citizenship + change_job_status_on_error_CF + Y + + error_description + + + + + + + + update_citizen_has_foreign_residence + change_job_status_on_error_CF Y error_description diff --git a/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_repeat.hpl index b2b184e..f536890 100644 --- a/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_repeat.hpl @@ -21,68 +21,96 @@ - Create job execution record - Table input + upsert_job_execution_record_CF + table_input_CF Y - Filter rows - Change job status on success + filter_last_row_CF + change_job_status_on_success_CF Y - Detect empty stream - Change job status on success + detect_empty_stream_CF + change_job_status_on_success_CF Y - Change job status on error - Abort + change_job_status_on_error_CF + abort_CF Y - Insert / update - Change job status on error + upsert_CF + change_job_status_on_error_CF Y - Update - Change job status on error - Y - - - Table input + table_input_CF Identify last row in a stream Y Identify last row in a stream - Insert / update + upsert_CF Y Identify last row in a stream - Detect empty stream + detect_empty_stream_CF + Y + + + upsert_CF + filter_last_row_CF + Y + + + filter_true_citizenship + update_citizen_has_foreign_citizenship + Y + + + filter_true_citizenship + update_citizen_has_foreign_residence Y Identify last row in a stream - sort_by_recruit_id + filter_true_citizenship Y - sort_by_recruit_id - Update + update_citizen_has_foreign_citizenship + change_job_status_on_error_CF Y - Insert / update - Filter rows + update_citizen_has_foreign_residence + change_job_status_on_error_CF Y - Abort + Identify last row in a stream + DetectLastRow + + N + + 1 + + none + + + last_row + + + 704 + 208 + + + + abort_CF Abort Y @@ -97,12 +125,12 @@ 0 - 1312 - 400 + 1408 + 352 - Change job status on error + change_job_status_on_error_CF ExecSql Y @@ -133,12 +161,12 @@ and recruitment_id = '${IDM_ID}'; - 1120 - 400 + 1216 + 352 - Change job status on success + change_job_status_on_success_CF ExecSql Y @@ -165,12 +193,291 @@ and recruitment_id = '${IDM_ID}'; - 1360 + 1408 64 - Create job execution record + detect_empty_stream_CF + DetectEmptyStream + + Y + + 1 + + none + + + + + 704 + 64 + + + + filter_last_row_CF + FilterRows + + Y + + 1 + + none + + + + + + + = + last_row + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + change_job_status_on_success_CF + + + 1408 + 208 + + + + filter_true_citizenship + FilterRows + + Y + + 1 + + none + + + + + + + = + has_foreign_citizenship + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + update_citizen_has_foreign_residence + update_citizen_has_foreign_citizenship + + + 704 + 352 + + + + table_input_CF + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + WITH filtered_data AS ( + SELECT + ri.recruit_id, + jsonb_array_elements(ri.info -> 'svedFL' -> 'svedIGPP' -> 'svedIG') AS data, + 'citizenship' AS type + FROM ervu_dashboard.recruits_info ri + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedIGPP' -> 'svedIG') = 'array' + AND coalesce(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + + UNION ALL + + SELECT + ri.recruit_id, + jsonb_array_elements(ri.info -> 'svedFL' -> 'svedIGPP' -> 'svedPP') AS data, + 'residence' AS type + FROM ervu_dashboard.recruits_info ri + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedIGPP' -> 'svedPP') = 'array' + AND coalesce(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' +) +SELECT + recruit_id, + coalesce(to_date(NULLIF(data ->> 'dataSved', ''), 'YYYY-MM-DD'), current_date) AS source_update_date, -- дата обновления сведений + data ->> CASE WHEN type = 'residence' THEN 'osnovPriobrPP' ELSE 'osnovPriobrIG' END AS basis, + to_date(data ->> CASE WHEN type = 'residence' THEN 'dataPriobrPP' ELSE 'dataPriobrIG' END, 'YYYY-MM-DD') AS acquisition_date, + data -> 'oksm' ->> 'kodOKSM' AS oksm_code, + type, + (type = 'residence') AS has_foreign_residence, + (type = 'citizenship') AS has_foreign_citizenship +FROM filtered_data; + + Y + + + 480 + 208 + + + + update_citizen_has_foreign_citizenship + Update + + N + + 1 + + none + + + 10000 + ervu-dashboard + Y + + + = + recruit_id + recruit_id + + ervu_dashboard + citizen
+ + has_foreign_citizenship + has_foreign_citizenship + + + N + Y + + + 944 + 352 + + + + update_citizen_has_foreign_residence + Update + + N + + 1 + + none + + + 10000 + ervu-dashboard + Y + + + = + recruit_id + recruit_id + + ervu_dashboard + citizen
+ + has_foreign_residence + has_foreign_residence + + + N + Y + + + 944 + 512 + + + + upsert_CF + InsertUpdate + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + = + recruit_id + recruit_id + + + = + source_id + source_id + + ervu_dashboard + citizenship_foreign
+ + recruit_id + recruit_id + N + + + source_update_date + source_update_date + Y + + + basis + basis + Y + + + acquisition_date + acquisition_date + Y + + + oksm_code + oksm_code + Y + + + type + type + Y + + + source_id + source_id + N + + + Y + + + 944 + 208 + + + + upsert_job_execution_record_CF ExecSql Y @@ -216,302 +523,10 @@ DO UPDATE SET 208 - - Detect empty stream - DetectEmptyStream - - Y - - 1 - - none - - - - - 704 - 64 - - - - Filter rows - FilterRows - - Y - - 1 - - none - - - - - - - = - last_row - N - - - - N - -1 - constant - -1 - Y - Boolean - - - - Change job status on success - - - 1360 - 208 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 704 - 208 - - - - Insert / update - InsertUpdate - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - = - recruit_id - recruit_id - - - - = - type - type - - - ervu_dashboard - citizenship_foreign
- - recruit_id - recruit_id - N - - - residense_right_basis - residense_right_basis - Y - - - residense_right_date - residense_right_date - Y - - - country_name - country_name - Y - - - citizenship_basis - citizenship_basis - Y - - - citizenship_date - citizenship_date - Y - - - type - type - Y - - - source_update_date - source_update_date - Y - - - Y - - - 912 - 208 - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - WITH base AS ( - SELECT - ri.recruit_id, - ri.info - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку - AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - WHERE jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedIG') = 'array' - OR jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedPP') = 'array' -), -citizen AS ( - SELECT - ri.recruit_id, - jsonb_array_elements(ri.info->'svedFL'->'svedIGPP'->'svedIG') AS data, - 'citizenship' AS type - FROM base ri - WHERE jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedIG') = 'array' - - UNION ALL - - SELECT - ri.recruit_id, - jsonb_array_elements(ri.info->'svedFL'->'svedIGPP'->'svedPP') AS data, - 'residense' AS type - FROM base ri WHERE jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedPP') = 'array' -) -SELECT - recruit_id, - to_date(data->>'dataSved', 'YYYY-MM-DD') AS source_update_date, -- дата обновления сведений - CASE - WHEN type='residense' THEN data->>'osnovPriobrPP' - END::text AS residense_right_basis, - CASE - WHEN type='residense' THEN to_date(NULLIF(data->>'dataPriobrPP',''), 'YYYY-MM-DD') - END::date AS residense_right_date, - data->'oksm'->>'naimOKSM' AS country_name, - CASE - WHEN type='citizenship' THEN data->>'osnovPriobrIG' - END::text AS citizenship_basis, - CASE - WHEN type='citizenship' THEN to_date(NULLIF(data->>'dataPriobrIG',''), 'YYYY-MM-DD') - END::date AS citizenship_date, - type, - CASE - WHEN type = 'residense' THEN true - ELSE false - END AS has_foreign_residence, - CASE - WHEN type = 'citizenship' THEN true - ELSE false - END AS has_foreign_citizenship -FROM citizen - Y - - - 480 - 208 - - - - Update - Update - - N - - 1 - - none - - - 10000 - ervu-dashboard - Y - - - = - recruit_id - recruit_id - - ervu_dashboard - citizen
- - has_foreign_residence - has_foreign_residence - - - has_foreign_citizenship - has_foreign_citizenship - - - N - Y - - - 912 - 400 - - - - sort_by_recruit_id - SortRows - - Y - - 1 - - none - - - ${java.io.tmpdir} - srt - 100000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - - - - 704 - 400 - - - Insert / update - Change job status on error + update_citizen_has_foreign_citizenship + change_job_status_on_error_CF Y error_description @@ -522,8 +537,20 @@ FROM citizen - Update - Change job status on error + update_citizen_has_foreign_residence + change_job_status_on_error_CF + Y + + error_description + + + + + + + + upsert_CF + change_job_status_on_error_CF Y error_description diff --git a/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_delta.hpl index 8e72b00..104a943 100644 --- a/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_delta.hpl @@ -21,38 +21,38 @@ - Table input + get_update_recrutiments_CF citizenship_foreign_flow_delta.hpl Y - get_max_source_update_date - Table input + get_max_source_update_date_CF + get_update_recrutiments_CF Y - Table input + get_update_recrutiments_CF citizenship_foreign_flow_delta.hpl 2 Y - Table input + get_update_recrutiments_CF citizenship_foreign_flow_delta.hpl 3 Y - Table input + get_update_recrutiments_CF citizenship_foreign_flow_delta.hpl 4 Y - Table input + get_update_recrutiments_CF citizenship_foreign_flow_delta.hpl 5 Y - Table input + get_update_recrutiments_CF TableInput Y @@ -89,7 +89,7 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') Y - 352 + 384 288 @@ -374,7 +374,7 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - get_max_source_update_date + get_max_source_update_date_CF TableInput Y diff --git a/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_on_error.hpl b/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_on_error.hpl index a0de7f8..1fa4c89 100644 --- a/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_on_error.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_on_error.hpl @@ -21,33 +21,33 @@ - Table input + get_missed_error_recruitments_CF citizenship_foreign_flow_repeat.hpl Y - Table input + get_missed_error_recruitments_CF citizenship_foreign_flow_repeat.hpl 2 Y - Table input + get_missed_error_recruitments_CF citizenship_foreign_flow_repeat.hpl 3 Y - Table input + get_missed_error_recruitments_CF citizenship_foreign_flow_repeat.hpl 4 Y - Table input + get_missed_error_recruitments_CF citizenship_foreign_flow_repeat.hpl 5 Y - Table input + get_missed_error_recruitments_CF TableInput Y @@ -223,7 +223,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING'); - 560 + 544 384 @@ -274,7 +274,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING'); - 560 + 544 480 @@ -325,7 +325,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING'); - 560 + 544 560 diff --git a/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitments_five_flow.hpl b/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitments_five_flow.hpl index 6ea0db8..133dacd 100644 --- a/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitments_five_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitments_five_flow.hpl @@ -21,33 +21,33 @@ - Get all recruitments ordered by created_date + get_all_recruitments_CF citizenship_foreign_flow.hpl Y - Get all recruitments ordered by created_date + get_all_recruitments_CF citizenship_foreign_flow.hpl 2 Y - Get all recruitments ordered by created_date + get_all_recruitments_CF citizenship_foreign_flow.hpl 3 Y - Get all recruitments ordered by created_date + get_all_recruitments_CF citizenship_foreign_flow.hpl 4 Y - Get all recruitments ordered by created_date + get_all_recruitments_CF citizenship_foreign_flow.hpl 5 Y - Get all recruitments ordered by created_date + get_all_recruitments_CF TableInput Y @@ -169,8 +169,8 @@ FROM ervu_dashboard.recruitment - 752 - 240 + 736 + 224 @@ -220,8 +220,8 @@ FROM ervu_dashboard.recruitment - 752 - 320 + 736 + 304 @@ -271,8 +271,8 @@ FROM ervu_dashboard.recruitment - 768 - 400 + 736 + 384 @@ -322,8 +322,8 @@ FROM ervu_dashboard.recruitment - 752 - 480 + 736 + 464 diff --git a/mappings/info_recruits/citizen_tables/deferment_liberation/deferment_decision_update_date_ervu_dashboard.hpl b/mappings/info_recruits/citizen_tables/deferment_liberation/deferment_decision_update_date_ervu_dashboard.hpl deleted file mode 100644 index 339243a..0000000 --- a/mappings/info_recruits/citizen_tables/deferment_liberation/deferment_decision_update_date_ervu_dashboard.hpl +++ /dev/null @@ -1,98 +0,0 @@ - - - - deferment_decision_update_date_ervu_dashboard - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/06/02 14:50:31.833 - - - 2025/06/02 14:50:31.833 - - - - - - Table input - Table output - Y - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT - created_at AS system_create_date, - CASE - WHEN updated_at is null THEN '0001-01-01 00:00:00' - ELSE updated_at - END as system_update_date, - current_timestamp AS record_created, - 'deferment_decision_table' AS workflow -FROM ervu_dashboard.deferment_liberation -WHERE updated_at = (SELECT MAX(updated_at) FROM ervu_dashboard.deferment_liberation) -limit 1 - N - - - 896 - 304 - - - - Table output - TableOutput - - Y - - 1 - - none - - - 1000 - ervu-dashboard - - - N - N - N - N - Y - N - public - N - recruit_create
- N - Y - N - Y - - - 1216 - 304 - - - - - - diff --git a/mappings/info_recruits/citizen_tables/deferment_liberation/job_last_update_date_deferment_decision.hwf b/mappings/info_recruits/citizen_tables/deferment_liberation/job_last_update_date_deferment_decision.hwf deleted file mode 100644 index 44acc2f..0000000 --- a/mappings/info_recruits/citizen_tables/deferment_liberation/job_last_update_date_deferment_decision.hwf +++ /dev/null @@ -1,126 +0,0 @@ - - - job_last_update_date_deferment_decision - Y - - - - - - 2025/06/02 14:48:05.666 - - - 2025/06/02 14:48:05.666 - - - - - Start - - SPECIAL - - 1 - 12 - 60 - 0 - 0 - N - 0 - 1 - N - 592 - 352 - - - - deferment_decision_update_date_ervu_dashboard.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation/deferment_decision_update_date_ervu_dashboard.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 848 - 352 - - - - row_last_deferment_decision_update_date.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation/row_last_deferment_decision_update_date.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 1248 - 352 - - - - Success - - SUCCESS - - N - 1568 - 352 - - - - - - Start - deferment_decision_update_date_ervu_dashboard.hpl - Y - Y - Y - - - deferment_decision_update_date_ervu_dashboard.hpl - row_last_deferment_decision_update_date.hpl - Y - Y - N - - - row_last_deferment_decision_update_date.hpl - Success - Y - Y - N - - - - - - diff --git a/mappings/info_recruits/citizen_tables/deferment_liberation/parallel/deferment_liberation_flow.hpl b/mappings/info_recruits/citizen_tables/deferment_liberation/parallel/deferment_liberation_flow.hpl index 8089625..58e1af8 100644 --- a/mappings/info_recruits/citizen_tables/deferment_liberation/parallel/deferment_liberation_flow.hpl +++ b/mappings/info_recruits/citizen_tables/deferment_liberation/parallel/deferment_liberation_flow.hpl @@ -45,11 +45,6 @@ Change job status on error Y - - Update - Change job status on error - Y - Table input Identify last row in a stream @@ -70,14 +65,34 @@ Detect empty stream Y + + Table output + Filter rows + Y + sort_by_recruit_id + Group by + Y + + + Change job status on error 2 + Abort 2 + Y + + + Group by + User defined Java expression + Y + + + User defined Java expression Update Y - Table output - Filter rows + Update + Change job status on error 2 Y @@ -98,7 +113,7 @@ 1408 - 464 + 400 @@ -134,7 +149,7 @@ and recruitment_id = '${IDM_ID}'; 1200 - 464 + 400 @@ -282,67 +297,32 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); postgres.decision-document-service N - WITH -part_recruits AS ( - SELECT - id - FROM public.recruit - WHERE '${IDM_ID}' != '' -- Проверка на пустую строку - AND (current_recruitment = '${IDM_ID}' or (current_recruitment is null and target_recruitment = '${IDM_ID}')) -), -p_deferment_decision AS ( - SELECT - d.id, - d.recruit_id, - d.base_deferment, - d.created_at, - d.updated_at, - d.postponement_date, - d.hidden, - d.fact_provision, - ddt.code AS type_code - FROM public.deferment_decision d - JOIN public.deferment_decision_type ddt ON d.type_id = ddt.id -) -SELECT - d.id AS deferment_id, - d.recruit_id, - d.base_deferment, --Основание на отсрочку от призыва/Основание на освобождение от призыва - d.created_at,--Дата начала действия отсрочки от призыва/Дата начала действия отвобождения от призыва - d.updated_at, - d.postponement_date, --Дата окончания действия отсрочки от призыва/Дата окончания действия освобождения от призыва - d.hidden, - 'deferment' AS type, - 'fact' AS permission_fact, -- ключ принадлежности к решению по отсрочке/освобождению - CASE - WHEN d.fact_provision = '1' THEN true - ELSE false - END AS has_deferment,--Факт предоставления отсрочки от призыва/Факт предоставления освобождения от призыва (1 - предоставлено, 2 - не предоставлено) - false AS has_liberation -FROM p_deferment_decision d -JOIN part_recruits r ON r.id = d.recruit_id -WHERE d.type_code = '1' -- отсрочка - -UNION ALL - -SELECT - d.id AS deferment_id, - d.recruit_id, - d.base_deferment, --Основание на отсрочку от призыва/Основание на освобождение от призыва - d.created_at,--Дата начала действия отсрочки от призыва/Дата начала действия отвобождения от призыва - d.updated_at, - d.postponement_date, --Дата окончания действия отсрочки от призыва/Дата окончания действия освобождения от призыва - d.hidden, - 'liberation' AS type, - 'fact' AS permission_fact, -- ключ принадлежности к решению по отсрочке/освобождению - false AS has_deferment, - CASE - WHEN d.fact_provision = '1' THEN true - ELSE false - END AS has_liberation-- Факт предоставления отсрочки от призыва/Факт предоставления освобождения от призыва (1 - предоставлено, 2 - не предоставлено) -FROM p_deferment_decision d -JOIN part_recruits r ON r.id = d.recruit_id -WHERE d.type_code = '2' -- освобождение + + SELECT + dd.id AS deferment_id, + dd.recruit_id, + dd.base_deferment, + dd.created_at, + dd.updated_at, + dd.postponement_date, + dd.hidden, + CASE ddt.code + WHEN '1' THEN 'deferment' + WHEN '2' THEN 'liberation' + END AS type, +-- 'fact' AS permission_fact, + dd.fact_provision AS has_deferment_raw, + -- true только если тип "освобождение" и факт предоставления = 1 + case when (ddt.code = '2' AND dd.fact_provision = '1') then 1 + else 0 end AS has_liberation_raw +FROM public.deferment_decision dd + JOIN public.recruit r + ON r.id = dd.recruit_id + AND (r.current_recruitment = '${IDM_ID}' + OR (r.current_recruitment IS NULL AND r.target_recruitment = '${IDM_ID}')) + JOIN public.deferment_decision_type ddt + ON dd.type_id = ddt.id +WHERE ddt.code IN ('1', '2'); Y @@ -455,8 +435,8 @@ WHERE d.type_code = '2' -- освобождение Y - 880 - 464 + 1152 + 544 @@ -490,7 +470,139 @@ WHERE d.type_code = '2' -- освобождение 688 - 464 + 544 + + + + Group by + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + has_deferment_raw + + MAX + + + + has_liberation_raw + + MAX + + + + N + + + recruit_id + + + N + + grp + + + 816 + 544 + + + + User defined Java expression + Janino + + Y + + 1 + + none + + + + has_deferment + has_deferment_raw == 1 + Boolean + -1 + -1 + + + + has_liberation + has_liberation_raw == 1 + Boolean + -1 + -1 + + + + + 976 + 544 + + + + Abort 2 + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 1520 + 544 + + + + Change job status on error 2 + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1312 + 544 @@ -508,7 +620,7 @@ WHERE d.type_code = '2' -- освобождение Update - Change job status on error + Change job status on error 2 Y error_description diff --git a/mappings/info_recruits/citizen_tables/deferment_liberation/parallel/deferment_liberation_flow_delta.hpl b/mappings/info_recruits/citizen_tables/deferment_liberation/parallel/deferment_liberation_flow_delta.hpl index 58fc303..8155b04 100644 --- a/mappings/info_recruits/citizen_tables/deferment_liberation/parallel/deferment_liberation_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/deferment_liberation/parallel/deferment_liberation_flow_delta.hpl @@ -45,11 +45,6 @@ Change job status on error Y - - Update - Change job status on error - Y - Table input Identify last row in a stream @@ -66,18 +61,38 @@ Y - Identify last row in a stream - sort_by_recruit_id + Insert / update + Filter rows Y - sort_by_recruit_id + Change job status on error 2 + Abort 2 + Y + + + Group by + User defined Java expression + Y + + + Update + Change job status on error 2 + Y + + + User defined Java expression Update Y - Insert / update - Filter rows + sort_by_recruit_id + Group by + Y + + + Identify last row in a stream + sort_by_recruit_id Y @@ -98,7 +113,7 @@ 1392 - 512 + 432 @@ -134,7 +149,7 @@ and recruitment_id = '${IDM_ID}'; 1184 - 512 + 432 @@ -354,68 +369,33 @@ and recruitment_id = '${IDM_ID}'; postgres.decision-document-service N - WITH -part_recruits AS ( - SELECT - id - FROM public.recruit - WHERE '${IDM_ID}' != '' -- Проверка на пустую строку - AND (current_recruitment = '${IDM_ID}' or (current_recruitment is null and target_recruitment = '${IDM_ID}')) -), -p_deferment_decision AS ( - SELECT - d.id, - d.recruit_id, - d.base_deferment, - d.created_at, - d.updated_at, - d.postponement_date, - d.hidden, - d.fact_provision, - ddt.code AS type_code - FROM public.deferment_decision d - JOIN public.deferment_decision_type ddt ON d.type_id = ddt.id - WHERE d.updated_at >= '${M_DEFDEC_UP_DATE}'::timestamp -) -SELECT - d.id AS deferment_id, - d.recruit_id, - d.base_deferment, --Основание на отсрочку от призыва/Основание на освобождение от призыва - d.created_at,--Дата начала действия отсрочки от призыва/Дата начала действия отвобождения от призыва - d.updated_at, - d.postponement_date, --Дата окончания действия отсрочки от призыва/Дата окончания действия освобождения от призыва - d.hidden, - 'deferment' AS type, - 'fact' AS permission_fact, -- ключ принадлежности к решению по отсрочке/освобождению - CASE - WHEN d.fact_provision = '1' THEN true - ELSE false - END AS has_deferment,--Факт предоставления отсрочки от призыва/Факт предоставления освобождения от призыва (1 - предоставлено, 2 - не предоставлено) - false AS has_liberation -FROM p_deferment_decision d -JOIN part_recruits r ON r.id = d.recruit_id -WHERE d.type_code = '1' -- отсрочка - -UNION ALL - -SELECT - d.id AS deferment_id, - d.recruit_id, - d.base_deferment, --Основание на отсрочку от призыва/Основание на освобождение от призыва - d.created_at,--Дата начала действия отсрочки от призыва/Дата начала действия отвобождения от призыва - d.updated_at, - d.postponement_date, --Дата окончания действия отсрочки от призыва/Дата окончания действия освобождения от призыва - d.hidden, - 'liberation' AS type, - 'fact' AS permission_fact, -- ключ принадлежности к решению по отсрочке/освобождению - false AS has_deferment, - CASE - WHEN d.fact_provision = '1' THEN true - ELSE false - END AS has_liberation-- Факт предоставления отсрочки от призыва/Факт предоставления освобождения от призыва (1 - предоставлено, 2 - не предоставлено) -FROM p_deferment_decision d -JOIN part_recruits r ON r.id = d.recruit_id -WHERE d.type_code = '2' -- освобождение + SELECT + dd.id AS deferment_id, + dd.recruit_id, + dd.base_deferment, + dd.created_at, + dd.updated_at, + dd.postponement_date, + dd.hidden, + CASE ddt.code + WHEN '1' THEN 'deferment' + WHEN '2' THEN 'liberation' + END AS type, + 'fact' AS permission_fact, + (dd.fact_provision = '1') AS has_deferment, + -- true только если тип "освобождение" и факт предоставления = 1 + (ddt.code = '2' AND dd.fact_provision = '1') AS has_liberation +FROM public.deferment_decision dd +JOIN public.recruit r + ON r.id = dd.recruit_id + AND (r.current_recruitment = '${IDM_ID}' + OR (r.current_recruitment IS NULL AND r.target_recruitment = '${IDM_ID}')) +JOIN public.deferment_decision_type ddt + ON dd.type_id = ddt.id +WHERE ddt.code IN ('1', '2') +and case when '${MAX_SOURCE_UPDATE_DATE}' <> '' + then dd.updated_at > to_date('${MAX_SOURCE_UPDATE_DATE}', 'YYYY-MM-DD') + else true end; Y @@ -423,6 +403,100 @@ WHERE d.type_code = '2' -- освобождение 352 + + Abort 2 + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 1584 + 544 + + + + Change job status on error 2 + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1376 + 544 + + + + Group by + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + has_deferment_raw + MAX + + + has_liberation_raw + MAX + + + N + + + recruit_id + + + N + grp + + + 880 + 544 + + Update Update @@ -458,8 +532,41 @@ WHERE d.type_code = '2' -- освобождение Y - 928 - 512 + 1216 + 544 + + + + User defined Java expression + Janino + + Y + + 1 + + none + + + + has_deferment + has_deferment_raw == 1 + Boolean + -1 + -1 + + + + has_liberation + has_liberation_raw == 1 + Boolean + -1 + -1 + + + + + 1040 + 544 @@ -493,7 +600,7 @@ WHERE d.type_code = '2' -- освобождение 752 - 512 + 544 @@ -511,7 +618,7 @@ WHERE d.type_code = '2' -- освобождение Update - Change job status on error + Change job status on error 2 Y error_description diff --git a/mappings/info_recruits/citizen_tables/deferment_liberation/parallel/deferment_liberation_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/deferment_liberation/parallel/deferment_liberation_flow_repeat.hpl index f4bb3dd..f3ceda1 100644 --- a/mappings/info_recruits/citizen_tables/deferment_liberation/parallel/deferment_liberation_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/deferment_liberation/parallel/deferment_liberation_flow_repeat.hpl @@ -45,11 +45,6 @@ Change job status on error Y - - Update - Change job status on error - Y - Table input Identify last row in a stream @@ -66,18 +61,38 @@ Y - Identify last row in a stream - sort_by_recruit_id + Insert / update + Filter rows Y - sort_by_recruit_id + Change job status on error 2 + Abort 2 + Y + + + Group by + User defined Java expression + Y + + + Update + Change job status on error 2 + Y + + + User defined Java expression Update Y - Insert / update - Filter rows + sort_by_recruit_id + Group by + Y + + + Identify last row in a stream + sort_by_recruit_id Y @@ -97,8 +112,8 @@ 0 - 1312 - 400 + 1264 + 304 @@ -133,8 +148,8 @@ and recruitment_id = '${IDM_ID}'; - 1120 - 400 + 1072 + 304 @@ -305,7 +320,6 @@ DO UPDATE SET = deferment_id deferment_id - ervu_dashboard deferment_liberation
@@ -370,67 +384,30 @@ DO UPDATE SET postgres.decision-document-service N - WITH -part_recruits AS ( - SELECT - id - FROM public.recruit - WHERE '${IDM_ID}' != '' -- Проверка на пустую строку - AND (current_recruitment = '${IDM_ID}' or (current_recruitment is null and target_recruitment = '${IDM_ID}')) -), -p_deferment_decision AS ( - SELECT - d.id, - d.recruit_id, - d.base_deferment, - d.created_at, - d.updated_at, - d.postponement_date, - d.hidden, - d.fact_provision, - ddt.code AS type_code - FROM public.deferment_decision d - JOIN public.deferment_decision_type ddt ON d.type_id = ddt.id -) -SELECT - d.id AS deferment_id, - d.recruit_id, - d.base_deferment, --Основание на отсрочку от призыва/Основание на освобождение от призыва - d.created_at,--Дата начала действия отсрочки от призыва/Дата начала действия отвобождения от призыва - d.updated_at, - d.postponement_date, --Дата окончания действия отсрочки от призыва/Дата окончания действия освобождения от призыва - d.hidden, - 'deferment' AS type, - 'fact' AS permission_fact, -- ключ принадлежности к решению по отсрочке/освобождению - CASE - WHEN d.fact_provision = '1' THEN true - ELSE false - END AS has_deferment,--Факт предоставления отсрочки от призыва/Факт предоставления освобождения от призыва (1 - предоставлено, 2 - не предоставлено) - false AS has_liberation -FROM p_deferment_decision d -JOIN part_recruits r ON r.id = d.recruit_id -WHERE d.type_code = '1' -- отсрочка - -UNION ALL - -SELECT - d.id AS deferment_id, - d.recruit_id, - d.base_deferment, --Основание на отсрочку от призыва/Основание на освобождение от призыва - d.created_at,--Дата начала действия отсрочки от призыва/Дата начала действия отвобождения от призыва - d.updated_at, - d.postponement_date, --Дата окончания действия отсрочки от призыва/Дата окончания действия освобождения от призыва - d.hidden, - 'liberation' AS type, - 'fact' AS permission_fact, -- ключ принадлежности к решению по отсрочке/освобождению - false AS has_deferment, - CASE - WHEN d.fact_provision = '1' THEN true - ELSE false - END AS has_liberation-- Факт предоставления отсрочки от призыва/Факт предоставления освобождения от призыва (1 - предоставлено, 2 - не предоставлено) -FROM p_deferment_decision d -JOIN part_recruits r ON r.id = d.recruit_id -WHERE d.type_code = '2' -- освобождение + SELECT + dd.id AS deferment_id, + dd.recruit_id, + dd.base_deferment, + dd.created_at, + dd.updated_at, + dd.postponement_date, + dd.hidden, + CASE ddt.code + WHEN '1' THEN 'deferment' + WHEN '2' THEN 'liberation' + END AS type, + 'fact' AS permission_fact, + (dd.fact_provision = '1') AS has_deferment, + -- true только если тип "освобождение" и факт предоставления = 1 + (ddt.code = '2' AND dd.fact_provision = '1') AS has_liberation +FROM public.deferment_decision dd +JOIN public.recruit r + ON r.id = dd.recruit_id + AND (r.current_recruitment = '${IDM_ID}' + OR (r.current_recruitment IS NULL AND r.target_recruitment = '${IDM_ID}')) +JOIN public.deferment_decision_type ddt + ON dd.type_id = ddt.id +WHERE ddt.code IN ('1', '2'); Y @@ -438,6 +415,100 @@ WHERE d.type_code = '2' -- освобождение 208 + + Abort 2 + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 1488 + 432 + + + + Change job status on error 2 + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1280 + 432 + + + + Group by + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + has_deferment_raw + MAX + + + has_liberation_raw + MAX + + + N + + + recruit_id + + + N + grp + + + 784 + 432 + + Update Update @@ -473,8 +544,41 @@ WHERE d.type_code = '2' -- освобождение Y - 832 - 400 + 1120 + 432 + + + + User defined Java expression + Janino + + Y + + 1 + + none + + + + has_deferment + has_deferment_raw == 1 + Boolean + -1 + -1 + + + + has_liberation + has_liberation_raw == 1 + Boolean + -1 + -1 + + + + + 944 + 432 @@ -508,7 +612,7 @@ WHERE d.type_code = '2' -- освобождение 656 - 400 + 432 @@ -526,7 +630,7 @@ WHERE d.type_code = '2' -- освобождение Update - Change job status on error + Change job status on error 2 Y error_description diff --git a/mappings/info_recruits/citizen_tables/deferment_liberation/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/deferment_liberation/recruitment_five_flow_delta.hpl index 11b96cf..47060c2 100644 --- a/mappings/info_recruits/citizen_tables/deferment_liberation/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/deferment_liberation/recruitment_five_flow_delta.hpl @@ -21,73 +21,56 @@ - Table input + Append streams + Sort rows + Y + + + failed_deferment_liberation_permission_idm_ids_on_delta + Append streams + Y + + + max_update_date_from_deferment_liberation + failed_deferment_liberation_permission_idm_ids_on_delta + Y + + + max_update_date_from_deferment_liberation + updated_idm_ids_fromrecruits_deferment + Y + + + updated_idm_ids_fromrecruits_deferment + Append streams + Y + + + Sort rows deferment_liberation_flow_delta.hpl Y - Table input + Sort rows deferment_liberation_flow_delta.hpl 2 Y - Table input + Sort rows deferment_liberation_flow_delta.hpl 3 Y - Table input + Sort rows deferment_liberation_flow_delta.hpl 4 Y - Table input + Sort rows deferment_liberation_flow_delta.hpl 5 Y - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - WITH mud AS (SELECT recruitment_id, - MAX(execution_datetime) AS max_upd_date - FROM etl.job_execution - WHERE job_name = '${JOB_NAME}' - AND status IN ('SUCCESS', 'DELTA_SUCCESS') - GROUP BY recruitment_id) -SELECT r.idm_id -FROM ervu_dashboard.recruitment r - JOIN mud ON mud.recruitment_id = r.idm_id - JOIN recruits_info ri - ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id - AND ri.updated_at > mud.max_upd_date - -union - -select r2.idm_id -from ervu_dashboard.recruitment r2 - join etl.job_execution je - on r2.idm_id = je.recruitment_id -where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - and je.job_name = '${JOB_NAME}';; - Y - - - 352 - 288 - - deferment_liberation_flow_delta.hpl PipelineExecutor @@ -135,8 +118,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - 640 - 128 + 1152 + 224 @@ -186,8 +169,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - 656 - 208 + 1152 + 304 @@ -237,8 +220,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - 672 - 288 + 1152 + 384 @@ -288,8 +271,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - 672 - 384 + 1152 + 464 @@ -339,8 +322,142 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - 640 - 464 + 1152 + 544 + + + + Append streams + Append + + Y + + 1 + + none + + + updated_idm_ids_fromrecruits_deferment + failed_deferment_liberation_permission_idm_ids_on_delta + + + 704 + 384 + + + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruitment_id + Y + N + N + 0 + N + + + + + 896 + 384 + + + + failed_deferment_liberation_permission_idm_ids_on_delta + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + max_update_date_from_deferment_liberation + select r.idm_id as recruitment_id, + ? as max_update_date +from ervu_dashboard.recruitment r + join etl.job_execution je + on r.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y + + + 496 + 480 + + + + max_update_date_from_deferment_liberation + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + 0 + select max(updated_at) as max_update_date + from deferment_liberation; + N + + + 304 + 384 + + + + updated_idm_ids_fromrecruits_deferment + TableInput + + Y + + 1 + + none + + + ervu_person_registry + N + 0 + max_update_date_from_deferment_liberation + WITH max_update_date(val) AS (SELECT CAST(? AS timestamp)) +SELECT DISTINCT COALESCE(r.current_recruitment, r.target_recruitment) AS recruitment_id, + mud.val as max_update_date +FROM public.deferment_decision dd + JOIN recruit r ON dd.recruit_id = r.id + JOIN max_update_date mud ON TRUE +WHERE (mud.val IS NULL OR dd.updated_at > mud.val); + N + + + 496 + 288 diff --git a/mappings/info_recruits/citizen_tables/deferment_liberation/row_last_deferment_decision_update_date.hpl b/mappings/info_recruits/citizen_tables/deferment_liberation/row_last_deferment_decision_update_date.hpl deleted file mode 100644 index feeba5a..0000000 --- a/mappings/info_recruits/citizen_tables/deferment_liberation/row_last_deferment_decision_update_date.hpl +++ /dev/null @@ -1,168 +0,0 @@ - - - - row_last_deferment_decision_update_date - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/06/02 14:51:59.611 - - - 2025/06/02 14:51:59.611 - - - - - - Copy rows to result - Set variables - N - - - Table input - Copy rows to result - N - - - Table input - Set variables - Y - - - Set variables - Write to log - N - - - - Copy rows to result - RowsToResult - - Y - - 1 - - none - - - - - 912 - 128 - - - - Set variables - SetVariable - - Y - - 1 - - none - - - - - - MAX_DEFERMENT_DECISION_UPDATE_DATE - M_DEFDEC_UP_DATE - GP_WORKFLOW - - - - MAX_RECRUITS_DEFERMENT_UPDATE_DATE - M_RECDEF_UP_DATE - GP_WORKFLOW - - - Y - - - 1232 - 320 - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - with base as( -SELECT - (SELECT system_update_date - FROM public.recruit_create - WHERE workflow = 'deferment_decision_table' - AND system_update_date = (SELECT MAX(system_update_date) FROM public.recruit_create) - ORDER BY record_created DESC - LIMIT 1) AS MAX_DEFERMENT_DECISION_UPDATE_DATE, - - (SELECT system_update_date - FROM public.recruit_create - WHERE workflow = 'recruits_deferment_table' - AND system_update_date = (SELECT MAX(system_update_date) FROM public.recruit_create) - ORDER BY record_created DESC - LIMIT 1) AS MAX_RECRUITS_DEFERMENT_UPDATE_DATE -) -select - CASE - WHEN MAX_DEFERMENT_DECISION_UPDATE_DATE is null THEN '0001-01-01 00:00:00' - ELSE MAX_DEFERMENT_DECISION_UPDATE_DATE - END AS MAX_DEFERMENT_DECISION_UPDATE_DATE, - CASE - WHEN MAX_RECRUITS_DEFERMENT_UPDATE_DATE is null THEN '0001-01-01 00:00:00' - ELSE MAX_RECRUITS_DEFERMENT_UPDATE_DATE - END AS MAX_RECRUITS_DEFERMENT_UPDATE_DATE -from base - N - - - 688 - 320 - - - - Write to log - WriteToLog - - Y - - 1 - - none - - - Y - - - N - 0 - Basic - ${M_RECDEF_UP_DATE} -${M_DEFDEC_UP_DATE} - - - 1440 - 176 - - - - - - diff --git a/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow.hpl b/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow.hpl index 516e4a2..f2381cc 100644 --- a/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow.hpl +++ b/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow.hpl @@ -260,29 +260,25 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu_person_registry N - with filtered_data as (SELECT - d.id AS deferment_id, - d.recruit_id, - rd.code AS base_deferment, -- коды основания права на отсрочку - d.postponement_date, -- Дата окончания действия права на отсрочку - d.date_of_information, -- Дата начала действия права на отсрочку - d.hidden, - d.system_create_date AS created_at, - d.system_update_date AS updated_at, - 'permission' AS permission_fact, -- ключ принадлежности к праву на отсрочку - NOT d.hidden AS defer_conscription, -- наличие права на отсрочку от призыва - CTID + SELECT d.id AS deferment_id, + d.recruit_id, + rd.code AS base_deferment, -- коды основания права на отсрочку + d.postponement_date, -- Дата окончания действия права на отсрочку + d.date_of_information, -- Дата начала действия права на отсрочку + false, + d.system_create_date AS created_at, + d.system_update_date AS updated_at, + 'permission' AS permission_fact, -- ключ принадлежности к праву на отсрочку + true AS defer_conscription -- наличие права на отсрочку от призыва FROM public.recruits r JOIN public.recruits_deferment d ON r.id = d.recruit_id JOIN public.ervu_reason_for_deferment rd ON d.ervu_reason_for_deferment_id = rd.id WHERE (r.current_recruitment_id = '${IDM_ID}' - OR (r.current_recruitment_id IS NULL AND r.target_recruitment_id = '${IDM_ID}') - )) -select fd.*, - fd.ctid = (select max(ctid) from filtered_data) as last_row - from filtered_data fd + OR (r.current_recruitment_id IS NULL AND r.target_recruitment_id = '${IDM_ID}') + ) +and rd.hidden is false; Y diff --git a/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_delta.hpl b/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_delta.hpl index 58cba80..1a91cef 100644 --- a/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_delta.hpl @@ -337,32 +337,28 @@ and recruitment_id = '${IDM_ID}'; ervu_person_registry N - with filtered_data as (SELECT - d.id AS deferment_id, - d.recruit_id, - rd.code AS base_deferment, -- коды основания права на отсрочку - d.postponement_date, -- Дата окончания действия права на отсрочку - d.date_of_information, -- Дата начала действия права на отсрочку - d.hidden, - d.system_create_date AS created_at, - d.system_update_date AS updated_at, - 'permission' AS permission_fact, -- ключ принадлежности к праву на отсрочку - NOT d.hidden AS defer_conscription, -- наличие права на отсрочку от призыва - CTID - FROM public.recruits r - JOIN public.recruits_deferment d - ON r.id = d.recruit_id - JOIN public.ervu_reason_for_deferment rd - ON d.ervu_reason_for_deferment_id = rd.id - WHERE (r.current_recruitment_id = '${IDM_ID}' - OR (r.current_recruitment_id IS NULL AND r.target_recruitment_id = '${IDM_ID}') - ) - and case when '${MAX_SOURCE_UPDATE_DATE}' <> '' + SELECT d.id AS deferment_id, + d.recruit_id, + rd.code AS base_deferment, -- коды основания права на отсрочку + d.postponement_date, -- Дата окончания действия права на отсрочку + d.date_of_information, -- Дата начала действия права на отсрочку + false, + d.system_create_date AS created_at, + d.system_update_date AS updated_at, + 'permission' AS permission_fact, -- ключ принадлежности к праву на отсрочку + true AS defer_conscription -- наличие права на отсрочку от призыва +FROM public.recruits r + JOIN public.recruits_deferment d + ON r.id = d.recruit_id + JOIN public.ervu_reason_for_deferment rd + ON d.ervu_reason_for_deferment_id = rd.id +WHERE (r.current_recruitment_id = '${IDM_ID}' + OR (r.current_recruitment_id IS NULL AND r.target_recruitment_id = '${IDM_ID}') + ) +and rd.hidden is false +and case when '${MAX_SOURCE_UPDATE_DATE}' <> '' then d.system_update_date > to_date('${MAX_SOURCE_UPDATE_DATE}', 'YYYY-MM-DD') - else true end ) -select fd.*, - fd.ctid = (select max(ctid) from filtered_data) as last_row -from filtered_data fd + else true end; Y diff --git a/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_repeat.hpl index cec0779..050143e 100644 --- a/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_repeat.hpl @@ -352,29 +352,25 @@ DO UPDATE SET ervu_person_registry N - with filtered_data as (SELECT - d.id AS deferment_id, - d.recruit_id, - rd.code AS base_deferment, -- коды основания права на отсрочку - d.postponement_date, -- Дата окончания действия права на отсрочку - d.date_of_information, -- Дата начала действия права на отсрочку - d.hidden, - d.system_create_date AS created_at, - d.system_update_date AS updated_at, - 'permission' AS permission_fact, -- ключ принадлежности к праву на отсрочку - NOT d.hidden AS defer_conscription, -- наличие права на отсрочку от призыва - CTID + SELECT d.id AS deferment_id, + d.recruit_id, + rd.code AS base_deferment, -- коды основания права на отсрочку + d.postponement_date, -- Дата окончания действия права на отсрочку + d.date_of_information, -- Дата начала действия права на отсрочку + false, + d.system_create_date AS created_at, + d.system_update_date AS updated_at, + 'permission' AS permission_fact, -- ключ принадлежности к праву на отсрочку + true AS defer_conscription -- наличие права на отсрочку от призыва FROM public.recruits r JOIN public.recruits_deferment d ON r.id = d.recruit_id JOIN public.ervu_reason_for_deferment rd ON d.ervu_reason_for_deferment_id = rd.id WHERE (r.current_recruitment_id = '${IDM_ID}' - OR (r.current_recruitment_id IS NULL AND r.target_recruitment_id = '${IDM_ID}') - )) -select fd.*, - fd.ctid = (select max(ctid) from filtered_data) as last_row - from filtered_data fd + OR (r.current_recruitment_id IS NULL AND r.target_recruitment_id = '${IDM_ID}') + ) + and rd.hidden is false Y diff --git a/mappings/info_recruits/citizen_tables/deferment_liberation_permission/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/deferment_liberation_permission/recruitment_five_flow_delta.hpl index d1a53ae..86e9b3d 100644 --- a/mappings/info_recruits/citizen_tables/deferment_liberation_permission/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/deferment_liberation_permission/recruitment_five_flow_delta.hpl @@ -82,6 +82,8 @@ none + updated_idm_ids_fromrecruits_deferment + failed_deferment_liberation_permission_idm_ids_on_delta 592 @@ -471,7 +473,7 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') max_update_date_from_deferment_liberation WITH max_update_date(val) AS (SELECT CAST(? AS timestamp)) SELECT DISTINCT COALESCE(r.current_recruitment_id, r.target_recruitment_id) AS recruitment_id, - ? as max_update_date + mud.val as max_update_date FROM recruits r JOIN recruits_deferment rd ON rd.recruit_id = r.id JOIN max_update_date mud ON TRUE diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow.hpl index fd55124..60c8dd5 100644 --- a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow.hpl +++ b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow.hpl @@ -134,7 +134,7 @@ WHERE code = 'educationLevel'; N 0 SELECT - key, + key as code, value ->> 'value' as value, NOT hidden as actual, updated_at as update_date diff --git a/mappings/info_recruits/citizen_tables/subpoena/job_last_update_date_subpoena.hwf b/mappings/info_recruits/citizen_tables/subpoena/job_last_update_date_subpoena.hwf deleted file mode 100644 index b30b7cc..0000000 --- a/mappings/info_recruits/citizen_tables/subpoena/job_last_update_date_subpoena.hwf +++ /dev/null @@ -1,126 +0,0 @@ - - - job_last_update_date_subpoena - Y - - - - - - 2025/06/02 14:48:05.666 - - - 2025/06/02 14:48:05.666 - - - - - Start - - SPECIAL - - 1 - 12 - 60 - 0 - 0 - N - 0 - 1 - N - 400 - 352 - - - - subpoena_update_date_ervu_dashboard.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/subpoena/subpoena_update_date_ervu_dashboard.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 784 - 352 - - - - row_last_subpoena_update_date.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/subpoena/row_last_subpoena_update_date.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 1264 - 352 - - - - Success - - SUCCESS - - N - 1600 - 352 - - - - - - Start - subpoena_update_date_ervu_dashboard.hpl - Y - Y - Y - - - subpoena_update_date_ervu_dashboard.hpl - row_last_subpoena_update_date.hpl - Y - Y - N - - - row_last_subpoena_update_date.hpl - Success - Y - Y - N - - - - - - diff --git a/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow.hpl b/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow.hpl index 4fab2d6..5bba8a1 100644 --- a/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow.hpl +++ b/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow.hpl @@ -45,11 +45,6 @@ Change job status on error Y - - Update - Change job status on error - Y - Table input Identify last row in a stream @@ -70,14 +65,34 @@ sort_by_recruit_id Y + + Table output + Filter rows + Y + sort_by_recruit_id + Group by + Y + + + Group by + User defined Java expression + Y + + + User defined Java expression Update Y - Table output - Filter rows + Change job status on error 2 + Abort 2 + Y + + + Update + Change job status on error 2 Y @@ -97,8 +112,8 @@ 0 - 1424 - 512 + 1408 + 432 @@ -133,8 +148,8 @@ and recruitment_id = '${IDM_ID}'; - 1200 - 512 + 1184 + 432 @@ -251,6 +266,61 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 320 + + Group by + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + has_sent_summons + has_sent_summons + MAX + + + + has_served_summons + has_served_summons + MAX + + + + appeared_to_recruitment_center + appeared_to_recruitment_center + MAX + + + + not_appeared_by_valid_excuse + not_appeared_by_valid_excuse + MAX + + + + N + + + recruit_id + + + N + + grp + + + 864 + 656 + + Identify last row in a stream DetectLastRow @@ -283,50 +353,43 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); postgres.subpoena N SELECT - s.id, --id повестки - CASE - WHEN s.send_date is not null THEN true - ELSE false - END AS has_sent_summons, --факт направления повестки///Признак наличия направленной повестки - CASE - WHEN s.system_stages->'subpoenaAwarded' != 'true' THEN true - ELSE false - END AS has_served_summons, --has_served_summons//Признак наличия врученной повестки - CASE - WHEN s.system_stages->'recruitAttendanceConfirm' != 'true' THEN true - ELSE false - END AS appeared_to_recruitment_center, -- appeared_to_recruitment_center//Признак явки в военный комиссариат - CASE - WHEN ss.code = '5.1' THEN true - ELSE false - END AS not_appeared_by_valid_excuse, -- not_appeared_by_valid_excuse//Признак неявки в военкомат по уважительной причине - s.recruit_id, - s.send_date, --дата направления повестки - s.series, --серия повестки - s.number, --новер повестки - s.reason_id AS subpoena_reason, --причина вызова по повестке /// код передавать или сразу наименование? - s.full_name_responsible_user, --фио военного комиссара(фио пользователя сформировавшео повестку) - s.sig_info, --сертификат ключа проверки ЭП - s.recruitment_name, --наименование ВК - s.recruitment_address, --адрес, по которому необходимо явиться - s.visit_date, --дата и время явки - --, --способ направления // искать в subpoena_send_info - --, --УН заказного почтового отправления // искать в subpoena_send_info - s.status_id, --статус // ссылка на справочник subpoena_status/// AS subpoena_status - s.create_date, --дата создания // для загрузки данных - s.status_change_date, --дата обновления // для загрузки данных (timestamp without time zone) - sr.type AS subpoena_reason_type --- поля для карточки, раздел "Сведения о направлении повестки военного комиссариата" - + s.id, --id повестки + CASE + WHEN (s.system_stages->>'subpoenaDirected')::text = 'true' THEN 1 + ELSE 0 + END AS has_sent_summons_raw, --факт направления повестки///Признак наличия направленной повестки + CASE + WHEN (s.system_stages->>'subpoenaAwarded') = 'true' THEN 1 + ELSE 0 + END AS has_served_summons_raw, --has_served_summons//Признак наличия врученной повестки + CASE + WHEN (s.system_stages->>'recruitAttendanceConfirm') = 'true' THEN 1 + ELSE 0 + END AS appeared_to_recruitment_center_raw, -- appeared_to_recruitment_center//Признак явки в военный комиссариат + CASE + WHEN ss.code = '5.1' THEN 1 + ELSE 0 + END AS not_appeared_by_valid_excuse_raw, -- not_appeared_by_valid_excuse//Признак неявки в военкомат по уважительной причине + s.recruit_id, + s.send_date, --дата направления повестки + s.series, --серия повестки + s.number, --новер повестки + s.reason_id AS subpoena_reason, --причина вызова по повестке /// код передавать или сразу наименование? + s.full_name_responsible_user, --фио военного комиссара(фио пользователя сформировавшео повестку) + s.sig_info, --сертификат ключа проверки ЭП + s.recruitment_name, --наименование ВК + s.recruitment_address, --адрес, по которому необходимо явиться + s.visit_date, --дата и время явки + --, --способ направления // искать в subpoena_send_info + --, --УН заказного почтового отправления // искать в subpoena_send_info + s.status_id, --статус // ссылка на справочник subpoena_status/// AS subpoena_status + s.create_date, --дата создания // для загрузки данных + s.status_change_date, --дата обновления // для загрузки данных (timestamp without time zone) + sr.type AS subpoena_reason_type -- поля для карточки, раздел "Сведения о направлении повестки военного комиссариата" FROM public.subpoena s -join public.subpoena_status ss on s.status_id = ss.id -join public.subpoena_reason sr on sr.id = s.reason_id - -WHERE - - '${IDM_ID}' != '' -- Проверка на пустую строку - - AND s.department_id = '${IDM_ID}' + join public.subpoena_status ss on s.status_id = ss.id + join public.subpoena_reason sr on sr.id = s.reason_id +WHERE s.department_id = '${IDM_ID}' Y @@ -424,7 +487,7 @@ WHERE Y - 912 + 1040 320 @@ -471,8 +534,8 @@ WHERE Y - 912 - 512 + 1232 + 656 @@ -492,7 +555,7 @@ WHERE N - Y + N recruit_id @@ -506,7 +569,112 @@ WHERE 704 - 512 + 656 + + + + User defined Java expression + Janino + + Y + + 1 + + none + + + + has_sent_summons + has_sent_summons_raw == 1 + Boolean + -1 + -1 + + + + has_served_summons + has_served_summons_raw == 1 + Boolean + -1 + -1 + + + + appeared_to_recruitment_center + appeared_to_recruitment_center_raw == 1 + Boolean + -1 + -1 + + + + not_appeared_by_valid_excuse + not_appeared_by_valid_excuse_raw == 1 + Boolean + -1 + -1 + + + + + 1056 + 656 + + + + Abort 2 + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 1632 + 656 + + + + Change job status on error 2 + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1408 + 656 @@ -524,7 +692,7 @@ WHERE Update - Change job status on error + Change job status on error 2 Y error_description diff --git a/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow_delta.hpl b/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow_delta.hpl index 74eef4b..b49d619 100644 --- a/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow_delta.hpl @@ -45,11 +45,6 @@ Change job status on error Y - - Update - Change job status on error - Y - Table input Identify last row in a stream @@ -66,18 +61,38 @@ Y - Identify last row in a stream - sort_by_recruit_id + Insert / update + Filter rows Y - sort_by_recruit_id + Change job status on error 2 + Abort 2 + Y + + + Group by + User defined Java expression + Y + + + Update + Change job status on error 2 + Y + + + User defined Java expression Update Y - Insert / update - Filter rows + sort_by_recruit_id + Group by + Y + + + Identify last row in a stream + sort_by_recruit_id Y @@ -97,8 +112,8 @@ 0 - 1488 - 880 + 1456 + 800 @@ -133,8 +148,8 @@ and recruitment_id = '${IDM_ID}'; - 1312 - 880 + 1280 + 800 @@ -390,47 +405,43 @@ and recruitment_id = '${IDM_ID}'; postgres.subpoena N SELECT - s.id, --id повестки - CASE - WHEN s.send_date is not null THEN true - ELSE false - END AS has_sent_summons, --факт направления повестки///Признак наличия направленной повестки - CASE - WHEN s.system_stages->'subpoenaAwarded' != 'true' THEN true - ELSE false - END AS has_served_summons, --has_served_summons//Признак наличия врученной повестки - CASE - WHEN s.system_stages->'recruitAttendanceConfirm' != 'true' THEN true - ELSE false - END AS appeared_to_recruitment_center, -- appeared_to_recruitment_center//Признак явки в военный комиссариат - CASE - WHEN ss.code = '5.1' THEN true - ELSE false - END AS not_appeared_by_valid_excuse, -- not_appeared_by_valid_excuse//Признак неявки в военкомат по уважительной причине - s.recruit_id, - s.send_date, --дата направления повестки - s.series, --серия повестки - s.number, --новер повестки - s.reason_id AS subpoena_reason, --причина вызова по повестке /// код передавать или сразу наименование? - s.full_name_responsible_user, --фио военного комиссара(фио пользователя сформировавшео повестку) - s.sig_info, --сертификат ключа проверки ЭП - s.recruitment_name, --наименование ВК - s.recruitment_address, --адрес, по которому необходимо явиться - s.visit_date, --дата и время явки - --, --способ направления // искать в subpoena_send_info - --, --УН заказного почтового отправления // искать в subpoena_send_info - s.status_id, --статус // ссылка на справочник subpoena_status/// AS subpoena_status - s.create_date, --дата создания // для загрузки данных - s.status_change_date, --дата обновления // для загрузки данных (timestamp without time zone) - sr.type AS subpoena_reason_type --- поля для карточки, раздел "Сведения о направлении повестки военного комиссариата" - + s.id, --id повестки + CASE + WHEN (s.system_stages->>'subpoenaDirected')::text = 'true' THEN 1 + ELSE 0 + END AS has_sent_summons_raw, --факт направления повестки///Признак наличия направленной повестки + CASE + WHEN (s.system_stages->>'subpoenaAwarded') = 'true' THEN 1 + ELSE 0 + END AS has_served_summons_raw, --has_served_summons//Признак наличия врученной повестки + CASE + WHEN (s.system_stages->>'recruitAttendanceConfirm') = 'true' THEN 1 + ELSE 0 + END AS appeared_to_recruitment_center_raw, -- appeared_to_recruitment_center//Признак явки в военный комиссариат + CASE + WHEN ss.code = '5.1' THEN 1 + ELSE 0 + END AS not_appeared_by_valid_excuse_raw, -- not_appeared_by_valid_excuse//Признак неявки в военкомат по уважительной причине + s.recruit_id, + s.send_date, --дата направления повестки + s.series, --серия повестки + s.number, --новер повестки + s.reason_id AS subpoena_reason, --причина вызова по повестке /// код передавать или сразу наименование? + s.full_name_responsible_user, --фио военного комиссара(фио пользователя сформировавшео повестку) + s.sig_info, --сертификат ключа проверки ЭП + s.recruitment_name, --наименование ВК + s.recruitment_address, --адрес, по которому необходимо явиться + s.visit_date, --дата и время явки + --, --способ направления // искать в subpoena_send_info + --, --УН заказного почтового отправления // искать в subpoena_send_info + s.status_id, --статус // ссылка на справочник subpoena_status/// AS subpoena_status + s.create_date, --дата создания // для загрузки данных + s.status_change_date, --дата обновления // для загрузки данных (timestamp without time zone) + sr.type AS subpoena_reason_type -- поля для карточки, раздел "Сведения о направлении повестки военного комиссариата" FROM public.subpoena s -join public.subpoena_status ss on s.status_id = ss.id -join public.subpoena_reason sr on sr.id = s.reason_id -WHERE - '${IDM_ID}' != '' -- Проверка на пустую строку - AND s.department_id = '${IDM_ID}' + join public.subpoena_status ss on s.status_id = ss.id + join public.subpoena_reason sr on sr.id = s.reason_id +WHERE s.department_id = '${IDM_ID}' AND s.status_change_date >= '${M_S_UP_DATE}'::timestamp Y @@ -439,6 +450,112 @@ WHERE 704 + + Abort 2 + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 1760 + 960 + + + + Change job status on error 2 + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1536 + 960 + + + + Group by + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + has_sent_summons + has_sent_summons + MAX + + + has_served_summons + has_served_summons + MAX + + + appeared_to_recruitment_center + appeared_to_recruitment_center + MAX + + + not_appeared_by_valid_excuse + not_appeared_by_valid_excuse + MAX + + + N + + + recruit_id + + + N + grp + + + 992 + 960 + + Update Update @@ -482,8 +599,57 @@ WHERE Y - 1056 - 880 + 1360 + 960 + + + + User defined Java expression + Janino + + Y + + 1 + + none + + + + has_sent_summons + has_sent_summons_raw == 1 + Boolean + -1 + -1 + + + + has_served_summons + has_served_summons_raw == 1 + Boolean + -1 + -1 + + + + appeared_to_recruitment_center + appeared_to_recruitment_center_raw == 1 + Boolean + -1 + -1 + + + + not_appeared_by_valid_excuse + not_appeared_by_valid_excuse_raw == 1 + Boolean + -1 + -1 + + + + + 1184 + 960 @@ -503,7 +669,7 @@ WHERE N - Y + N recruit_id @@ -517,7 +683,7 @@ WHERE 848 - 880 + 960 @@ -535,7 +701,7 @@ WHERE Update - Change job status on error + Change job status on error 2 Y error_description diff --git a/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow_repeat.hpl index 463874c..80bb5ee 100644 --- a/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/subpoena/parallel/subpoena_flow_repeat.hpl @@ -45,11 +45,6 @@ Change job status on error Y - - Update - Change job status on error - Y - Table input Identify last row in a stream @@ -66,18 +61,38 @@ Y - Identify last row in a stream - sort_by_recruit_id + Insert / update + Filter rows + Y + + + Group by + User defined Java expression + Y + + + Update + Change job status on error 2 Y sort_by_recruit_id + Group by + Y + + + User defined Java expression Update Y - Insert / update - Filter rows + Change job status on error 2 + Abort 2 + Y + + + Identify last row in a stream + sort_by_recruit_id Y @@ -97,8 +112,28 @@ 0 - 1472 - 752 + 1424 + 656 + + + + Abort 2 + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 1760 + 816 @@ -133,8 +168,44 @@ and recruitment_id = '${IDM_ID}'; - 1280 - 752 + 1232 + 656 + + + + Change job status on error 2 + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1536 + 816 @@ -269,6 +340,56 @@ DO UPDATE SET 560 + + Group by + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + has_sent_summons + has_sent_summons + MAX + + + has_served_summons + has_served_summons + MAX + + + appeared_to_recruitment_center + appeared_to_recruitment_center + MAX + + + not_appeared_by_valid_excuse + not_appeared_by_valid_excuse + MAX + + + N + + + recruit_id + + + N + grp + + + 992 + 816 + + Identify last row in a stream DetectLastRow @@ -305,7 +426,6 @@ DO UPDATE SET = id id - ervu_dashboard subpoena
@@ -405,49 +525,45 @@ DO UPDATE SET postgres.subpoena N + SELECT - s.id, --id повестки - CASE - WHEN s.send_date is not null THEN true - ELSE false - END AS has_sent_summons, --факт направления повестки///Признак наличия направленной повестки - CASE - WHEN s.system_stages->'subpoenaAwarded' != 'true' THEN true - ELSE false - END AS has_served_summons, --has_served_summons//Признак наличия врученной повестки - CASE - WHEN s.system_stages->'recruitAttendanceConfirm' != 'true' THEN true - ELSE false - END AS appeared_to_recruitment_center, -- appeared_to_recruitment_center//Признак явки в военный комиссариат - CASE - WHEN ss.code = '5.1' THEN true - ELSE false - END AS not_appeared_by_valid_excuse, -- not_appeared_by_valid_excuse//Признак неявки в военкомат по уважительной причине - s.recruit_id, - s.send_date, --дата направления повестки - s.series, --серия повестки - s.number, --новер повестки - s.reason_id AS subpoena_reason, --причина вызова по повестке /// код передавать или сразу наименование? - s.full_name_responsible_user, --фио военного комиссара(фио пользователя сформировавшео повестку) - s.sig_info, --сертификат ключа проверки ЭП - s.recruitment_name, --наименование ВК - s.recruitment_address, --адрес, по которому необходимо явиться - s.visit_date, --дата и время явки - --, --способ направления // искать в subpoena_send_info - --, --УН заказного почтового отправления // искать в subpoena_send_info - s.status_id, --статус // ссылка на справочник subpoena_status/// AS subpoena_status - s.create_date, --дата создания // для загрузки данных - s.status_change_date, --дата обновления // для загрузки данных (timestamp without time zone) - sr.type AS subpoena_reason_type --- поля для карточки, раздел "Сведения о направлении повестки военного комиссариата" - + s.id, --id повестки + CASE + WHEN (s.system_stages->>'subpoenaDirected')::text = 'true' THEN 1 + ELSE 0 + END AS has_sent_summons_raw, --факт направления повестки///Признак наличия направленной повестки + CASE + WHEN (s.system_stages->>'subpoenaAwarded') = 'true' THEN 1 + ELSE 0 + END AS has_served_summons_raw, --has_served_summons//Признак наличия врученной повестки + CASE + WHEN (s.system_stages->>'recruitAttendanceConfirm') = 'true' THEN 1 + ELSE 0 + END AS appeared_to_recruitment_center_raw, -- appeared_to_recruitment_center//Признак явки в военный комиссариат + CASE + WHEN ss.code = '5.1' THEN 1 + ELSE 0 + END AS not_appeared_by_valid_excuse_raw, -- not_appeared_by_valid_excuse//Признак неявки в военкомат по уважительной причине + s.recruit_id, + s.send_date, --дата направления повестки + s.series, --серия повестки + s.number, --новер повестки + s.reason_id AS subpoena_reason, --причина вызова по повестке /// код передавать или сразу наименование? + s.full_name_responsible_user, --фио военного комиссара(фио пользователя сформировавшео повестку) + s.sig_info, --сертификат ключа проверки ЭП + s.recruitment_name, --наименование ВК + s.recruitment_address, --адрес, по которому необходимо явиться + s.visit_date, --дата и время явки + --, --способ направления // искать в subpoena_send_info + --, --УН заказного почтового отправления // искать в subpoena_send_info + s.status_id, --статус // ссылка на справочник subpoena_status/// AS subpoena_status + s.create_date, --дата создания // для загрузки данных + s.status_change_date, --дата обновления // для загрузки данных (timestamp without time zone) + sr.type AS subpoena_reason_type -- поля для карточки, раздел "Сведения о направлении повестки военного комиссариата" FROM public.subpoena s -join public.subpoena_status ss on s.status_id = ss.id -join public.subpoena_reason sr on sr.id = s.reason_id -WHERE - '${IDM_ID}' != '' -- Проверка на пустую строку - AND s.department_id = '${IDM_ID}' - + join public.subpoena_status ss on s.status_id = ss.id + join public.subpoena_reason sr on sr.id = s.reason_id +WHERE s.department_id = '${IDM_ID}' Y @@ -498,8 +614,57 @@ WHERE Y - 1056 - 752 + 1360 + 816 + + + + User defined Java expression + Janino + + Y + + 1 + + none + + + + has_sent_summons + has_sent_summons_raw == 1 + Boolean + -1 + -1 + + + + has_served_summons + has_served_summons_raw == 1 + Boolean + -1 + -1 + + + + appeared_to_recruitment_center + appeared_to_recruitment_center_raw == 1 + Boolean + -1 + -1 + + + + not_appeared_by_valid_excuse + not_appeared_by_valid_excuse_raw == 1 + Boolean + -1 + -1 + + + + + 1184 + 816 @@ -519,7 +684,7 @@ WHERE N - Y + N recruit_id @@ -533,7 +698,7 @@ WHERE 848 - 752 + 816 @@ -551,7 +716,7 @@ WHERE Update - Change job status on error + Change job status on error 2 Y error_description diff --git a/mappings/info_recruits/citizen_tables/subpoena/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/subpoena/recruitment_five_flow_delta.hpl index 9d7c020..4c5a018 100644 --- a/mappings/info_recruits/citizen_tables/subpoena/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/subpoena/recruitment_five_flow_delta.hpl @@ -21,33 +21,111 @@ - Table input + failed_subpoena_idm_ids_on_delta + Append streams + Y + + + max_update_date_from_subpoena + failed_subpoena_idm_ids_on_delta + Y + + + max_update_date_from_subpoena + updated_idm_ids_from_subpoena + Y + + + updated_idm_ids_from_subpoena + Append streams + Y + + + Append streams + Sort rows + Y + + + Sort rows subpoena_flow_delta.hpl Y - Table input + Sort rows subpoena_flow_delta.hpl 2 Y - Table input + Sort rows subpoena_flow_delta.hpl 3 Y - Table input + Sort rows subpoena_flow_delta.hpl 4 Y - Table input + Sort rows subpoena_flow_delta.hpl 5 Y - Table input + Append streams + Append + + Y + + 1 + + none + + + failed_subpoena_idm_ids_on_delta + updated_idm_ids_from_subpoena + + + 704 + 304 + + + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruitment_id + Y + N + N + 0 + N + + + + + 864 + 304 + + + + failed_subpoena_idm_ids_on_delta TableInput Y @@ -60,32 +138,42 @@ ervu-dashboard N 0 - WITH mud AS (SELECT recruitment_id, - MAX(execution_datetime) AS max_upd_date - FROM etl.job_execution - WHERE job_name = '${JOB_NAME}' - AND status IN ('SUCCESS', 'DELTA_SUCCESS') - GROUP BY recruitment_id) -SELECT r.idm_id -FROM ervu_dashboard.recruitment r - JOIN mud ON mud.recruitment_id = r.idm_id - JOIN recruits_info ri - ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id - AND ri.updated_at > mud.max_upd_date - -union - -select r2.idm_id -from ervu_dashboard.recruitment r2 + max_update_date_from_subpoena + select r.idm_id as recruitment_id, + ? as max_update_date +from ervu_dashboard.recruitment r join etl.job_execution je - on r2.idm_id = je.recruitment_id + on r.idm_id = je.recruitment_id where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') and je.job_name = '${JOB_NAME}'; Y - 352 - 288 + 496 + 400 + + + + max_update_date_from_subpoena + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + 0 + select max(status_change_date) as max_update_date + from subpoena; + N + + + 304 + 304 @@ -112,6 +200,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') idm_id + + MAX_UPDATE_DATE + max_update_date + + Y @@ -135,8 +228,36 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - 640 - 128 + 1088 + 144 + + + + updated_idm_ids_from_subpoena + TableInput + + Y + + 1 + + none + + + postgres.subpoena + N + 0 + max_update_date_from_subpoena + WITH max_update_date(val) AS (SELECT CAST(? AS timestamp)) +SELECT DISTINCT department_id, + ? as max_update_date +FROM subpoena s +JOIN max_update_date mud ON TRUE +WHERE (mud.val IS NULL OR s.status_change_date > mud.val); + N + + + 496 + 208 @@ -163,6 +284,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') idm_id + + MAX_UPDATE_DATE + max_update_date + + Y @@ -186,8 +312,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - 640 - 208 + 1088 + 224 @@ -214,6 +340,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') idm_id + + MAX_UPDATE_DATE + max_update_date + + Y @@ -237,8 +368,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - 640 - 288 + 1088 + 304 @@ -265,6 +396,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') idm_id + + MAX_UPDATE_DATE + max_update_date + + Y @@ -288,8 +424,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - 640 - 368 + 1088 + 384 @@ -316,6 +452,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') idm_id + + MAX_UPDATE_DATE + max_update_date + + Y @@ -339,8 +480,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') - 640 - 448 + 1088 + 464 diff --git a/mappings/info_recruits/citizen_tables/subpoena/row_last_subpoena_update_date.hpl b/mappings/info_recruits/citizen_tables/subpoena/row_last_subpoena_update_date.hpl deleted file mode 100644 index bf1d17c..0000000 --- a/mappings/info_recruits/citizen_tables/subpoena/row_last_subpoena_update_date.hpl +++ /dev/null @@ -1,142 +0,0 @@ - - - - row_last_subpoena_update_date - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/06/02 14:51:59.611 - - - 2025/06/02 14:51:59.611 - - - - - - Copy rows to result - Set variables - N - - - Table input - Copy rows to result - N - - - Table input - Set variables - Y - - - Set variables - Write to log - N - - - - Copy rows to result - RowsToResult - - Y - - 1 - - none - - - - - 912 - 176 - - - - Set variables - SetVariable - - Y - - 1 - - none - - - - - - MAX_SUBPOENA_UPDATE_DATE - M_S_UP_DATE - GP_WORKFLOW - - - Y - - - 1232 - 320 - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT - system_update_date AS MAX_SUBPOENA_UPDATE_DATE--, ---record_created -FROM public.recruit_create -WHERE system_update_date = (select max(system_update_date) from public.recruit_create where workflow = 'subpoena_table') -order by record_created desc -limit 1 - N - - - 688 - 320 - - - - Write to log - WriteToLog - - Y - - 1 - - none - - - Y - - - N - 0 - Basic - ${M_S_UP_DATE} - - - 1360 - 192 - - - - - - diff --git a/mappings/info_recruits/citizen_tables/subpoena/subpoena_update_date_ervu_dashboard.hpl b/mappings/info_recruits/citizen_tables/subpoena/subpoena_update_date_ervu_dashboard.hpl deleted file mode 100644 index 939d780..0000000 --- a/mappings/info_recruits/citizen_tables/subpoena/subpoena_update_date_ervu_dashboard.hpl +++ /dev/null @@ -1,93 +0,0 @@ - - - - subpoena_update_date_ervu_dashboard - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/06/02 14:50:31.833 - - - 2025/06/02 14:50:31.833 - - - - - - Table input - Table output - Y - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT - COALESCE(MAX(create_date), '0001-01-01 00:00:00'::timestamp) AS system_create_date, - COALESCE(MAX(status_change_date), '0001-01-01 00:00:00'::timestamp) AS system_update_date, - current_timestamp AS record_created, - 'subpoena_table' AS workflow -FROM ervu_dashboard.subpoena - N - - - 896 - 304 - - - - Table output - TableOutput - - Y - - 1 - - none - - - 1000 - ervu-dashboard - - - N - N - N - N - Y - N - public - N - recruit_create
- N - Y - N - Y - - - 1216 - 304 - - - - - - diff --git a/mappings/info_recruits/job_general_info_recruits.hwf b/mappings/info_recruits/job_general_info_recruits.hwf index fc74a71..aa1a970 100644 --- a/mappings/info_recruits/job_general_info_recruits.hwf +++ b/mappings/info_recruits/job_general_info_recruits.hwf @@ -279,7 +279,7 @@ IF job_parallel_recruits_info_citizen SUCCESS job_parallel_subpoena_decision_archive_person.hwf - N + Y Y N diff --git a/mappings/info_recruits/job_parallel_subpoena_decision_archive_person.hwf b/mappings/info_recruits/job_parallel_subpoena_decision_archive_person.hwf index 0975e46..6264971 100644 --- a/mappings/info_recruits/job_parallel_subpoena_decision_archive_person.hwf +++ b/mappings/info_recruits/job_parallel_subpoena_decision_archive_person.hwf @@ -25,119 +25,23 @@ N 0 1 - Y - 688 + N + 784 1232 - - citizen_liability_fz53_job.hwf - - WORKFLOW - - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_liability_fz53/citizen_liability_fz53_job.hwf - Nothing - - - M_R_CR_DATE - ${M_R_CR_DATE} - - - M_INFR_UP_DATE - ${M_INFR_UP_DATE} - - Y - - N - local - N - N - Y - N - 1264 - 1040 - - - - citizen_criminal_liability_fz53_job.hwf - - WORKFLOW - - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_criminal_liability_fz53/citizen_criminal_liability_fz53_job.hwf - Nothing - - - M_R_CR_DATE - ${M_R_CR_DATE} - - - M_INFR_UP_DATE - ${M_INFR_UP_DATE} - - Y - - N - local - N - N - Y - N - 1264 - 976 - - - - citizen_appealing_violations_fz53_job.hwf - - WORKFLOW - - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_appealing_violations_fz53/citizen_appealing_violations_fz53_job.hwf - Nothing - - - M_R_CR_DATE - ${M_R_CR_DATE} - - - M_R_UP_DATE - ${M_R_UP_DATE} - - Y - - N - local - N - N - Y - N - 1264 - 912 - - Success SUCCESS N - 1664 + 1408 1232 - deferment_liberation_job.hwf + job_reference_data.hwf WORKFLOW @@ -145,17 +49,9 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation/deferment_liberation_job.hwf + ${PROJECT_HOME}/info_recruits/reference_data/job_reference_data.hwf Nothing - - M_DEFDEC_UP_DATE - ${M_DEFDEC_UP_DATE} - - - M_R_CR_DATE - ${M_R_CR_DATE} - Y N @@ -164,76 +60,12 @@ N Y N - 1264 - 1104 - - - - deferment_liberation_permission_job.hwf - - WORKFLOW - - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation_permission/deferment_liberation_permission_job.hwf - Nothing - - - M_RECDEF_UP_DATE - ${M_RECDEF_UP_DATE} - - - M_R_CR_DATE - ${M_R_CR_DATE} - - Y - - N - local - N - N - Y - N - 1264 - 1168 - - - - recruit_archive_job.hwf - - WORKFLOW - - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/recruit_archive/recruit_archive_job.hwf - Nothing - - - M_RECARCH_UP_DATE - ${M_RECARCH_UP_DATE} - - - M_R_CR_DATE - ${M_R_CR_DATE} - - Y - - N - local - N - N - Y - N - 1264 + 960 1232 - subpoena_job.hwf + job_deferment_fz53.hwf WORKFLOW @@ -241,340 +73,8 @@ N N N - ${PROJECT_HOME}/info_recruits/citizen_tables/subpoena/subpoena_job.hwf + ${PROJECT_HOME}/info_recruits/raw_data/job_deferment_fz53.hwf Nothing - - - M_S_UP_DATE - ${M_S_UP_DATE} - - - M_R_CR_DATE - ${M_R_CR_DATE} - - Y - - N - local - N - N - Y - N - 1264 - 1488 - - - - restriction_document_job.hwf - - WORKFLOW - - N - N - N - N - ${PROJECT_HOME}/info_recruits/raw_data/restriction_document/restriction_document_job.hwf - Nothing - - - M_RESTRDOC_UP_DATE - ${M_RESTRDOC_UP_DATE} - - - M_R_CR_DATE - ${M_R_CR_DATE} - - Y - - N - local - N - N - Y - N - 1264 - 1296 - - - - restriction_document_item_job.hwf - - WORKFLOW - - N - N - N - N - ${PROJECT_HOME}/info_recruits/raw_data/restriction_document_item/restriction_document_item_job.hwf - Nothing - - - M_RDI_UP_DATE - ${M_RDI_UP_DATE} - - - M_R_CR_DATE - ${M_R_CR_DATE} - - Y - - N - local - N - N - Y - N - 1264 - 1360 - - - - subpoena_appearance_job.hwf - - WORKFLOW - - N - N - N - N - ${PROJECT_HOME}/info_recruits/raw_data/subpoena_appearance/subpoena_appearance_job.hwf - Nothing - - - M_SA_UP_DATE - ${M_SA_UP_DATE} - - - M_R_CR_DATE - ${M_R_CR_DATE} - - Y - - N - local - N - N - Y - N - 1264 - 1424 - - - - subpoena_send_info_job.hwf - - WORKFLOW - - N - N - N - N - ${PROJECT_HOME}/info_recruits/raw_data/subpoena_send_info/subpoena_send_info_job.hwf - Nothing - - - M_S_UP_DATE - ${M_S_UP_DATE} - - - M_R_CR_DATE - ${M_R_CR_DATE} - - Y - - N - local - N - N - Y - N - 1264 - 1552 - - - - job_last_update_date_subpoena.hwf - - WORKFLOW - - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/subpoena/job_last_update_date_subpoena.hwf - Nothing - - Y - - N - local - N - N - Y - Y - 944 - 1488 - - - - job_last_update_date_appeal_document.hwf - - WORKFLOW - - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_appealing_violations_fz53/job_last_update_date_appeal_document.hwf - Nothing - - Y - - N - local - N - N - Y - Y - 944 - 912 - - - - job_last_update_date_deferment_decision.hwf - - WORKFLOW - - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation/job_last_update_date_deferment_decision.hwf - Nothing - - Y - - N - local - N - N - Y - Y - 944 - 1104 - - - - job_last_update_date_restriction_document.hwf - - WORKFLOW - - N - N - N - N - ${PROJECT_HOME}/info_recruits/raw_data/restriction_document/job_last_update_date_restriction_document.hwf - Nothing - - Y - - N - local - N - N - Y - Y - 944 - 1296 - - - - job_last_update_date_restriction_document_item.hwf - - WORKFLOW - - N - N - N - N - ${PROJECT_HOME}/info_recruits/raw_data/restriction_document_item/job_last_update_date_restriction_document_item.hwf - Nothing - - Y - - N - local - N - N - Y - Y - 944 - 1360 - - - - job_last_update_date_subpoena_appearance.hwf - - WORKFLOW - - N - N - N - N - ${PROJECT_HOME}/info_recruits/raw_data/subpoena_appearance/job_last_update_date_subpoena_appearance.hwf - Nothing - - Y - - N - local - N - N - Y - Y - 944 - 1424 - - - - row_last_infringement_update_date.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/citizen_liability_fz53/row_last_infringement_update_date.hpl - Basic - - Y - - N - local - N - N - Y - Y - 944 - 976 - - - - row_last_recruit_archive_update_date.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/recruit_archive/row_last_recruit_archive_update_date.hpl - Basic Y @@ -584,271 +84,73 @@ N Y N - 944 - 1232 + 1200 + 1136 + + + + subpoena_restriction_archive_job.hwf + + WORKFLOW + + N + N + N + N + ${PROJECT_HOME}/info_recruits/raw_data/subpoena_restriction_archive_job.hwf + Nothing + + Y + + N + local + N + N + Y + N + 1200 + 1328 - - citizen_liability_fz53_job.hwf - Success - Y - Y - N - - - citizen_criminal_liability_fz53_job.hwf - Success - Y - Y - N - - - citizen_appealing_violations_fz53_job.hwf - Success - Y - Y - N - - - deferment_liberation_job.hwf - Success - Y - Y - N - - - deferment_liberation_permission_job.hwf - Success - Y - Y - N - - - recruit_archive_job.hwf - Success - Y - Y - N - - - subpoena_job.hwf - Success - Y - Y - N - - - restriction_document_job.hwf - Success - Y - Y - N - - - restriction_document_item_job.hwf - Success - Y - Y - N - - - subpoena_appearance_job.hwf - Success - Y - Y - N - - - subpoena_send_info_job.hwf - Success - Y - Y - N - Start - job_last_update_date_subpoena.hwf - N - Y - Y - - - job_last_update_date_subpoena.hwf - subpoena_send_info_job.hwf - Y - Y - N - - - job_last_update_date_subpoena.hwf - subpoena_job.hwf - Y - Y - N - - - Start - job_last_update_date_appeal_document.hwf - N - Y - Y - - - job_last_update_date_appeal_document.hwf - citizen_appealing_violations_fz53_job.hwf - N - Y - N - - - job_last_update_date_deferment_decision.hwf - deferment_liberation_job.hwf - Y - Y - N - - - Start - job_last_update_date_deferment_decision.hwf - N - Y - Y - - - Start - job_last_update_date_restriction_document.hwf - N - Y - Y - - - job_last_update_date_restriction_document.hwf - restriction_document_job.hwf - Y - Y - N - - - Start - job_last_update_date_restriction_document_item.hwf - N - Y - Y - - - job_last_update_date_restriction_document_item.hwf - restriction_document_item_job.hwf - Y - Y - N - - - job_last_update_date_subpoena_appearance.hwf - subpoena_appearance_job.hwf - Y - Y - N - - - Start - job_last_update_date_subpoena_appearance.hwf - N - Y - Y - - - job_last_update_date_deferment_decision.hwf - deferment_liberation_permission_job.hwf - Y - Y - N - - - Start - citizen_appealing_violations_fz53_job.hwf - N - Y - Y - - - Start - row_last_infringement_update_date.hpl + job_reference_data.hwf Y Y Y - row_last_infringement_update_date.hpl - citizen_criminal_liability_fz53_job.hwf + job_reference_data.hwf + job_deferment_fz53.hwf Y Y N - row_last_infringement_update_date.hpl - citizen_liability_fz53_job.hwf + job_reference_data.hwf + subpoena_restriction_archive_job.hwf Y Y N - Start - row_last_recruit_archive_update_date.hpl + job_deferment_fz53.hwf + Success Y Y - Y + N - row_last_recruit_archive_update_date.hpl - recruit_archive_job.hwf + subpoena_restriction_archive_job.hwf + Success Y Y N - - 251 - 232 - 201 - 90 - 58 - 14 - N - 90 - 58 - 14 - N - Segoe UI - 9 - 26 - 832 - 848 - в следующий пайплайн(check_status_job_info_recruits_from_json.hpl) родительского воркфлоу вписать новый созданный воркфлоу, который создан тут - 849 - - - 251 - 232 - 201 - 90 - 58 - 14 - N - 90 - 58 - 14 - N - Segoe UI - 9 - 26 - 1600 - 1376 - rdi - 24 - diff --git a/mappings/info_recruits/raw_data/subpoena_appearance/job_last_update_date_subpoena_appearance.hwf b/mappings/info_recruits/raw_data/subpoena_appearance/job_last_update_date_subpoena_appearance.hwf deleted file mode 100644 index 9bab539..0000000 --- a/mappings/info_recruits/raw_data/subpoena_appearance/job_last_update_date_subpoena_appearance.hwf +++ /dev/null @@ -1,126 +0,0 @@ - - - job_last_update_date_subpoena_appearance - Y - - - - - - 2025/06/02 14:48:05.666 - - - 2025/06/02 14:48:05.666 - - - - - Start - - SPECIAL - - 1 - 12 - 60 - 0 - 0 - N - 0 - 1 - N - 384 - 352 - - - - subpoena_appearance_update_date_ervu_dashboard.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/raw_data/subpoena_appearance/subpoena_appearance_update_date_ervu_dashboard.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 720 - 352 - - - - row_last_subpoena_appearance_update_date.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/raw_data/subpoena_appearance/row_last_subpoena_appearance_update_date.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 1232 - 352 - - - - Success - - SUCCESS - - N - 1584 - 352 - - - - - - Start - subpoena_appearance_update_date_ervu_dashboard.hpl - Y - Y - Y - - - subpoena_appearance_update_date_ervu_dashboard.hpl - row_last_subpoena_appearance_update_date.hpl - Y - Y - N - - - row_last_subpoena_appearance_update_date.hpl - Success - Y - Y - N - - - - - - diff --git a/mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow.hpl b/mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow.hpl index 212fc2f..00f203b 100644 --- a/mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow.hpl +++ b/mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow.hpl @@ -265,14 +265,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); postgres.subpoena N + select sa.* from public.subpoena_appearance sa join public.subpoena s on s.id = sa.subpoena_id -WHERE - '${IDM_ID}' != '' -- Проверка на пустую строку - AND s.department_id = '${IDM_ID}' - AND '${M_R_CR_DATE}' >= sa.created_date_time -${LIMIT_FW} +WHERE s.department_id = '${IDM_ID}'; Y @@ -327,14 +324,11 @@ ${LIMIT_FW} N N N - Y - N ervu_dashboard Y subpoena_appearance
- N Y N diff --git a/mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow_delta.hpl b/mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow_delta.hpl index 9919ce4..1ae98bf 100644 --- a/mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow_delta.hpl +++ b/mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow_delta.hpl @@ -341,12 +341,8 @@ and recruitment_id = '${IDM_ID}'; select sa.* from public.subpoena_appearance sa join public.subpoena s on s.id = sa.subpoena_id -WHERE - '${IDM_ID}' != '' -- Проверка на пустую строку - AND s.department_id = '${IDM_ID}' - AND sa.change_date_time >= '${M_SA_UP_DATE}'::timestamp - -${LIMIT_FW} +WHERE s.department_id = '${IDM_ID}'; + AND sa.change_date_time >= '${MAX_UPDATE_DATE}'; Y diff --git a/mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow_repeat.hpl b/mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow_repeat.hpl index 1e9c4a1..c69e200 100644 --- a/mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow_repeat.hpl +++ b/mappings/info_recruits/raw_data/subpoena_appearance/parallel/subpoena_appearance_flow_repeat.hpl @@ -288,13 +288,11 @@ DO UPDATE SET = id id - = created_date_time created_date_time - ervu_dashboard subpoena_appearance
@@ -354,14 +352,11 @@ DO UPDATE SET postgres.subpoena N + select sa.* from public.subpoena_appearance sa join public.subpoena s on s.id = sa.subpoena_id -WHERE - '${IDM_ID}' != '' -- Проверка на пустую строку - AND s.department_id = '${IDM_ID}' - AND '${M_R_CR_DATE}' >= sa.created_date_time -${LIMIT_FW} +WHERE s.department_id = '${IDM_ID}'; Y diff --git a/mappings/info_recruits/raw_data/subpoena_appearance/recruitment_five_flow_delta.hpl b/mappings/info_recruits/raw_data/subpoena_appearance/recruitment_five_flow_delta.hpl index 82c9e21..79355db 100644 --- a/mappings/info_recruits/raw_data/subpoena_appearance/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/raw_data/subpoena_appearance/recruitment_five_flow_delta.hpl @@ -21,33 +21,111 @@ - Table input + Append streams + Sort rows + Y + + + failed_subpoena_appearance_idm_ids_on_delta + Append streams + Y + + + max_update_date_from_subpoena_appearance + failed_subpoena_appearance_idm_ids_on_delta + Y + + + max_update_date_from_subpoena_appearance + updated_idm_ids_from_subpoena_appearance + Y + + + updated_idm_ids_from_subpoena_appearance + Append streams + Y + + + Sort rows subpoena_appearance_flow_delta.hpl Y - Table input + Sort rows subpoena_appearance_flow_delta.hpl 2 Y - Table input + Sort rows subpoena_appearance_flow_delta.hpl 3 Y - Table input + Sort rows subpoena_appearance_flow_delta.hpl 4 Y - Table input + Sort rows subpoena_appearance_flow_delta.hpl 5 Y - Table input + Append streams + Append + + Y + + 1 + + none + + + updated_idm_ids_from_subpoena_appearance + failed_subpoena_appearance_idm_ids_on_delta + + + 784 + 432 + + + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruitment_id + Y + N + N + 0 + N + + + + + 976 + 432 + + + + failed_subpoena_appearance_idm_ids_on_delta TableInput Y @@ -60,18 +138,42 @@ ervu-dashboard N 0 - SELECT - r.idm_id as recruitment_id -FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'subpoena_appearance_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); + max_update_date_from_subpoena_appearance + select r.idm_id as recruitment_id, + ? as max_update_date +from ervu_dashboard.recruitment r + join etl.job_execution je + on r.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y + + + 576 + 528 + + + + max_update_date_from_subpoena_appearance + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + 0 + select max(change_date_time) as max_update_date + from subpoena_appearance; N - 352 - 288 + 384 + 432 @@ -98,6 +200,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING recruitment_id + + MAX_UPDATE_DATE + max_update_date + + Y @@ -121,8 +228,8 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 608 - 128 + 1216 + 240 @@ -172,8 +279,8 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 608 - 208 + 1216 + 320 @@ -223,8 +330,8 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 608 - 288 + 1216 + 400 @@ -274,8 +381,8 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 608 - 368 + 1216 + 480 @@ -325,8 +432,37 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 608 - 448 + 1216 + 560 + + + + updated_idm_ids_from_subpoena_appearance + TableInput + + Y + + 1 + + none + + + postgres.subpoena + N + 0 + max_update_date_from_subpoena_appearance + WITH max_update_date(val) AS (SELECT CAST(? AS timestamp)) +SELECT DISTINCT s.department_id AS recruitment_id, + mud.val as max_update_date +FROM subpoena_appearance sa + JOIN subpoena s ON s.id = sa.subpoena_id + JOIN max_update_date mud ON TRUE +WHERE (mud.val IS NULL OR sa.change_date_time > mud.val); + N + + + 576 + 336 diff --git a/mappings/info_recruits/raw_data/subpoena_appearance/row_last_subpoena_appearance_update_date.hpl b/mappings/info_recruits/raw_data/subpoena_appearance/row_last_subpoena_appearance_update_date.hpl deleted file mode 100644 index ce8f1d8..0000000 --- a/mappings/info_recruits/raw_data/subpoena_appearance/row_last_subpoena_appearance_update_date.hpl +++ /dev/null @@ -1,142 +0,0 @@ - - - - row_last_subpoena_appearance_update_date - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/06/02 14:51:59.611 - - - 2025/06/02 14:51:59.611 - - - - - - Copy rows to result - Set variables - N - - - Table input - Copy rows to result - N - - - Table input - Set variables - Y - - - Set variables - Write to log - N - - - - Copy rows to result - RowsToResult - - Y - - 1 - - none - - - - - 896 - 176 - - - - Set variables - SetVariable - - Y - - 1 - - none - - - - - - MAX_SUBPOENA_APPEARANCE_UPDATE_DATE - M_SA_UP_DATE - GP_WORKFLOW - - - Y - - - 1232 - 320 - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT - system_update_date AS MAX_SUBPOENA_APPEARANCE_UPDATE_DATE--, ---record_created -FROM public.recruit_create -WHERE system_update_date = (select max(system_update_date) from public.recruit_create where workflow = 'subpoena_appearance_table') -order by record_created desc -limit 1 - N - - - 688 - 320 - - - - Write to log - WriteToLog - - Y - - 1 - - none - - - Y - - - N - 0 - Basic - ${M_SA_UP_DATE} - - - 1424 - 240 - - - - - - diff --git a/mappings/info_recruits/raw_data/subpoena_appearance/subpoena_appearance_update_date_ervu_dashboard.hpl b/mappings/info_recruits/raw_data/subpoena_appearance/subpoena_appearance_update_date_ervu_dashboard.hpl deleted file mode 100644 index 486215a..0000000 --- a/mappings/info_recruits/raw_data/subpoena_appearance/subpoena_appearance_update_date_ervu_dashboard.hpl +++ /dev/null @@ -1,93 +0,0 @@ - - - - subpoena_appearance_update_date_ervu_dashboard - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/06/02 14:50:31.833 - - - 2025/06/02 14:50:31.833 - - - - - - Table input - Table output - Y - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT - COALESCE(MAX(created_date_time), '0001-01-01 00:00:00'::timestamp) AS system_create_date, - COALESCE(MAX(change_date_time), '0001-01-01 00:00:00'::timestamp) AS system_update_date, - current_timestamp AS record_created, - 'subpoena_appearance_table' AS workflow -FROM ervu_dashboard.subpoena_appearance - N - - - 896 - 304 - - - - Table output - TableOutput - - Y - - 1 - - none - - - 1000 - ervu-dashboard - - - N - N - N - N - Y - N - public - N - recruit_create
- N - Y - N - Y - - - 1216 - 304 - - - - - - diff --git a/mappings/info_recruits/raw_data/subpoena_send_info/check_if_need_to_repeat_job.hpl b/mappings/info_recruits/raw_data/subpoena_send_info/check_if_need_to_repeat_job.hpl deleted file mode 100644 index fc7dea6..0000000 --- a/mappings/info_recruits/raw_data/subpoena_send_info/check_if_need_to_repeat_job.hpl +++ /dev/null @@ -1,86 +0,0 @@ - - - - check_if_need_to_repeat_job - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/08/08 12:27:51.630 - - - 2025/08/08 12:27:51.630 - - - - - - Table input - Set variables - Y - - - - Set variables - SetVariable - - Y - - 1 - - none - - - - - need_to_repeat_job - NEED_TO_REPEAT_JOB - ROOT_WORKFLOW - - - Y - - - 624 - 288 - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT EXISTS (SELECT 1 -FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'subpoena_send_info_job' -where je.status is null or je.status in ('ERROR', 'PROCESSING') -limit 1 -) as need_to_repeat_job; - N - - - 336 - 288 - - - - - - diff --git a/mappings/info_recruits/raw_data/subpoena_send_info/checkpoints/check_if_subpoena_send_info_job_exists.hpl b/mappings/info_recruits/raw_data/subpoena_send_info/checkpoints/check_if_subpoena_send_info_job_exists.hpl deleted file mode 100644 index 2f4bd39..0000000 --- a/mappings/info_recruits/raw_data/subpoena_send_info/checkpoints/check_if_subpoena_send_info_job_exists.hpl +++ /dev/null @@ -1,113 +0,0 @@ - - - - check_if_subpoena_send_info_job_exists - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/08/05 13:28:46.661 - - - 2025/08/05 13:28:46.661 - - - - - - Set variables - Write to log - N - - - Table input - Set variables - Y - - - - Set variables - SetVariable - - Y - - 1 - - none - - - - - job_executed_flag - JOB_EXECUTED_FLAG - PARENT_WORKFLOW - - - Y - - - 672 - 336 - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT EXISTS ( - SELECT 1 - FROM etl.job_execution - WHERE job_name = 'subpoena_send_info_job' - limit 1 - ) AS job_executed_flag; - N - - - 384 - 336 - - - - Write to log - WriteToLog - - Y - - 1 - - none - - - Y - - - N - 0 - Basic - ${STATUS} - - - 944 - 224 - - - - - - diff --git a/mappings/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow.hpl b/mappings/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow.hpl index 84b1d61..bfed15c 100644 --- a/mappings/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow.hpl +++ b/mappings/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow.hpl @@ -265,14 +265,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); postgres.subpoena N - select ssi.* + select ssi.*, + current_timestamp as update_date from public.subpoena_send_info ssi -join public.subpoena s on s.id = ssi.subpoena_id -WHERE - '${IDM_ID}' != '' -- Проверка на пустую строку - AND s.department_id = '${IDM_ID}' - AND s.create_date >= '${M_R_UP_DATE}'::timestamp -${LIMIT_FW} + join public.subpoena s on s.id = ssi.subpoena_id +WHERE s.department_id = '${IDM_ID}'; Y @@ -355,14 +352,11 @@ ${LIMIT_FW} N N N - Y - N ervu_dashboard Y subpoena_send_info
- N Y N diff --git a/mappings/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow_delta.hpl b/mappings/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow_delta.hpl index 80dcf27..2a599d1 100644 --- a/mappings/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow_delta.hpl +++ b/mappings/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow_delta.hpl @@ -368,14 +368,12 @@ and recruitment_id = '${IDM_ID}'; postgres.subpoena N - select ssi.* + select ssi.*, + current_timestamp as update_date from public.subpoena_send_info ssi -join public.subpoena s on s.id = ssi.subpoena_id -WHERE - '${IDM_ID}' != '' -- Проверка на пустую строку - AND s.department_id = '${IDM_ID}' - AND s.status_change_date >= '${M_S_UP_DATE}'::timestamp -${LIMIT_FW} + join public.subpoena s on s.id = ssi.subpoena_id +WHERE s.department_id = '${IDM_ID}' +and s.status_change_date > '${MAX_UPDATE_DATE}'; Y diff --git a/mappings/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow_repeat.hpl b/mappings/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow_repeat.hpl index 13d2b39..476580a 100644 --- a/mappings/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow_repeat.hpl +++ b/mappings/info_recruits/raw_data/subpoena_send_info/parallel/subpoena_send_info_flow_repeat.hpl @@ -288,7 +288,6 @@ DO UPDATE SET = id id - ervu_dashboard subpoena_send_info
@@ -383,14 +382,12 @@ DO UPDATE SET postgres.subpoena N - select ssi.* + + select ssi.*, + current_timestamp as update_date from public.subpoena_send_info ssi -join public.subpoena s on s.id = ssi.subpoena_id -WHERE - '${IDM_ID}' != '' -- Проверка на пустую строку - AND s.department_id = '${IDM_ID}' - AND s.create_date >= '${M_R_UP_DATE}'::timestamp -${LIMIT_FW} + join public.subpoena s on s.id = ssi.subpoena_id +WHERE s.department_id = '${IDM_ID}'; Y diff --git a/mappings/info_recruits/raw_data/subpoena_send_info/recruitment_five_flow_delta.hpl b/mappings/info_recruits/raw_data/subpoena_send_info/recruitment_five_flow_delta.hpl index e1e6863..1f26c80 100644 --- a/mappings/info_recruits/raw_data/subpoena_send_info/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/raw_data/subpoena_send_info/recruitment_five_flow_delta.hpl @@ -21,33 +21,111 @@ - Table input + failed_subpoena_appearance_idm_ids_on_delta + Append streams + Y + + + max_update_date_from_subpoena_appearance + failed_subpoena_appearance_idm_ids_on_delta + Y + + + max_update_date_from_subpoena_appearance + updated_idm_ids_from_subpoena_appearance + Y + + + updated_idm_ids_from_subpoena_appearance + Append streams + Y + + + Append streams + Sort rows + Y + + + Sort rows subpoena_send_info_flow_delta.hpl Y - Table input + Sort rows subpoena_send_info_flow_delta.hpl 2 Y - Table input + Sort rows subpoena_send_info_flow_delta.hpl 3 Y - Table input + Sort rows subpoena_send_info_flow_delta.hpl 4 Y - Table input + Sort rows subpoena_send_info_flow_delta.hpl 5 Y - Table input + Append streams + Append + + Y + + 1 + + none + + + updated_idm_ids_from_subpoena_appearance + failed_subpoena_appearance_idm_ids_on_delta + + + 928 + 352 + + + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruitment_id + Y + N + N + 0 + N + + + + + 1104 + 352 + + + + failed_subpoena_appearance_idm_ids_on_delta TableInput Y @@ -60,18 +138,42 @@ ervu-dashboard N 0 - SELECT - r.idm_id as recruitment_id -FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = 'subpoena_send_info_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); + max_update_date_from_subpoena_appearance + select r.idm_id as recruitment_id, + ? as max_update_date +from ervu_dashboard.recruitment r + join etl.job_execution je + on r.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y + + + 720 + 448 + + + + max_update_date_from_subpoena_appearance + TableInput + + N + + 1 + + none + + + ervu-dashboard + N + 0 + select max(update_date) as max_update_date + from subpoena_send_info; N - 352 - 288 + 528 + 352 @@ -98,6 +200,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING recruitment_id + + MAX_UPDATE_DATE + max_update_date + + Y @@ -121,8 +228,36 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 624 - 128 + 1360 + 192 + + + + updated_idm_ids_from_subpoena_appearance + TableInput + + Y + + 1 + + none + + + postgres.subpoena + N + 0 + max_update_date_from_subpoena_appearance + WITH max_update_date(val) AS (SELECT CAST(? AS timestamp)) +SELECT DISTINCT s.department_id AS recruitment_id, + mud.val as max_update_date +FROM subpoena s + JOIN max_update_date mud ON TRUE +WHERE (mud.val IS NULL OR s.status_change_date> mud.val); + N + + + 720 + 256 @@ -149,6 +284,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING recruitment_id + + MAX_UPDATE_DATE + max_update_date + + Y @@ -172,8 +312,8 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 624 - 208 + 1360 + 272 @@ -200,6 +340,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING recruitment_id + + MAX_UPDATE_DATE + max_update_date + + Y @@ -223,8 +368,8 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 624 - 288 + 1360 + 352 @@ -251,6 +396,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING recruitment_id + + MAX_UPDATE_DATE + max_update_date + + Y @@ -274,8 +424,8 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 624 - 368 + 1360 + 432 @@ -302,6 +452,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING recruitment_id + + MAX_UPDATE_DATE + max_update_date + + Y @@ -325,8 +480,8 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 624 - 448 + 1360 + 512 diff --git a/mappings/info_recruits/recruitment/job_recruitment.hwf b/mappings/info_recruits/recruitment/job_recruitment.hwf index f7585aa..66616d6 100644 --- a/mappings/info_recruits/recruitment/job_recruitment.hwf +++ b/mappings/info_recruits/recruitment/job_recruitment.hwf @@ -70,32 +70,6 @@ 224 - - recruitment_param_create_date.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/recruitment/recruitment_param_create_date.hpl - Basic - - Y - - N - local - N - N - Y - N - 336 - 320 - - @@ -105,20 +79,6 @@ Y N - - Start - recruitment_param_create_date.hpl - N - Y - Y - - - recruitment_param_create_date.hpl - recruitment.hpl - N - Y - N - Start recruitment.hpl diff --git a/mappings/info_recruits/reference_data/job_reference_data.hwf b/mappings/info_recruits/reference_data/job_reference_data.hwf index 9b96309..f335438 100644 --- a/mappings/info_recruits/reference_data/job_reference_data.hwf +++ b/mappings/info_recruits/reference_data/job_reference_data.hwf @@ -27,7 +27,7 @@ 1 N 432 - 352 + 416 @@ -224,8 +224,6 @@ N N ${PROJECT_HOME}/info_recruits/reference_data/deferment_reason.hpl - - Basic Y diff --git a/mappings/job_general_info_recruits.hwf b/mappings/job_general_info_recruits.hwf new file mode 100644 index 0000000..fc74a71 --- /dev/null +++ b/mappings/job_general_info_recruits.hwf @@ -0,0 +1,325 @@ + + + job_general_info_recruits + Y + + + + 0 + - + 2025/05/21 13:22:33.264 + - + 2025/05/21 13:22:33.264 + + + M_R_CR_DATE + + + + + M_R_UP_DATE + + + + + + + Start + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 720 + 672 + + + + job_recruitment.hwf 2 + + WORKFLOW + + N + N + N + N + ${PROJECT_HOME}/info_recruits/recruitment/job_recruitment.hwf + Nothing + + Y + + N + local + N + N + Y + N + 976 + 672 + + + + job_parallel_recruits_info_citizen.hwf + + WORKFLOW + + N + N + N + N + ${PROJECT_HOME}/info_recruits/job_parallel_recruits_info_citizen.hwf + Nothing + + + M_R_CR_DATE + ${M_R_CR_DATE} + + Y + + N + local + N + N + Y + N + 1216 + 672 + + + + job_info_recruits_from_json.hwf + + WORKFLOW + + N + N + N + N + ${PROJECT_HOME}/info_recruits/job_info_recruits_from_json.hwf + Nothing + + + M_R_CR_DATE + ${M_R_CR_DATE} + + + M_R_UP_DATE + ${M_R_UP_DATE} + + Y + + N + local + N + N + Y + N + 2400 + 768 + + + + check_status_parallel_recruits_info_citizen.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/supportive_jobs/check_status_parallel_recruits_info_citizen.hpl + Basic + + Y + + N + local + N + N + Y + N + 1664 + 672 + + + + IF job_parallel_recruits_info_citizen ERROR + + SIMPLE_EVAL + + ERROR + boolean + true + equal + equal + N + variable + JOB_PARALLEL_ERROR_EXITS + N + 2048 + 528 + + + + IF job_parallel_recruits_info_citizen SUCCESS + + SIMPLE_EVAL + + SUCCESS + boolean + false + equal + equal + N + variable + JOB_PARALLEL_ERROR_EXITS + N + 2048 + 768 + + + + Abort parallel_recruits_info_citizen + + ABORT + + N + N + 2400 + 528 + + + + job_parallel_subpoena_decision_archive_person.hwf + + WORKFLOW + + N + N + N + N + ${PROJECT_HOME}/info_recruits/job_parallel_subpoena_decision_archive_person.hwf + Nothing + + + M_R_CR_DATE + ${M_R_CR_DATE} + + Y + + N + local + N + N + Y + N + 2400 + 880 + + + + Success_recruits_info_from_json + + SUCCESS + + N + 2720 + 768 + + + + Success_subpoena_decisions + + SUCCESS + + N + 2720 + 880 + + + + + + job_parallel_recruits_info_citizen.hwf + check_status_parallel_recruits_info_citizen.hpl + Y + Y + N + + + IF job_parallel_recruits_info_citizen SUCCESS + job_info_recruits_from_json.hwf + Y + Y + N + + + check_status_parallel_recruits_info_citizen.hpl + IF job_parallel_recruits_info_citizen ERROR + Y + N + Y + + + check_status_parallel_recruits_info_citizen.hpl + IF job_parallel_recruits_info_citizen SUCCESS + Y + N + Y + + + IF job_parallel_recruits_info_citizen SUCCESS + job_parallel_subpoena_decision_archive_person.hwf + N + Y + N + + + Start + job_recruitment.hwf 2 + Y + Y + Y + + + job_recruitment.hwf 2 + job_parallel_recruits_info_citizen.hwf + Y + Y + N + + + IF job_parallel_recruits_info_citizen ERROR + Abort parallel_recruits_info_citizen + Y + Y + N + + + job_info_recruits_from_json.hwf + Success_recruits_info_from_json + Y + Y + N + + + job_parallel_subpoena_decision_archive_person.hwf + Success_subpoena_decisions + Y + Y + N + + + + + +