From 66840c2c287431c1b8a0a55c97c28fc964f2bc73 Mon Sep 17 00:00:00 2001 From: "adel.ka" Date: Fri, 14 Nov 2025 16:17:39 +0300 Subject: [PATCH] rename --- .../punishment/parallel/punishment_flow.hpl | 416 ++++++++-------- .../parallel/punishment_flow_delta.hpl | 103 ++-- .../parallel/punishment_flow_repeat.hpl | 413 ++++++++-------- .../snils/parallel/snils_flow.hpl | 64 ++- .../snils/parallel/snils_flow_delta.hpl | 443 +++++++++--------- .../snils/parallel/snils_flow_repeat.hpl | 363 +++++++------- 6 files changed, 926 insertions(+), 876 deletions(-) diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl index 693b56f..f85a492 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl @@ -21,68 +21,68 @@ - Create job execution record - punishment_input + create_job_execution_record_punishment + table_input_punishment Y - punishment_output - punishment_error_job_status + table_output_punishment + change_job_status_on_error_punishment Y - punishment_error_job_status - Abort + change_job_status_on_error_punishment + abort_punishment Y - Detect empty stream - punishment_success_job_status + detect_empty_stream_punishment + change_job_status_on_success_punishment Y - punishment_input - punishment_output + table_input_punishment + table_output_punishment Y - punishment_update_flags - punishment_error_job_status + update_flags_punishment + change_job_status_on_error_punishment Y - punishment_sort_rows - punishment_update_flags + sort_rows_punishment + update_flags_punishment Y - punishment_input - punishment_sort_rows + table_input_punishment + sort_rows_punishment Y - punishment_input - Detect empty stream + table_input_punishment + detect_empty_stream_punishment Y - punishment_output - Identify last row in a stream + table_output_punishment + identify_last_row_in_a_stream_punishment Y - Identify last row in a stream - Filter rows + identify_last_row_in_a_stream_punishment + filter_rows_punishment Y - Filter rows - punishment_success_job_status + filter_rows_punishment + change_job_status_on_success_punishment Y - Abort + abort_punishment Abort Y @@ -94,44 +94,16 @@ ABORT_WITH_ERROR Y + 0 - 1168 - 448 + 1472 + 544 - Create job execution record - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - N - N - Y - N - N - INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id) -VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - - - - - 80 - 320 - - - - Detect empty stream + detect_empty_stream_punishment DetectEmptyStream Y @@ -148,7 +120,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Filter rows + filter_rows_punishment FilterRows Y @@ -176,15 +148,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - punishment_success_job_status + change_job_status_on_success_punishment - 720 + 928 144 - Identify last row in a stream + identify_last_row_in_a_stream_punishment DetectLastRow Y @@ -197,12 +169,41 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); last_row - 720 - 224 + 928 + 272 - punishment_error_job_status + create_job_execution_record_punishment + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + N + N + Y + N + N + INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id) +VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); + + + + + 80 + 416 + + + + change_job_status_on_error_punishment ExecSql Y @@ -218,8 +219,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard + Y + N + Y Y N @@ -231,14 +235,139 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + - 1008 - 448 + 1216 + 544 - punishment_input + sort_rows_punishment + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 100000 + + N + + Y + + + recruit_id + Y + N + N + 0 + N + + + + + 336 + 720 + + + + change_job_status_on_success_punishment + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + + Y + + N + + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS', + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}' +and status = 'PROCESSING'; + + + + + + 688 + 144 + + + + update_flags_punishment + Update + + N + + 1 + + none + + + 10000 + ervu-dashboard + Y + + + + = + recruit_id + recruit_id + + + + <> + has_punishment_execution_info + has_punishment_execution_info + + + + <> + has_sentence_info + has_sentence_info + + + ervu_dashboard + citizen
+ + has_punishment_execution_info + has_punishment_execution_info + + + has_sentence_info + has_sentence_info + + + N + Y + + + 1216 + 720 + + + + table_input_punishment TableInput N @@ -250,7 +379,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH punishment_data AS ( SELECT ri.recruit_id, @@ -299,11 +427,11 @@ GROUP BY recruit_id, punishment_info; 336 - 320 + 416 - punishment_output + table_output_punishment TableOutput N @@ -408,142 +536,14 @@ GROUP BY recruit_id, punishment_info; Y - 720 - 320 - - - - punishment_sort_rows - SortRows - - Y - - 1 - - none - - - ${java.io.tmpdir} - out - 100000 - - N - - Y - - - recruit_id - Y - N - N - 0 - N - - - - - 336 - 624 - - - - punishment_success_job_status - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'SUCCESS', - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'PROCESSING'; - - - - - 512 - 144 - - - - punishment_update_flags - Update - - N - - 1 - - none - - - 10000 - ervu-dashboard - Y - - - = - recruit_id - recruit_id - - - <> - has_punishment_execution_info - has_punishment_execution_info - - - <> - has_sentence_info - has_sentence_info - - ervu_dashboard - citizen
- - has_punishment_execution_info - has_punishment_execution_info - - - has_sentence_info - has_sentence_info - - - N - Y - - - 1008 - 624 + 928 + 416 - punishment_output - punishment_error_job_status - Y - - error_description - - - - - - - - punishment_update_flags - punishment_error_job_status + update_flags_punishment + change_job_status_on_error_punishment Y error_decription @@ -553,6 +553,18 @@ and status = 'PROCESSING'; + + table_output_punishment + change_job_status_on_error_punishment + Y + + error_description + + + + + + diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl index b8801b1..a12b013 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl @@ -40,68 +40,68 @@ - Create job execution record + create_job_execution_record_punishment punishment_input Y - punishment_insert_or_update - punishment_error_job_status + insert_or_update_punishment + change_job_status_on_error_punishment Y - Detect empty stream - punishment_success_job_status + detect_empty_stream_punishment + change_job_status_on_success_punishment Y - punishment_error_job_status - Abort + change_job_status_on_error_punishment + abort_punishment Y punishment_input - punishment_insert_or_update + insert_or_update_punishment Y - punishment_update_flags - punishment_error_job_status + update_flags_punishment + change_job_status_on_error_punishment Y punishment_input - punishment_sort_rows + sort_rows_punishment Y punishment_input - Detect empty stream + detect_empty_stream_punishment Y - punishment_sort_rows - punishment_update_flags + sort_rows_punishment + update_flags_punishment Y - punishment_insert_or_update + insert_or_update_punishment Identify last row in a stream Y Identify last row in a stream - Filter rows + filter_rows_punishment Y - Filter rows - punishment_success_job_status + filter_rows_punishment + change_job_status_on_success_punishment Y - Abort + abort_punishment Abort Y @@ -113,15 +113,16 @@ ABORT_WITH_ERROR Y + 0 - 1120 - 384 + 1392 + 368 - Create job execution record + create_job_execution_record_punishment ExecSql Y @@ -134,8 +135,11 @@ ervu-dashboard + N + N + Y N N @@ -146,6 +150,7 @@ SET error_description = NULL where job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + 128 @@ -153,7 +158,7 @@ and recruitment_id = '${IDM_ID}'; - Detect empty stream + detect_empty_stream_punishment DetectEmptyStream Y @@ -170,7 +175,7 @@ and recruitment_id = '${IDM_ID}'; - Filter rows + filter_rows_punishment FilterRows Y @@ -198,10 +203,10 @@ and recruitment_id = '${IDM_ID}'; - punishment_success_job_status + change_job_status_on_success_punishment - 784 + 944 0 @@ -219,12 +224,12 @@ and recruitment_id = '${IDM_ID}'; last_row - 784 - 96 + 944 + 80 - punishment_error_job_status + change_job_status_on_error_punishment ExecSql Y @@ -240,8 +245,11 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard + Y + N + Y Y N @@ -253,10 +261,11 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + - 960 - 384 + 1120 + 368 @@ -325,7 +334,7 @@ GROUP BY recruit_id, punishment_info; - punishment_insert_or_update + insert_or_update_punishment InsertUpdate N @@ -342,11 +351,13 @@ GROUP BY recruit_id, punishment_info; = recruit_id recruit_id + = punishment_external_id punishment_external_id + ervu_dashboard punishment
@@ -449,12 +460,12 @@ GROUP BY recruit_id, punishment_info; N - 784 + 944 224 - punishment_sort_rows + sort_rows_punishment SortRows Y @@ -488,7 +499,7 @@ GROUP BY recruit_id, punishment_info; - punishment_success_job_status + change_job_status_on_success_punishment ExecSql Y @@ -501,8 +512,11 @@ GROUP BY recruit_id, punishment_info; ervu-dashboard + Y + N + Y N N @@ -514,14 +528,15 @@ and recruitment_id = '${IDM_ID}' and status = 'DELTA_PROCESSING'; + - 576 + 688 0 - punishment_update_flags + update_flags_punishment Update N @@ -534,21 +549,25 @@ and status = 'DELTA_PROCESSING'; 10000 ervu-dashboard Y + = recruit_id recruit_id + <> has_punishment_execution_info has_punishment_execution_info + <> has_sentence_info has_sentence_info + ervu_dashboard citizen
@@ -565,14 +584,14 @@ and status = 'DELTA_PROCESSING'; Y - 960 + 1120 608 - punishment_insert_or_update - punishment_error_job_status + insert_or_update_punishment + change_job_status_on_error_punishment Y error_description @@ -583,8 +602,8 @@ and status = 'DELTA_PROCESSING'; - punishment_update_flags - punishment_error_job_status + update_flags_punishment + change_job_status_on_error_punishment Y error_description diff --git a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl index 2295a8e..279e272 100644 --- a/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl @@ -21,52 +21,52 @@ - Create job execution record - punishment_input + create_job_execution_record_punishment + table_input_punishment Y - punishment_error_job_status - Abort + change_job_status_on_error_punishment + abort_punishment Y - Detect empty stream - punishment_success_job_status + detect_empty_stream_punishment + change_job_status_on_success_punishment Y - punishment_update_flags - punishment_error_job_status + update_flags_punishment + change_job_status_on_error_punishment Y - punishment_insert_or_update - punishment_error_job_status + insert_or_update_punishment + change_job_status_on_error_punishment Y - punishment_input - punishment_insert_or_update + table_input_punishment + insert_or_update_punishment Y - punishment_input - punishment_sort_rows + table_input_punishment + sort_rows_punishment Y - punishment_sort_rows - punishment_update_flags + sort_rows_punishment + update_flags_punishment Y - punishment_input - Detect empty stream + table_input_punishment + detect_empty_stream_punishment Y - punishment_insert_or_update + insert_or_update_punishment Identify last row in a stream Y @@ -77,12 +77,66 @@ Filter rows - punishment_success_job_status + change_job_status_on_success_punishment Y - Abort + Filter rows + FilterRows + + Y + + 1 + + none + + + + + + + = + last_row + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + change_job_status_on_success_punishment + + + 992 + 64 + + + + Identify last row in a stream + DetectLastRow + + Y + + 1 + + none + + + last_row + + + 992 + 144 + + + + abort_punishment Abort Y @@ -97,12 +151,81 @@ 0 - 1152 + 1264 432 - Create job execution record + change_job_status_on_error_punishment + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ?, + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 992 + 432 + + + + change_job_status_on_success_punishment + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS', + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}' +and status = 'PROCESSING'; + + + + + 688 + 64 + + + + create_job_execution_record_punishment ExecSql Y @@ -148,7 +271,7 @@ DO UPDATE SET - Detect empty stream + detect_empty_stream_punishment DetectEmptyStream Y @@ -165,162 +288,7 @@ DO UPDATE SET - Filter rows - FilterRows - - Y - - 1 - - none - - - - - - - = - last_row - N - - - - N - -1 - constant - -1 - Y - Boolean - - - - punishment_success_job_status - - - 800 - 64 - - - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 800 - 144 - - - - punishment_error_job_status - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 992 - 432 - - - - punishment_input - TableInput - - N - - 1 - - none - - - ervu-dashboard - N - - WITH punishment_data AS ( - SELECT - ri.recruit_id, - jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info - FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id - AND '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null' -) -SELECT recruit_id, - (punishment_info ->> 'id')::uuid AS punishment_external_id, - -- Блок "Сведения о вынесенном приговоре" - punishment_info ->> 'vidNakaz' AS punishment_type, -- Вид наказания - punishment_info ->> 'kodVidNakaz' AS punishment_code, -- Код вида наказания - TO_DATE(punishment_info ->> 'dataOsuzhd', 'YYYY-MM-DD') AS sentense_date, -- Дата осуждения - punishment_info ->> 'naimSud' AS authority_name, -- Наименование суда - punishment_info ->> 'srokNakaz' AS sentence, -- Срок наказания - CASE - WHEN punishment_info ->> 'kodIzmSrokNakaz' = '1' THEN 'Изменений срока наказания не было' - WHEN punishment_info ->> 'kodIzmSrokNakaz' = '2' THEN 'Были изменения срока наказания' - ELSE punishment_info ->> 'kodIzmSrokNakaz' - END AS sentence_changed, -- Признак изменения срока - STRING_AGG(TRIM(BOTH ' "' FROM law_articles), ', ') AS law_point, -- Статьи УК - -- Блок "Сведения об отбывании наказания" - punishment_info ->> 'katUcheta' AS registration_category, -- Категория учета - punishment_info ->> 'naimSubekt' AS region_name, -- Наименование субъекта - punishment_info ->> 'naimOrg' AS facility_name, -- Название учреждения - punishment_info ->> 'adresOrg' AS facility_address, -- Адрес учреждения - punishment_info ->> 'nomerUD' AS criminal_case_number, -- Номер уголовного дела - TO_DATE(punishment_info ->> 'dataNachSrok', 'YYYY-MM-DD') AS start_date, -- Дата начала срока - TO_DATE(punishment_info ->> 'dataOkonchSrok', 'YYYY-MM-DD') AS planned_date, -- Планируемая дата освобождения - CASE - WHEN punishment_info ->> 'kodPerekvalif' = '1' THEN 'Изменений статьи не было' - WHEN punishment_info ->> 'kodPerekvalif' = '2' THEN 'Были изменения статьи' - ELSE punishment_info ->> 'kodPerekvalif' - END AS article_modification_flag, -- Признак переквалификации - punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения - TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения - true AS has_punishment_execution_info, - true AS has_sentence_info -FROM punishment_data -LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true -GROUP BY recruit_id, punishment_info; - Y - - - 416 - 272 - - - - punishment_insert_or_update + insert_or_update_punishment InsertUpdate N @@ -444,12 +412,12 @@ GROUP BY recruit_id, punishment_info; Y - 800 + 992 272 - punishment_sort_rows + sort_rows_punishment SortRows Y @@ -483,40 +451,71 @@ GROUP BY recruit_id, punishment_info; - punishment_success_job_status - ExecSql + table_input_punishment + TableInput - Y + N 1 none - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'SUCCESS', - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'PROCESSING'; - - + N + WITH punishment_data AS ( + SELECT + ri.recruit_id, + jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info + FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id + AND '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null' +) +SELECT recruit_id, + (punishment_info ->> 'id')::uuid AS punishment_external_id, + -- Блок "Сведения о вынесенном приговоре" + punishment_info ->> 'vidNakaz' AS punishment_type, -- Вид наказания + punishment_info ->> 'kodVidNakaz' AS punishment_code, -- Код вида наказания + TO_DATE(punishment_info ->> 'dataOsuzhd', 'YYYY-MM-DD') AS sentense_date, -- Дата осуждения + punishment_info ->> 'naimSud' AS authority_name, -- Наименование суда + punishment_info ->> 'srokNakaz' AS sentence, -- Срок наказания + CASE + WHEN punishment_info ->> 'kodIzmSrokNakaz' = '1' THEN 'Изменений срока наказания не было' + WHEN punishment_info ->> 'kodIzmSrokNakaz' = '2' THEN 'Были изменения срока наказания' + ELSE punishment_info ->> 'kodIzmSrokNakaz' + END AS sentence_changed, -- Признак изменения срока + STRING_AGG(TRIM(BOTH ' "' FROM law_articles), ', ') AS law_point, -- Статьи УК + -- Блок "Сведения об отбывании наказания" + punishment_info ->> 'katUcheta' AS registration_category, -- Категория учета + punishment_info ->> 'naimSubekt' AS region_name, -- Наименование субъекта + punishment_info ->> 'naimOrg' AS facility_name, -- Название учреждения + punishment_info ->> 'adresOrg' AS facility_address, -- Адрес учреждения + punishment_info ->> 'nomerUD' AS criminal_case_number, -- Номер уголовного дела + TO_DATE(punishment_info ->> 'dataNachSrok', 'YYYY-MM-DD') AS start_date, -- Дата начала срока + TO_DATE(punishment_info ->> 'dataOkonchSrok', 'YYYY-MM-DD') AS planned_date, -- Планируемая дата освобождения + CASE + WHEN punishment_info ->> 'kodPerekvalif' = '1' THEN 'Изменений статьи не было' + WHEN punishment_info ->> 'kodPerekvalif' = '2' THEN 'Были изменения статьи' + ELSE punishment_info ->> 'kodPerekvalif' + END AS article_modification_flag, -- Признак переквалификации + punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения + TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения + true AS has_punishment_execution_info, + true AS has_sentence_info +FROM punishment_data +LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true +GROUP BY recruit_id, punishment_info; + Y - 608 - 64 + 416 + 272 - punishment_update_flags + update_flags_punishment Update N @@ -566,8 +565,8 @@ and status = 'PROCESSING'; - punishment_insert_or_update - punishment_error_job_status + insert_or_update_punishment + change_job_status_on_error_punishment Y error_description @@ -578,8 +577,8 @@ and status = 'PROCESSING'; - punishment_update_flags - punishment_error_job_status + update_flags_punishment + change_job_status_on_error_punishment Y error_description diff --git a/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow.hpl b/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow.hpl index dade5bb..a98505f 100644 --- a/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow.hpl +++ b/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow.hpl @@ -21,53 +21,53 @@ - snils_create_job - snils_input + create_job_execution_record_snils + table_input_snils Y - Filter rows + filter_rows_snils snils_success_job Y - Detect empty stream + detect_empty_stream_snils snils_success_job Y - snils_failure_job - Abort + change_job_status_on_error_snils + abort_snils Y - snils_output - snils_failure_job + table_output_snils + change_job_status_on_error_snils Y - snils_input - Detect empty stream + table_input_snils + detect_empty_stream_snils Y - snils_output - Identify last row in a stream + table_output_snils + identify_last_row_in_a_stream_snils Y - Identify last row in a stream - Filter rows + identify_last_row_in_a_stream_snils + filter_rows_snils Y - snils_input - snils_output + table_input_snils + table_output_snils Y - Abort + abort_snils Abort Y @@ -79,6 +79,7 @@ ABORT_WITH_ERROR Y + 0 @@ -87,7 +88,7 @@ - Detect empty stream + detect_empty_stream_snils DetectEmptyStream Y @@ -104,7 +105,7 @@ - Filter rows + filter_rows_snils FilterRows Y @@ -140,7 +141,7 @@ - Identify last row in a stream + identify_last_row_in_a_stream_snils DetectLastRow Y @@ -158,7 +159,7 @@ - snils_create_job + create_job_execution_record_snils ExecSql Y @@ -171,8 +172,11 @@ ervu-dashboard + N + N + Y N N @@ -180,6 +184,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); + 288 @@ -187,7 +192,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - snils_failure_job + change_job_status_on_error_snils ExecSql Y @@ -203,8 +208,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard + Y + N + Y Y N @@ -216,6 +224,7 @@ WHERE job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; + 1024 @@ -223,7 +232,7 @@ and recruitment_id = '${IDM_ID}'; - snils_input + table_input_snils TableInput N @@ -280,7 +289,7 @@ WHERE LENGTH(pred_snils_cleaned) = 11 - snils_output + table_output_snils TableOutput N @@ -318,11 +327,14 @@ WHERE LENGTH(pred_snils_cleaned) = 11 N N N + Y + N ervu_dashboard Y snils
+ N Y N @@ -367,8 +379,8 @@ and recruitment_id = '${IDM_ID}'; - snils_output - snils_failure_job + table_output_snils + change_job_status_on_error_snils Y error_description diff --git a/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_delta.hpl b/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_delta.hpl index 9113b18..b7af8c8 100644 --- a/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_delta.hpl @@ -21,53 +21,53 @@ - snils_create_job - snils_input + create_job_execution_record_snils + table_input_snils Y - Detect empty stream - snils_success_job + detect_empty_stream_snils + change_job_status_on_success_snils Y - Filter rows - snils_success_job + filter_rows_snils + change_job_status_on_success_snils Y - snils_failure_job - Abort + change_job_status_on_error_snils + abort_snils Y - snils_insert_or_update - snils_failure_job + insert_or_update_snils + change_job_status_on_error_snils Y - snils_insert_or_update - Identify last row in a stream 2 + insert_or_update_snils + identify_last_row_in_a_stream_snils Y - Identify last row in a stream 2 - Filter rows + identify_last_row_in_a_stream_snils + filter_rows_snils Y - snils_input - snils_insert_or_update + table_input_snils + insert_or_update_snils Y - snils_input - Detect empty stream + table_input_snils + detect_empty_stream_snils Y - Abort + abort_snils Abort Y @@ -79,6 +79,7 @@ ABORT_WITH_ERROR Y + 0 @@ -87,7 +88,114 @@ - snils_failure_job + detect_empty_stream_snils + DetectEmptyStream + + Y + + 1 + + none + + + + + 544 + 240 + + + + filter_rows_snils + FilterRows + + Y + + 1 + + none + + + + + + + = + last_row + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + change_job_status_on_success_snils + + + 1104 + 64 + + + + identify_last_row_in_a_stream_snils + DetectLastRow + + Y + + 1 + + none + + + last_row + + + 1104 + 240 + + + + create_job_execution_record_snils + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + + N + + N + + Y + N + N + UPDATE etl.job_execution +SET + status = 'DELTA_PROCESSING', + execution_datetime = DEFAULT, + error_description = NULL +where job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + 336 + 432 + + + + change_job_status_on_error_snils ExecSql Y @@ -127,205 +235,7 @@ and recruitment_id = '${IDM_ID}'; - snils_success_job - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - - Y - - N - - Y - N - N - UPDATE etl.job_execution -SET status = 'DELTA_SUCCESS', - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - - 544 - 64 - - - - snils_create_job - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - - N - - N - - Y - N - N - UPDATE etl.job_execution -SET - status = 'DELTA_PROCESSING', - execution_datetime = DEFAULT, - error_description = NULL -where job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - 336 - 432 - - - - Detect empty stream - DetectEmptyStream - - Y - - 1 - - none - - - - - 544 - 288 - - - - Filter rows - FilterRows - - Y - - 1 - - none - - - - - - - = - last_row - N - - - - N - -1 - constant - -1 - Y - Boolean - - - - snils_success_job - - - 1104 - 64 - - - - Identify last row in a stream 2 - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 1104 - 240 - - - - snils_insert_or_update - InsertUpdate - - N - - 1 - - none - - - 10000 - ervu-dashboard - - - = - source_id - source_id - - - ervu_dashboard - snils
- - recruit_id - recruit_id - N - - - snils - snils - Y - - - actual - actual - Y - - - source_update_date - source_update_date - Y - - - source_id - source_id - N - - - N - - - 1104 - 432 - - - - snils_input + table_input_snils TableInput N @@ -397,10 +307,101 @@ FROM snils_data; 432 + + insert_or_update_snils + InsertUpdate + + N + + 1 + + none + + + 10000 + ervu-dashboard + + + = + source_id + source_id + + + ervu_dashboard + snils
+ + recruit_id + recruit_id + N + + + snils + snils + Y + + + actual + actual + Y + + + source_update_date + source_update_date + Y + + + source_id + source_id + N + + + N + + + 1104 + 432 + + + + change_job_status_on_success_snils + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + + Y + + N + + Y + N + N + UPDATE etl.job_execution +SET status = 'DELTA_SUCCESS', + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + + 544 + 64 + + - snils_insert_or_update - snils_failure_job + insert_or_update_snils + change_job_status_on_error_snils Y error_description diff --git a/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_repeat.hpl index 4432488..6a8892f 100644 --- a/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_repeat.hpl @@ -21,53 +21,53 @@ - snils_create_job - snils_input + create_job_execution_record_snils + table_input_snils Y - Filter rows - snils_success_job + filter_rows_snils + change_job_status_on_success_snils Y - Detect empty stream - snils_success_job + detect_empty_stream_snils + change_job_status_on_success_snils Y - snils_failure_job - Abort + change_job_status_on_error_snils + abort_snils Y - snils_insert_or_update - snils_failure_job + insert_or_update_snils + change_job_status_on_error_snils Y - snils_input - snils_insert_or_update + table_input_snils + insert_or_update_snils Y - snils_input - Detect empty stream + table_input_snils + detect_empty_stream_snils Y - snils_insert_or_update - Identify last row in a stream + insert_or_update_snils + identify_last_row_in_a_stream_snils Y - Identify last row in a stream - Filter rows + identify_last_row_in_a_stream_snils + filter_rows_snils Y - Abort + abort_snils Abort Y @@ -79,6 +79,7 @@ ABORT_WITH_ERROR Y + 0 @@ -87,7 +88,129 @@ - snils_failure_job + detect_empty_stream_snils + DetectEmptyStream + + Y + + 1 + + none + + + + + 480 + 208 + + + + filter_rows_snils + FilterRows + + Y + + 1 + + none + + + + + + + = + last_row + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + change_job_status_on_success_snils + + + 1008 + 48 + + + + identify_last_row_in_a_stream_snils + DetectLastRow + + Y + + 1 + + none + + + last_row + + + 1008 + 224 + + + + create_job_execution_record_snils + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + + N + + N + + Y + N + N + INSERT INTO etl.job_execution ( + id, + job_name, + status, + execution_datetime, + error_description, + recruitment_id +) +VALUES ( + DEFAULT, + '${JOB_NAME}', + 'PROCESSING', + DEFAULT, + NULL, + '${IDM_ID}' +) +ON CONFLICT (job_name, recruitment_id) +DO UPDATE SET + status = 'PROCESSING', + execution_datetime = DEFAULT, + error_description = NULL + + + + + 272 + 400 + + + + change_job_status_on_error_snils ExecSql Y @@ -127,161 +250,7 @@ and recruitment_id = '${IDM_ID}'; - snils_success_job - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - - Y - - N - - Y - N - N - UPDATE etl.job_execution -SET status = 'SUCCESS', - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - - 480 - 48 - - - - Detect empty stream - DetectEmptyStream - - Y - - 1 - - none - - - - - 480 - 208 - - - - Filter rows - FilterRows - - Y - - 1 - - none - - - - - - - = - last_row - N - - - - N - -1 - constant - -1 - Y - Boolean - - - - snils_success_job - - - 1008 - 48 - - - - Identify last row in a stream - DetectLastRow - - Y - - 1 - - none - - - last_row - - - 1008 - 224 - - - - snils_create_job - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - N - N - Y - N - N - INSERT INTO etl.job_execution ( - id, - job_name, - status, - execution_datetime, - error_description, - recruitment_id -) -VALUES ( - DEFAULT, - '${JOB_NAME}', - 'PROCESSING', - DEFAULT, - NULL, - '${IDM_ID}' -) -ON CONFLICT (job_name, recruitment_id) -DO UPDATE SET - status = 'PROCESSING', - execution_datetime = DEFAULT, - error_description = NULL - - - - 272 - 400 - - - - snils_input + table_input_snils TableInput N @@ -293,6 +262,7 @@ DO UPDATE SET ervu-dashboard N + WITH base as ( SELECT @@ -337,7 +307,7 @@ WHERE LENGTH(pred_snils_cleaned) = 11 - snils_insert_or_update + insert_or_update_snils InsertUpdate N @@ -354,6 +324,7 @@ WHERE LENGTH(pred_snils_cleaned) = 11 = source_id source_id + ervu_dashboard snils
@@ -390,10 +361,46 @@ WHERE LENGTH(pred_snils_cleaned) = 11 400 + + change_job_status_on_success_snils + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + + Y + + N + + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS', + execution_end_datetime = current_timestamp +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + + 480 + 48 + + - snils_insert_or_update - snils_failure_job + insert_or_update_snils + change_job_status_on_error_snils Y error_description