diff --git a/mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/border_crossing/recruitment_five_flow_delta.hpl similarity index 81% rename from mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow_repeat.hpl rename to mappings/info_recruits/citizen_tables/border_crossing/recruitment_five_flow_delta.hpl index fca1560..01ec515 100644 --- a/mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/border_crossing/recruitment_five_flow_delta.hpl @@ -1,7 +1,7 @@ - recruitment_five_flow_repeat + recruitment_five_flow_delta Y @@ -13,36 +13,36 @@ 1000 100 - - 2025/09/01 17:02:32.905 + 2025/08/11 13:16:11.204 - - 2025/09/01 17:02:32.905 + 2025/08/11 13:16:11.204 Table input - russian_diploma_flow_repeat.hpl + border_crossing_flow_delta.hpl Y Table input - russian_diploma_flow_repeat.hpl 2 + border_crossing_flow_delta.hpl 2 Y Table input - russian_diploma_flow_repeat.hpl 3 + border_crossing_flow_delta.hpl 3 Y Table input - russian_diploma_flow_repeat.hpl 4 + border_crossing_flow_delta.hpl 4 Y Table input - russian_diploma_flow_repeat.hpl 5 + border_crossing_flow_delta.hpl 5 Y @@ -57,25 +57,25 @@ none - ervu-dashboard-test + ervu-dashboard N 0 - SELECT r.idm_id AS recruitment_id + SELECT + r.idm_id as recruitment_id FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - AND je.job_name = 'russian_diploma_job' -WHERE je.id IS NULL - OR je.status IN ('ERROR', 'PROCESSING'); + and job_name = 'border_crossing_job' +where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); N - 368 - 352 + 352 + 288 - russian_diploma_flow_repeat.hpl + border_crossing_flow_delta.hpl PipelineExecutor Y @@ -86,7 +86,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl N 1 @@ -94,10 +94,15 @@ WHERE je.id IS NULL - + IDM_ID recruitment_id + + JOB_NAME + + border_crossing_job + Y @@ -121,12 +126,12 @@ WHERE je.id IS NULL - 576 - 192 + 640 + 128 - russian_diploma_flow_repeat.hpl 2 + border_crossing_flow_delta.hpl 2 PipelineExecutor Y @@ -137,7 +142,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl N 1 @@ -145,10 +150,15 @@ WHERE je.id IS NULL - + IDM_ID recruitment_id + + JOB_NAME + + border_crossing_job + Y @@ -172,12 +182,12 @@ WHERE je.id IS NULL - 576 - 272 + 656 + 224 - russian_diploma_flow_repeat.hpl 3 + border_crossing_flow_delta.hpl 3 PipelineExecutor Y @@ -188,7 +198,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl N 1 @@ -196,10 +206,15 @@ WHERE je.id IS NULL - + IDM_ID recruitment_id + + JOB_NAME + + border_crossing_job + Y @@ -223,12 +238,12 @@ WHERE je.id IS NULL - 576 - 352 + 656 + 304 - russian_diploma_flow_repeat.hpl 4 + border_crossing_flow_delta.hpl 4 PipelineExecutor Y @@ -239,7 +254,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl N 1 @@ -247,10 +262,15 @@ WHERE je.id IS NULL - + IDM_ID recruitment_id + + JOB_NAME + + border_crossing_job + Y @@ -274,12 +294,12 @@ WHERE je.id IS NULL - 576 - 432 + 656 + 384 - russian_diploma_flow_repeat.hpl 5 + border_crossing_flow_delta.hpl 5 PipelineExecutor Y @@ -290,7 +310,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl N 1 @@ -298,10 +318,15 @@ WHERE je.id IS NULL - + IDM_ID recruitment_id + + JOB_NAME + + border_crossing_job + Y @@ -325,8 +350,8 @@ WHERE je.id IS NULL - 576 - 512 + 656 + 480 diff --git a/mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/child/recruitment_five_flow_delta.hpl similarity index 84% rename from mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_repeat.hpl rename to mappings/info_recruits/citizen_tables/child/recruitment_five_flow_delta.hpl index 818600f..71e650a 100644 --- a/mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/child/recruitment_five_flow_delta.hpl @@ -1,7 +1,7 @@ - recruitment_five_flow_repeat + recruitment_five_flow_delta Y @@ -13,36 +13,36 @@ 1000 100 - - 2025/09/05 08:56:57.814 + 2025/08/11 13:16:11.204 - - 2025/09/05 08:56:57.814 + 2025/08/11 13:16:11.204 Table input - foreign_diploma_flow_repeat.hpl + child_flow_delta.hpl Y Table input - foreign_diploma_flow_repeat.hpl 2 + child_flow_delta.hpl 2 Y Table input - foreign_diploma_flow_repeat.hpl 3 + child_flow_delta.hpl 3 Y Table input - foreign_diploma_flow_repeat.hpl 4 + child_flow_delta.hpl 4 Y Table input - foreign_diploma_flow_repeat.hpl 5 + child_flow_delta.hpl 5 Y @@ -57,25 +57,25 @@ none - ervu-dashboard-test + ervu-dashboard N 0 - SELECT r.idm_id AS recruitment_id + SELECT + r.idm_id as recruitment_id FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - AND je.job_name = 'foreign_diploma_job' -WHERE je.id IS NULL - OR je.status IN ('ERROR', 'PROCESSING'); + and job_name = 'child_job' +where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); N - 624 - 512 + 352 + 288 - foreign_diploma_flow_repeat.hpl + child_flow_delta.hpl PipelineExecutor Y @@ -86,7 +86,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl N 1 @@ -98,6 +98,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + child_job + Y @@ -121,12 +126,12 @@ WHERE je.id IS NULL - 832 - 352 + 544 + 128 - foreign_diploma_flow_repeat.hpl 2 + child_flow_delta.hpl 2 PipelineExecutor Y @@ -137,7 +142,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl N 1 @@ -149,6 +154,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + child_job + Y @@ -172,12 +182,12 @@ WHERE je.id IS NULL - 832 - 432 + 544 + 208 - foreign_diploma_flow_repeat.hpl 3 + child_flow_delta.hpl 3 PipelineExecutor Y @@ -188,7 +198,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl N 1 @@ -200,6 +210,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + child_job + Y @@ -223,12 +238,12 @@ WHERE je.id IS NULL - 832 - 512 + 544 + 288 - foreign_diploma_flow_repeat.hpl 4 + child_flow_delta.hpl 4 PipelineExecutor Y @@ -239,7 +254,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl N 1 @@ -251,6 +266,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + child_job + Y @@ -274,12 +294,12 @@ WHERE je.id IS NULL - 832 - 592 + 544 + 368 - foreign_diploma_flow_repeat.hpl 5 + child_flow_delta.hpl 5 PipelineExecutor Y @@ -290,7 +310,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl N 1 @@ -302,6 +322,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + child_job + Y @@ -325,8 +350,8 @@ WHERE je.id IS NULL - 832 - 672 + 544 + 448 diff --git a/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl similarity index 64% rename from mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl rename to mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl index 1d1c16f..fbffb7c 100644 --- a/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl @@ -1,7 +1,7 @@ - foreign_diploma_flow + citizen_address_flow Y @@ -13,21 +13,36 @@ 1000 100 - - 2025/09/04 20:53:34.643 + 2025/08/05 14:31:22.799 - - 2025/09/04 20:53:34.643 + 2025/08/05 14:31:22.799 + + Create job execution record + Table input + Y + + + Table output + Change job status on error + Y + Change job status on error Abort Y - Create job execution record - foerign_diploma_input + Table output + Detect empty stream + Y + + + Detect empty stream + Change job status on success Y @@ -36,33 +51,18 @@ Y - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - citizen_education_diploma_output - Y - - - citizen_education_diploma_output - Change job status on error - Y - - - citizen_education_diploma_output - Filter rows - Y - - - foerign_diploma_input + Table input Identify last row in a stream Y - Detect empty stream - Change job status on success + Identify last row in a stream + Table output + Y + + + Table output + Filter rows Y @@ -77,13 +77,13 @@ none - ABORT + ABORT_WITH_ERROR Y 0 - 1248 - 608 + 1328 + 704 @@ -117,8 +117,8 @@ and recruitment_id = '${IDM_ID}'; - 1024 - 608 + 832 + 704 @@ -143,13 +143,14 @@ and recruitment_id = '${IDM_ID}'; UPDATE etl.job_execution SET status = 'SUCCESS' WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; +and recruitment_id = '${IDM_ID}' +and status = 'PROCESSING'; - 1248 - 256 + 1328 + 96 @@ -177,8 +178,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 320 - 432 + 240 + 320 @@ -194,8 +195,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 736 - 256 + 832 + 96 @@ -230,15 +231,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1248 - 432 + 1328 + 320 Identify last row in a stream DetectLastRow - N + Y 1 @@ -248,106 +249,12 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); last_row - 736 - 432 + 640 + 320 - citizen_education_diploma_output - TableOutput - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - recruit_id - recruit_id - - - source_id - source_id - - - source_update_date - source_update_date - - - education_level_code - education_level_code - - - educational_institution_name - educational_institution_name - - - foreign_doc_name - foreign_doc_name - - - ser_num_doc - ser_num_doc - - - reg_num - reg_num - - - recognition_certificate_number - recognition_certificate_number - - - recognition_date - recognition_date - - - speciality_name - speciality_name - - - minobr_speciality_code - minobr_speciality_code - - - minprosv_speciality_code - minprosv_speciality_code - - - oksm_code - oksm_code - - - vus_spec - vus_spec - - - N - N - N - N - Y - N - ervu_dashboard - Y - foreign_diploma
- N - Y - N - Y - - - 1024 - 432 - -
- - foerign_diploma_input + Table input TableInput Y @@ -360,46 +267,125 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr - FROM recruits_info ri - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' -) -SELECT fd.recruit_id, - diploma_elem ->> 'id' AS source_id, - to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, - diploma_elem ->> 'kodUrObr' as education_level_code, - diploma_elem ->> 'naimUchZaved' as educational_institution_name, - diploma_elem ->> 'naimInoDok' as foreign_doc_name, - diploma_elem ->> 'serNomDok' as ser_num_doc, - diploma_elem ->> 'regNomDok' as reg_num, - diploma_elem ->> 'nomSved' as recognition_certificate_number, - to_date(diploma_elem ->> 'dataPriz', 'YYYY-MM-DD') as recognition_date, - diploma_elem -> 'svedSpecz' ->> 'naimSpecz' as speciality_name, - diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, - diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, - diploma_elem ->> 'kodOKSM' as oksm_code, - coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec -FROM filteredData fd - CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem; - + SELECT + ri.recruit_id::uuid, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, + + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, + CASE + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' + ELSE NULL + END AS place_stay_registration_type, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date +FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id + CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr +WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date +GROUP BY ri.recruit_id; Y - 544 - 432 + 464 + 320 + + + + Table output + TableOutput + + N + + 1 + + none + + + 1000 + ervu-dashboard + + + recruit_id + recruit_id + + + residence_address + residence_address + + + residence_registration_date + residence_registration_date + + + residence_deregistration_date + residence_deregistration_date + + + place_stay_address + place_stay_address + + + place_stay_registration_date + place_stay_registration_date + + + place_stay_deregistration_date + place_stay_deregistration_date + + + place_stay_registration_type + place_stay_registration_type + + + factual_address + factual_address + + + factual_info_received_date + factual_info_received_date + + + factual_info_expiry_date + factual_info_expiry_date + + + N + N + N + N + Y + N + ervu_dashboard + Y + citizen_address
+ N + Y + N + Y + + + 832 + 320
- citizen_education_diploma_output + Table output Change job status on error Y error_description - + error_code diff --git a/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl similarity index 65% rename from mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl rename to mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl index 61e43e6..262fee7 100644 --- a/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl @@ -1,7 +1,7 @@ - russian_diploma_flow_delta + citizen_address_flow_delta Y @@ -13,41 +13,16 @@ 1000 100 - - 2025/09/01 17:09:06.132 + 2025/08/11 13:22:56.048 - - 2025/09/01 17:09:06.132 + 2025/08/11 13:22:56.048 - - Change job status on error - Abort - Y - Create job execution record - education_input - Y - - - Filter rows - Change job status on success - Y - - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - Insert / update - Y - - - education_input - Identify last row in a stream + Table input Y @@ -55,14 +30,39 @@ Change job status on error Y + + Detect empty stream + Change job status on success + Y + + + Filter rows + Change job status on success + Y + + + Change job status on error + Abort + Y + + + Table input + Insert / update + Y + Insert / update + Identify last row in a stream + Y + + + Identify last row in a stream Filter rows Y - Detect empty stream - Change job status on success + Insert / update + Detect empty stream Y @@ -77,13 +77,13 @@ none - ABORT + ABORT_WITH_ERROR Y 0 - 1392 - 656 + 1008 + 464 @@ -117,8 +117,8 @@ and recruitment_id = '${IDM_ID}'; - 1168 - 656 + 752 + 464 @@ -143,13 +143,14 @@ and recruitment_id = '${IDM_ID}'; UPDATE etl.job_execution SET status = 'DELTA_SUCCESS' WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; +and recruitment_id = '${IDM_ID}' +and status = 'DELTA_PROCESSING'; - 1392 - 288 + 1120 + 16 @@ -174,14 +175,14 @@ and recruitment_id = '${IDM_ID}'; UPDATE etl.job_execution SET status = 'DELTA_PROCESSING', - execution_datetime = current_timestamp, + execution_datetime = DEFAULT, error_description = NULL where job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - 464 - 464 + 128 + 224 @@ -197,8 +198,8 @@ and recruitment_id = '${IDM_ID}'; - 880 - 288 + 752 + 16 @@ -233,15 +234,15 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1392 - 464 + 1120 + 224 Identify last row in a stream DetectLastRow - N + Y 1 @@ -251,121 +252,96 @@ and recruitment_id = '${IDM_ID}'; last_row - 880 - 464 + 944 + 224 Insert / update InsertUpdate - Y + N 1 none - 10000 + 1000 ervu-dashboard = - source_id - source_id + recruit_id + recruit_id ervu_dashboard - russian_diploma
+ citizen_address
recruit_id recruit_id N - source_id - source_id - N - - - source_update_date - source_update_date + residence_address + residence_address Y - education_level_code - education_level_code + residence_registration_date + residence_registration_date Y - document_type - document_type + residence_deregistration_date + residence_deregistration_date Y - minobr_document_type_code - minobr_document_type_code + place_stay_address + place_stay_address Y - minprosv_document_type_code - minprosv_document_type_code + place_stay_registration_date + place_stay_registration_date Y - seria - seria + place_stay_deregistration_date + place_stay_deregistration_date Y - number - number + place_stay_registration_type + place_stay_registration_type Y - reg_number - reg_number + factual_address + factual_address Y - issue_date - issue_date + factual_info_received_date + factual_info_received_date Y - educational_institution_name - educational_institution_name - Y - - - speciality_name - speciality_name - Y - - - minobr_speciality_code - minobr_speciality_code - Y - - - minprosv_speciality_code - minprosv_speciality_code - Y - - - vus_spec - vus_spec + factual_info_expiry_date + factual_info_expiry_date Y
N - 1168 - 464 + 752 + 224
- education_input + Table input TableInput Y @@ -378,35 +354,37 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr - FROM recruits_info ri - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedRObr' -> 'robr') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}') -SELECT fd.recruit_id, - diploma_elem ->> 'id' AS source_id, - to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, - diploma_elem ->> 'kodUrObr' as education_level_code, - diploma_elem -> 'vidDok' ->> 'naimVidDok' as document_type, - diploma_elem -> 'vidDokMinobr' ->> 'kodVidDok' as minobr_document_type_code, - diploma_elem -> 'vidDokMinprosv' ->> 'kodVidDok' as minprosv_document_type_code, - diploma_elem ->> 'seriyaBlank' as seria, - diploma_elem ->> 'nomerBlank' as number, - diploma_elem ->> 'regNomer' as reg_number, - to_date(diploma_elem ->> 'dataVyd', 'YYYY-MM-DD') as issue_date, - diploma_elem ->> 'naimOrg' as educational_institution_name, - diploma_elem -> 'svedSpecz' ->> 'naimSpecz' as speciality_name, - diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, - diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, - coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec -FROM filteredData fd - CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem -where to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'; + SELECT + ri.recruit_id::uuid, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, + + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, + CASE + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' + ELSE NULL + END AS place_stay_registration_type, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date +FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id + CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr +WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date + AND c.update_date >= '${M_R_UP_DATE}'::timestamp +GROUP BY ri.recruit_id; Y - 688 - 464 + 400 + 224 @@ -417,7 +395,7 @@ where to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_ error_description - + error_code diff --git a/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl similarity index 64% rename from mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl rename to mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl index 0dbeced..6722b66 100644 --- a/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl @@ -1,7 +1,7 @@ - russian_diploma_flow_repeat + citizen_address_flow_repeat Y @@ -13,41 +13,16 @@ 1000 100 - - 2025/09/01 17:03:13.520 + 2025/08/08 13:25:52.319 - - 2025/09/01 17:03:13.520 + 2025/08/08 13:25:52.319 - - Change job status on error - Abort - Y - Create job execution record - education_input - Y - - - Filter rows - Change job status on success - Y - - - Identify last row in a stream - Detect empty stream - Y - - - education_input - Identify last row in a stream - Y - - - Identify last row in a stream - Insert / update + Table input Y @@ -57,7 +32,7 @@ Insert / update - Filter rows + Detect empty stream Y @@ -65,6 +40,31 @@ Change job status on success Y + + Filter rows + Change job status on success + Y + + + Change job status on error + Abort + Y + + + Insert / update + Identify last row in a stream + Y + + + Identify last row in a stream + Filter rows + Y + + + Table input + Insert / update + Y + Abort @@ -77,12 +77,12 @@ none - ABORT + ABORT_WITH_ERROR Y 0 - 1136 + 1232 544 @@ -117,7 +117,7 @@ and recruitment_id = '${IDM_ID}'; - 912 + 768 544 @@ -143,13 +143,14 @@ and recruitment_id = '${IDM_ID}'; UPDATE etl.job_execution SET status = 'SUCCESS' WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; +and recruitment_id = '${IDM_ID}' +and status = 'PROCESSING'; - 1136 - 208 + 1120 + 16 @@ -171,17 +172,31 @@ and recruitment_id = '${IDM_ID}'; Y N N - INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id) -VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}') + INSERT INTO etl.job_execution ( + id, + job_name, + status, + execution_datetime, + error_description, + recruitment_id +) +VALUES ( + DEFAULT, + '${JOB_NAME}', + 'PROCESSING', + DEFAULT, + NULL, + '${IDM_ID}' +) ON CONFLICT (job_name, recruitment_id) -DO UPDATE -SET status = 'PROCESSING', - error_description = null, - execution_datetime = current_timestamp; +DO UPDATE SET + status = 'PROCESSING', + execution_datetime = DEFAULT, + error_description = NULL; - 208 - 384 + 176 + 272 @@ -197,8 +212,8 @@ SET status = 'PROCESSING', - 624 - 208 + 768 + 16 @@ -233,15 +248,15 @@ SET status = 'PROCESSING', Change job status on success - 1136 - 384 + 1120 + 272 Identify last row in a stream DetectLastRow - N + Y 1 @@ -251,121 +266,96 @@ SET status = 'PROCESSING', last_row - 624 - 384 + 960 + 272 Insert / update InsertUpdate - Y + N 1 none - 10000 + 1000 ervu-dashboard = - source_id - source_id + recruit_id + recruit_id ervu_dashboard - russian_diploma
+ citizen_address
recruit_id recruit_id N - source_id - source_id - N - - - source_update_date - source_update_date + residence_address + residence_address Y - education_level_code - education_level_code + residence_registration_date + residence_registration_date Y - document_type - document_type + residence_deregistration_date + residence_deregistration_date Y - minobr_document_type_code - minobr_document_type_code + place_stay_address + place_stay_address Y - minprosv_document_type_code - minprosv_document_type_code + place_stay_registration_date + place_stay_registration_date Y - seria - seria + place_stay_deregistration_date + place_stay_deregistration_date Y - number - number + place_stay_registration_type + place_stay_registration_type Y - reg_number - reg_number + factual_address + factual_address Y - issue_date - issue_date + factual_info_received_date + factual_info_received_date Y - educational_institution_name - educational_institution_name - Y - - - speciality_name - speciality_name - Y - - - minobr_speciality_code - minobr_speciality_code - Y - - - minprosv_speciality_code - minprosv_speciality_code - Y - - - vus_spec - vus_spec + factual_info_expiry_date + factual_info_expiry_date Y
N - 912 - 384 + 768 + 272
- education_input + Table input TableInput Y @@ -378,35 +368,36 @@ SET status = 'PROCESSING', ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr - FROM recruits_info ri - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedRObr' -> 'robr') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' -) -SELECT fd.recruit_id, - diploma_elem ->> 'id' AS source_id, - to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, - diploma_elem ->> 'kodUrObr' as education_level_code, - diploma_elem -> 'vidDok' ->> 'naimVidDok' as document_type, - diploma_elem -> 'vidDokMinobr' ->> 'kodVidDok' as minobr_document_type_code, - diploma_elem -> 'vidDokMinprosv' ->> 'kodVidDok' as minprosv_document_type_code, - diploma_elem ->> 'seriyaBlank' as seria, - diploma_elem ->> 'nomerBlank' as number, - diploma_elem ->> 'regNomer' as reg_number, - to_date(diploma_elem ->> 'dataVyd', 'YYYY-MM-DD') as issue_date, - diploma_elem ->> 'naimOrg' as educational_institution_name, - diploma_elem -> 'svedSpecz' ->> 'naimSpecz' as speciality_name, - diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, - diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, - coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec -FROM filteredData fd - CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem; + SELECT + ri.recruit_id::uuid, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, + MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, + + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, + MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, + CASE + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' + WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' + ELSE NULL + END AS place_stay_registration_type, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, + MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date +FROM ervu_dashboard.recruits_info ri + JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id + CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr +WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date +GROUP BY ri.recruit_id; Y - 432 - 384 + 480 + 272 @@ -417,7 +408,7 @@ FROM filteredData fd error_description - + error_code diff --git a/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl similarity index 64% rename from mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl rename to mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl index fda88dd..ebfe4cd 100644 --- a/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl @@ -1,7 +1,7 @@ - russian_diploma_flow + citizen_spouse_flow Y @@ -13,21 +13,21 @@ 1000 100 - - 2025/09/01 16:13:06.345 + 2025/08/05 14:31:22.799 - - 2025/09/01 16:13:06.345 + 2025/08/05 14:31:22.799 - Change job status on error - Abort + Create job execution record + Table input Y - Create job execution record - education_input + Table output + Change job status on error Y @@ -36,35 +36,35 @@ Y - Identify last row in a stream + Table output Detect empty stream Y - - citizen_education_diploma_output - Change job status on error - Y - - - education_input - Identify last row in a stream - Y - - - Identify last row in a stream - citizen_education_diploma_output - Y - - - citizen_education_diploma_output - Filter rows - Y - Detect empty stream Change job status on success Y + + Change job status on error + Abort + Y + + + Table input + Identify last row in a stream + Y + + + Identify last row in a stream + Table output + Y + + + Table output + Filter rows + Y + Abort @@ -77,13 +77,13 @@ none - ABORT + ABORT_WITH_ERROR Y 0 - 1152 - 560 + 1136 + 528 @@ -117,8 +117,8 @@ and recruitment_id = '${IDM_ID}'; - 928 - 560 + 720 + 528 @@ -143,13 +143,14 @@ and recruitment_id = '${IDM_ID}'; UPDATE etl.job_execution SET status = 'SUCCESS' WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; +and recruitment_id = '${IDM_ID}' +and status = 'PROCESSING'; - 1152 - 224 + 1120 + 64 @@ -177,8 +178,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 224 - 400 + 80 + 320 @@ -194,8 +195,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 640 - 224 + 1120 + 320 @@ -230,15 +231,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1152 - 400 + 720 + 64 Identify last row in a stream DetectLastRow - N + Y 1 @@ -248,113 +249,12 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); last_row - 640 - 400 + 576 + 320 - citizen_education_diploma_output - TableOutput - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - recruit_id - recruit_id - - - source_id - source_id - - - source_update_date - source_update_date - - - education_level_code - education_level_code - - - document_type - document_type - - - minobr_document_type_code - minobr_document_type_code - - - minprosv_document_type_code - minprosv_document_type_code - - - seria - seria - - - number - number - - - reg_number - reg_number - - - issue_date - issue_date - - - educational_institution_name - educational_institution_name - - - speciality_name - speciality_name - - - minobr_speciality_code - minobr_speciality_code - - - minprosv_speciality_code - minprosv_speciality_code - - - vus_spec - vus_spec - - - N - N - N - N - - Y - - N - ervu_dashboard - Y - russian_diploma
- - N - Y - N - Y - - - 928 - 400 - -
- - education_input + Table input TableInput Y @@ -367,46 +267,159 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr - FROM recruits_info ri - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedRObr' -> 'robr') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' -) -SELECT fd.recruit_id, - diploma_elem ->> 'id' AS source_id, - to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, - diploma_elem ->> 'kodUrObr' as education_level_code, - diploma_elem -> 'vidDok' ->> 'naimVidDok' as document_type, - diploma_elem -> 'vidDokMinobr' ->> 'kodVidDok' as minobr_document_type_code, - diploma_elem -> 'vidDokMinprosv' ->> 'kodVidDok' as minprosv_document_type_code, - diploma_elem ->> 'seriyaBlank' as seria, - diploma_elem ->> 'nomerBlank' as number, - diploma_elem ->> 'regNomer' as reg_number, - to_date(diploma_elem ->> 'dataVyd', 'YYYY-MM-DD') as issue_date, - diploma_elem ->> 'naimOrg' as educational_institution_name, - diploma_elem -> 'svedSpecz' ->> 'naimSpecz' as speciality_name, - diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, - diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, - coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec -FROM filteredData fd - CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem; + SELECT + ri.recruit_id AS recruit_id, + (supr->>'id')::uuid AS spouse_external_id, + NULLIF(supr->>'idERN', '') AS spouse_id_ern, + NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, + supr->'svedFLBS'->'fio'->>'familiya' AS last_name, + supr->'svedFLBS'->'fio'->>'imya' AS first_name, + supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + supr->'svedFLBS'->'fio'->>'familiya', + supr->'svedFLBS'->'fio'->>'imya', + supr->'svedFLBS'->'fio'->>'otchestvo' + ) AS full_name, + MAKE_DATE( + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, + supr->'svedSmert'->>'nomerZapis' AS death_az_number, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, + CASE + WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true + ELSE false + END AS information_excluded +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id +CROSS JOIN LATERAL ( + SELECT supri + FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri + WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' +) AS supr(supr) +WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date; Y - 448 - 400 + 368 + 320 + + + + Table output + TableOutput + + N + + 1 + + none + + + 1000 + ervu-dashboard + + + recruit_id + recruit_id + + + spouse_external_id + spouse_external_id + + + spouse_id_ern + spouse_id_ern + + + kinship_type + kinship_type + + + last_name + last_name + + + first_name + first_name + + + middle_name + middle_name + + + full_name + full_name + + + birth_date + birth_date + + + death_date + death_date + + + death_az_number + death_az_number + + + marriage_az_number + marriage_az_number + + + marriage_date + marriage_date + + + divorce_az_number + divorce_az_number + + + divorce_date + divorce_date + + + information_excluded + information_excluded + + + N + N + N + N + Y + N + ervu_dashboard + Y + citizen_spouse
+ N + Y + N + Y + + + 720 + 320
- citizen_education_diploma_output + Table output Change job status on error Y error_description - + error_code diff --git a/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl similarity index 63% rename from mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl rename to mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl index 9095203..99e9f11 100644 --- a/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl @@ -1,7 +1,7 @@ - foreign_diploma_flow_delta + citizen_spouse_flow_delta Y @@ -13,21 +13,31 @@ 1000 100 - - 2025/09/05 08:36:17.176 + 2025/08/11 13:22:56.048 - - 2025/09/05 08:36:17.176 + 2025/08/11 13:22:56.048 + + Create job execution record + Table input + Y + + + Insert / update + Change job status on error + Y + Change job status on error Abort Y - Create job execution record - foerign_diploma_input + Insert / update + Filter rows Y @@ -36,35 +46,25 @@ Y - Identify last row in a stream + Insert / update Detect empty stream Y - - Identify last row in a stream - foreign_diploma_upsert - Y - - - foerign_diploma_input - Identify last row in a stream - Y - - - foreign_diploma_upsert - Change job status on error - Y - - - foreign_diploma_upsert - Filter rows - Y - Detect empty stream Change job status on success Y + + Table input + Identify last row in a stream + Y + + + Identify last row in a stream + Insert / update + Y + Abort @@ -77,13 +77,13 @@ none - ABORT + ABORT_WITH_ERROR Y 0 - 1280 - 688 + 976 + 480 @@ -101,6 +101,9 @@ error_description + + error_code + ervu-dashboard Y @@ -117,8 +120,8 @@ and recruitment_id = '${IDM_ID}'; - 1056 - 688 + 736 + 480 @@ -141,15 +144,16 @@ and recruitment_id = '${IDM_ID}'; N N UPDATE etl.job_execution -SET status = 'DELTA_SUCCESS' +SET status = 'DELTA_SUCCESS', + error_description = null WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - + and recruitment_id = '${IDM_ID}' + and status = 'DELTA_PROCESSING'; - 1280 - 320 + 1200 + 80 @@ -174,14 +178,13 @@ and recruitment_id = '${IDM_ID}'; UPDATE etl.job_execution SET status = 'DELTA_PROCESSING', - execution_datetime = current_timestamp, - error_description = NULL + execution_datetime = DEFAULT where job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - 352 - 496 + 96 + 224 @@ -197,8 +200,8 @@ and recruitment_id = '${IDM_ID}'; - 768 - 320 + 1200 + 224 @@ -233,15 +236,15 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1280 - 496 + 736 + 80 Identify last row in a stream DetectLastRow - N + Y 1 @@ -251,15 +254,129 @@ and recruitment_id = '${IDM_ID}'; last_row - 768 - 496 + 576 + 224 - foerign_diploma_input + Insert / update + InsertUpdate + + N + + 1 + + none + + + 100 + ervu-dashboard + + + = + recruit_id + recruit_id + + + = + spouse_external_id + spouse_external_id + + ervu_dashboard + citizen_spouse
+ + recruit_id + recruit_id + N + + + spouse_external_id + spouse_external_id + N + + + spouse_id_ern + spouse_id_ern + Y + + + kinship_type + kinship_type + Y + + + last_name + last_name + Y + + + first_name + first_name + Y + + + middle_name + middle_name + Y + + + full_name + full_name + Y + + + birth_date + birth_date + Y + + + death_date + death_date + Y + + + death_az_number + death_az_number + Y + + + marriage_az_number + marriage_az_number + Y + + + marriage_date + marriage_date + Y + + + divorce_az_number + divorce_az_number + Y + + + divorce_date + divorce_date + Y + + + information_excluded + information_excluded + Y + +
+ N + + + 736 + 224 + +
+ + Table input TableInput - Y + N 1 @@ -269,151 +386,63 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, - diploma_elem - FROM recruits_info ri - CROSS JOIN LATERAL jsonb_array_elements( - ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr') AS diploma_elem - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - and to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' -) -SELECT fd.recruit_id, - diploma_elem ->> 'id' AS source_id, - to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, - diploma_elem ->> 'kodUrObr' as education_level_code, - diploma_elem ->> 'naimUchZaved' as educational_institution_name, - diploma_elem ->> 'naimInoDok' as foreign_doc_name, - diploma_elem ->> 'serNomDok' as ser_num_doc, - diploma_elem ->> 'regNomDok' as reg_num, - diploma_elem ->> 'nomSved' as recognition_certificate_number, - to_date(diploma_elem ->> 'dataPriz', 'YYYY-MM-DD') as recognition_date, - diploma_elem -> 'svedSpecz' ->> 'naimSpecz' as speciality_name, - diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, - diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, - diploma_elem ->> 'kodOKSM' as oksm_code, - coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec -FROM filteredData fd; + SELECT + ri.recruit_id AS recruit_id, + (supr->>'id')::uuid AS spouse_external_id, + NULLIF(supr->>'idERN', '') AS spouse_id_ern, + NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, + supr->'svedFLBS'->'fio'->>'familiya' AS last_name, + supr->'svedFLBS'->'fio'->>'imya' AS first_name, + supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, + CONCAT_WS(' ', + supr->'svedFLBS'->'fio'->>'familiya', + supr->'svedFLBS'->'fio'->>'imya', + supr->'svedFLBS'->'fio'->>'otchestvo' + ) AS full_name, + MAKE_DATE( + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, + NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int + ) AS birth_date, + (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, + supr->'svedSmert'->>'nomerZapis' AS death_az_number, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, + ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, + (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, + CASE + WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true + ELSE false + END AS information_excluded +FROM ervu_dashboard.recruits_info ri +JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id +CROSS JOIN LATERAL ( + SELECT supri + FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri + WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' +) AS supr(supr) +WHERE + '${IDM_ID}' != '' + AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date + AND c.update_date >= '${M_R_UP_DATE}'::timestamp; + Y - 576 - 496 - - - - foreign_diploma_upsert - InsertUpdate - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - = - source_id - source_id - - ervu_dashboard - foreign_diploma
- - recruit_id - recruit_id - N - - - source_id - source_id - N - - - source_update_date - source_update_date - Y - - - education_level_code - education_level_code - Y - - - educational_institution_name - educational_institution_name - Y - - - foreign_doc_name - foreign_doc_name - Y - - - ser_num_doc - ser_num_doc - Y - - - reg_num - reg_num - Y - - - recognition_certificate_number - recognition_certificate_number - Y - - - recognition_date - recognition_date - Y - - - speciality_name - speciality_name - Y - - - minobr_speciality_code - minobr_speciality_code - Y - - - minprosv_speciality_code - minprosv_speciality_code - Y - - - oksm_code - oksm_code - Y - - - vus_spec - vus_spec - Y - -
- N - - - 1056 - 496 + 368 + 224
- foreign_diploma_upsert + Insert / update Change job status on error Y error_description - + error_code diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow.hpl b/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_delta.hpl similarity index 78% rename from mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow.hpl rename to mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_delta.hpl index 47fa430..864793c 100644 --- a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_delta.hpl @@ -1,7 +1,7 @@ - recruitment_five_flow + recruitment_five_flow_delta Y @@ -13,41 +13,41 @@ 1000 100 - - 2025/08/28 08:16:44.450 + 2025/08/11 13:16:11.204 - - 2025/08/28 08:16:44.450 + 2025/08/11 13:16:11.204 - Get all recruitments - citizen_education_flow.hpl + Table input + citizenship_foreign_flow_delta.hpl Y - Get all recruitments - citizen_education_flow.hpl 2 + Table input + citizenship_foreign_flow_delta.hpl 2 Y - Get all recruitments - citizen_education_flow.hpl 3 + Table input + citizenship_foreign_flow_delta.hpl 3 Y - Get all recruitments - citizen_education_flow.hpl 4 + Table input + citizenship_foreign_flow_delta.hpl 4 Y - Get all recruitments - citizen_education_flow.hpl 5 + Table input + citizenship_foreign_flow_delta.hpl 5 Y - Get all recruitments + Table input TableInput Y @@ -57,20 +57,25 @@ none - ervu-dashboard-test + ervu-dashboard N - SELECT -idm_id -FROM ervu_dashboard.recruitment; + 0 + SELECT + r.idm_id as recruitment_id +FROM ervu_dashboard.recruitment r + LEFT JOIN etl.job_execution je + ON r.idm_id = je.recruitment_id + and job_name = 'citizenship_foreign_job' +where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); N 352 - 352 + 288 - citizen_education_flow.hpl + citizenship_foreign_flow_delta.hpl PipelineExecutor Y @@ -81,7 +86,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl N 1 @@ -90,9 +95,14 @@ FROM ervu_dashboard.recruitment; IDM_ID - idm_id + recruitment_id + + JOB_NAME + + citizenship_foreign_job + Y @@ -116,12 +126,12 @@ FROM ervu_dashboard.recruitment; - 624 - 192 + 640 + 128 - citizen_education_flow.hpl 2 + citizenship_foreign_flow_delta.hpl 2 PipelineExecutor Y @@ -132,7 +142,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl N 1 @@ -141,9 +151,14 @@ FROM ervu_dashboard.recruitment; IDM_ID - idm_id + recruitment_id + + JOB_NAME + + citizenship_foreign_job + Y @@ -167,12 +182,12 @@ FROM ervu_dashboard.recruitment; - 624 - 272 + 656 + 208 - citizen_education_flow.hpl 3 + citizenship_foreign_flow_delta.hpl 3 PipelineExecutor Y @@ -183,7 +198,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl N 1 @@ -192,9 +207,14 @@ FROM ervu_dashboard.recruitment; IDM_ID - idm_id + recruitment_id + + JOB_NAME + + citizenship_foreign_job + Y @@ -218,12 +238,12 @@ FROM ervu_dashboard.recruitment; - 624 - 352 + 656 + 288 - citizen_education_flow.hpl 4 + citizenship_foreign_flow_delta.hpl 4 PipelineExecutor Y @@ -234,7 +254,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl N 1 @@ -243,9 +263,14 @@ FROM ervu_dashboard.recruitment; IDM_ID - idm_id + recruitment_id + + JOB_NAME + + citizenship_foreign_job + Y @@ -269,12 +294,12 @@ FROM ervu_dashboard.recruitment; - 624 - 432 + 656 + 368 - citizen_education_flow.hpl 5 + citizenship_foreign_flow_delta.hpl 5 PipelineExecutor Y @@ -285,7 +310,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl N 1 @@ -294,9 +319,14 @@ FROM ervu_dashboard.recruitment; IDM_ID - idm_id + recruitment_id + + JOB_NAME + + citizenship_foreign_job + Y @@ -320,8 +350,8 @@ FROM ervu_dashboard.recruitment; - 624 - 512 + 656 + 448 diff --git a/mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/deferment_liberation_permission/recruitment_five_flow_delta.hpl similarity index 77% rename from mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_delta.hpl rename to mappings/info_recruits/citizen_tables/deferment_liberation_permission/recruitment_five_flow_delta.hpl index ea8c53a..e55fa8d 100644 --- a/mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/deferment_liberation_permission/recruitment_five_flow_delta.hpl @@ -13,41 +13,36 @@ 1000 100 - - 2025/08/28 21:23:43.331 + 2025/08/11 13:16:11.204 - - 2025/08/28 21:23:43.331 + 2025/08/11 13:16:11.204 Table input - foreign_diploma_flow_delta.hpl - Y - - - get_max_source_update_date - Table input + deferment_liberation_permission_flow_delta.hpl Y Table input - foreign_diploma_flow_delta.hpl 2 + deferment_liberation_permission_flow_delta.hpl 2 Y Table input - foreign_diploma_flow_delta.hpl 3 + deferment_liberation_permission_flow_delta.hpl 3 Y Table input - foreign_diploma_flow_delta.hpl 4 + deferment_liberation_permission_flow_delta.hpl 4 Y Table input - foreign_diploma_flow_delta.hpl 5 + deferment_liberation_permission_flow_delta.hpl 5 Y @@ -62,35 +57,249 @@ none - ervu-dashboard-test + ervu-dashboard N 0 - get_max_source_update_date SELECT - r.idm_id, - ? max_source_update_date + r.idm_id as recruitment_id FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - and job_name = '${JOB_NAME}' - JOIN recruits_info ri - ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id - AND ri.updated_at > ( - SELECT MAX(execution_datetime) - FROM etl.job_execution - WHERE job_name = '${JOB_NAME}' - AND recruitment_id = r.idm_id - ) + and job_name = 'deferment_liberation_permission_job' where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); N - 608 + 352 + 288 + + + + deferment_liberation_permission_flow_delta.hpl + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + deferment_liberation_permission_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 640 + 128 + + + + deferment_liberation_permission_flow_delta.hpl 2 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + deferment_liberation_permission_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 640 + 208 + + + + deferment_liberation_permission_flow_delta.hpl 3 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + deferment_liberation_permission_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 640 + 288 + + + + deferment_liberation_permission_flow_delta.hpl 4 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_delta.hpl + N + + 1 + + + + + IDM_ID + recruitment_id + + + + JOB_NAME + + deferment_liberation_permission_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 640 384 - foreign_diploma_flow_delta.hpl + deferment_liberation_permission_flow_delta.hpl 5 PipelineExecutor Y @@ -101,7 +310,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_delta.hpl N 1 @@ -110,13 +319,13 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - idm_id + recruitment_id - MAX_SOURCE_UPDATE_DATE - max_source_update_date - + JOB_NAME + + deferment_liberation_permission_job Y @@ -141,255 +350,8 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 800 - 224 - - - - foreign_diploma_flow_delta.hpl 2 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 800 - 304 - - - - foreign_diploma_flow_delta.hpl 3 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 800 - 384 - - - - foreign_diploma_flow_delta.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 800 - 464 - - - - foreign_diploma_flow_delta.hpl 5 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 800 - 544 - - - - get_max_source_update_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - select max(source_update_date) -from foreign_diploma; - N - - - 432 - 384 + 640 + 480 diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow.hpl deleted file mode 100644 index cd730ef..0000000 --- a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow.hpl +++ /dev/null @@ -1,362 +0,0 @@ - - - - education_constants_flow - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/21 20:19:05.913 - - - 2025/09/21 20:19:05.913 - - - - - - education_level_input - education_level_output - Y - - - minprosv_speciality_input - minprosv_speciality_output - Y - - - minobr_speciality_input - minobr_speciality_output - Y - - - educational_institution_status_input - educational_institution_status_output - Y - - - - education_level_input - TableInput - - Y - - 1 - - none - - - nsi - N - 0 - SELECT - key as code, - value ->> 'value' as value, - NOT hidden as actual, - updated_at as update_date -FROM classifier_records -WHERE code = 'educationLevel'; - N - - - 368 - 208 - - - - education_level_output - TableOutput - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - code - code - - - value - value - - - actual - actual - - - update_date - update_date - - - N - N - N - N - Y - N - ervu_dashboard - Y - education_level
- N - Y - N - Y - - - 624 - 208 - -
- - educational_institution_status_input - TableInput - - Y - - 1 - - none - - - nsi - N - 0 - SELECT - key, - value ->> 'value' as value, - NOT hidden as actual, - updated_at as update_date -FROM classifier_records -WHERE code = 'organizationStatus'; - N - - - 368 - 304 - - - - educational_institution_status_output - TableOutput - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - code - code - - - value - value - - - actual - actual - - - update_date - update_date - - - N - N - N - N - Y - N - ervu_dashboard - Y - educational_institution_status
- N - Y - N - Y - - - 624 - 304 - -
- - minobr_speciality_input - TableInput - - Y - - 1 - - none - - - nsi - N - 0 - SELECT - value -> 'extraInfo' ->> 'kodSpecz' as code, - value -> 'extraInfo' ->> 'naimSpecz' as value, - NOT hidden as actual, - updated_at as update_date -FROM classifier_records -WHERE code = 'educationSpecialityMinobr' - AND value -> 'extraInfo' ->> 'kodSpecz' != 'null'; - N - - - 368 - 400 - - - - minobr_speciality_output - TableOutput - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - code - code - - - value - value - - - actual - actual - - - update_date - update_date - - - N - N - N - N - Y - N - ervu_dashboard - Y - minobr_education_speciality
- N - Y - N - Y - - - 624 - 400 - -
- - minprosv_speciality_input - TableInput - - Y - - 1 - - none - - - nsi - N - 0 - SELECT - key, - value -> 'extraInfo' ->> 'kodSpecz' as code, - value -> 'extraInfo' ->> 'naimSpecz' as value, - NOT hidden as actual, - updated_at as update_date -FROM classifier_records -WHERE code = 'educationSpecialityMinprosv' - AND value -> 'extraInfo' ->> 'kodSpecz' != 'null'; - N - - - 368 - 496 - - - - minprosv_speciality_output - TableOutput - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - code - code - - - value - value - - - actual - actual - - - update_date - update_date - - - N - N - N - N - Y - N - ervu_dashboard - Y - minprosv_education_speciality
- N - Y - N - Y - - - 624 - 496 - -
- - - -
diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow_repeat.hpl deleted file mode 100644 index ebffbdf..0000000 --- a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow_repeat.hpl +++ /dev/null @@ -1,508 +0,0 @@ - - - - education_constants_flow_repeat - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/21 20:33:27.321 - - - 2025/09/21 20:33:27.321 - - - - - - get_max_education_level_update_date - education_level_input - Y - - - get_max_educational_institution_status_update_date - educational_institution_status_input - Y - - - get_max_minobr_speciality_update_date - minobr_speciality_input - Y - - - get_max_minprosv_speciality_update_date - minprosv_speciality_input - Y - - - education_level_input - education_level_upsert - Y - - - educational_institution_status_input - educational_institution_status_upsert - Y - - - minobr_speciality_input - minobr_speciality_upsert - Y - - - minprosv_speciality_input - minprosv_speciality_upsert - Y - - - - education_level_input - TableInput - - Y - - 1 - - none - - - nsi - N - 0 - get_max_education_level_update_date - SELECT - key as code, - value ->> 'value' as value, - NOT hidden as actual, - updated_at as update_date -FROM classifier_records -WHERE code = 'educationLevel' -AND updated_at > ?; - N - - - 480 - 256 - - - - education_level_upsert - InsertUpdate - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - = - code - code - - - - <> - actual - actual - - - ervu_dashboard - education_level
- - code - code - N - - - value - value - Y - - - actual - actual - Y - - - update_date - update_date - Y - -
- N - - - 768 - 256 - -
- - educational_institution_status_input - TableInput - - Y - - 1 - - none - - - nsi - N - 0 - get_max_educational_institution_status_update_date - SELECT - key, - value ->> 'value' as value, - NOT hidden as actual, - updated_at as update_date -FROM classifier_records -WHERE code = 'organizationStatus' -AND updated_at > ?; - N - - - 480 - 352 - - - - educational_institution_status_upsert - InsertUpdate - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - = - code - code - - - <> - actual - actual - - ervu_dashboard - educational_institution_status
- - code - code - N - - - value - value - Y - - - actual - actual - Y - - - update_date - update_date - Y - -
- N - - - 768 - 352 - -
- - get_max_education_level_update_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT - max(update_date) as max_update_date -FROM education_level; - - N - - - 128 - 256 - - - - get_max_educational_institution_status_update_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT - max(update_date) as max_update_date -FROM educational_institution_status; - - N - - - 128 - 352 - - - - get_max_minobr_speciality_update_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT - max(update_date) as max_update_date -FROM minobr_education_speciality; - - N - - - 128 - 448 - - - - get_max_minprosv_speciality_update_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT - max(update_date) as max_update_date -FROM minprosv_education_speciality - - N - - - 128 - 544 - - - - minobr_speciality_input - TableInput - - Y - - 1 - - none - - - nsi - N - 0 - get_max_minobr_speciality_update_date - SELECT - value -> 'extraInfo' ->> 'kodSpecz' as code, - value -> 'extraInfo' ->> 'naimSpecz' as value, - NOT hidden as actual, - updated_at as update_date -FROM classifier_records -WHERE code = 'educationSpecialityMinobr' - AND value -> 'extraInfo' ->> 'kodSpecz' != 'null' -AND updated_at > ?; - N - - - 480 - 448 - - - - minobr_speciality_upsert - InsertUpdate - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - = - code - code - - - <> - actual - actual - - ervu_dashboard - minobr_education_speciality
- - code - code - N - - - value - value - Y - - - actual - actual - Y - - - update_date - update_date - Y - -
- N - - - 768 - 448 - -
- - minprosv_speciality_input - TableInput - - Y - - 1 - - none - - - nsi - N - 0 - get_max_minprosv_speciality_update_date - SELECT - key, - value -> 'extraInfo' ->> 'kodSpecz' as code, - value -> 'extraInfo' ->> 'naimSpecz' as value, - NOT hidden as actual, - updated_at as update_date -FROM classifier_records -WHERE code = 'educationSpecialityMinprosv' - AND value -> 'extraInfo' ->> 'kodSpecz' != 'null' -AND updated_at > ?; - N - - - 480 - 544 - - - - minprosv_speciality_upsert - InsertUpdate - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - = - code - code - - - <> - actual - actual - - ervu_dashboard - minprosv_education_speciality
- - code - code - N - - - value - value - Y - - - actual - actual - Y - - - update_date - update_date - Y - -
- N - - - 768 - 544 - -
- - - -
diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/support/check_if_constants_exists.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/support/check_if_constants_exists.hpl deleted file mode 100644 index 906090d..0000000 --- a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/support/check_if_constants_exists.hpl +++ /dev/null @@ -1,88 +0,0 @@ - - - - check_if_constants_exists - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/21 20:58:53.188 - - - 2025/09/21 20:58:53.188 - - - - - - Table input - Set variables - Y - - - - Set variables - SetVariable - - Y - - 1 - - none - - - - - - constasnts_exists - CONSTANTS_EXISTS - PARENT_WORKFLOW - - - Y - - - 608 - 304 - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT EXISTS ( - SELECT 1 FROM education_level - UNION ALL - SELECT 1 FROM educational_institution_status - UNION ALL - SELECT 1 FROM minobr_education_speciality - UNION ALL - SELECT 1 FROM minprosv_education_speciality -) AS constasnts_exists - Y - - - 320 - 304 - - - - - - diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/educational_institution_learning_job.hwf b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/educational_institution_learning_job.hwf deleted file mode 100644 index 405c135..0000000 --- a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/educational_institution_learning_job.hwf +++ /dev/null @@ -1,410 +0,0 @@ - - - educational_institution_learning_job - Y - - - - - - 2025/08/27 21:14:40.330 - - - 2025/08/27 21:14:40.330 - - - - - Start - - SPECIAL - - 1 - 12 - 60 - 0 - 0 - N - 0 - 1 - N - 384 - 288 - - - - check_if_job_execution_exists.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_job_execution_exists.hpl - Basic - - Y - - N - local - N - N - Y - N - 1392 - 304 - - - - employer_job_execution_exists_check - - SIMPLE_EVAL - - boolean - false - equal - equal - N - variable - JOB_EXECUTED_FLAG - N - 1696 - 304 - - - - recruitment_five_flow.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow.hpl - Basic - - Y - - N - local - N - N - Y - Y - 1968 - 304 - - - - recruitment_five_flow_repeat.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_repeat.hpl - Basic - - Y - - N - local - N - N - Y - N - 1968 - 608 - - - - check_if_need_to_repeat.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_need_to_repeat.hpl - Basic - - Y - - N - local - N - N - Y - N - 1696 - 448 - - - - Simple evaluation - - SIMPLE_EVAL - - boolean - true - equal - equal - N - variable - NEED_TO_REPEAT_JOB - N - 1696 - 608 - - - - recruitment_five_flow_delta.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_delta.hpl - Basic - - Y - - N - local - N - N - Y - N - 1696 - 800 - - - - check_if_constants_exists.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/constants/support/check_if_constants_exists.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 576 - 288 - - - - constants_exists - - SIMPLE_EVAL - - boolean - true - equal - equal - N - variable - CONSTANTS_EXISTS - N - 752 - 288 - - - - education_constants_flow.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow.hpl - Basic - - Y - - N - local - N - N - Y - N - 944 - 208 - - - - education_constants_flow_repeat.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow_repeat.hpl - Basic - - Y - - N - local - N - N - Y - N - 944 - 384 - - - - init_job_name - - SET_VARIABLES - - - - JOB_NAME - CURRENT_WORKFLOW - educational_institution_learning_job - - - CURRENT_WORKFLOW - N - N - 1184 - 304 - - - - - - check_if_job_execution_exists.hpl - employer_job_execution_exists_check - Y - Y - N - - - employer_job_execution_exists_check - check_if_need_to_repeat.hpl - Y - N - N - - - check_if_need_to_repeat.hpl - Simple evaluation - Y - Y - N - - - check_if_constants_exists.hpl - constants_exists - Y - Y - N - - - constants_exists - education_constants_flow.hpl - Y - N - N - - - constants_exists - education_constants_flow_repeat.hpl - Y - Y - N - - - Start - check_if_constants_exists.hpl - Y - Y - Y - - - education_constants_flow.hpl - init_job_name - Y - Y - N - - - education_constants_flow_repeat.hpl - init_job_name - Y - Y - N - - - init_job_name - check_if_job_execution_exists.hpl - Y - Y - N - - - employer_job_execution_exists_check - recruitment_five_flow.hpl - Y - Y - N - - - Simple evaluation - recruitment_five_flow_repeat.hpl - Y - Y - N - - - Simple evaluation - recruitment_five_flow_delta.hpl - Y - N - N - - - - - - diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl deleted file mode 100644 index 8bc8c1b..0000000 --- a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl +++ /dev/null @@ -1,1022 +0,0 @@ - - - - citizen_education_flow - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/08/28 08:20:00.808 - - - 2025/08/28 08:20:00.808 - - - - - - Change job status on error - Abort - Y - - - Create job execution record - education_input - Y - - - Filter rows - Change job status on success - Y - - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - Filter_null_parental_leave_dates - Y - - - education_input - Identify last row in a stream - Y - - - citizen_education_output - Change job status on error - Y - - - citizen_education_output - Filter rows - Y - - - education_leave_output - Change job status on error - Y - - - Filter_null_parental_leave_dates - Block until transforms finish - Y - - - Block until transforms finish - education_leave_output - Y - - - Identify last row in a stream - Group by - Y - - - Group by - citizen_education_output - Y - - - Detect empty stream - Change job status on success - Y - - - - Abort - Abort - - Y - - 1 - - none - - - ABORT - Y - 0 - - - 1488 - 640 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - citizen_education_output - - - - - 880 - 640 - - - - Change job status on error - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ? -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 1264 - 640 - - - - Change job status on success - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'SUCCESS' -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 1488 - 256 - - - - Create job execution record - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - N - N - Y - N - N - INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id) -VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - - - - - 208 - 432 - - - - Detect empty stream - DetectEmptyStream - - Y - - 1 - - none - - - - - 624 - 256 - - - - Filter rows - FilterRows - - Y - - 1 - - none - - - - - - - = - last_row - N - - - - N - -1 - constant - -1 - Y - Boolean - - - - Change job status on success - - - 1488 - 432 - - - - Filter_null_parental_leave_dates - FilterRows - - Y - - 1 - - none - - - - - - - IS NOT NULL - academ_reason - N - - - - - Block until transforms finish - - - 624 - 640 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - recruit_id - recruit_id - FIRST_INCL_NULL - - - source_update_date - source_update_date - FIRST_INCL_NULL - - - educational_institution_type - educational_institution_type - FIRST_INCL_NULL - - - org_type - org_type - FIRST_INCL_NULL - - - inn - inn - FIRST_INCL_NULL - - - kpp - kpp - FIRST_INCL_NULL - - - ogrn - ogrn - FIRST_INCL_NULL - - - educational_institution_address - educational_institution_address - FIRST_INCL_NULL - - - educational_organization_separate_unit_type - educational_organization_separate_unit_type - FIRST_INCL_NULL - - - educational_organization_separate_unit_address - educational_organization_separate_unit_address - FIRST_INCL_NULL - - - student_status - student_status - FIRST_INCL_NULL - - - grade_number - grade_number - FIRST_INCL_NULL - - - education_level_code - education_level_code - FIRST_INCL_NULL - - - admission_date - admission_date - FIRST_INCL_NULL - - - enrollment_date - enrollment_date - FIRST_INCL_NULL - - - enrollment_order_number - enrollment_order_number - FIRST_INCL_NULL - - - enrollment_order_date - enrollment_order_date - FIRST_INCL_NULL - - - completion_date - completion_date - FIRST_INCL_NULL - - - education_form - education_form - FIRST_INCL_NULL - - - education_program_start_date - education_program_start_date - FIRST_INCL_NULL - - - education_start_date - education_start_date - FIRST_INCL_NULL - - - planned_completion_date - planned_completion_date - FIRST_INCL_NULL - - - minobr_speciality_code - minobr_speciality_code - FIRST_INCL_NULL - - - minprosv_speciality_code - minprosv_speciality_code - FIRST_INCL_NULL - - - recovery_info - recovery_info - FIRST_INCL_NULL - - - recovery_date - recovery_date - FIRST_INCL_NULL - - - recovery_order_date - recovery_order_date - FIRST_INCL_NULL - - - recovery_order_number - recovery_order_number - FIRST_INCL_NULL - - - expulsion_info - expulsion_info - FIRST_INCL_NULL - - - expulsion_date - expulsion_date - FIRST_INCL_NULL - - - expulsion_order_date - expulsion_order_date - FIRST_INCL_NULL - - - expulsion_order_number - expulsion_order_number - FIRST_INCL_NULL - - - academical_leave_info - academical_leave_info - FIRST_INCL_NULL - - - first_education - first_education - FIRST_INCL_NULL - - - vus_spec - vus_spec - FIRST_INCL_NULL - - - military_center_education_info - military_center_education_info - FIRST_INCL_NULL - - - military_center_education_start_date - military_center_education_start_date - FIRST_INCL_NULL - - - military_center_education_end_date - military_center_education_end_date - FIRST_INCL_NULL - - - military_center_education_period - military_center_education_period - FIRST_INCL_NULL - - - military_center_completion - military_center_completion - FIRST_INCL_NULL - - - military_department_education_info - military_department_education_info - FIRST_INCL_NULL - - - military_department_start_date - military_department_start_date - FIRST_INCL_NULL - - - military_department_end_date - military_department_end_date - FIRST_INCL_NULL - - - last_row - last_row - LAST_INCL_NULL - - - academ_start_date - academ_start_date - FIRST_INCL_NULL - - - academ_end_date - academ_end_date - FIRST_INCL_NULL - - - academ_reason - academ_reason - FIRST_INCL_NULL - - - N - - - source_id - - - N - grp - - - 944 - 432 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 624 - 432 - - - - citizen_education_output - TableOutput - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - source_id - source_id - - - recruit_id - recruit_id - - - source_update_date - source_update_date - - - educational_institution_type - educational_institution_type - - - org_type - org_type - - - inn - inn - - - kpp - kpp - - - ogrn - ogrn - - - educational_institution_address - educational_institution_address - - - educational_organization_separate_unit_type - educational_organization_separate_unit_type - - - educational_organization_separate_unit_address - educational_organization_separate_unit_address - - - student_status - student_status - - - grade_number - grade_number - - - education_level_code - education_level_code - - - admission_date - admission_date - - - enrollment_date - enrollment_date - - - enrollment_order_number - enrollment_order_number - - - enrollment_order_date - enrollment_order_date - - - completion_date - completion_date - - - education_form - education_form - - - education_program_start_date - education_program_start_date - - - education_start_date - education_start_date - - - planned_completion_date - planned_completion_date - - - minobr_speciality_code - minobr_speciality_code - - - minprosv_speciality_code - minprosv_speciality_code - - - recovery_info - recovery_info - - - recovery_date - recovery_date - - - recovery_order_date - recovery_order_date - - - recovery_order_number - recovery_order_number - - - expulsion_info - expulsion_info - - - expulsion_date - expulsion_date - - - expulsion_order_date - expulsion_order_date - - - expulsion_order_number - expulsion_order_number - - - academical_leave_info - academical_leave_info - - - first_education - first_education - - - vus_spec - vus_spec - - - military_center_education_info - military_center_education_info - - - military_center_education_start_date - military_center_education_start_date - - - military_center_education_end_date - military_center_education_end_date - - - military_center_education_period - military_center_education_period - - - military_center_completion - military_center_completion - - - military_department_education_info - military_department_education_info - - - military_department_start_date - military_department_start_date - - - military_department_end_date - military_department_end_date - - - N - N - N - N - Y - N - ervu_dashboard - Y - educational_institution_learning
- N - Y - N - Y - - - 1264 - 432 - -
- - education_input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - WITH filteredData AS (SELECT ri.recruit_id, - education_elem, - academ_elem, - 'Образовательная организация высшего образования' as educational_institution_type - FROM recruits_info ri - CROSS JOIN LATERAL jsonb_array_elements( - ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') AS education_elem - LEFT JOIN LATERAL jsonb_array_elements( - CASE - WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array' - THEN education_elem -> 'svedOtpusk' - ELSE '[]'::jsonb - END - ) AS academ_elem ON true - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - - union all - - SELECT ri.recruit_id, - education_elem, - academ_elem, - 'Профессиональная образовательная организация' as educational_institution_type - FROM recruits_info ri - CROSS JOIN LATERAL jsonb_array_elements( - ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') AS education_elem - LEFT JOIN LATERAL jsonb_array_elements( - CASE - WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array' - THEN education_elem -> 'svedOtpusk' - ELSE '[]'::jsonb - END - ) AS academ_elem ON true - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - - union all - - SELECT ri.recruit_id, - education_elem, - academ_elem, - 'Общеобразовательная организация' as educational_institution_type - FROM recruits_info ri - CROSS JOIN LATERAL jsonb_array_elements( - ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') AS education_elem - LEFT JOIN LATERAL jsonb_array_elements( - CASE - WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array' - THEN education_elem -> 'svedOtpusk' - ELSE '[]'::jsonb - END - ) AS academ_elem ON true - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}') -SELECT fd.recruit_id, - education_elem ->> 'id' AS source_id, - to_date(education_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, - educational_institution_type, - education_elem -> 'svedOrg' ->> 'naimTipOrg' as org_type, - education_elem -> 'svedOrg' ->> 'inn' as inn, - education_elem -> 'svedOrg' ->> 'kpp' as kpp, - coalesce(education_elem -> 'svedOrg' ->> 'ogrn', education_elem -> 'svedOrg' ->> - 'ogrnip') as ogrn, - education_elem -> 'svedOrg' ->> 'adresOrg' as educational_institution_address, - education_elem -> 'svedOrg' -> 'extend' ->> 'statusOrg' as educational_institution_address, - education_elem -> 'svedOrg' -> 'extend' ->> 'vidPodrazdOrg' as educational_organization_separate_unit_type, - education_elem -> 'svedOrg' -> 'extend' ->> 'adresPodrazdOrg' as educational_organization_separate_unit_address, - education_elem ->> 'naimStatus' as student_status, - case - when educational_institution_type = 'Образовательная организация высшего образования' - then education_elem ->> 'nomerKursa' - when educational_institution_type = 'Профессиональная образовательная организация' - then education_elem -> 'extend' ->> 'klass' - when educational_institution_type = 'Общеобразовательная организация' then education_elem ->> 'klass' - end as grade_number, - education_elem ->> 'kodUrovObr' as education_level_code, - to_date(coalesce(education_elem -> 'extend' ->> 'dataPostup', education_elem ->> 'dataPostup'), - 'YYYY-MM-DD') as admission_date, - to_date(education_elem ->> 'dataZachisl', 'YYYY-MM-DD') as enrollment_date, - education_elem ->> 'nomPrikazZachisl' as enrollment_order_number, - to_date(education_elem ->> 'dataPrikazZachisl', 'YYYY-MM-DD') as enrollment_order_date, - to_date(coalesce(education_elem -> 'extend' ->> 'dataZavershObuch', education_elem ->> 'dataOkonch'), - 'YYYY-MM-DD') as completion_date, - education_elem -> 'svedObrProg' ->> 'naimFormaObuch' as education_form, - education_elem -> 'svedObrProg' ->> 'dataNachObuch' as education_program_start_date, - to_date(education_elem -> 'extend' ->> 'dataNachObuch', 'YYYY-MM-DD') as education_start_date, - to_date(coalesce(education_elem -> 'svedObrProg' ->> 'planDataOkonch', education_elem ->> 'dataOzhid'), - 'YYYY-MM-DD') as planned_completion_date, - education_elem -> 'svedObrProg' -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, - education_elem -> 'svedObrProg' -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, - coalesce(education_elem ->> 'svedVosst' <> 'null', false) as recovery_info, - to_date(education_elem -> 'svedVosst' ->> 'dataVosst', 'YYYY-MM-DD') as recovery_date, - to_date(education_elem -> 'svedVosst' ->> 'dataPrikazVosst', - 'YYYY-MM-DD') as recovery_order_date, - education_elem -> 'svedVosst' ->> 'nomPrikazVosst' as recovery_order_number, - coalesce(education_elem ->> 'svedOtchisl' <> 'null', false) as expulsion_info, - to_date(education_elem -> 'svedOtchisl' ->> 'dataOtchisl', 'YYYY-MM-DD') as expulsion_date, - to_date(education_elem -> 'svedOtchisl' ->> 'dataPrikazOtchisl', - 'YYYY-MM-DD') as expulsion_order_date, - education_elem -> 'svedOtchisl' ->> 'nomPrikazOtchisl' as expulsion_order_number, - coalesce(education_elem ->> 'svedOtpusk' <> 'null', false) as academical_leave_info, - coalesce(education_elem ->> 'prPervObuch' = '1', false) as first_education, - coalesce(education_elem -> 'svedObrProg' ->> 'prVUSpecz' = '1', false) as vus_spec, - coalesce(education_elem ->> 'svedVoenObuch' <> 'null', false) as military_center_education_info, - to_date(education_elem -> 'svedVoenObuch' ->> 'dataNach', 'YYYY-MM-DD') as military_center_education_start_date, - to_date(education_elem -> 'svedVoenObuch' ->> 'dataOkonch', - 'YYYY-MM-DD') as military_center_education_end_date, - education_elem -> 'svedVoenObuch' -> 'extend' ->> 'srokObuch' as military_center_education_period, - coalesce(education_elem -> 'svedVoenObuch' -> 'extend' ->> 'svedUspeshZaverObuch' = '1', - false) as military_center_completion, - coalesce(education_elem -> 'extend' ->> 'svedVoenKaf' <> 'null', false) as military_department_education_info, - to_date(education_elem -> 'extend' -> 'svedVoenKaf' ->> 'dataNach', - 'YYYY-MM-DD') as military_department_start_date, - to_date(education_elem -> 'extend' -> 'svedVoenKaf' ->> 'dataOkonch', - 'YYYY-MM-DD') as military_department_end_date, - to_date(academ_elem ->> 'dataNach', 'YYYY-MM-DD') as academ_start_date, - to_date(academ_elem ->> 'dataOkonch', 'YYYY-MM-DD') as academ_end_date, - academ_elem -> 'naimPrichOtpusk' as academ_reason -FROM filteredData fd -ORDER BY source_id; - Y - - - 432 - 432 - - - - education_leave_output - TableOutput - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - start_date - academ_start_date - - - end_date - academ_end_date - - - reason - academ_reason - - - citizen_education_source_id - source_id - - - N - N - N - N - Y - N - ervu_dashboard - Y - educational_leave
- N - Y - N - Y - - - 1088 - 640 - -
- - - citizen_education_output - Change job status on error - Y - - error_description - - - - - - - - education_leave_output - Change job status on error - Y - - error_description - - - - - - - - -
diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl deleted file mode 100644 index cb2ca00..0000000 --- a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl +++ /dev/null @@ -1,1028 +0,0 @@ - - - - citizen_education_flow_delta - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/08/28 21:27:15.946 - - - 2025/08/28 21:27:15.946 - - - - - - Block until transforms finish - education_leave_output - Y - - - Change job status on error - Abort - Y - - - Create job execution record - education_input - Y - - - Filter rows - Change job status on success - Y - - - Filter_null_parental_leave_dates - Block until transforms finish - Y - - - Group by - citizen_education_output - Y - - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - Filter_null_parental_leave_dates - Y - - - Identify last row in a stream - Group by - Y - - - citizen_education_output - Change job status on error - Y - - - citizen_education_output - Filter rows - Y - - - education_input - Identify last row in a stream - Y - - - education_leave_output - Change job status on error - Y - - - Detect empty stream - Change job status on success - Y - - - - Abort - Abort - - Y - - 1 - - none - - - ABORT - Y - 0 - - - 1696 - 672 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - citizen_education_output - - - - - 1088 - 672 - - - - Change job status on error - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ? -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 1472 - 672 - - - - Change job status on success - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'DELTA_SUCCESS' -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 1696 - 288 - - - - Create job execution record - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - N - N - Y - N - N - UPDATE etl.job_execution -SET - status = 'DELTA_PROCESSING', - execution_datetime = current_timestamp, - error_description = NULL -where job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - 416 - 464 - - - - Detect empty stream - DetectEmptyStream - - Y - - 1 - - none - - - - - 832 - 288 - - - - Filter rows - FilterRows - - Y - - 1 - - none - - - - - - - = - last_row - N - - - - N - -1 - constant - -1 - Y - Boolean - - - - Change job status on success - - - 1696 - 464 - - - - Filter_null_parental_leave_dates - FilterRows - - Y - - 1 - - none - - - - - - - IS NOT NULL - academ_reason - N - - - - - Block until transforms finish - - - 832 - 672 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - recruit_id - recruit_id - FIRST_INCL_NULL - - - source_update_date - source_update_date - FIRST_INCL_NULL - - - educational_institution_type - educational_institution_type - FIRST_INCL_NULL - - - org_type - org_type - FIRST_INCL_NULL - - - inn - inn - FIRST_INCL_NULL - - - kpp - kpp - FIRST_INCL_NULL - - - ogrn - ogrn - FIRST_INCL_NULL - - - educational_institution_address - educational_institution_address - FIRST_INCL_NULL - - - educational_institution_address_1 - educational_institution_address_1 - FIRST_INCL_NULL - - - educational_organization_separate_unit_type - educational_organization_separate_unit_type - FIRST_INCL_NULL - - - educational_organization_separate_unit_address - educational_organization_separate_unit_address - FIRST_INCL_NULL - - - student_status - student_status - FIRST_INCL_NULL - - - grade_number - grade_number - FIRST_INCL_NULL - - - education_level_code - education_level_code - FIRST_INCL_NULL - - - admission_date - admission_date - FIRST_INCL_NULL - - - enrollment_date - enrollment_date - FIRST_INCL_NULL - - - enrollment_order_number - enrollment_order_number - FIRST_INCL_NULL - - - enrollment_order_date - enrollment_order_date - FIRST_INCL_NULL - - - completion_date - completion_date - FIRST_INCL_NULL - - - education_form - education_form - FIRST_INCL_NULL - - - education_program_start_date - education_program_start_date - FIRST_INCL_NULL - - - education_start_date - education_start_date - FIRST_INCL_NULL - - - planned_completion_date - planned_completion_date - FIRST_INCL_NULL - - - minobr_speciality_code - minobr_speciality_code - FIRST_INCL_NULL - - - minprosv_speciality_code - minprosv_speciality_code - FIRST_INCL_NULL - - - recovery_info - recovery_info - FIRST_INCL_NULL - - - recovery_date - recovery_date - FIRST_INCL_NULL - - - recovery_order_date - recovery_order_date - FIRST_INCL_NULL - - - recovery_order_number - recovery_order_number - FIRST_INCL_NULL - - - expulsion_info - expulsion_info - FIRST_INCL_NULL - - - expulsion_date - expulsion_date - FIRST_INCL_NULL - - - expulsion_order_date - expulsion_order_date - FIRST_INCL_NULL - - - expulsion_order_number - expulsion_order_number - FIRST_INCL_NULL - - - academical_leave_info - academical_leave_info - FIRST_INCL_NULL - - - first_education - first_education - FIRST_INCL_NULL - - - vus_spec - vus_spec - FIRST_INCL_NULL - - - military_center_education_info - military_center_education_info - FIRST_INCL_NULL - - - military_center_education_start_date - military_center_education_start_date - FIRST_INCL_NULL - - - military_center_education_end_date - military_center_education_end_date - FIRST_INCL_NULL - - - military_center_education_period - military_center_education_period - FIRST_INCL_NULL - - - military_center_completion - military_center_completion - FIRST_INCL_NULL - - - military_department_education_info - military_department_education_info - FIRST_INCL_NULL - - - military_department_start_date - military_department_start_date - FIRST_INCL_NULL - - - military_department_end_date - military_department_end_date - FIRST_INCL_NULL - - - last_row - last_row - LAST_INCL_NULL - - - academ_start_date - academ_start_date - FIRST_INCL_NULL - - - academ_end_date - academ_end_date - FIRST_INCL_NULL - - - academ_reason - academ_reason - FIRST_INCL_NULL - - - N - - - source_id - - - N - grp - - - 1152 - 464 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 832 - 464 - - - - citizen_education_output - TableOutput - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - source_id - source_id - - - recruit_id - recruit_id - - - source_update_date - source_update_date - - - educational_institution_type - educational_institution_type - - - org_type - org_type - - - inn - inn - - - kpp - kpp - - - ogrn - ogrn - - - educational_institution_address - educational_institution_address - - - educational_institution_address_1 - educational_institution_address_1 - - - educational_organization_separate_unit_type - educational_organization_separate_unit_type - - - educational_organization_separate_unit_address - educational_organization_separate_unit_address - - - student_status - student_status - - - grade_number - grade_number - - - education_level_code - education_level_code - - - admission_date - admission_date - - - enrollment_date - enrollment_date - - - enrollment_order_number - enrollment_order_number - - - enrollment_order_date - enrollment_order_date - - - completion_date - completion_date - - - education_form - education_form - - - education_program_start_date - education_program_start_date - - - education_start_date - education_start_date - - - planned_completion_date - planned_completion_date - - - minobr_speciality_code - minobr_speciality_code - - - minprosv_speciality_code - minprosv_speciality_code - - - recovery_info - recovery_info - - - recovery_date - recovery_date - - - recovery_order_date - recovery_order_date - - - recovery_order_number - recovery_order_number - - - expulsion_info - expulsion_info - - - expulsion_date - expulsion_date - - - expulsion_order_date - expulsion_order_date - - - expulsion_order_number - expulsion_order_number - - - academical_leave_info - academical_leave_info - - - first_education - first_education - - - vus_spec - vus_spec - - - military_center_education_info - military_center_education_info - - - military_center_education_start_date - military_center_education_start_date - - - military_center_education_end_date - military_center_education_end_date - - - military_center_education_period - military_center_education_period - - - military_center_completion - military_center_completion - - - military_department_education_info - military_department_education_info - - - military_department_start_date - military_department_start_date - - - military_department_end_date - military_department_end_date - - - N - N - N - N - Y - N - ervu_dashboard - Y - educational_institution_learning
- N - Y - N - Y - - - 1472 - 464 - -
- - education_input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - WITH filteredData AS (SELECT ri.recruit_id, - education_elem, - academ_elem, - 'Образовательная организация высшего образования' as educational_institution_type - FROM recruits_info ri - CROSS JOIN LATERAL jsonb_array_elements( - ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') AS education_elem - LEFT JOIN LATERAL jsonb_array_elements( - CASE - WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array' - THEN education_elem -> 'svedOtpusk' - ELSE '[]'::jsonb - END - ) AS academ_elem ON true - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND to_date(education_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' - - union all - - SELECT ri.recruit_id, - education_elem, - academ_elem, - 'Профессиональная образовательная организация' as educational_institution_type - FROM recruits_info ri - CROSS JOIN LATERAL jsonb_array_elements( - ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') AS education_elem - LEFT JOIN LATERAL jsonb_array_elements( - CASE - WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array' - THEN education_elem -> 'svedOtpusk' - ELSE '[]'::jsonb - END - ) AS academ_elem ON true - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND to_date(education_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' - - union all - - SELECT ri.recruit_id, - education_elem, - academ_elem, - 'Общеобразовательная организация' as educational_institution_type - FROM recruits_info ri - CROSS JOIN LATERAL jsonb_array_elements( - ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') AS education_elem - LEFT JOIN LATERAL jsonb_array_elements( - CASE - WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array' - THEN education_elem -> 'svedOtpusk' - ELSE '[]'::jsonb - END - ) AS academ_elem ON true - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND to_date(education_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' -) -SELECT fd.recruit_id, - education_elem ->> 'id' AS source_id, - education_elem ->> 'dataSved' as source_update_date, - educational_institution_type, - education_elem -> 'svedOrg' ->> 'naimTipOrg' as org_type, - education_elem -> 'svedOrg' ->> 'inn' as inn, - education_elem -> 'svedOrg' ->> 'kpp' as kpp, - coalesce(education_elem -> 'svedOrg' ->> 'ogrn', education_elem -> 'svedOrg' ->> 'ogrnip') as ogrn, - education_elem -> 'svedOrg' ->> 'adresOrg' as educational_institution_address, - education_elem -> 'svedOrg' -> 'extend' ->> 'statusOrg' as educational_institution_address, - education_elem -> 'svedOrg' -> 'extend' ->> 'vidPodrazdOrg' as educational_organization_separate_unit_type, - education_elem -> 'svedOrg' -> 'extend' ->> 'adresPodrazdOrg' as educational_organization_separate_unit_address, - education_elem ->> 'naimStatus' as student_status, - case - when educational_institution_type = 'Образовательная организация высшего образования' - then education_elem ->> 'nomerKursa' - when educational_institution_type = 'Профессиональная образовательная организация' - then education_elem -> 'extend' ->> 'klass' - when educational_institution_type = 'Общеобразовательная организация' then education_elem ->> 'klass' - end as grade_number, - education_elem ->> 'kodUrovObr' as education_level_code, - coalesce(education_elem -> 'extend' ->> 'dataPostup', education_elem ->> 'dataPostup') as admission_date, - education_elem ->> 'dataZachisl' as enrollment_date, - education_elem ->> 'nomPrikazZachisl' as enrollment_order_number, - education_elem ->> 'dataPrikazZachisl' as enrollment_order_date, - coalesce(education_elem -> 'extend' ->> 'dataZavershObuch', education_elem ->> 'dataOkonch') as completion_date, - education_elem -> 'svedObrProg' ->> 'naimFormaObuch' as education_form, - education_elem -> 'svedObrProg' ->> 'dataNachObuch' as education_program_start_date, - education_elem -> 'extend' ->> 'dataNachObuch' as education_start_date, - coalesce(education_elem -> 'svedObrProg' ->> 'planDataOkonch', education_elem ->>'dataOzhid') as planned_completion_date, - education_elem -> 'svedObrProg' -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, - education_elem -> 'svedObrProg' -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, - education_elem ->> 'svedVosst' != 'null' as recovery_info, - education_elem -> 'svedVosst' ->> 'dataVosst' as recovery_date, - education_elem -> 'svedVosst' ->> 'dataPrikazVosst' as recovery_order_date, - education_elem -> 'svedVosst' ->> 'nomPrikazVosst' as recovery_order_number, - education_elem ->> 'svedOtchisl' != 'null' as expulsion_info, - education_elem -> 'svedOtchisl' ->> 'dataOtchisl' as expulsion_date, - education_elem -> 'svedOtchisl' ->> 'dataPrikazOtchisl' as expulsion_order_date, - education_elem -> 'svedOtchisl' ->> 'nomPrikazOtchisl' as expulsion_order_number, - education_elem ->> 'svedOtpusk' != 'null' as academical_leave_info, - education_elem ->> 'prPervObuch' = '1' as first_education, - education_elem -> 'svedObrProg' ->> 'prVUSpecz' = '1' as vus_spec, - education_elem -> 'svedVoenObuch' != 'null' as military_center_education_info, - education_elem -> 'svedVoenObuch' ->> 'dataNach' as military_center_education_start_date, - education_elem -> 'svedVoenObuch' ->> 'dataOkonch' as military_center_education_end_date, - education_elem -> 'svedVoenObuch' -> 'extend' ->> 'srokObuch' as military_center_education_period, - education_elem -> 'svedVoenObuch' -> 'extend' ->> 'svedUspeshZaverObuch' ='1' as military_center_completion, - education_elem -> 'extend' ->> 'svedVoenKaf' != 'null' as military_department_education_info, - education_elem -> 'extend' -> 'svedVoenKaf' ->> 'dataNach' as military_department_start_date, - education_elem -> 'extend' -> 'svedVoenKaf' ->> 'dataOkonch' as military_department_end_date, - academ_elem -> 'dataNach' as academ_start_date, - academ_elem -> 'dataOkonch' as academ_end_date, - academ_elem -> 'naimPrichOtpusk' as academ_reason -FROM filteredData fd -ORDER BY source_id; - Y - - - 640 - 464 - - - - education_leave_output - TableOutput - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - start_date - academ_start_date - - - end_date - academ_end_date - - - reason - academ_reason - - - citizen_education_source_id - source_id - - - N - N - N - N - Y - N - ervu_dashboard - Y - educational_leave
- N - Y - N - Y - - - 1296 - 672 - -
- - - citizen_education_output - Change job status on error - Y - - error_description - - - - - - - - education_leave_output - Change job status on error - Y - - error_description - - - - - - - - -
diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl deleted file mode 100644 index 4dff655..0000000 --- a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl +++ /dev/null @@ -1,1025 +0,0 @@ - - - - citizen_education_flow_repeat - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/08/28 21:16:35.770 - - - 2025/08/28 21:16:35.770 - - - - - - Block until transforms finish - education_leave_output - Y - - - Change job status on error - Abort - Y - - - Create job execution record - education_input - Y - - - Filter rows - Change job status on success - Y - - - Filter_null_parental_leave_dates - Block until transforms finish - Y - - - Group by - citizen_education_output - Y - - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - Filter_null_parental_leave_dates - Y - - - Identify last row in a stream - Group by - Y - - - citizen_education_output - Change job status on error - Y - - - citizen_education_output - Filter rows - Y - - - education_input - Identify last row in a stream - Y - - - education_leave_output - Change job status on error - Y - - - Detect empty stream - Change job status on success - Y - - - - Abort - Abort - - Y - - 1 - - none - - - ABORT - Y - 0 - - - 1536 - 704 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - citizen_education_output - - - - - 928 - 704 - - - - Change job status on error - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ? -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 1312 - 704 - - - - Change job status on success - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - Y - N - Y - N - N - UPDATE etl.job_execution -SET status = 'SUCCESS' -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 1552 - 320 - - - - Create job execution record - ExecSql - - Y - - 1 - - none - - - - - ervu-dashboard - N - N - Y - N - N - INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id) -VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}') -ON CONFLICT (job_name, recruitment_id) -DO UPDATE -SET status = 'PROCESSING', - error_description = null, - execution_datetime = current_timestamp; - - - 256 - 496 - - - - Detect empty stream - DetectEmptyStream - - Y - - 1 - - none - - - - - 672 - 320 - - - - Filter rows - FilterRows - - Y - - 1 - - none - - - - - - - = - last_row - N - - - - N - -1 - constant - -1 - Y - Boolean - - - - Change job status on success - - - 1552 - 496 - - - - Filter_null_parental_leave_dates - FilterRows - - Y - - 1 - - none - - - - - - - IS NOT NULL - academ_reason - N - - - - - Block until transforms finish - - - 672 - 704 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - recruit_id - recruit_id - FIRST_INCL_NULL - - - source_update_date - source_update_date - FIRST_INCL_NULL - - - educational_institution_type - educational_institution_type - FIRST_INCL_NULL - - - org_type - org_type - FIRST_INCL_NULL - - - inn - inn - FIRST_INCL_NULL - - - kpp - kpp - FIRST_INCL_NULL - - - ogrn - ogrn - FIRST_INCL_NULL - - - educational_institution_address - educational_institution_address - FIRST_INCL_NULL - - - educational_institution_address_1 - educational_institution_address_1 - FIRST_INCL_NULL - - - educational_organization_separate_unit_type - educational_organization_separate_unit_type - FIRST_INCL_NULL - - - educational_organization_separate_unit_address - educational_organization_separate_unit_address - FIRST_INCL_NULL - - - student_status - student_status - FIRST_INCL_NULL - - - grade_number - grade_number - FIRST_INCL_NULL - - - education_level_code - education_level_code - FIRST_INCL_NULL - - - admission_date - admission_date - FIRST_INCL_NULL - - - enrollment_date - enrollment_date - FIRST_INCL_NULL - - - enrollment_order_number - enrollment_order_number - FIRST_INCL_NULL - - - enrollment_order_date - enrollment_order_date - FIRST_INCL_NULL - - - completion_date - completion_date - FIRST_INCL_NULL - - - education_form - education_form - FIRST_INCL_NULL - - - education_program_start_date - education_program_start_date - FIRST_INCL_NULL - - - education_start_date - education_start_date - FIRST_INCL_NULL - - - planned_completion_date - planned_completion_date - FIRST_INCL_NULL - - - minobr_speciality_code - minobr_speciality_code - FIRST_INCL_NULL - - - minprosv_speciality_code - minprosv_speciality_code - FIRST_INCL_NULL - - - recovery_info - recovery_info - FIRST_INCL_NULL - - - recovery_date - recovery_date - FIRST_INCL_NULL - - - recovery_order_date - recovery_order_date - FIRST_INCL_NULL - - - recovery_order_number - recovery_order_number - FIRST_INCL_NULL - - - expulsion_info - expulsion_info - FIRST_INCL_NULL - - - expulsion_date - expulsion_date - FIRST_INCL_NULL - - - expulsion_order_date - expulsion_order_date - FIRST_INCL_NULL - - - expulsion_order_number - expulsion_order_number - FIRST_INCL_NULL - - - academical_leave_info - academical_leave_info - FIRST_INCL_NULL - - - first_education - first_education - FIRST_INCL_NULL - - - vus_spec - vus_spec - FIRST_INCL_NULL - - - military_center_education_info - military_center_education_info - FIRST_INCL_NULL - - - military_center_education_start_date - military_center_education_start_date - FIRST_INCL_NULL - - - military_center_education_end_date - military_center_education_end_date - FIRST_INCL_NULL - - - military_center_education_period - military_center_education_period - FIRST_INCL_NULL - - - military_center_completion - military_center_completion - FIRST_INCL_NULL - - - military_department_education_info - military_department_education_info - FIRST_INCL_NULL - - - military_department_start_date - military_department_start_date - FIRST_INCL_NULL - - - military_department_end_date - military_department_end_date - FIRST_INCL_NULL - - - last_row - last_row - LAST_INCL_NULL - - - academ_start_date - academ_start_date - FIRST_INCL_NULL - - - academ_end_date - academ_end_date - FIRST_INCL_NULL - - - academ_reason - academ_reason - FIRST_INCL_NULL - - - N - - - source_id - - - N - grp - - - 992 - 496 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 672 - 496 - - - - citizen_education_output - TableOutput - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - source_id - source_id - - - recruit_id - recruit_id - - - source_update_date - source_update_date - - - educational_institution_type - educational_institution_type - - - org_type - org_type - - - inn - inn - - - kpp - kpp - - - ogrn - ogrn - - - educational_institution_address - educational_institution_address - - - educational_institution_address_1 - educational_institution_address_1 - - - educational_organization_separate_unit_type - educational_organization_separate_unit_type - - - educational_organization_separate_unit_address - educational_organization_separate_unit_address - - - student_status - student_status - - - grade_number - grade_number - - - education_level_code - education_level_code - - - admission_date - admission_date - - - enrollment_date - enrollment_date - - - enrollment_order_number - enrollment_order_number - - - enrollment_order_date - enrollment_order_date - - - completion_date - completion_date - - - education_form - education_form - - - education_program_start_date - education_program_start_date - - - education_start_date - education_start_date - - - planned_completion_date - planned_completion_date - - - minobr_speciality_code - minobr_speciality_code - - - minprosv_speciality_code - minprosv_speciality_code - - - recovery_info - recovery_info - - - recovery_date - recovery_date - - - recovery_order_date - recovery_order_date - - - recovery_order_number - recovery_order_number - - - expulsion_info - expulsion_info - - - expulsion_date - expulsion_date - - - expulsion_order_date - expulsion_order_date - - - expulsion_order_number - expulsion_order_number - - - academical_leave_info - academical_leave_info - - - first_education - first_education - - - vus_spec - vus_spec - - - military_center_education_info - military_center_education_info - - - military_center_education_start_date - military_center_education_start_date - - - military_center_education_end_date - military_center_education_end_date - - - military_center_education_period - military_center_education_period - - - military_center_completion - military_center_completion - - - military_department_education_info - military_department_education_info - - - military_department_start_date - military_department_start_date - - - military_department_end_date - military_department_end_date - - - N - N - N - N - Y - N - ervu_dashboard - Y - educational_institution_learning
- N - Y - N - Y - - - 1312 - 496 - -
- - education_input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - WITH filteredData AS (SELECT ri.recruit_id, - education_elem, - academ_elem, - 'Образовательная организация высшего образования' as educational_institution_type - FROM recruits_info ri - CROSS JOIN LATERAL jsonb_array_elements( - ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') AS education_elem - LEFT JOIN LATERAL jsonb_array_elements( - CASE - WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array' - THEN education_elem -> 'svedOtpusk' - ELSE '[]'::jsonb - END - ) AS academ_elem ON true - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - - union all - - SELECT ri.recruit_id, - education_elem, - academ_elem, - 'Профессиональная образовательная организация' as educational_institution_type - FROM recruits_info ri - CROSS JOIN LATERAL jsonb_array_elements( - ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') AS education_elem - LEFT JOIN LATERAL jsonb_array_elements( - CASE - WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array' - THEN education_elem -> 'svedOtpusk' - ELSE '[]'::jsonb - END - ) AS academ_elem ON true - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - - union all - - SELECT ri.recruit_id, - education_elem, - academ_elem, - 'Общеобразовательная организация' as educational_institution_type - FROM recruits_info ri - CROSS JOIN LATERAL jsonb_array_elements( - ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') AS education_elem - LEFT JOIN LATERAL jsonb_array_elements( - CASE - WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array' - THEN education_elem -> 'svedOtpusk' - ELSE '[]'::jsonb - END - ) AS academ_elem ON true - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - ) -SELECT fd.recruit_id, - education_elem ->> 'id' AS source_id, - education_elem ->> 'dataSved' as source_update_date, - educational_institution_type, - education_elem -> 'svedOrg' ->> 'naimTipOrg' as org_type, - education_elem -> 'svedOrg' ->> 'inn' as inn, - education_elem -> 'svedOrg' ->> 'kpp' as kpp, - coalesce(education_elem -> 'svedOrg' ->> 'ogrn', education_elem -> 'svedOrg' ->> 'ogrnip') as ogrn, - education_elem -> 'svedOrg' ->> 'adresOrg' as educational_institution_address, - education_elem -> 'svedOrg' -> 'extend' ->> 'statusOrg' as educational_institution_address, - education_elem -> 'svedOrg' -> 'extend' ->> 'vidPodrazdOrg' as educational_organization_separate_unit_type, - education_elem -> 'svedOrg' -> 'extend' ->> 'adresPodrazdOrg' as educational_organization_separate_unit_address, - education_elem ->> 'naimStatus' as student_status, - case - when educational_institution_type = 'Образовательная организация высшего образования' - then education_elem ->> 'nomerKursa' - when educational_institution_type = 'Профессиональная образовательная организация' - then education_elem -> 'extend' ->> 'klass' - when educational_institution_type = 'Общеобразовательная организация' then education_elem ->> 'klass' - end as grade_number, - education_elem ->> 'kodUrovObr' as education_level_code, - coalesce(education_elem -> 'extend' ->> 'dataPostup', education_elem ->> 'dataPostup') as admission_date, - education_elem ->> 'dataZachisl' as enrollment_date, - education_elem ->> 'nomPrikazZachisl' as enrollment_order_number, - education_elem ->> 'dataPrikazZachisl' as enrollment_order_date, - coalesce(education_elem -> 'extend' ->> 'dataZavershObuch', education_elem ->> 'dataOkonch') as completion_date, - education_elem -> 'svedObrProg' ->> 'naimFormaObuch' as education_form, - education_elem -> 'svedObrProg' ->> 'dataNachObuch' as education_program_start_date, - education_elem -> 'extend' ->> 'dataNachObuch' as education_start_date, - coalesce(education_elem -> 'svedObrProg' ->> 'planDataOkonch', education_elem ->>'dataOzhid') as planned_completion_date, - education_elem -> 'svedObrProg' -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, - education_elem -> 'svedObrProg' -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, - education_elem ->> 'svedVosst' != 'null' as recovery_info, - education_elem -> 'svedVosst' ->> 'dataVosst' as recovery_date, - education_elem -> 'svedVosst' ->> 'dataPrikazVosst' as recovery_order_date, - education_elem -> 'svedVosst' ->> 'nomPrikazVosst' as recovery_order_number, - education_elem ->> 'svedOtchisl' != 'null' as expulsion_info, - education_elem -> 'svedOtchisl' ->> 'dataOtchisl' as expulsion_date, - education_elem -> 'svedOtchisl' ->> 'dataPrikazOtchisl' as expulsion_order_date, - education_elem -> 'svedOtchisl' ->> 'nomPrikazOtchisl' as expulsion_order_number, - education_elem ->> 'svedOtpusk' != 'null' as academical_leave_info, - education_elem ->> 'prPervObuch' = '1' as first_education, - education_elem -> 'svedObrProg' ->> 'prVUSpecz' = '1' as vus_spec, - education_elem -> 'svedVoenObuch' != 'null' as military_center_education_info, - education_elem -> 'svedVoenObuch' ->> 'dataNach' as military_center_education_start_date, - education_elem -> 'svedVoenObuch' ->> 'dataOkonch' as military_center_education_end_date, - education_elem -> 'svedVoenObuch' -> 'extend' ->> 'srokObuch' as military_center_education_period, - education_elem -> 'svedVoenObuch' -> 'extend' ->> 'svedUspeshZaverObuch' ='1' as military_center_completion, - education_elem -> 'extend' ->> 'svedVoenKaf' != 'null' as military_department_education_info, - education_elem -> 'extend' -> 'svedVoenKaf' ->> 'dataNach' as military_department_start_date, - education_elem -> 'extend' -> 'svedVoenKaf' ->> 'dataOkonch' as military_department_end_date, - academ_elem -> 'dataNach' as academ_start_date, - academ_elem -> 'dataOkonch' as academ_end_date, - academ_elem -> 'naimPrichOtpusk' as academ_reason -FROM filteredData fd -ORDER BY source_id; - Y - - - 480 - 496 - - - - education_leave_output - TableOutput - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - start_date - academ_start_date - - - end_date - academ_end_date - - - reason - academ_reason - - - citizen_education_source_id - source_id - - - N - N - N - N - Y - N - ervu_dashboard - Y - educational_leave
- N - Y - N - Y - - - 1136 - 704 - -
- - - citizen_education_output - Change job status on error - Y - - error_description - - - - - - - - education_leave_output - Change job status on error - Y - - error_description - - - - - - - - -
diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_delta.hpl deleted file mode 100644 index bce8e0b..0000000 --- a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_delta.hpl +++ /dev/null @@ -1,399 +0,0 @@ - - - - recruitment_five_flow_delta - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/08/28 21:23:43.331 - - - 2025/08/28 21:23:43.331 - - - - - - Table input - citizen_education_flow_delta.hpl - Y - - - get_max_source_update_date - Table input - Y - - - Table input - citizen_education_flow_delta.hpl 2 - Y - - - Table input - citizen_education_flow_delta.hpl 3 - Y - - - Table input - citizen_education_flow_delta.hpl 4 - Y - - - Table input - citizen_education_flow_delta.hpl 5 - Y - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard-test - N - 0 - get_max_source_update_date - WITH mud AS ( - SELECT - recruitment_id, - MAX(execution_datetime) AS max_upd_date - FROM etl.job_execution - WHERE job_name = '${JOB_NAME}' - AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING') - GROUP BY recruitment_id -) -SELECT - r.idm_id, - ? max_source_update_date -FROM ervu_dashboard.recruitment r - JOIN mud ON mud.recruitment_id = r.idm_id - JOIN recruits_info ri - ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id - AND ri.updated_at > mud.max_upd_date; - Y - - - 608 - 384 - - - - citizen_education_flow_delta.hpl - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 800 - 224 - - - - citizen_education_flow_delta.hpl 2 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 800 - 304 - - - - citizen_education_flow_delta.hpl 3 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 800 - 384 - - - - citizen_education_flow_delta.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 800 - 464 - - - - citizen_education_flow_delta.hpl 5 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 800 - 544 - - - - get_max_source_update_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - select max(source_update_date) -from education_institution_learning; - N - - - 432 - 384 - - - - - - diff --git a/mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/foreign_diploma_constants_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/foreign_diploma_constants_flow_repeat.hpl deleted file mode 100644 index 52729cf..0000000 --- a/mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/foreign_diploma_constants_flow_repeat.hpl +++ /dev/null @@ -1,148 +0,0 @@ - - - - foreign_diploma_constants_flow_repeat - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/21 21:27:15.216 - - - 2025/09/21 21:27:15.216 - - - - - - get_max_country_oksm_update_date - country_oksm_input - Y - - - country_oksm_input - country_oksm - Y - - - - country_oksm_input - TableInput - - Y - - 1 - - none - - - nsi - N - 0 - get_max_country_oksm_update_date - SELECT - key as code, - value ->> 'value' as value, - NOT hidden as actual, - updated_at as update_date -FROM classifier_records -WHERE code = 'countriesOKSM' -AND updated_at > ?; - N - - - 464 - 240 - - - - get_max_country_oksm_update_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT - max(update_date) as max_update_date -FROM country_oksm; - - N - - - 240 - 240 - - - - country_oksm - InsertUpdate - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - = - code - code - - - - <> - actual - actual - - - ervu_dashboard - country_oksm
- - code - code - N - - - value - value - Y - - - actual - actual - Y - - - update_date - update_date - Y - -
- N - - - 688 - 240 - -
- - - -
diff --git a/mappings/info_recruits/citizen_tables/education/foreign_diploma/foreign_diploma_job.hwf b/mappings/info_recruits/citizen_tables/education/foreign_diploma/foreign_diploma_job.hwf deleted file mode 100644 index 0868d20..0000000 --- a/mappings/info_recruits/citizen_tables/education/foreign_diploma/foreign_diploma_job.hwf +++ /dev/null @@ -1,408 +0,0 @@ - - - foreign_diploma_job - Y - - - - - - 2025/09/04 20:50:35.289 - - - 2025/09/04 20:50:35.289 - - - - - Start - - SPECIAL - - 1 - 12 - 60 - 0 - 0 - N - 0 - 1 - N - 336 - 224 - - - - check_if_job_execution_exists.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_job_execution_exists.hpl - Basic - - Y - - N - local - N - N - Y - N - 1360 - 240 - - - - employer_job_execution_exists_check - - SIMPLE_EVAL - - boolean - false - equal - equal - N - variable - JOB_EXECUTED_FLAG - N - 1664 - 240 - - - - recruitment_five_flow.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow.hpl - Basic - - Y - - N - local - N - N - Y - Y - 1936 - 240 - - - - recruitment_five_flow_repeat.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_repeat.hpl - Basic - - Y - - N - local - N - N - Y - N - 1936 - 544 - - - - check_if_need_to_repeat.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_need_to_repeat.hpl - Basic - - Y - - N - local - N - N - Y - N - 1664 - 384 - - - - Simple evaluation - - SIMPLE_EVAL - - boolean - true - equal - equal - N - variable - NEED_TO_REPEAT_JOB - N - 1664 - 544 - - - - recruitment_five_flow_delta.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_delta.hpl - Basic - - Y - - N - local - N - N - Y - N - 1664 - 704 - - - - check_if_constants_exists.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/constants/support/check_if_constants_exists.hpl - Basic - - Y - - N - local - N - N - Y - N - 512 - 224 - - - - constants_exists - - SIMPLE_EVAL - - boolean - true - equal - equal - N - variable - CONSTANTS_EXISTS - N - 688 - 224 - - - - foreign_diploma_constants_flow.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/constants/foreign_diploma_constants_flow.hpl - Basic - - Y - - N - local - N - N - Y - N - 880 - 144 - - - - foreign_diploma_constants_flow_repeat.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/constants/foreign_diploma_constants_flow_repeat.hpl - Basic - - Y - - N - local - N - N - Y - N - 880 - 320 - - - - init_job_name - - SET_VARIABLES - - - - JOB_NAME - CURRENT_WORKFLOW - foreign_diploma_job - - - CURRENT_WORKFLOW - N - N - 1120 - 240 - - - - - - check_if_job_execution_exists.hpl - employer_job_execution_exists_check - Y - Y - N - - - employer_job_execution_exists_check - check_if_need_to_repeat.hpl - Y - N - N - - - check_if_need_to_repeat.hpl - Simple evaluation - Y - Y - N - - - check_if_constants_exists.hpl - constants_exists - Y - Y - N - - - constants_exists - foreign_diploma_constants_flow.hpl - Y - N - N - - - constants_exists - foreign_diploma_constants_flow_repeat.hpl - Y - Y - N - - - foreign_diploma_constants_flow.hpl - init_job_name - Y - Y - N - - - foreign_diploma_constants_flow_repeat.hpl - init_job_name - Y - Y - N - - - init_job_name - check_if_job_execution_exists.hpl - Y - Y - N - - - Start - check_if_constants_exists.hpl - Y - Y - Y - - - employer_job_execution_exists_check - recruitment_five_flow.hpl - Y - Y - N - - - Simple evaluation - recruitment_five_flow_repeat.hpl - Y - Y - N - - - Simple evaluation - recruitment_five_flow_delta.hpl - Y - N - N - - - - - - diff --git a/mappings/info_recruits/citizen_tables/education/russian_diploma/constants/russian_diploma_constants_flow.hpl b/mappings/info_recruits/citizen_tables/education/russian_diploma/constants/russian_diploma_constants_flow.hpl deleted file mode 100644 index aee3c4e..0000000 --- a/mappings/info_recruits/citizen_tables/education/russian_diploma/constants/russian_diploma_constants_flow.hpl +++ /dev/null @@ -1,199 +0,0 @@ - - - - russian_diploma_constants_flow - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/22 08:32:44.247 - - - 2025/09/22 08:32:44.247 - - - - - - minobr_doc_type_input - minobr_doc_type_output - Y - - - minprosv_doc_type_input - minprosv_doc_type_output - Y - - - - minobr_doc_type_input - TableInput - - Y - - 1 - - none - - - nsi - N - 0 - SELECT - key as code, - value ->> 'value' as value, - NOT hidden as actual, - updated_at as update_date -FROM classifier_records -WHERE code = 'educationNaimVidDocMinobr'; - N - - - 384 - 272 - - - - minprosv_doc_type_input - TableInput - - Y - - 1 - - none - - - nsi - N - 0 - SELECT - key as code, - value ->> 'value' as value, - NOT hidden as actual, - updated_at as update_date -FROM classifier_records -WHERE code = 'educationNaimVidDocMinprosv'; - N - - - 384 - 384 - - - - minobr_doc_type_output - TableOutput - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - code - code - - - value - value - - - actual - actual - - - update_date - update_date - - - N - N - N - N - - Y - - N - ervu_dashboard - Y - minobr_doc_type
- - N - Y - N - Y - - - 624 - 272 - -
- - minprosv_doc_type_output - TableOutput - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - code - code - - - value - value - - - actual - actual - - - update_date - update_date - - - N - N - N - N - - Y - - N - ervu_dashboard - Y - minprosv_doc_type
- - N - Y - N - Y - - - 624 - 384 - -
- - - -
diff --git a/mappings/info_recruits/citizen_tables/education/russian_diploma/constants/russian_diploma_constants_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/education/russian_diploma/constants/russian_diploma_constants_flow_repeat.hpl deleted file mode 100644 index 54dedda..0000000 --- a/mappings/info_recruits/citizen_tables/education/russian_diploma/constants/russian_diploma_constants_flow_repeat.hpl +++ /dev/null @@ -1,255 +0,0 @@ - - - - foreign_diploma_constants_flow - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/22 08:32:45.452 - - - 2025/09/22 08:32:45.452 - - - - - - minobr_doc_type_input - minobr_doc_type_upsert - Y - - - minprosv_doc_type_input - minprosv_doc_type_upsert - Y - - - get_max_minobr_doc_type_update_date - minobr_doc_type_input - Y - - - get_max_minprosv_doc_type_date - minprosv_doc_type_input - Y - - - - minobr_doc_type_input - TableInput - - Y - - 1 - - none - - - nsi - N - 0 - get_max_minobr_doc_type_update_date - SELECT - key as code, - value ->> 'value' as value, - NOT hidden as actual -FROM classifier_records -WHERE code = 'educationNaimVidDocMinobr' -AND updated_at > ?; - N - - - 720 - 256 - - - - minobr_doc_type_upsert - InsertUpdate - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - = - code - code - - - ervu_dashboard - minobr_doc_type
- - code - code - N - - - value - value - Y - - - actual - actual - Y - - - update_date - update_date - Y - -
- N - - - 992 - 256 - -
- - minprosv_doc_type_input - TableInput - - Y - - 1 - - none - - - nsi - N - 0 - get_max_minprosv_doc_type_date - SELECT - key as code, - value ->> 'value' as value, - NOT hidden as actual -FROM classifier_records -WHERE code = 'educationNaimVidDocMinprosv' -AND updated_at > ?; - N - - - 720 - 416 - - - - minprosv_doc_type_upsert - InsertUpdate - - Y - - 1 - - none - - - 10000 - ervu-dashboard - - - = - code - code - - - ervu_dashboard - minprosv_doc_type
- - code - code - N - - - value - value - Y - - - actual - actual - Y - - - update_date - update_date - Y - -
- N - - - 992 - 416 - -
- - get_max_minobr_doc_type_update_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT - max(update_date) as max_update_date -FROM minobr_doc_type; - - N - - - 464 - 256 - - - - get_max_minprosv_doc_type_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT - max(update_date) as max_update_date -FROM minprosv_doc_type; - - N - - - 464 - 416 - - - - - -
diff --git a/mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow_delta.hpl deleted file mode 100644 index 38479ff..0000000 --- a/mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow_delta.hpl +++ /dev/null @@ -1,423 +0,0 @@ - - - - recruitment_five_flow_delta - Y - - - - Normal - - - N - 1000 - 100 - - - 2025/09/01 17:07:38.569 - - - 2025/09/01 17:07:38.569 - - - - - - Table input - russian_diploma_flow_delta.hpl - Y - - - get_max_source_update_date - Table input - Y - - - Table input - russian_diploma_flow_delta.hpl 2 - Y - - - Table input - russian_diploma_flow_delta.hpl 3 - Y - - - Table input - russian_diploma_flow_delta.hpl 4 - Y - - - Table input - russian_diploma_flow_delta.hpl 5 - Y - - - - Table input - TableInput - - Y - - 1 - - none - - - ervu-dashboard-test - N - 0 - get_max_source_update_date - SELECT - r.idm_id, - ? max_source_update_date -FROM ervu_dashboard.recruitment r - LEFT JOIN etl.job_execution je - ON r.idm_id = je.recruitment_id - and job_name = '${JOB_NAME}' - JOIN recruits_info ri - ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id - AND ri.updated_at > ( - SELECT MAX(execution_datetime) - FROM etl.job_execution - WHERE job_name = '${JOB_NAME}' - AND recruitment_id = r.idm_id - ) -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N - - - 688 - 368 - - - - get_max_source_update_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - select max(source_update_date) -from russian_diploma; - N - - - 512 - 368 - - - - russian_diploma_flow_delta.hpl - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - JOB_NAME - - russian_diploma_job - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 880 - 208 - - - - russian_diploma_flow_delta.hpl 2 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - JOB_NAME - - russian_diploma_job - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 880 - 288 - - - - russian_diploma_flow_delta.hpl 3 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - JOB_NAME - - russian_diploma_job - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 880 - 368 - - - - russian_diploma_flow_delta.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - JOB_NAME - - russian_diploma_job - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 880 - 448 - - - - russian_diploma_flow_delta.hpl 5 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - - JOB_NAME - - russian_diploma_job - - - MAX_SOURCE_UPDATE_DATE - max_source_update_date - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 880 - 528 - - - - - - diff --git a/mappings/info_recruits/citizen_tables/education/russian_diploma/russian_diploma_job.hwf b/mappings/info_recruits/citizen_tables/education/russian_diploma/russian_diploma_job.hwf deleted file mode 100644 index cc7726c..0000000 --- a/mappings/info_recruits/citizen_tables/education/russian_diploma/russian_diploma_job.hwf +++ /dev/null @@ -1,419 +0,0 @@ - - - russian_diploma_job - Y - - - - - - 2025/09/01 15:59:15.492 - - - 2025/09/01 15:59:15.492 - - - - - Start - - SPECIAL - - 1 - 12 - 60 - 0 - 0 - N - 0 - 1 - N - 272 - 208 - - - - check_if_job_execution_exists.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_job_execution_exists.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 1200 - 224 - - - - employer_job_execution_exists_check - - SIMPLE_EVAL - - boolean - false - equal - equal - N - variable - JOB_EXECUTED_FLAG - N - 1504 - 224 - - - - recruitment_five_flow.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow.hpl - Basic - - Y - - N - local - N - N - Y - Y - 1744 - 224 - - - - recruitment_five_flow_repeat.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow_repeat.hpl - Basic - - Y - - N - local - N - N - Y - N - 1728 - 528 - - - - check_if_need_to_repeat.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_need_to_repeat.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 1504 - 368 - - - - Simple evaluation - - SIMPLE_EVAL - - boolean - true - equal - equal - N - variable - NEED_TO_REPEAT_JOB - N - 1504 - 528 - - - - recruitment_five_flow_delta.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow_delta.hpl - Basic - - Y - - N - local - N - N - Y - N - 1504 - 672 - - - - check_if_constants_exists.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/constants/support/check_if_constants_exists.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 432 - 208 - - - - constants_exists - - SIMPLE_EVAL - - boolean - true - equal - equal - N - variable - CONSTANTS_EXISTS - N - 608 - 208 - - - - russian_diploma_constants_flow.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/constants/russian_diploma_constants_flow.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 800 - 128 - - - - russian_diploma_constants_flow_repeat.hpl - - PIPELINE - - N - N - N - N - N - N - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/constants/russian_diploma_constants_flow_repeat.hpl - - - Basic - - Y - - N - local - N - N - Y - N - 800 - 304 - - - - init_job_name - - SET_VARIABLES - - - - JOB_NAME - CURRENT_WORKFLOW - russian_diploma_job - - - CURRENT_WORKFLOW - - N - N - 992 - 224 - - - - - - check_if_job_execution_exists.hpl - employer_job_execution_exists_check - Y - Y - N - - - employer_job_execution_exists_check - check_if_need_to_repeat.hpl - Y - N - N - - - check_if_need_to_repeat.hpl - Simple evaluation - Y - Y - N - - - check_if_constants_exists.hpl - constants_exists - Y - Y - N - - - constants_exists - russian_diploma_constants_flow.hpl - Y - N - N - - - constants_exists - russian_diploma_constants_flow_repeat.hpl - Y - Y - N - - - russian_diploma_constants_flow.hpl - init_job_name - Y - Y - N - - - russian_diploma_constants_flow_repeat.hpl - init_job_name - Y - Y - N - - - Start - check_if_constants_exists.hpl - Y - Y - Y - - - init_job_name - check_if_job_execution_exists.hpl - Y - Y - N - - - employer_job_execution_exists_check - recruitment_five_flow.hpl - Y - Y - N - - - Simple evaluation - recruitment_five_flow_repeat.hpl - Y - Y - N - - - Simple evaluation - recruitment_five_flow_delta.hpl - Y - N - N - - - - - - diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/passport/recruitment_five_flow_delta.hpl similarity index 83% rename from mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_repeat.hpl rename to mappings/info_recruits/citizen_tables/passport/recruitment_five_flow_delta.hpl index a120f61..8259a10 100644 --- a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/passport/recruitment_five_flow_delta.hpl @@ -1,7 +1,7 @@ - recruitment_five_flow_repeat + recruitment_five_flow_delta Y @@ -13,36 +13,36 @@ 1000 100 - - 2025/08/28 21:16:24.371 + 2025/08/11 13:16:11.204 - - 2025/08/28 21:16:24.371 + 2025/08/11 13:16:11.204 Table input - citizen_education_flow_repeat.hpl + passport_flow_delta.hpl Y Table input - citizen_education_flow_repeat.hpl 2 + passport_flow_delta.hpl 2 Y Table input - citizen_education_flow_repeat.hpl 3 + passport_flow_delta.hpl 3 Y Table input - citizen_education_flow_repeat.hpl 4 + passport_flow_delta.hpl 4 Y Table input - citizen_education_flow_repeat.hpl 5 + passport_flow_delta.hpl 5 Y @@ -57,25 +57,25 @@ none - ervu-dashboard-test + ervu-dashboard N - 0 - SELECT r.idm_id AS recruitment_id + + SELECT + r.idm_id as recruitment_id FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - AND je.job_name = '${JOB_NAME}' -WHERE je.id IS NULL - OR je.status IN ('ERROR', 'PROCESSING'); - Y + and job_name = 'passport_job' +where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); + N - 672 - 496 + 352 + 288 - citizen_education_flow_repeat.hpl + passport_flow_delta.hpl PipelineExecutor Y @@ -86,7 +86,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl N 1 @@ -98,6 +98,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + passport_job + Y @@ -121,12 +126,12 @@ WHERE je.id IS NULL - 880 - 336 + 544 + 128 - citizen_education_flow_repeat.hpl 2 + passport_flow_delta.hpl 2 PipelineExecutor Y @@ -137,7 +142,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl N 1 @@ -149,6 +154,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + passport_job + Y @@ -172,12 +182,12 @@ WHERE je.id IS NULL - 880 - 416 + 544 + 208 - citizen_education_flow_repeat.hpl 3 + passport_flow_delta.hpl 3 PipelineExecutor Y @@ -188,7 +198,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl N 1 @@ -200,6 +210,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + passport_job + Y @@ -223,12 +238,12 @@ WHERE je.id IS NULL - 880 - 496 + 544 + 288 - citizen_education_flow_repeat.hpl 4 + passport_flow_delta.hpl 4 PipelineExecutor Y @@ -239,7 +254,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl N 1 @@ -251,6 +266,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + passport_job + Y @@ -274,12 +294,12 @@ WHERE je.id IS NULL - 880 - 576 + 544 + 368 - citizen_education_flow_repeat.hpl 5 + passport_flow_delta.hpl 5 PipelineExecutor Y @@ -290,7 +310,7 @@ WHERE je.id IS NULL local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl N 1 @@ -302,6 +322,11 @@ WHERE je.id IS NULL recruitment_id + + JOB_NAME + + passport_job + Y @@ -325,8 +350,8 @@ WHERE je.id IS NULL - 880 - 656 + 544 + 448 diff --git a/mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow.hpl b/mappings/info_recruits/citizen_tables/prosecution/recruitments_five_flow.hpl similarity index 81% rename from mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow.hpl rename to mappings/info_recruits/citizen_tables/prosecution/recruitments_five_flow.hpl index 06fe914..2d2c4ac 100644 --- a/mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow.hpl +++ b/mappings/info_recruits/citizen_tables/prosecution/recruitments_five_flow.hpl @@ -1,7 +1,7 @@ - recruitment_five_flow + recruitments_five_flow Y @@ -13,41 +13,41 @@ 1000 100 - - 2025/09/04 20:54:08.982 + 2025/04/18 09:48:01.970 - - 2025/09/04 20:54:08.982 + 2025/04/18 09:48:01.970 - Get all recruitments - foreign_diploma_flow.hpl + Get all recruitments ordered by created_date + prosecution_flow.hpl Y - Get all recruitments - foreign_diploma_flow.hpl 2 + Get all recruitments ordered by created_date + prosecution_flow.hpl 2 Y - Get all recruitments - foreign_diploma_flow.hpl 3 + Get all recruitments ordered by created_date + prosecution_flow.hpl 3 Y - Get all recruitments - foreign_diploma_flow.hpl 4 + Get all recruitments ordered by created_date + prosecution_flow.hpl 4 Y - Get all recruitments - foreign_diploma_flow.hpl 5 + Get all recruitments ordered by created_date + prosecution_flow.hpl 5 Y - Get all recruitments + Get all recruitments ordered by created_date TableInput Y @@ -57,71 +57,21 @@ none - ervu-dashboard-test + ervu-dashboard N + SELECT -idm_id +idm_id AS recruitment FROM ervu_dashboard.recruitment; N - 448 - 464 - - - - foreign_diploma_flow.hpl - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 720 + 432 304 - foreign_diploma_flow.hpl 2 + prosecution_flow.hpl PipelineExecutor Y @@ -132,7 +82,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl N 1 @@ -141,9 +91,14 @@ FROM ervu_dashboard.recruitment; IDM_ID - idm_id + recruitment + + JOB_NAME + + prosecution_job + Y @@ -167,12 +122,180 @@ FROM ervu_dashboard.recruitment; - 720 + 704 + 144 + + + + prosecution_flow.hpl 2 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl + N + + 1 + + + + + IDM_ID + recruitment + + + + JOB_NAME + + prosecution_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 704 + 224 + + + + prosecution_flow.hpl 3 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl + N + + 1 + + + + + IDM_ID + recruitment + + + + JOB_NAME + + prosecution_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 704 + 304 + + + + prosecution_flow.hpl 4 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl + N + + 1 + + + + + IDM_ID + recruitment + + + + JOB_NAME + + prosecution_job + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 704 384 - foreign_diploma_flow.hpl 3 + prosecution_flow.hpl 5 PipelineExecutor Y @@ -183,7 +306,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl N 1 @@ -192,9 +315,14 @@ FROM ervu_dashboard.recruitment; IDM_ID - idm_id + recruitment + + JOB_NAME + + prosecution_job + Y @@ -218,112 +346,10 @@ FROM ervu_dashboard.recruitment; - 720 + 704 464 - - foreign_diploma_flow.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 720 - 544 - - - - foreign_diploma_flow.hpl 5 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl - N - - 1 - - - - - IDM_ID - idm_id - - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 720 - 624 - - diff --git a/mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow.hpl b/mappings/info_recruits/citizen_tables/punishment/recruitments_five_flow.hpl similarity index 81% rename from mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow.hpl rename to mappings/info_recruits/citizen_tables/punishment/recruitments_five_flow.hpl index 3860444..33ee4da 100644 --- a/mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow.hpl +++ b/mappings/info_recruits/citizen_tables/punishment/recruitments_five_flow.hpl @@ -1,7 +1,7 @@ - recruitment_five_flow + recruitments_five_flow Y @@ -13,41 +13,41 @@ 1000 100 - - 2025/09/01 16:12:33.364 + 2025/04/18 09:48:01.970 - - 2025/09/01 16:12:33.364 + 2025/04/18 09:48:01.970 - Get all recruitments - russian_diploma_flow.hpl + Get all recruitments ordered by created_date + punishment_flow.hpl Y - Get all recruitments - russian_diploma_flow.hpl 2 + Get all recruitments ordered by created_date + punishment_flow.hpl 2 Y - Get all recruitments - russian_diploma_flow.hpl 3 + Get all recruitments ordered by created_date + punishment_flow.hpl 3 Y - Get all recruitments - russian_diploma_flow.hpl 4 + Get all recruitments ordered by created_date + punishment_flow.hpl 4 Y - Get all recruitments - russian_diploma_flow.hpl 5 + Get all recruitments ordered by created_date + punishment_flow.hpl 5 Y - Get all recruitments + Get all recruitments ordered by created_date TableInput Y @@ -57,20 +57,21 @@ none - ervu-dashboard-test + ervu-dashboard N + SELECT -idm_id +idm_id AS recruitment FROM ervu_dashboard.recruitment; N 432 - 512 + 304 - russian_diploma_flow.hpl + punishment_flow.hpl PipelineExecutor Y @@ -81,7 +82,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl N 1 @@ -90,9 +91,14 @@ FROM ervu_dashboard.recruitment; IDM_ID - idm_id + recruitment + + JOB_NAME + + punishment_job + Y @@ -117,11 +123,11 @@ FROM ervu_dashboard.recruitment; 704 - 352 + 144 - russian_diploma_flow.hpl 2 + punishment_flow.hpl 2 PipelineExecutor Y @@ -132,7 +138,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl N 1 @@ -141,9 +147,14 @@ FROM ervu_dashboard.recruitment; IDM_ID - idm_id + recruitment + + JOB_NAME + + punishment_job + Y @@ -168,11 +179,11 @@ FROM ervu_dashboard.recruitment; 704 - 432 + 224 - russian_diploma_flow.hpl 3 + punishment_flow.hpl 3 PipelineExecutor Y @@ -183,7 +194,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl N 1 @@ -192,9 +203,14 @@ FROM ervu_dashboard.recruitment; IDM_ID - idm_id + recruitment + + JOB_NAME + + punishment_job + Y @@ -219,11 +235,11 @@ FROM ervu_dashboard.recruitment; 704 - 512 + 304 - russian_diploma_flow.hpl 4 + punishment_flow.hpl 4 PipelineExecutor Y @@ -234,7 +250,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl N 1 @@ -243,9 +259,14 @@ FROM ervu_dashboard.recruitment; IDM_ID - idm_id + recruitment + + JOB_NAME + + punishment_job + Y @@ -270,11 +291,11 @@ FROM ervu_dashboard.recruitment; 704 - 592 + 384 - russian_diploma_flow.hpl 5 + punishment_flow.hpl 5 PipelineExecutor Y @@ -285,7 +306,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl N 1 @@ -294,9 +315,14 @@ FROM ervu_dashboard.recruitment; IDM_ID - idm_id + recruitment + + JOB_NAME + + punishment_job + Y @@ -321,7 +347,7 @@ FROM ervu_dashboard.recruitment; 704 - 672 + 464 diff --git a/mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/support/check_if_constants_exists.hpl b/mappings/info_recruits/citizen_tables/work/self_employed/support/check_if_self_employed_job_execution_exists.hpl similarity index 75% rename from mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/support/check_if_constants_exists.hpl rename to mappings/info_recruits/citizen_tables/work/self_employed/support/check_if_self_employed_job_execution_exists.hpl index 7a4fb03..494e70f 100644 --- a/mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/support/check_if_constants_exists.hpl +++ b/mappings/info_recruits/citizen_tables/work/self_employed/support/check_if_self_employed_job_execution_exists.hpl @@ -1,7 +1,7 @@ - check_if_constants_exists + check_if_self_employed_job_execution_exists Y @@ -13,9 +13,9 @@ 1000 100 - - 2025/09/21 21:25:15.712 + 2025/08/18 20:34:56.618 - - 2025/09/21 21:25:15.712 + 2025/08/18 20:34:56.618 @@ -39,16 +39,16 @@ - constasnts_exists - CONSTANTS_EXISTS + job_executed_flag + JOB_EXECUTED_FLAG PARENT_WORKFLOW Y - 576 - 272 + 592 + 192 @@ -66,13 +66,16 @@ N 0 SELECT EXISTS ( - SELECT 1 FROM country_oksm -) AS constasnts_exists - Y + SELECT 1 + FROM etl.job_execution + WHERE job_name = 'self_employed_job' + limit 1 + ) AS job_executed_flag; + N - 288 - 272 + 304 + 192 diff --git a/mappings/info_recruits/citizen_tables/education/russian_diploma/constants/support/check_if_constants_exists.hpl b/mappings/info_recruits/citizen_tables/work/work_activity/support/check_if_work_activity_job_exists.hpl similarity index 76% rename from mappings/info_recruits/citizen_tables/education/russian_diploma/constants/support/check_if_constants_exists.hpl rename to mappings/info_recruits/citizen_tables/work/work_activity/support/check_if_work_activity_job_exists.hpl index fd54f04..a4b4134 100644 --- a/mappings/info_recruits/citizen_tables/education/russian_diploma/constants/support/check_if_constants_exists.hpl +++ b/mappings/info_recruits/citizen_tables/work/work_activity/support/check_if_work_activity_job_exists.hpl @@ -1,7 +1,7 @@ - check_if_constants_exists + check_if_work_activity_job_exists Y @@ -13,9 +13,9 @@ 1000 100 - - 2025/09/22 08:47:26.114 + 2025/08/05 13:28:46.661 - - 2025/09/22 08:47:26.114 + 2025/08/05 13:28:46.661 @@ -39,17 +39,16 @@ - - constants_exists - CONSTANTS_EXISTS + job_executed_flag + JOB_EXECUTED_FLAG PARENT_WORKFLOW Y - 480 - 208 + 672 + 336 @@ -67,15 +66,15 @@ N 0 SELECT EXISTS ( - SELECT 1 FROM minobr_doc_type - UNION ALL - SELECT 1 FROM minprosv_doc_type -) AS constants_exists - Y + SELECT 1 + FROM etl.job_execution + WHERE job_name = 'work_activity_job' + ) AS job_executed_flag; + N - 192 - 208 + 384 + 336 diff --git a/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl b/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_repeat.hpl similarity index 63% rename from mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl rename to mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_repeat.hpl index 83f0d80..809cfea 100644 --- a/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl +++ b/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_repeat.hpl @@ -1,7 +1,7 @@ - foreign_diploma_flow_repeat + restriction_document_flow_repeat Y @@ -13,21 +13,21 @@ 1000 100 - - 2025/09/04 21:35:58.671 + 2025/08/08 13:25:52.319 - - 2025/09/04 21:35:58.671 + 2025/08/08 13:25:52.319 - Change job status on error - Abort + Create job execution record + Table input Y - Create job execution record - foerign_diploma_input + Table input + Identify last row in a stream Y @@ -36,33 +36,33 @@ Y - Identify last row in a stream - Detect empty stream + Detect empty stream + Change job status on success Y - foerign_diploma_input - Identify last row in a stream + Change job status on error + Abort Y Identify last row in a stream - foreign_diploma_upsert + Insert / update Y - foreign_diploma_upsert + Insert / update Filter rows Y - foreign_diploma_upsert - Change job status on error + Insert / update + Detect empty stream Y - Detect empty stream - Change job status on success + Insert / update + Change job status on error Y @@ -77,13 +77,13 @@ none - ABORT + ABORT_WITH_ERROR Y 0 1248 - 624 + 400 @@ -117,8 +117,8 @@ and recruitment_id = '${IDM_ID}'; - 1024 - 624 + 960 + 400 @@ -148,8 +148,8 @@ and recruitment_id = '${IDM_ID}'; - 1248 - 272 + 1440 + 48 @@ -171,17 +171,32 @@ and recruitment_id = '${IDM_ID}'; Y N N - UPDATE etl.job_execution -SET + INSERT INTO etl.job_execution ( + id, + job_name, + status, + execution_datetime, + error_description, + recruitment_id +) +VALUES ( + DEFAULT, + '${JOB_NAME}', + 'PROCESSING', + DEFAULT, + NULL, + '${IDM_ID}' +) +ON CONFLICT (job_name, recruitment_id) +DO UPDATE SET status = 'PROCESSING', - execution_datetime = current_timestamp, + execution_datetime = DEFAULT, error_description = NULL -where job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; + - 320 - 448 + 272 + 208 @@ -197,8 +212,8 @@ and recruitment_id = '${IDM_ID}'; - 736 - 272 + 1440 + 208 @@ -233,15 +248,15 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1248 - 448 + 960 + 48 Identify last row in a stream DetectLastRow - N + Y 1 @@ -251,166 +266,151 @@ and recruitment_id = '${IDM_ID}'; last_row - 736 - 448 + 720 + 208 - foerign_diploma_input - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr - FROM recruits_info ri - WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr') = 'array' - and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' -) -SELECT fd.recruit_id, - diploma_elem ->> 'id' AS source_id, - to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, - diploma_elem ->> 'kodUrObr' as education_level_code, - diploma_elem ->> 'naimUchZaved' as educational_institution_name, - diploma_elem ->> 'naimInoDok' as foreign_doc_name, - diploma_elem ->> 'serNomDok' as ser_num_doc, - diploma_elem ->> 'regNomDok' as reg_num, - diploma_elem ->> 'nomSved' as recognition_certificate_number, - to_date(diploma_elem ->> 'dataPriz', 'YYYY-MM-DD') as recognition_date, - diploma_elem -> 'svedSpecz' ->> 'naimSpecz' as speciality_name, - diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, - diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, - diploma_elem ->> 'kodOKSM' as oksm_code, - coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec -FROM filteredData fd - CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem; - - Y - - - 544 - 448 - - - - foreign_diploma_upsert + Insert / update InsertUpdate - Y + N 1 none - 10000 + 100 ervu-dashboard = - source_id - source_id + id + id + + + + = + created_at + created_at + ervu_dashboard - foreign_diploma
+ restriction_document
- recruit_id - recruit_id + id + id N - source_id - source_id + subpoena_id + subpoena_id + Y + + + created_at + created_at N - source_update_date - source_update_date + updated_at + updated_at Y - education_level_code - education_level_code + vk_id + vk_id Y - educational_institution_name - educational_institution_name + user_id + user_id Y - foreign_doc_name - foreign_doc_name + user_name + user_name Y - ser_num_doc - ser_num_doc + status + status Y - reg_num - reg_num + decision_number + decision_number Y - recognition_certificate_number - recognition_certificate_number + decision_date + decision_date Y - recognition_date - recognition_date + decision_reason + decision_reason Y - speciality_name - speciality_name + extra_info + extra_info Y - minobr_speciality_code - minobr_speciality_code + type + type Y - minprosv_speciality_code - minprosv_speciality_code - Y - - - oksm_code - oksm_code - Y - - - vus_spec - vus_spec + recruitment_name + recruitment_name Y
N - 1024 - 448 + 960 + 208 + +
+ + Table input + TableInput + + N + + 1 + + none + + + postgres.subpoena + N + SELECT +* +FROM public.restriction_document +WHERE + '${IDM_ID}' != '' -- Проверка на пустую строку + AND vk_id = '${IDM_ID}' + AND '${M_R_CR_DATE}' >= created_at +${LIMIT_FW} + Y + + + 480 + 208 - foreign_diploma_upsert + Insert / update Change job status on error Y - error_description + diff --git a/mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/foreign_diploma_constants_flow.hpl b/mappings_new/info_recruits/citizen_tables/passport/recruit_update_date_ervu_dashboard.hpl similarity index 62% rename from mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/foreign_diploma_constants_flow.hpl rename to mappings_new/info_recruits/citizen_tables/passport/recruit_update_date_ervu_dashboard.hpl index e2c8de2..0236547 100644 --- a/mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/foreign_diploma_constants_flow.hpl +++ b/mappings_new/info_recruits/citizen_tables/passport/recruit_update_date_ervu_dashboard.hpl @@ -1,7 +1,7 @@ - foreign_diploma_constants_flow + recruit_update_date_ervu_dashboard Y @@ -13,21 +13,21 @@ 1000 100 - - 2025/09/22 08:32:45.452 + 2025/06/02 14:50:31.833 - - 2025/09/22 08:32:45.452 + 2025/06/02 14:50:31.833 - country_oksm_input - country_oksm_output + Table input + Table output Y - country_oksm_input + Table input TableInput Y @@ -37,26 +37,26 @@ none - nsi + ervu-dashboard N 0 - SELECT - key as code, - value ->> 'value' as value, - NOT hidden as actual, - updated_at as update_date -FROM classifier_records -WHERE code = 'countriesOKSM'; - + SELECT + created_at AS system_create_date, + updated_at AS system_update_date, + current_timestamp AS record_created, + 'job_recruits_info' AS workflow +FROM recruits_info +WHERE updated_at = (SELECT MAX(updated_at) FROM recruits_info) +limit 1 N - 416 - 192 + 896 + 304 - country_oksm_output + Table output TableOutput Y @@ -66,26 +66,10 @@ WHERE code = 'countriesOKSM'; none - 10000 + 1000 ervu-dashboard - - code - code - - - value - value - - - actual - actual - - - update_date - update_date - - + N N N @@ -94,9 +78,9 @@ WHERE code = 'countriesOKSM'; Y N - ervu_dashboard - Y - country_oksm
+ public + N + recruit_create
N Y @@ -104,8 +88,8 @@ WHERE code = 'countriesOKSM'; Y - 640 - 192 + 1216 + 304