diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow.hpl new file mode 100644 index 0000000..cd730ef --- /dev/null +++ b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow.hpl @@ -0,0 +1,362 @@ + + + + education_constants_flow + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/09/21 20:19:05.913 + - + 2025/09/21 20:19:05.913 + + + + + + education_level_input + education_level_output + Y + + + minprosv_speciality_input + minprosv_speciality_output + Y + + + minobr_speciality_input + minobr_speciality_output + Y + + + educational_institution_status_input + educational_institution_status_output + Y + + + + education_level_input + TableInput + + Y + + 1 + + none + + + nsi + N + 0 + SELECT + key as code, + value ->> 'value' as value, + NOT hidden as actual, + updated_at as update_date +FROM classifier_records +WHERE code = 'educationLevel'; + N + + + 368 + 208 + + + + education_level_output + TableOutput + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + code + code + + + value + value + + + actual + actual + + + update_date + update_date + + + N + N + N + N + Y + N + ervu_dashboard + Y + education_level
+ N + Y + N + Y + + + 624 + 208 + + + + educational_institution_status_input + TableInput + + Y + + 1 + + none + + + nsi + N + 0 + SELECT + key, + value ->> 'value' as value, + NOT hidden as actual, + updated_at as update_date +FROM classifier_records +WHERE code = 'organizationStatus'; + N + + + 368 + 304 + + + + educational_institution_status_output + TableOutput + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + code + code + + + value + value + + + actual + actual + + + update_date + update_date + + + N + N + N + N + Y + N + ervu_dashboard + Y + educational_institution_status
+ N + Y + N + Y + + + 624 + 304 + + + + minobr_speciality_input + TableInput + + Y + + 1 + + none + + + nsi + N + 0 + SELECT + value -> 'extraInfo' ->> 'kodSpecz' as code, + value -> 'extraInfo' ->> 'naimSpecz' as value, + NOT hidden as actual, + updated_at as update_date +FROM classifier_records +WHERE code = 'educationSpecialityMinobr' + AND value -> 'extraInfo' ->> 'kodSpecz' != 'null'; + N + + + 368 + 400 + + + + minobr_speciality_output + TableOutput + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + code + code + + + value + value + + + actual + actual + + + update_date + update_date + + + N + N + N + N + Y + N + ervu_dashboard + Y + minobr_education_speciality
+ N + Y + N + Y + + + 624 + 400 + + + + minprosv_speciality_input + TableInput + + Y + + 1 + + none + + + nsi + N + 0 + SELECT + key, + value -> 'extraInfo' ->> 'kodSpecz' as code, + value -> 'extraInfo' ->> 'naimSpecz' as value, + NOT hidden as actual, + updated_at as update_date +FROM classifier_records +WHERE code = 'educationSpecialityMinprosv' + AND value -> 'extraInfo' ->> 'kodSpecz' != 'null'; + N + + + 368 + 496 + + + + minprosv_speciality_output + TableOutput + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + code + code + + + value + value + + + actual + actual + + + update_date + update_date + + + N + N + N + N + Y + N + ervu_dashboard + Y + minprosv_education_speciality
+ N + Y + N + Y + + + 624 + 496 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow_repeat.hpl new file mode 100644 index 0000000..ebffbdf --- /dev/null +++ b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow_repeat.hpl @@ -0,0 +1,508 @@ + + + + education_constants_flow_repeat + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/09/21 20:33:27.321 + - + 2025/09/21 20:33:27.321 + + + + + + get_max_education_level_update_date + education_level_input + Y + + + get_max_educational_institution_status_update_date + educational_institution_status_input + Y + + + get_max_minobr_speciality_update_date + minobr_speciality_input + Y + + + get_max_minprosv_speciality_update_date + minprosv_speciality_input + Y + + + education_level_input + education_level_upsert + Y + + + educational_institution_status_input + educational_institution_status_upsert + Y + + + minobr_speciality_input + minobr_speciality_upsert + Y + + + minprosv_speciality_input + minprosv_speciality_upsert + Y + + + + education_level_input + TableInput + + Y + + 1 + + none + + + nsi + N + 0 + get_max_education_level_update_date + SELECT + key as code, + value ->> 'value' as value, + NOT hidden as actual, + updated_at as update_date +FROM classifier_records +WHERE code = 'educationLevel' +AND updated_at > ?; + N + + + 480 + 256 + + + + education_level_upsert + InsertUpdate + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + = + code + code + + + + <> + actual + actual + + + ervu_dashboard + education_level
+ + code + code + N + + + value + value + Y + + + actual + actual + Y + + + update_date + update_date + Y + + + N + + + 768 + 256 + + + + educational_institution_status_input + TableInput + + Y + + 1 + + none + + + nsi + N + 0 + get_max_educational_institution_status_update_date + SELECT + key, + value ->> 'value' as value, + NOT hidden as actual, + updated_at as update_date +FROM classifier_records +WHERE code = 'organizationStatus' +AND updated_at > ?; + N + + + 480 + 352 + + + + educational_institution_status_upsert + InsertUpdate + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + = + code + code + + + <> + actual + actual + + ervu_dashboard + educational_institution_status
+ + code + code + N + + + value + value + Y + + + actual + actual + Y + + + update_date + update_date + Y + + + N + + + 768 + 352 + + + + get_max_education_level_update_date + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT + max(update_date) as max_update_date +FROM education_level; + + N + + + 128 + 256 + + + + get_max_educational_institution_status_update_date + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT + max(update_date) as max_update_date +FROM educational_institution_status; + + N + + + 128 + 352 + + + + get_max_minobr_speciality_update_date + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT + max(update_date) as max_update_date +FROM minobr_education_speciality; + + N + + + 128 + 448 + + + + get_max_minprosv_speciality_update_date + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT + max(update_date) as max_update_date +FROM minprosv_education_speciality + + N + + + 128 + 544 + + + + minobr_speciality_input + TableInput + + Y + + 1 + + none + + + nsi + N + 0 + get_max_minobr_speciality_update_date + SELECT + value -> 'extraInfo' ->> 'kodSpecz' as code, + value -> 'extraInfo' ->> 'naimSpecz' as value, + NOT hidden as actual, + updated_at as update_date +FROM classifier_records +WHERE code = 'educationSpecialityMinobr' + AND value -> 'extraInfo' ->> 'kodSpecz' != 'null' +AND updated_at > ?; + N + + + 480 + 448 + + + + minobr_speciality_upsert + InsertUpdate + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + = + code + code + + + <> + actual + actual + + ervu_dashboard + minobr_education_speciality
+ + code + code + N + + + value + value + Y + + + actual + actual + Y + + + update_date + update_date + Y + + + N + + + 768 + 448 + + + + minprosv_speciality_input + TableInput + + Y + + 1 + + none + + + nsi + N + 0 + get_max_minprosv_speciality_update_date + SELECT + key, + value -> 'extraInfo' ->> 'kodSpecz' as code, + value -> 'extraInfo' ->> 'naimSpecz' as value, + NOT hidden as actual, + updated_at as update_date +FROM classifier_records +WHERE code = 'educationSpecialityMinprosv' + AND value -> 'extraInfo' ->> 'kodSpecz' != 'null' +AND updated_at > ?; + N + + + 480 + 544 + + + + minprosv_speciality_upsert + InsertUpdate + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + = + code + code + + + <> + actual + actual + + ervu_dashboard + minprosv_education_speciality
+ + code + code + N + + + value + value + Y + + + actual + actual + Y + + + update_date + update_date + Y + + + N + + + 768 + 544 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/support/check_if_constants_exists.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/support/check_if_constants_exists.hpl new file mode 100644 index 0000000..906090d --- /dev/null +++ b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/support/check_if_constants_exists.hpl @@ -0,0 +1,88 @@ + + + + check_if_constants_exists + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/09/21 20:58:53.188 + - + 2025/09/21 20:58:53.188 + + + + + + Table input + Set variables + Y + + + + Set variables + SetVariable + + Y + + 1 + + none + + + + + + constasnts_exists + CONSTANTS_EXISTS + PARENT_WORKFLOW + + + Y + + + 608 + 304 + + + + Table input + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT EXISTS ( + SELECT 1 FROM education_level + UNION ALL + SELECT 1 FROM educational_institution_status + UNION ALL + SELECT 1 FROM minobr_education_speciality + UNION ALL + SELECT 1 FROM minprosv_education_speciality +) AS constasnts_exists + Y + + + 320 + 304 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/educational_institution_learning_job.hwf b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/educational_institution_learning_job.hwf new file mode 100644 index 0000000..405c135 --- /dev/null +++ b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/educational_institution_learning_job.hwf @@ -0,0 +1,410 @@ + + + educational_institution_learning_job + Y + + + + - + 2025/08/27 21:14:40.330 + - + 2025/08/27 21:14:40.330 + + + + + Start + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 384 + 288 + + + + check_if_job_execution_exists.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_job_execution_exists.hpl + Basic + + Y + + N + local + N + N + Y + N + 1392 + 304 + + + + employer_job_execution_exists_check + + SIMPLE_EVAL + + boolean + false + equal + equal + N + variable + JOB_EXECUTED_FLAG + N + 1696 + 304 + + + + recruitment_five_flow.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow.hpl + Basic + + Y + + N + local + N + N + Y + Y + 1968 + 304 + + + + recruitment_five_flow_repeat.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_repeat.hpl + Basic + + Y + + N + local + N + N + Y + N + 1968 + 608 + + + + check_if_need_to_repeat.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_need_to_repeat.hpl + Basic + + Y + + N + local + N + N + Y + N + 1696 + 448 + + + + Simple evaluation + + SIMPLE_EVAL + + boolean + true + equal + equal + N + variable + NEED_TO_REPEAT_JOB + N + 1696 + 608 + + + + recruitment_five_flow_delta.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_delta.hpl + Basic + + Y + + N + local + N + N + Y + N + 1696 + 800 + + + + check_if_constants_exists.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/constants/support/check_if_constants_exists.hpl + + + Basic + + Y + + N + local + N + N + Y + N + 576 + 288 + + + + constants_exists + + SIMPLE_EVAL + + boolean + true + equal + equal + N + variable + CONSTANTS_EXISTS + N + 752 + 288 + + + + education_constants_flow.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow.hpl + Basic + + Y + + N + local + N + N + Y + N + 944 + 208 + + + + education_constants_flow_repeat.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow_repeat.hpl + Basic + + Y + + N + local + N + N + Y + N + 944 + 384 + + + + init_job_name + + SET_VARIABLES + + + + JOB_NAME + CURRENT_WORKFLOW + educational_institution_learning_job + + + CURRENT_WORKFLOW + N + N + 1184 + 304 + + + + + + check_if_job_execution_exists.hpl + employer_job_execution_exists_check + Y + Y + N + + + employer_job_execution_exists_check + check_if_need_to_repeat.hpl + Y + N + N + + + check_if_need_to_repeat.hpl + Simple evaluation + Y + Y + N + + + check_if_constants_exists.hpl + constants_exists + Y + Y + N + + + constants_exists + education_constants_flow.hpl + Y + N + N + + + constants_exists + education_constants_flow_repeat.hpl + Y + Y + N + + + Start + check_if_constants_exists.hpl + Y + Y + Y + + + education_constants_flow.hpl + init_job_name + Y + Y + N + + + education_constants_flow_repeat.hpl + init_job_name + Y + Y + N + + + init_job_name + check_if_job_execution_exists.hpl + Y + Y + N + + + employer_job_execution_exists_check + recruitment_five_flow.hpl + Y + Y + N + + + Simple evaluation + recruitment_five_flow_repeat.hpl + Y + Y + N + + + Simple evaluation + recruitment_five_flow_delta.hpl + Y + N + N + + + + + + diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl new file mode 100644 index 0000000..8bc8c1b --- /dev/null +++ b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl @@ -0,0 +1,1022 @@ + + + + citizen_education_flow + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/08/28 08:20:00.808 + - + 2025/08/28 08:20:00.808 + + + + + + Change job status on error + Abort + Y + + + Create job execution record + education_input + Y + + + Filter rows + Change job status on success + Y + + + Identify last row in a stream + Detect empty stream + Y + + + Identify last row in a stream + Filter_null_parental_leave_dates + Y + + + education_input + Identify last row in a stream + Y + + + citizen_education_output + Change job status on error + Y + + + citizen_education_output + Filter rows + Y + + + education_leave_output + Change job status on error + Y + + + Filter_null_parental_leave_dates + Block until transforms finish + Y + + + Block until transforms finish + education_leave_output + Y + + + Identify last row in a stream + Group by + Y + + + Group by + citizen_education_output + Y + + + Detect empty stream + Change job status on success + Y + + + + Abort + Abort + + Y + + 1 + + none + + + ABORT + Y + 0 + + + 1488 + 640 + + + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + citizen_education_output + + + + + 880 + 640 + + + + Change job status on error + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ? +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1264 + 640 + + + + Change job status on success + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS' +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1488 + 256 + + + + Create job execution record + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + N + N + Y + N + N + INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id) +VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); + + + + + 208 + 432 + + + + Detect empty stream + DetectEmptyStream + + Y + + 1 + + none + + + + + 624 + 256 + + + + Filter rows + FilterRows + + Y + + 1 + + none + + + + + + + = + last_row + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + Change job status on success + + + 1488 + 432 + + + + Filter_null_parental_leave_dates + FilterRows + + Y + + 1 + + none + + + + + + + IS NOT NULL + academ_reason + N + - + + + Block until transforms finish + + + 624 + 640 + + + + Group by + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + recruit_id + recruit_id + FIRST_INCL_NULL + + + source_update_date + source_update_date + FIRST_INCL_NULL + + + educational_institution_type + educational_institution_type + FIRST_INCL_NULL + + + org_type + org_type + FIRST_INCL_NULL + + + inn + inn + FIRST_INCL_NULL + + + kpp + kpp + FIRST_INCL_NULL + + + ogrn + ogrn + FIRST_INCL_NULL + + + educational_institution_address + educational_institution_address + FIRST_INCL_NULL + + + educational_organization_separate_unit_type + educational_organization_separate_unit_type + FIRST_INCL_NULL + + + educational_organization_separate_unit_address + educational_organization_separate_unit_address + FIRST_INCL_NULL + + + student_status + student_status + FIRST_INCL_NULL + + + grade_number + grade_number + FIRST_INCL_NULL + + + education_level_code + education_level_code + FIRST_INCL_NULL + + + admission_date + admission_date + FIRST_INCL_NULL + + + enrollment_date + enrollment_date + FIRST_INCL_NULL + + + enrollment_order_number + enrollment_order_number + FIRST_INCL_NULL + + + enrollment_order_date + enrollment_order_date + FIRST_INCL_NULL + + + completion_date + completion_date + FIRST_INCL_NULL + + + education_form + education_form + FIRST_INCL_NULL + + + education_program_start_date + education_program_start_date + FIRST_INCL_NULL + + + education_start_date + education_start_date + FIRST_INCL_NULL + + + planned_completion_date + planned_completion_date + FIRST_INCL_NULL + + + minobr_speciality_code + minobr_speciality_code + FIRST_INCL_NULL + + + minprosv_speciality_code + minprosv_speciality_code + FIRST_INCL_NULL + + + recovery_info + recovery_info + FIRST_INCL_NULL + + + recovery_date + recovery_date + FIRST_INCL_NULL + + + recovery_order_date + recovery_order_date + FIRST_INCL_NULL + + + recovery_order_number + recovery_order_number + FIRST_INCL_NULL + + + expulsion_info + expulsion_info + FIRST_INCL_NULL + + + expulsion_date + expulsion_date + FIRST_INCL_NULL + + + expulsion_order_date + expulsion_order_date + FIRST_INCL_NULL + + + expulsion_order_number + expulsion_order_number + FIRST_INCL_NULL + + + academical_leave_info + academical_leave_info + FIRST_INCL_NULL + + + first_education + first_education + FIRST_INCL_NULL + + + vus_spec + vus_spec + FIRST_INCL_NULL + + + military_center_education_info + military_center_education_info + FIRST_INCL_NULL + + + military_center_education_start_date + military_center_education_start_date + FIRST_INCL_NULL + + + military_center_education_end_date + military_center_education_end_date + FIRST_INCL_NULL + + + military_center_education_period + military_center_education_period + FIRST_INCL_NULL + + + military_center_completion + military_center_completion + FIRST_INCL_NULL + + + military_department_education_info + military_department_education_info + FIRST_INCL_NULL + + + military_department_start_date + military_department_start_date + FIRST_INCL_NULL + + + military_department_end_date + military_department_end_date + FIRST_INCL_NULL + + + last_row + last_row + LAST_INCL_NULL + + + academ_start_date + academ_start_date + FIRST_INCL_NULL + + + academ_end_date + academ_end_date + FIRST_INCL_NULL + + + academ_reason + academ_reason + FIRST_INCL_NULL + + + N + + + source_id + + + N + grp + + + 944 + 432 + + + + Identify last row in a stream + DetectLastRow + + N + + 1 + + none + + + last_row + + + 624 + 432 + + + + citizen_education_output + TableOutput + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + source_id + source_id + + + recruit_id + recruit_id + + + source_update_date + source_update_date + + + educational_institution_type + educational_institution_type + + + org_type + org_type + + + inn + inn + + + kpp + kpp + + + ogrn + ogrn + + + educational_institution_address + educational_institution_address + + + educational_organization_separate_unit_type + educational_organization_separate_unit_type + + + educational_organization_separate_unit_address + educational_organization_separate_unit_address + + + student_status + student_status + + + grade_number + grade_number + + + education_level_code + education_level_code + + + admission_date + admission_date + + + enrollment_date + enrollment_date + + + enrollment_order_number + enrollment_order_number + + + enrollment_order_date + enrollment_order_date + + + completion_date + completion_date + + + education_form + education_form + + + education_program_start_date + education_program_start_date + + + education_start_date + education_start_date + + + planned_completion_date + planned_completion_date + + + minobr_speciality_code + minobr_speciality_code + + + minprosv_speciality_code + minprosv_speciality_code + + + recovery_info + recovery_info + + + recovery_date + recovery_date + + + recovery_order_date + recovery_order_date + + + recovery_order_number + recovery_order_number + + + expulsion_info + expulsion_info + + + expulsion_date + expulsion_date + + + expulsion_order_date + expulsion_order_date + + + expulsion_order_number + expulsion_order_number + + + academical_leave_info + academical_leave_info + + + first_education + first_education + + + vus_spec + vus_spec + + + military_center_education_info + military_center_education_info + + + military_center_education_start_date + military_center_education_start_date + + + military_center_education_end_date + military_center_education_end_date + + + military_center_education_period + military_center_education_period + + + military_center_completion + military_center_completion + + + military_department_education_info + military_department_education_info + + + military_department_start_date + military_department_start_date + + + military_department_end_date + military_department_end_date + + + N + N + N + N + Y + N + ervu_dashboard + Y + educational_institution_learning
+ N + Y + N + Y + + + 1264 + 432 + + + + education_input + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + WITH filteredData AS (SELECT ri.recruit_id, + education_elem, + academ_elem, + 'Образовательная организация высшего образования' as educational_institution_type + FROM recruits_info ri + CROSS JOIN LATERAL jsonb_array_elements( + ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') AS education_elem + LEFT JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array' + THEN education_elem -> 'svedOtpusk' + ELSE '[]'::jsonb + END + ) AS academ_elem ON true + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') = 'array' + and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + + union all + + SELECT ri.recruit_id, + education_elem, + academ_elem, + 'Профессиональная образовательная организация' as educational_institution_type + FROM recruits_info ri + CROSS JOIN LATERAL jsonb_array_elements( + ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') AS education_elem + LEFT JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array' + THEN education_elem -> 'svedOtpusk' + ELSE '[]'::jsonb + END + ) AS academ_elem ON true + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') = 'array' + and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + + union all + + SELECT ri.recruit_id, + education_elem, + academ_elem, + 'Общеобразовательная организация' as educational_institution_type + FROM recruits_info ri + CROSS JOIN LATERAL jsonb_array_elements( + ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') AS education_elem + LEFT JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array' + THEN education_elem -> 'svedOtpusk' + ELSE '[]'::jsonb + END + ) AS academ_elem ON true + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') = 'array' + and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}') +SELECT fd.recruit_id, + education_elem ->> 'id' AS source_id, + to_date(education_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, + educational_institution_type, + education_elem -> 'svedOrg' ->> 'naimTipOrg' as org_type, + education_elem -> 'svedOrg' ->> 'inn' as inn, + education_elem -> 'svedOrg' ->> 'kpp' as kpp, + coalesce(education_elem -> 'svedOrg' ->> 'ogrn', education_elem -> 'svedOrg' ->> + 'ogrnip') as ogrn, + education_elem -> 'svedOrg' ->> 'adresOrg' as educational_institution_address, + education_elem -> 'svedOrg' -> 'extend' ->> 'statusOrg' as educational_institution_address, + education_elem -> 'svedOrg' -> 'extend' ->> 'vidPodrazdOrg' as educational_organization_separate_unit_type, + education_elem -> 'svedOrg' -> 'extend' ->> 'adresPodrazdOrg' as educational_organization_separate_unit_address, + education_elem ->> 'naimStatus' as student_status, + case + when educational_institution_type = 'Образовательная организация высшего образования' + then education_elem ->> 'nomerKursa' + when educational_institution_type = 'Профессиональная образовательная организация' + then education_elem -> 'extend' ->> 'klass' + when educational_institution_type = 'Общеобразовательная организация' then education_elem ->> 'klass' + end as grade_number, + education_elem ->> 'kodUrovObr' as education_level_code, + to_date(coalesce(education_elem -> 'extend' ->> 'dataPostup', education_elem ->> 'dataPostup'), + 'YYYY-MM-DD') as admission_date, + to_date(education_elem ->> 'dataZachisl', 'YYYY-MM-DD') as enrollment_date, + education_elem ->> 'nomPrikazZachisl' as enrollment_order_number, + to_date(education_elem ->> 'dataPrikazZachisl', 'YYYY-MM-DD') as enrollment_order_date, + to_date(coalesce(education_elem -> 'extend' ->> 'dataZavershObuch', education_elem ->> 'dataOkonch'), + 'YYYY-MM-DD') as completion_date, + education_elem -> 'svedObrProg' ->> 'naimFormaObuch' as education_form, + education_elem -> 'svedObrProg' ->> 'dataNachObuch' as education_program_start_date, + to_date(education_elem -> 'extend' ->> 'dataNachObuch', 'YYYY-MM-DD') as education_start_date, + to_date(coalesce(education_elem -> 'svedObrProg' ->> 'planDataOkonch', education_elem ->> 'dataOzhid'), + 'YYYY-MM-DD') as planned_completion_date, + education_elem -> 'svedObrProg' -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, + education_elem -> 'svedObrProg' -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, + coalesce(education_elem ->> 'svedVosst' <> 'null', false) as recovery_info, + to_date(education_elem -> 'svedVosst' ->> 'dataVosst', 'YYYY-MM-DD') as recovery_date, + to_date(education_elem -> 'svedVosst' ->> 'dataPrikazVosst', + 'YYYY-MM-DD') as recovery_order_date, + education_elem -> 'svedVosst' ->> 'nomPrikazVosst' as recovery_order_number, + coalesce(education_elem ->> 'svedOtchisl' <> 'null', false) as expulsion_info, + to_date(education_elem -> 'svedOtchisl' ->> 'dataOtchisl', 'YYYY-MM-DD') as expulsion_date, + to_date(education_elem -> 'svedOtchisl' ->> 'dataPrikazOtchisl', + 'YYYY-MM-DD') as expulsion_order_date, + education_elem -> 'svedOtchisl' ->> 'nomPrikazOtchisl' as expulsion_order_number, + coalesce(education_elem ->> 'svedOtpusk' <> 'null', false) as academical_leave_info, + coalesce(education_elem ->> 'prPervObuch' = '1', false) as first_education, + coalesce(education_elem -> 'svedObrProg' ->> 'prVUSpecz' = '1', false) as vus_spec, + coalesce(education_elem ->> 'svedVoenObuch' <> 'null', false) as military_center_education_info, + to_date(education_elem -> 'svedVoenObuch' ->> 'dataNach', 'YYYY-MM-DD') as military_center_education_start_date, + to_date(education_elem -> 'svedVoenObuch' ->> 'dataOkonch', + 'YYYY-MM-DD') as military_center_education_end_date, + education_elem -> 'svedVoenObuch' -> 'extend' ->> 'srokObuch' as military_center_education_period, + coalesce(education_elem -> 'svedVoenObuch' -> 'extend' ->> 'svedUspeshZaverObuch' = '1', + false) as military_center_completion, + coalesce(education_elem -> 'extend' ->> 'svedVoenKaf' <> 'null', false) as military_department_education_info, + to_date(education_elem -> 'extend' -> 'svedVoenKaf' ->> 'dataNach', + 'YYYY-MM-DD') as military_department_start_date, + to_date(education_elem -> 'extend' -> 'svedVoenKaf' ->> 'dataOkonch', + 'YYYY-MM-DD') as military_department_end_date, + to_date(academ_elem ->> 'dataNach', 'YYYY-MM-DD') as academ_start_date, + to_date(academ_elem ->> 'dataOkonch', 'YYYY-MM-DD') as academ_end_date, + academ_elem -> 'naimPrichOtpusk' as academ_reason +FROM filteredData fd +ORDER BY source_id; + Y + + + 432 + 432 + + + + education_leave_output + TableOutput + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + start_date + academ_start_date + + + end_date + academ_end_date + + + reason + academ_reason + + + citizen_education_source_id + source_id + + + N + N + N + N + Y + N + ervu_dashboard + Y + educational_leave
+ N + Y + N + Y + + + 1088 + 640 + + + + + citizen_education_output + Change job status on error + Y + + error_description + + + + + + + + education_leave_output + Change job status on error + Y + + error_description + + + + + + + + + diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl new file mode 100644 index 0000000..cb2ca00 --- /dev/null +++ b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl @@ -0,0 +1,1028 @@ + + + + citizen_education_flow_delta + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/08/28 21:27:15.946 + - + 2025/08/28 21:27:15.946 + + + + + + Block until transforms finish + education_leave_output + Y + + + Change job status on error + Abort + Y + + + Create job execution record + education_input + Y + + + Filter rows + Change job status on success + Y + + + Filter_null_parental_leave_dates + Block until transforms finish + Y + + + Group by + citizen_education_output + Y + + + Identify last row in a stream + Detect empty stream + Y + + + Identify last row in a stream + Filter_null_parental_leave_dates + Y + + + Identify last row in a stream + Group by + Y + + + citizen_education_output + Change job status on error + Y + + + citizen_education_output + Filter rows + Y + + + education_input + Identify last row in a stream + Y + + + education_leave_output + Change job status on error + Y + + + Detect empty stream + Change job status on success + Y + + + + Abort + Abort + + Y + + 1 + + none + + + ABORT + Y + 0 + + + 1696 + 672 + + + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + citizen_education_output + + + + + 1088 + 672 + + + + Change job status on error + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ? +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1472 + 672 + + + + Change job status on success + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'DELTA_SUCCESS' +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1696 + 288 + + + + Create job execution record + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + N + N + Y + N + N + UPDATE etl.job_execution +SET + status = 'DELTA_PROCESSING', + execution_datetime = current_timestamp, + error_description = NULL +where job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + 416 + 464 + + + + Detect empty stream + DetectEmptyStream + + Y + + 1 + + none + + + + + 832 + 288 + + + + Filter rows + FilterRows + + Y + + 1 + + none + + + + + + + = + last_row + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + Change job status on success + + + 1696 + 464 + + + + Filter_null_parental_leave_dates + FilterRows + + Y + + 1 + + none + + + + + + + IS NOT NULL + academ_reason + N + - + + + Block until transforms finish + + + 832 + 672 + + + + Group by + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + recruit_id + recruit_id + FIRST_INCL_NULL + + + source_update_date + source_update_date + FIRST_INCL_NULL + + + educational_institution_type + educational_institution_type + FIRST_INCL_NULL + + + org_type + org_type + FIRST_INCL_NULL + + + inn + inn + FIRST_INCL_NULL + + + kpp + kpp + FIRST_INCL_NULL + + + ogrn + ogrn + FIRST_INCL_NULL + + + educational_institution_address + educational_institution_address + FIRST_INCL_NULL + + + educational_institution_address_1 + educational_institution_address_1 + FIRST_INCL_NULL + + + educational_organization_separate_unit_type + educational_organization_separate_unit_type + FIRST_INCL_NULL + + + educational_organization_separate_unit_address + educational_organization_separate_unit_address + FIRST_INCL_NULL + + + student_status + student_status + FIRST_INCL_NULL + + + grade_number + grade_number + FIRST_INCL_NULL + + + education_level_code + education_level_code + FIRST_INCL_NULL + + + admission_date + admission_date + FIRST_INCL_NULL + + + enrollment_date + enrollment_date + FIRST_INCL_NULL + + + enrollment_order_number + enrollment_order_number + FIRST_INCL_NULL + + + enrollment_order_date + enrollment_order_date + FIRST_INCL_NULL + + + completion_date + completion_date + FIRST_INCL_NULL + + + education_form + education_form + FIRST_INCL_NULL + + + education_program_start_date + education_program_start_date + FIRST_INCL_NULL + + + education_start_date + education_start_date + FIRST_INCL_NULL + + + planned_completion_date + planned_completion_date + FIRST_INCL_NULL + + + minobr_speciality_code + minobr_speciality_code + FIRST_INCL_NULL + + + minprosv_speciality_code + minprosv_speciality_code + FIRST_INCL_NULL + + + recovery_info + recovery_info + FIRST_INCL_NULL + + + recovery_date + recovery_date + FIRST_INCL_NULL + + + recovery_order_date + recovery_order_date + FIRST_INCL_NULL + + + recovery_order_number + recovery_order_number + FIRST_INCL_NULL + + + expulsion_info + expulsion_info + FIRST_INCL_NULL + + + expulsion_date + expulsion_date + FIRST_INCL_NULL + + + expulsion_order_date + expulsion_order_date + FIRST_INCL_NULL + + + expulsion_order_number + expulsion_order_number + FIRST_INCL_NULL + + + academical_leave_info + academical_leave_info + FIRST_INCL_NULL + + + first_education + first_education + FIRST_INCL_NULL + + + vus_spec + vus_spec + FIRST_INCL_NULL + + + military_center_education_info + military_center_education_info + FIRST_INCL_NULL + + + military_center_education_start_date + military_center_education_start_date + FIRST_INCL_NULL + + + military_center_education_end_date + military_center_education_end_date + FIRST_INCL_NULL + + + military_center_education_period + military_center_education_period + FIRST_INCL_NULL + + + military_center_completion + military_center_completion + FIRST_INCL_NULL + + + military_department_education_info + military_department_education_info + FIRST_INCL_NULL + + + military_department_start_date + military_department_start_date + FIRST_INCL_NULL + + + military_department_end_date + military_department_end_date + FIRST_INCL_NULL + + + last_row + last_row + LAST_INCL_NULL + + + academ_start_date + academ_start_date + FIRST_INCL_NULL + + + academ_end_date + academ_end_date + FIRST_INCL_NULL + + + academ_reason + academ_reason + FIRST_INCL_NULL + + + N + + + source_id + + + N + grp + + + 1152 + 464 + + + + Identify last row in a stream + DetectLastRow + + N + + 1 + + none + + + last_row + + + 832 + 464 + + + + citizen_education_output + TableOutput + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + source_id + source_id + + + recruit_id + recruit_id + + + source_update_date + source_update_date + + + educational_institution_type + educational_institution_type + + + org_type + org_type + + + inn + inn + + + kpp + kpp + + + ogrn + ogrn + + + educational_institution_address + educational_institution_address + + + educational_institution_address_1 + educational_institution_address_1 + + + educational_organization_separate_unit_type + educational_organization_separate_unit_type + + + educational_organization_separate_unit_address + educational_organization_separate_unit_address + + + student_status + student_status + + + grade_number + grade_number + + + education_level_code + education_level_code + + + admission_date + admission_date + + + enrollment_date + enrollment_date + + + enrollment_order_number + enrollment_order_number + + + enrollment_order_date + enrollment_order_date + + + completion_date + completion_date + + + education_form + education_form + + + education_program_start_date + education_program_start_date + + + education_start_date + education_start_date + + + planned_completion_date + planned_completion_date + + + minobr_speciality_code + minobr_speciality_code + + + minprosv_speciality_code + minprosv_speciality_code + + + recovery_info + recovery_info + + + recovery_date + recovery_date + + + recovery_order_date + recovery_order_date + + + recovery_order_number + recovery_order_number + + + expulsion_info + expulsion_info + + + expulsion_date + expulsion_date + + + expulsion_order_date + expulsion_order_date + + + expulsion_order_number + expulsion_order_number + + + academical_leave_info + academical_leave_info + + + first_education + first_education + + + vus_spec + vus_spec + + + military_center_education_info + military_center_education_info + + + military_center_education_start_date + military_center_education_start_date + + + military_center_education_end_date + military_center_education_end_date + + + military_center_education_period + military_center_education_period + + + military_center_completion + military_center_completion + + + military_department_education_info + military_department_education_info + + + military_department_start_date + military_department_start_date + + + military_department_end_date + military_department_end_date + + + N + N + N + N + Y + N + ervu_dashboard + Y + educational_institution_learning
+ N + Y + N + Y + + + 1472 + 464 + + + + education_input + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + WITH filteredData AS (SELECT ri.recruit_id, + education_elem, + academ_elem, + 'Образовательная организация высшего образования' as educational_institution_type + FROM recruits_info ri + CROSS JOIN LATERAL jsonb_array_elements( + ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') AS education_elem + LEFT JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array' + THEN education_elem -> 'svedOtpusk' + ELSE '[]'::jsonb + END + ) AS academ_elem ON true + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') = 'array' + and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND to_date(education_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' + + union all + + SELECT ri.recruit_id, + education_elem, + academ_elem, + 'Профессиональная образовательная организация' as educational_institution_type + FROM recruits_info ri + CROSS JOIN LATERAL jsonb_array_elements( + ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') AS education_elem + LEFT JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array' + THEN education_elem -> 'svedOtpusk' + ELSE '[]'::jsonb + END + ) AS academ_elem ON true + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') = 'array' + and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND to_date(education_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' + + union all + + SELECT ri.recruit_id, + education_elem, + academ_elem, + 'Общеобразовательная организация' as educational_institution_type + FROM recruits_info ri + CROSS JOIN LATERAL jsonb_array_elements( + ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') AS education_elem + LEFT JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array' + THEN education_elem -> 'svedOtpusk' + ELSE '[]'::jsonb + END + ) AS academ_elem ON true + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') = 'array' + and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + AND to_date(education_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' +) +SELECT fd.recruit_id, + education_elem ->> 'id' AS source_id, + education_elem ->> 'dataSved' as source_update_date, + educational_institution_type, + education_elem -> 'svedOrg' ->> 'naimTipOrg' as org_type, + education_elem -> 'svedOrg' ->> 'inn' as inn, + education_elem -> 'svedOrg' ->> 'kpp' as kpp, + coalesce(education_elem -> 'svedOrg' ->> 'ogrn', education_elem -> 'svedOrg' ->> 'ogrnip') as ogrn, + education_elem -> 'svedOrg' ->> 'adresOrg' as educational_institution_address, + education_elem -> 'svedOrg' -> 'extend' ->> 'statusOrg' as educational_institution_address, + education_elem -> 'svedOrg' -> 'extend' ->> 'vidPodrazdOrg' as educational_organization_separate_unit_type, + education_elem -> 'svedOrg' -> 'extend' ->> 'adresPodrazdOrg' as educational_organization_separate_unit_address, + education_elem ->> 'naimStatus' as student_status, + case + when educational_institution_type = 'Образовательная организация высшего образования' + then education_elem ->> 'nomerKursa' + when educational_institution_type = 'Профессиональная образовательная организация' + then education_elem -> 'extend' ->> 'klass' + when educational_institution_type = 'Общеобразовательная организация' then education_elem ->> 'klass' + end as grade_number, + education_elem ->> 'kodUrovObr' as education_level_code, + coalesce(education_elem -> 'extend' ->> 'dataPostup', education_elem ->> 'dataPostup') as admission_date, + education_elem ->> 'dataZachisl' as enrollment_date, + education_elem ->> 'nomPrikazZachisl' as enrollment_order_number, + education_elem ->> 'dataPrikazZachisl' as enrollment_order_date, + coalesce(education_elem -> 'extend' ->> 'dataZavershObuch', education_elem ->> 'dataOkonch') as completion_date, + education_elem -> 'svedObrProg' ->> 'naimFormaObuch' as education_form, + education_elem -> 'svedObrProg' ->> 'dataNachObuch' as education_program_start_date, + education_elem -> 'extend' ->> 'dataNachObuch' as education_start_date, + coalesce(education_elem -> 'svedObrProg' ->> 'planDataOkonch', education_elem ->>'dataOzhid') as planned_completion_date, + education_elem -> 'svedObrProg' -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, + education_elem -> 'svedObrProg' -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, + education_elem ->> 'svedVosst' != 'null' as recovery_info, + education_elem -> 'svedVosst' ->> 'dataVosst' as recovery_date, + education_elem -> 'svedVosst' ->> 'dataPrikazVosst' as recovery_order_date, + education_elem -> 'svedVosst' ->> 'nomPrikazVosst' as recovery_order_number, + education_elem ->> 'svedOtchisl' != 'null' as expulsion_info, + education_elem -> 'svedOtchisl' ->> 'dataOtchisl' as expulsion_date, + education_elem -> 'svedOtchisl' ->> 'dataPrikazOtchisl' as expulsion_order_date, + education_elem -> 'svedOtchisl' ->> 'nomPrikazOtchisl' as expulsion_order_number, + education_elem ->> 'svedOtpusk' != 'null' as academical_leave_info, + education_elem ->> 'prPervObuch' = '1' as first_education, + education_elem -> 'svedObrProg' ->> 'prVUSpecz' = '1' as vus_spec, + education_elem -> 'svedVoenObuch' != 'null' as military_center_education_info, + education_elem -> 'svedVoenObuch' ->> 'dataNach' as military_center_education_start_date, + education_elem -> 'svedVoenObuch' ->> 'dataOkonch' as military_center_education_end_date, + education_elem -> 'svedVoenObuch' -> 'extend' ->> 'srokObuch' as military_center_education_period, + education_elem -> 'svedVoenObuch' -> 'extend' ->> 'svedUspeshZaverObuch' ='1' as military_center_completion, + education_elem -> 'extend' ->> 'svedVoenKaf' != 'null' as military_department_education_info, + education_elem -> 'extend' -> 'svedVoenKaf' ->> 'dataNach' as military_department_start_date, + education_elem -> 'extend' -> 'svedVoenKaf' ->> 'dataOkonch' as military_department_end_date, + academ_elem -> 'dataNach' as academ_start_date, + academ_elem -> 'dataOkonch' as academ_end_date, + academ_elem -> 'naimPrichOtpusk' as academ_reason +FROM filteredData fd +ORDER BY source_id; + Y + + + 640 + 464 + + + + education_leave_output + TableOutput + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + start_date + academ_start_date + + + end_date + academ_end_date + + + reason + academ_reason + + + citizen_education_source_id + source_id + + + N + N + N + N + Y + N + ervu_dashboard + Y + educational_leave
+ N + Y + N + Y + + + 1296 + 672 + + + + + citizen_education_output + Change job status on error + Y + + error_description + + + + + + + + education_leave_output + Change job status on error + Y + + error_description + + + + + + + + + diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl new file mode 100644 index 0000000..4dff655 --- /dev/null +++ b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl @@ -0,0 +1,1025 @@ + + + + citizen_education_flow_repeat + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/08/28 21:16:35.770 + - + 2025/08/28 21:16:35.770 + + + + + + Block until transforms finish + education_leave_output + Y + + + Change job status on error + Abort + Y + + + Create job execution record + education_input + Y + + + Filter rows + Change job status on success + Y + + + Filter_null_parental_leave_dates + Block until transforms finish + Y + + + Group by + citizen_education_output + Y + + + Identify last row in a stream + Detect empty stream + Y + + + Identify last row in a stream + Filter_null_parental_leave_dates + Y + + + Identify last row in a stream + Group by + Y + + + citizen_education_output + Change job status on error + Y + + + citizen_education_output + Filter rows + Y + + + education_input + Identify last row in a stream + Y + + + education_leave_output + Change job status on error + Y + + + Detect empty stream + Change job status on success + Y + + + + Abort + Abort + + Y + + 1 + + none + + + ABORT + Y + 0 + + + 1536 + 704 + + + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + citizen_education_output + + + + + 928 + 704 + + + + Change job status on error + ExecSql + + Y + + 1 + + none + + + + + error_description + + + ervu-dashboard + Y + N + Y + Y + N + UPDATE etl.job_execution +SET status = 'ERROR', + error_description = ? +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1312 + 704 + + + + Change job status on success + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + Y + N + Y + N + N + UPDATE etl.job_execution +SET status = 'SUCCESS' +WHERE job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; + + + + + 1552 + 320 + + + + Create job execution record + ExecSql + + Y + + 1 + + none + + + + + ervu-dashboard + N + N + Y + N + N + INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id) +VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}') +ON CONFLICT (job_name, recruitment_id) +DO UPDATE +SET status = 'PROCESSING', + error_description = null, + execution_datetime = current_timestamp; + + + 256 + 496 + + + + Detect empty stream + DetectEmptyStream + + Y + + 1 + + none + + + + + 672 + 320 + + + + Filter rows + FilterRows + + Y + + 1 + + none + + + + + + + = + last_row + N + - + + N + -1 + constant + -1 + Y + Boolean + + + + Change job status on success + + + 1552 + 496 + + + + Filter_null_parental_leave_dates + FilterRows + + Y + + 1 + + none + + + + + + + IS NOT NULL + academ_reason + N + - + + + Block until transforms finish + + + 672 + 704 + + + + Group by + GroupBy + + Y + + 1 + + none + + + N + N + ${java.io.tmpdir} + + + recruit_id + recruit_id + FIRST_INCL_NULL + + + source_update_date + source_update_date + FIRST_INCL_NULL + + + educational_institution_type + educational_institution_type + FIRST_INCL_NULL + + + org_type + org_type + FIRST_INCL_NULL + + + inn + inn + FIRST_INCL_NULL + + + kpp + kpp + FIRST_INCL_NULL + + + ogrn + ogrn + FIRST_INCL_NULL + + + educational_institution_address + educational_institution_address + FIRST_INCL_NULL + + + educational_institution_address_1 + educational_institution_address_1 + FIRST_INCL_NULL + + + educational_organization_separate_unit_type + educational_organization_separate_unit_type + FIRST_INCL_NULL + + + educational_organization_separate_unit_address + educational_organization_separate_unit_address + FIRST_INCL_NULL + + + student_status + student_status + FIRST_INCL_NULL + + + grade_number + grade_number + FIRST_INCL_NULL + + + education_level_code + education_level_code + FIRST_INCL_NULL + + + admission_date + admission_date + FIRST_INCL_NULL + + + enrollment_date + enrollment_date + FIRST_INCL_NULL + + + enrollment_order_number + enrollment_order_number + FIRST_INCL_NULL + + + enrollment_order_date + enrollment_order_date + FIRST_INCL_NULL + + + completion_date + completion_date + FIRST_INCL_NULL + + + education_form + education_form + FIRST_INCL_NULL + + + education_program_start_date + education_program_start_date + FIRST_INCL_NULL + + + education_start_date + education_start_date + FIRST_INCL_NULL + + + planned_completion_date + planned_completion_date + FIRST_INCL_NULL + + + minobr_speciality_code + minobr_speciality_code + FIRST_INCL_NULL + + + minprosv_speciality_code + minprosv_speciality_code + FIRST_INCL_NULL + + + recovery_info + recovery_info + FIRST_INCL_NULL + + + recovery_date + recovery_date + FIRST_INCL_NULL + + + recovery_order_date + recovery_order_date + FIRST_INCL_NULL + + + recovery_order_number + recovery_order_number + FIRST_INCL_NULL + + + expulsion_info + expulsion_info + FIRST_INCL_NULL + + + expulsion_date + expulsion_date + FIRST_INCL_NULL + + + expulsion_order_date + expulsion_order_date + FIRST_INCL_NULL + + + expulsion_order_number + expulsion_order_number + FIRST_INCL_NULL + + + academical_leave_info + academical_leave_info + FIRST_INCL_NULL + + + first_education + first_education + FIRST_INCL_NULL + + + vus_spec + vus_spec + FIRST_INCL_NULL + + + military_center_education_info + military_center_education_info + FIRST_INCL_NULL + + + military_center_education_start_date + military_center_education_start_date + FIRST_INCL_NULL + + + military_center_education_end_date + military_center_education_end_date + FIRST_INCL_NULL + + + military_center_education_period + military_center_education_period + FIRST_INCL_NULL + + + military_center_completion + military_center_completion + FIRST_INCL_NULL + + + military_department_education_info + military_department_education_info + FIRST_INCL_NULL + + + military_department_start_date + military_department_start_date + FIRST_INCL_NULL + + + military_department_end_date + military_department_end_date + FIRST_INCL_NULL + + + last_row + last_row + LAST_INCL_NULL + + + academ_start_date + academ_start_date + FIRST_INCL_NULL + + + academ_end_date + academ_end_date + FIRST_INCL_NULL + + + academ_reason + academ_reason + FIRST_INCL_NULL + + + N + + + source_id + + + N + grp + + + 992 + 496 + + + + Identify last row in a stream + DetectLastRow + + N + + 1 + + none + + + last_row + + + 672 + 496 + + + + citizen_education_output + TableOutput + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + source_id + source_id + + + recruit_id + recruit_id + + + source_update_date + source_update_date + + + educational_institution_type + educational_institution_type + + + org_type + org_type + + + inn + inn + + + kpp + kpp + + + ogrn + ogrn + + + educational_institution_address + educational_institution_address + + + educational_institution_address_1 + educational_institution_address_1 + + + educational_organization_separate_unit_type + educational_organization_separate_unit_type + + + educational_organization_separate_unit_address + educational_organization_separate_unit_address + + + student_status + student_status + + + grade_number + grade_number + + + education_level_code + education_level_code + + + admission_date + admission_date + + + enrollment_date + enrollment_date + + + enrollment_order_number + enrollment_order_number + + + enrollment_order_date + enrollment_order_date + + + completion_date + completion_date + + + education_form + education_form + + + education_program_start_date + education_program_start_date + + + education_start_date + education_start_date + + + planned_completion_date + planned_completion_date + + + minobr_speciality_code + minobr_speciality_code + + + minprosv_speciality_code + minprosv_speciality_code + + + recovery_info + recovery_info + + + recovery_date + recovery_date + + + recovery_order_date + recovery_order_date + + + recovery_order_number + recovery_order_number + + + expulsion_info + expulsion_info + + + expulsion_date + expulsion_date + + + expulsion_order_date + expulsion_order_date + + + expulsion_order_number + expulsion_order_number + + + academical_leave_info + academical_leave_info + + + first_education + first_education + + + vus_spec + vus_spec + + + military_center_education_info + military_center_education_info + + + military_center_education_start_date + military_center_education_start_date + + + military_center_education_end_date + military_center_education_end_date + + + military_center_education_period + military_center_education_period + + + military_center_completion + military_center_completion + + + military_department_education_info + military_department_education_info + + + military_department_start_date + military_department_start_date + + + military_department_end_date + military_department_end_date + + + N + N + N + N + Y + N + ervu_dashboard + Y + educational_institution_learning
+ N + Y + N + Y + + + 1312 + 496 + + + + education_input + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + WITH filteredData AS (SELECT ri.recruit_id, + education_elem, + academ_elem, + 'Образовательная организация высшего образования' as educational_institution_type + FROM recruits_info ri + CROSS JOIN LATERAL jsonb_array_elements( + ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') AS education_elem + LEFT JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array' + THEN education_elem -> 'svedOtpusk' + ELSE '[]'::jsonb + END + ) AS academ_elem ON true + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') = 'array' + and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + + union all + + SELECT ri.recruit_id, + education_elem, + academ_elem, + 'Профессиональная образовательная организация' as educational_institution_type + FROM recruits_info ri + CROSS JOIN LATERAL jsonb_array_elements( + ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') AS education_elem + LEFT JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array' + THEN education_elem -> 'svedOtpusk' + ELSE '[]'::jsonb + END + ) AS academ_elem ON true + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') = 'array' + and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + + union all + + SELECT ri.recruit_id, + education_elem, + academ_elem, + 'Общеобразовательная организация' as educational_institution_type + FROM recruits_info ri + CROSS JOIN LATERAL jsonb_array_elements( + ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') AS education_elem + LEFT JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array' + THEN education_elem -> 'svedOtpusk' + ELSE '[]'::jsonb + END + ) AS academ_elem ON true + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') = 'array' + and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + ) +SELECT fd.recruit_id, + education_elem ->> 'id' AS source_id, + education_elem ->> 'dataSved' as source_update_date, + educational_institution_type, + education_elem -> 'svedOrg' ->> 'naimTipOrg' as org_type, + education_elem -> 'svedOrg' ->> 'inn' as inn, + education_elem -> 'svedOrg' ->> 'kpp' as kpp, + coalesce(education_elem -> 'svedOrg' ->> 'ogrn', education_elem -> 'svedOrg' ->> 'ogrnip') as ogrn, + education_elem -> 'svedOrg' ->> 'adresOrg' as educational_institution_address, + education_elem -> 'svedOrg' -> 'extend' ->> 'statusOrg' as educational_institution_address, + education_elem -> 'svedOrg' -> 'extend' ->> 'vidPodrazdOrg' as educational_organization_separate_unit_type, + education_elem -> 'svedOrg' -> 'extend' ->> 'adresPodrazdOrg' as educational_organization_separate_unit_address, + education_elem ->> 'naimStatus' as student_status, + case + when educational_institution_type = 'Образовательная организация высшего образования' + then education_elem ->> 'nomerKursa' + when educational_institution_type = 'Профессиональная образовательная организация' + then education_elem -> 'extend' ->> 'klass' + when educational_institution_type = 'Общеобразовательная организация' then education_elem ->> 'klass' + end as grade_number, + education_elem ->> 'kodUrovObr' as education_level_code, + coalesce(education_elem -> 'extend' ->> 'dataPostup', education_elem ->> 'dataPostup') as admission_date, + education_elem ->> 'dataZachisl' as enrollment_date, + education_elem ->> 'nomPrikazZachisl' as enrollment_order_number, + education_elem ->> 'dataPrikazZachisl' as enrollment_order_date, + coalesce(education_elem -> 'extend' ->> 'dataZavershObuch', education_elem ->> 'dataOkonch') as completion_date, + education_elem -> 'svedObrProg' ->> 'naimFormaObuch' as education_form, + education_elem -> 'svedObrProg' ->> 'dataNachObuch' as education_program_start_date, + education_elem -> 'extend' ->> 'dataNachObuch' as education_start_date, + coalesce(education_elem -> 'svedObrProg' ->> 'planDataOkonch', education_elem ->>'dataOzhid') as planned_completion_date, + education_elem -> 'svedObrProg' -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, + education_elem -> 'svedObrProg' -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, + education_elem ->> 'svedVosst' != 'null' as recovery_info, + education_elem -> 'svedVosst' ->> 'dataVosst' as recovery_date, + education_elem -> 'svedVosst' ->> 'dataPrikazVosst' as recovery_order_date, + education_elem -> 'svedVosst' ->> 'nomPrikazVosst' as recovery_order_number, + education_elem ->> 'svedOtchisl' != 'null' as expulsion_info, + education_elem -> 'svedOtchisl' ->> 'dataOtchisl' as expulsion_date, + education_elem -> 'svedOtchisl' ->> 'dataPrikazOtchisl' as expulsion_order_date, + education_elem -> 'svedOtchisl' ->> 'nomPrikazOtchisl' as expulsion_order_number, + education_elem ->> 'svedOtpusk' != 'null' as academical_leave_info, + education_elem ->> 'prPervObuch' = '1' as first_education, + education_elem -> 'svedObrProg' ->> 'prVUSpecz' = '1' as vus_spec, + education_elem -> 'svedVoenObuch' != 'null' as military_center_education_info, + education_elem -> 'svedVoenObuch' ->> 'dataNach' as military_center_education_start_date, + education_elem -> 'svedVoenObuch' ->> 'dataOkonch' as military_center_education_end_date, + education_elem -> 'svedVoenObuch' -> 'extend' ->> 'srokObuch' as military_center_education_period, + education_elem -> 'svedVoenObuch' -> 'extend' ->> 'svedUspeshZaverObuch' ='1' as military_center_completion, + education_elem -> 'extend' ->> 'svedVoenKaf' != 'null' as military_department_education_info, + education_elem -> 'extend' -> 'svedVoenKaf' ->> 'dataNach' as military_department_start_date, + education_elem -> 'extend' -> 'svedVoenKaf' ->> 'dataOkonch' as military_department_end_date, + academ_elem -> 'dataNach' as academ_start_date, + academ_elem -> 'dataOkonch' as academ_end_date, + academ_elem -> 'naimPrichOtpusk' as academ_reason +FROM filteredData fd +ORDER BY source_id; + Y + + + 480 + 496 + + + + education_leave_output + TableOutput + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + start_date + academ_start_date + + + end_date + academ_end_date + + + reason + academ_reason + + + citizen_education_source_id + source_id + + + N + N + N + N + Y + N + ervu_dashboard + Y + educational_leave
+ N + Y + N + Y + + + 1136 + 704 + + + + + citizen_education_output + Change job status on error + Y + + error_description + + + + + + + + education_leave_output + Change job status on error + Y + + error_description + + + + + + + + + diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow.hpl new file mode 100644 index 0000000..47fa430 --- /dev/null +++ b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow.hpl @@ -0,0 +1,330 @@ + + + + recruitment_five_flow + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/08/28 08:16:44.450 + - + 2025/08/28 08:16:44.450 + + + + + + Get all recruitments + citizen_education_flow.hpl + Y + + + Get all recruitments + citizen_education_flow.hpl 2 + Y + + + Get all recruitments + citizen_education_flow.hpl 3 + Y + + + Get all recruitments + citizen_education_flow.hpl 4 + Y + + + Get all recruitments + citizen_education_flow.hpl 5 + Y + + + + Get all recruitments + TableInput + + Y + + 1 + + none + + + ervu-dashboard-test + N + SELECT +idm_id +FROM ervu_dashboard.recruitment; + N + + + 352 + 352 + + + + citizen_education_flow.hpl + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 624 + 192 + + + + citizen_education_flow.hpl 2 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 624 + 272 + + + + citizen_education_flow.hpl 3 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 624 + 352 + + + + citizen_education_flow.hpl 4 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 624 + 432 + + + + citizen_education_flow.hpl 5 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 624 + 512 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_delta.hpl new file mode 100644 index 0000000..bce8e0b --- /dev/null +++ b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_delta.hpl @@ -0,0 +1,399 @@ + + + + recruitment_five_flow_delta + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/08/28 21:23:43.331 + - + 2025/08/28 21:23:43.331 + + + + + + Table input + citizen_education_flow_delta.hpl + Y + + + get_max_source_update_date + Table input + Y + + + Table input + citizen_education_flow_delta.hpl 2 + Y + + + Table input + citizen_education_flow_delta.hpl 3 + Y + + + Table input + citizen_education_flow_delta.hpl 4 + Y + + + Table input + citizen_education_flow_delta.hpl 5 + Y + + + + Table input + TableInput + + Y + + 1 + + none + + + ervu-dashboard-test + N + 0 + get_max_source_update_date + WITH mud AS ( + SELECT + recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING') + GROUP BY recruitment_id +) +SELECT + r.idm_id, + ? max_source_update_date +FROM ervu_dashboard.recruitment r + JOIN mud ON mud.recruitment_id = r.idm_id + JOIN recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > mud.max_upd_date; + Y + + + 608 + 384 + + + + citizen_education_flow_delta.hpl + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + + MAX_SOURCE_UPDATE_DATE + max_source_update_date + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 800 + 224 + + + + citizen_education_flow_delta.hpl 2 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + + MAX_SOURCE_UPDATE_DATE + max_source_update_date + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 800 + 304 + + + + citizen_education_flow_delta.hpl 3 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + + MAX_SOURCE_UPDATE_DATE + max_source_update_date + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 800 + 384 + + + + citizen_education_flow_delta.hpl 4 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + + MAX_SOURCE_UPDATE_DATE + max_source_update_date + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 800 + 464 + + + + citizen_education_flow_delta.hpl 5 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + + MAX_SOURCE_UPDATE_DATE + max_source_update_date + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 800 + 544 + + + + get_max_source_update_date + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + select max(source_update_date) +from education_institution_learning; + N + + + 432 + 384 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/passport/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_repeat.hpl similarity index 83% rename from mappings/info_recruits/citizen_tables/passport/recruitment_five_flow_delta.hpl rename to mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_repeat.hpl index 8259a10..a120f61 100644 --- a/mappings/info_recruits/citizen_tables/passport/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_repeat.hpl @@ -1,7 +1,7 @@ - recruitment_five_flow_delta + recruitment_five_flow_repeat Y @@ -13,36 +13,36 @@ 1000 100 - - 2025/08/11 13:16:11.204 + 2025/08/28 21:16:24.371 - - 2025/08/11 13:16:11.204 + 2025/08/28 21:16:24.371 Table input - passport_flow_delta.hpl + citizen_education_flow_repeat.hpl Y Table input - passport_flow_delta.hpl 2 + citizen_education_flow_repeat.hpl 2 Y Table input - passport_flow_delta.hpl 3 + citizen_education_flow_repeat.hpl 3 Y Table input - passport_flow_delta.hpl 4 + citizen_education_flow_repeat.hpl 4 Y Table input - passport_flow_delta.hpl 5 + citizen_education_flow_repeat.hpl 5 Y @@ -57,25 +57,25 @@ none - ervu-dashboard + ervu-dashboard-test N - - SELECT - r.idm_id as recruitment_id + 0 + SELECT r.idm_id AS recruitment_id FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - and job_name = 'passport_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); - N + AND je.job_name = '${JOB_NAME}' +WHERE je.id IS NULL + OR je.status IN ('ERROR', 'PROCESSING'); + Y - 352 - 288 + 672 + 496 - passport_flow_delta.hpl + citizen_education_flow_repeat.hpl PipelineExecutor Y @@ -86,7 +86,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING local - ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl N 1 @@ -98,11 +98,6 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING recruitment_id - - JOB_NAME - - passport_job - Y @@ -126,12 +121,12 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 544 - 128 + 880 + 336 - passport_flow_delta.hpl 2 + citizen_education_flow_repeat.hpl 2 PipelineExecutor Y @@ -142,7 +137,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING local - ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl N 1 @@ -154,11 +149,6 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING recruitment_id - - JOB_NAME - - passport_job - Y @@ -182,12 +172,12 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 544 - 208 + 880 + 416 - passport_flow_delta.hpl 3 + citizen_education_flow_repeat.hpl 3 PipelineExecutor Y @@ -198,7 +188,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING local - ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl N 1 @@ -210,11 +200,6 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING recruitment_id - - JOB_NAME - - passport_job - Y @@ -238,12 +223,12 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 544 - 288 + 880 + 496 - passport_flow_delta.hpl 4 + citizen_education_flow_repeat.hpl 4 PipelineExecutor Y @@ -254,7 +239,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING local - ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl N 1 @@ -266,11 +251,6 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING recruitment_id - - JOB_NAME - - passport_job - Y @@ -294,12 +274,12 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 544 - 368 + 880 + 576 - passport_flow_delta.hpl 5 + citizen_education_flow_repeat.hpl 5 PipelineExecutor Y @@ -310,7 +290,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING local - ${PROJECT_HOME}/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl N 1 @@ -322,11 +302,6 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING recruitment_id - - JOB_NAME - - passport_job - Y @@ -350,8 +325,8 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 544 - 448 + 880 + 656 diff --git a/mappings_new/info_recruits/citizen_tables/passport/recruit_update_date_ervu_dashboard.hpl b/mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/foreign_diploma_constants_flow.hpl similarity index 62% rename from mappings_new/info_recruits/citizen_tables/passport/recruit_update_date_ervu_dashboard.hpl rename to mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/foreign_diploma_constants_flow.hpl index 0236547..e2c8de2 100644 --- a/mappings_new/info_recruits/citizen_tables/passport/recruit_update_date_ervu_dashboard.hpl +++ b/mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/foreign_diploma_constants_flow.hpl @@ -1,7 +1,7 @@ - recruit_update_date_ervu_dashboard + foreign_diploma_constants_flow Y @@ -13,21 +13,21 @@ 1000 100 - - 2025/06/02 14:50:31.833 + 2025/09/22 08:32:45.452 - - 2025/06/02 14:50:31.833 + 2025/09/22 08:32:45.452 - Table input - Table output + country_oksm_input + country_oksm_output Y - Table input + country_oksm_input TableInput Y @@ -37,26 +37,26 @@ none - ervu-dashboard + nsi N 0 - SELECT - created_at AS system_create_date, - updated_at AS system_update_date, - current_timestamp AS record_created, - 'job_recruits_info' AS workflow -FROM recruits_info -WHERE updated_at = (SELECT MAX(updated_at) FROM recruits_info) -limit 1 + SELECT + key as code, + value ->> 'value' as value, + NOT hidden as actual, + updated_at as update_date +FROM classifier_records +WHERE code = 'countriesOKSM'; + N - 896 - 304 + 416 + 192 - Table output + country_oksm_output TableOutput Y @@ -66,10 +66,26 @@ limit 1 none - 1000 + 10000 ervu-dashboard - + + code + code + + + value + value + + + actual + actual + + + update_date + update_date + + N N N @@ -78,9 +94,9 @@ limit 1 Y N - public - N - recruit_create
+ ervu_dashboard + Y + country_oksm
N Y @@ -88,8 +104,8 @@ limit 1 Y - 1216 - 304 + 640 + 192 diff --git a/mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/foreign_diploma_constants_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/foreign_diploma_constants_flow_repeat.hpl new file mode 100644 index 0000000..52729cf --- /dev/null +++ b/mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/foreign_diploma_constants_flow_repeat.hpl @@ -0,0 +1,148 @@ + + + + foreign_diploma_constants_flow_repeat + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/09/21 21:27:15.216 + - + 2025/09/21 21:27:15.216 + + + + + + get_max_country_oksm_update_date + country_oksm_input + Y + + + country_oksm_input + country_oksm + Y + + + + country_oksm_input + TableInput + + Y + + 1 + + none + + + nsi + N + 0 + get_max_country_oksm_update_date + SELECT + key as code, + value ->> 'value' as value, + NOT hidden as actual, + updated_at as update_date +FROM classifier_records +WHERE code = 'countriesOKSM' +AND updated_at > ?; + N + + + 464 + 240 + + + + get_max_country_oksm_update_date + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT + max(update_date) as max_update_date +FROM country_oksm; + + N + + + 240 + 240 + + + + country_oksm + InsertUpdate + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + = + code + code + + + + <> + actual + actual + + + ervu_dashboard + country_oksm
+ + code + code + N + + + value + value + Y + + + actual + actual + Y + + + update_date + update_date + Y + + + N + + + 688 + 240 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/work/work_activity/support/check_if_work_activity_job_exists.hpl b/mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/support/check_if_constants_exists.hpl similarity index 76% rename from mappings/info_recruits/citizen_tables/work/work_activity/support/check_if_work_activity_job_exists.hpl rename to mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/support/check_if_constants_exists.hpl index a4b4134..7a4fb03 100644 --- a/mappings/info_recruits/citizen_tables/work/work_activity/support/check_if_work_activity_job_exists.hpl +++ b/mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/support/check_if_constants_exists.hpl @@ -1,7 +1,7 @@ - check_if_work_activity_job_exists + check_if_constants_exists Y @@ -13,9 +13,9 @@ 1000 100 - - 2025/08/05 13:28:46.661 + 2025/09/21 21:25:15.712 - - 2025/08/05 13:28:46.661 + 2025/09/21 21:25:15.712 @@ -39,16 +39,16 @@ - job_executed_flag - JOB_EXECUTED_FLAG + constasnts_exists + CONSTANTS_EXISTS PARENT_WORKFLOW Y - 672 - 336 + 576 + 272 @@ -66,15 +66,13 @@ N 0 SELECT EXISTS ( - SELECT 1 - FROM etl.job_execution - WHERE job_name = 'work_activity_job' - ) AS job_executed_flag; - N + SELECT 1 FROM country_oksm +) AS constasnts_exists + Y - 384 - 336 + 288 + 272 diff --git a/mappings/info_recruits/citizen_tables/education/foreign_diploma/foreign_diploma_job.hwf b/mappings/info_recruits/citizen_tables/education/foreign_diploma/foreign_diploma_job.hwf new file mode 100644 index 0000000..0868d20 --- /dev/null +++ b/mappings/info_recruits/citizen_tables/education/foreign_diploma/foreign_diploma_job.hwf @@ -0,0 +1,408 @@ + + + foreign_diploma_job + Y + + + + - + 2025/09/04 20:50:35.289 + - + 2025/09/04 20:50:35.289 + + + + + Start + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 336 + 224 + + + + check_if_job_execution_exists.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_job_execution_exists.hpl + Basic + + Y + + N + local + N + N + Y + N + 1360 + 240 + + + + employer_job_execution_exists_check + + SIMPLE_EVAL + + boolean + false + equal + equal + N + variable + JOB_EXECUTED_FLAG + N + 1664 + 240 + + + + recruitment_five_flow.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow.hpl + Basic + + Y + + N + local + N + N + Y + Y + 1936 + 240 + + + + recruitment_five_flow_repeat.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_repeat.hpl + Basic + + Y + + N + local + N + N + Y + N + 1936 + 544 + + + + check_if_need_to_repeat.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_need_to_repeat.hpl + Basic + + Y + + N + local + N + N + Y + N + 1664 + 384 + + + + Simple evaluation + + SIMPLE_EVAL + + boolean + true + equal + equal + N + variable + NEED_TO_REPEAT_JOB + N + 1664 + 544 + + + + recruitment_five_flow_delta.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_delta.hpl + Basic + + Y + + N + local + N + N + Y + N + 1664 + 704 + + + + check_if_constants_exists.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/constants/support/check_if_constants_exists.hpl + Basic + + Y + + N + local + N + N + Y + N + 512 + 224 + + + + constants_exists + + SIMPLE_EVAL + + boolean + true + equal + equal + N + variable + CONSTANTS_EXISTS + N + 688 + 224 + + + + foreign_diploma_constants_flow.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/constants/foreign_diploma_constants_flow.hpl + Basic + + Y + + N + local + N + N + Y + N + 880 + 144 + + + + foreign_diploma_constants_flow_repeat.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/constants/foreign_diploma_constants_flow_repeat.hpl + Basic + + Y + + N + local + N + N + Y + N + 880 + 320 + + + + init_job_name + + SET_VARIABLES + + + + JOB_NAME + CURRENT_WORKFLOW + foreign_diploma_job + + + CURRENT_WORKFLOW + N + N + 1120 + 240 + + + + + + check_if_job_execution_exists.hpl + employer_job_execution_exists_check + Y + Y + N + + + employer_job_execution_exists_check + check_if_need_to_repeat.hpl + Y + N + N + + + check_if_need_to_repeat.hpl + Simple evaluation + Y + Y + N + + + check_if_constants_exists.hpl + constants_exists + Y + Y + N + + + constants_exists + foreign_diploma_constants_flow.hpl + Y + N + N + + + constants_exists + foreign_diploma_constants_flow_repeat.hpl + Y + Y + N + + + foreign_diploma_constants_flow.hpl + init_job_name + Y + Y + N + + + foreign_diploma_constants_flow_repeat.hpl + init_job_name + Y + Y + N + + + init_job_name + check_if_job_execution_exists.hpl + Y + Y + N + + + Start + check_if_constants_exists.hpl + Y + Y + Y + + + employer_job_execution_exists_check + recruitment_five_flow.hpl + Y + Y + N + + + Simple evaluation + recruitment_five_flow_repeat.hpl + Y + Y + N + + + Simple evaluation + recruitment_five_flow_delta.hpl + Y + N + N + + + + + + diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl b/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl similarity index 64% rename from mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl rename to mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl index fbffb7c..1d1c16f 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl +++ b/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl @@ -1,7 +1,7 @@ - citizen_address_flow + foreign_diploma_flow Y @@ -13,36 +13,21 @@ 1000 100 - - 2025/08/05 14:31:22.799 + 2025/09/04 20:53:34.643 - - 2025/08/05 14:31:22.799 + 2025/09/04 20:53:34.643 - - Create job execution record - Table input - Y - - - Table output - Change job status on error - Y - Change job status on error Abort Y - Table output - Detect empty stream - Y - - - Detect empty stream - Change job status on success + Create job execution record + foerign_diploma_input Y @@ -51,20 +36,35 @@ Y - Table input - Identify last row in a stream + Identify last row in a stream + Detect empty stream Y Identify last row in a stream - Table output + citizen_education_diploma_output Y - Table output + citizen_education_diploma_output + Change job status on error + Y + + + citizen_education_diploma_output Filter rows Y + + foerign_diploma_input + Identify last row in a stream + Y + + + Detect empty stream + Change job status on success + Y + Abort @@ -77,13 +77,13 @@ none - ABORT_WITH_ERROR + ABORT Y 0 - 1328 - 704 + 1248 + 608 @@ -117,8 +117,8 @@ and recruitment_id = '${IDM_ID}'; - 832 - 704 + 1024 + 608 @@ -143,14 +143,13 @@ and recruitment_id = '${IDM_ID}'; UPDATE etl.job_execution SET status = 'SUCCESS' WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'PROCESSING'; +and recruitment_id = '${IDM_ID}'; - 1328 - 96 + 1248 + 256 @@ -178,8 +177,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 240 - 320 + 320 + 432 @@ -195,8 +194,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 832 - 96 + 736 + 256 @@ -231,15 +230,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1328 - 320 + 1248 + 432 Identify last row in a stream DetectLastRow - Y + N 1 @@ -249,12 +248,106 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); last_row - 640 - 320 + 736 + 432 - Table input + citizen_education_diploma_output + TableOutput + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + recruit_id + recruit_id + + + source_id + source_id + + + source_update_date + source_update_date + + + education_level_code + education_level_code + + + educational_institution_name + educational_institution_name + + + foreign_doc_name + foreign_doc_name + + + ser_num_doc + ser_num_doc + + + reg_num + reg_num + + + recognition_certificate_number + recognition_certificate_number + + + recognition_date + recognition_date + + + speciality_name + speciality_name + + + minobr_speciality_code + minobr_speciality_code + + + minprosv_speciality_code + minprosv_speciality_code + + + oksm_code + oksm_code + + + vus_spec + vus_spec + + + N + N + N + N + Y + N + ervu_dashboard + Y + foreign_diploma
+ N + Y + N + Y + + + 1024 + 432 + + + + foerign_diploma_input TableInput Y @@ -267,125 +360,46 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - SELECT - ri.recruit_id::uuid, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, - - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, - CASE - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' - ELSE NULL - END AS place_stay_registration_type, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date -FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr -WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date -GROUP BY ri.recruit_id; + WITH filteredData AS (SELECT ri.recruit_id, + ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr + FROM recruits_info ri + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr') = 'array' + and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' +) +SELECT fd.recruit_id, + diploma_elem ->> 'id' AS source_id, + to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, + diploma_elem ->> 'kodUrObr' as education_level_code, + diploma_elem ->> 'naimUchZaved' as educational_institution_name, + diploma_elem ->> 'naimInoDok' as foreign_doc_name, + diploma_elem ->> 'serNomDok' as ser_num_doc, + diploma_elem ->> 'regNomDok' as reg_num, + diploma_elem ->> 'nomSved' as recognition_certificate_number, + to_date(diploma_elem ->> 'dataPriz', 'YYYY-MM-DD') as recognition_date, + diploma_elem -> 'svedSpecz' ->> 'naimSpecz' as speciality_name, + diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, + diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, + diploma_elem ->> 'kodOKSM' as oksm_code, + coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec +FROM filteredData fd + CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem; + Y - 464 - 320 - - - - Table output - TableOutput - - N - - 1 - - none - - - 1000 - ervu-dashboard - - - recruit_id - recruit_id - - - residence_address - residence_address - - - residence_registration_date - residence_registration_date - - - residence_deregistration_date - residence_deregistration_date - - - place_stay_address - place_stay_address - - - place_stay_registration_date - place_stay_registration_date - - - place_stay_deregistration_date - place_stay_deregistration_date - - - place_stay_registration_type - place_stay_registration_type - - - factual_address - factual_address - - - factual_info_received_date - factual_info_received_date - - - factual_info_expiry_date - factual_info_expiry_date - - - N - N - N - N - Y - N - ervu_dashboard - Y - citizen_address
- N - Y - N - Y - - - 832 - 320 + 544 + 432 - Table output + citizen_education_diploma_output Change job status on error Y error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl b/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl similarity index 63% rename from mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl rename to mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl index 99e9f11..9095203 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl @@ -1,7 +1,7 @@ - citizen_spouse_flow_delta + foreign_diploma_flow_delta Y @@ -13,31 +13,21 @@ 1000 100 - - 2025/08/11 13:22:56.048 + 2025/09/05 08:36:17.176 - - 2025/08/11 13:22:56.048 + 2025/09/05 08:36:17.176 - - Create job execution record - Table input - Y - - - Insert / update - Change job status on error - Y - Change job status on error Abort Y - Insert / update - Filter rows + Create job execution record + foerign_diploma_input Y @@ -46,25 +36,35 @@ Y - Insert / update + Identify last row in a stream Detect empty stream Y + + Identify last row in a stream + foreign_diploma_upsert + Y + + + foerign_diploma_input + Identify last row in a stream + Y + + + foreign_diploma_upsert + Change job status on error + Y + + + foreign_diploma_upsert + Filter rows + Y + Detect empty stream Change job status on success Y - - Table input - Identify last row in a stream - Y - - - Identify last row in a stream - Insert / update - Y - Abort @@ -77,13 +77,13 @@ none - ABORT_WITH_ERROR + ABORT Y 0 - 976 - 480 + 1280 + 688 @@ -101,9 +101,6 @@ error_description - - error_code - ervu-dashboard Y @@ -120,8 +117,8 @@ and recruitment_id = '${IDM_ID}'; - 736 - 480 + 1056 + 688 @@ -144,16 +141,15 @@ and recruitment_id = '${IDM_ID}'; N N UPDATE etl.job_execution -SET status = 'DELTA_SUCCESS', - error_description = null +SET status = 'DELTA_SUCCESS' WHERE job_name = '${JOB_NAME}' - and recruitment_id = '${IDM_ID}' - and status = 'DELTA_PROCESSING'; +and recruitment_id = '${IDM_ID}'; + - 1200 - 80 + 1280 + 320 @@ -178,13 +174,14 @@ WHERE job_name = '${JOB_NAME}' UPDATE etl.job_execution SET status = 'DELTA_PROCESSING', - execution_datetime = DEFAULT + execution_datetime = current_timestamp, + error_description = NULL where job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - 96 - 224 + 352 + 496 @@ -200,8 +197,8 @@ and recruitment_id = '${IDM_ID}'; - 1200 - 224 + 768 + 320 @@ -236,15 +233,15 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 736 - 80 + 1280 + 496 Identify last row in a stream DetectLastRow - Y + N 1 @@ -254,129 +251,15 @@ and recruitment_id = '${IDM_ID}'; last_row - 576 - 224 + 768 + 496 - Insert / update - InsertUpdate - - N - - 1 - - none - - - 100 - ervu-dashboard - - - = - recruit_id - recruit_id - - - = - spouse_external_id - spouse_external_id - - ervu_dashboard - citizen_spouse
- - recruit_id - recruit_id - N - - - spouse_external_id - spouse_external_id - N - - - spouse_id_ern - spouse_id_ern - Y - - - kinship_type - kinship_type - Y - - - last_name - last_name - Y - - - first_name - first_name - Y - - - middle_name - middle_name - Y - - - full_name - full_name - Y - - - birth_date - birth_date - Y - - - death_date - death_date - Y - - - death_az_number - death_az_number - Y - - - marriage_az_number - marriage_az_number - Y - - - marriage_date - marriage_date - Y - - - divorce_az_number - divorce_az_number - Y - - - divorce_date - divorce_date - Y - - - information_excluded - information_excluded - Y - - - N - - - 736 - 224 - - - - Table input + foerign_diploma_input TableInput - N + Y 1 @@ -386,63 +269,151 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - SELECT - ri.recruit_id AS recruit_id, - (supr->>'id')::uuid AS spouse_external_id, - NULLIF(supr->>'idERN', '') AS spouse_id_ern, - NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, - supr->'svedFLBS'->'fio'->>'familiya' AS last_name, - supr->'svedFLBS'->'fio'->>'imya' AS first_name, - supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - supr->'svedFLBS'->'fio'->>'familiya', - supr->'svedFLBS'->'fio'->>'imya', - supr->'svedFLBS'->'fio'->>'otchestvo' - ) AS full_name, - MAKE_DATE( - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, - supr->'svedSmert'->>'nomerZapis' AS death_az_number, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, - CASE - WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true - ELSE false - END AS information_excluded -FROM ervu_dashboard.recruits_info ri -JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id -CROSS JOIN LATERAL ( - SELECT supri - FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri - WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' -) AS supr(supr) -WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date - AND c.update_date >= '${M_R_UP_DATE}'::timestamp; - + WITH filteredData AS (SELECT ri.recruit_id, + diploma_elem + FROM recruits_info ri + CROSS JOIN LATERAL jsonb_array_elements( + ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr') AS diploma_elem + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr') = 'array' + and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' + and to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' +) +SELECT fd.recruit_id, + diploma_elem ->> 'id' AS source_id, + to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, + diploma_elem ->> 'kodUrObr' as education_level_code, + diploma_elem ->> 'naimUchZaved' as educational_institution_name, + diploma_elem ->> 'naimInoDok' as foreign_doc_name, + diploma_elem ->> 'serNomDok' as ser_num_doc, + diploma_elem ->> 'regNomDok' as reg_num, + diploma_elem ->> 'nomSved' as recognition_certificate_number, + to_date(diploma_elem ->> 'dataPriz', 'YYYY-MM-DD') as recognition_date, + diploma_elem -> 'svedSpecz' ->> 'naimSpecz' as speciality_name, + diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, + diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, + diploma_elem ->> 'kodOKSM' as oksm_code, + coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec +FROM filteredData fd; Y - 368 - 224 + 576 + 496 + + + + foreign_diploma_upsert + InsertUpdate + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + = + source_id + source_id + + ervu_dashboard + foreign_diploma
+ + recruit_id + recruit_id + N + + + source_id + source_id + N + + + source_update_date + source_update_date + Y + + + education_level_code + education_level_code + Y + + + educational_institution_name + educational_institution_name + Y + + + foreign_doc_name + foreign_doc_name + Y + + + ser_num_doc + ser_num_doc + Y + + + reg_num + reg_num + Y + + + recognition_certificate_number + recognition_certificate_number + Y + + + recognition_date + recognition_date + Y + + + speciality_name + speciality_name + Y + + + minobr_speciality_code + minobr_speciality_code + Y + + + minprosv_speciality_code + minprosv_speciality_code + Y + + + oksm_code + oksm_code + Y + + + vus_spec + vus_spec + Y + + + N + + + 1056 + 496 - Insert / update + foreign_diploma_upsert Change job status on error Y error_description - error_code + diff --git a/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl similarity index 63% rename from mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_repeat.hpl rename to mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl index 809cfea..83f0d80 100644 --- a/mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl @@ -1,7 +1,7 @@ - restriction_document_flow_repeat + foreign_diploma_flow_repeat Y @@ -13,21 +13,21 @@ 1000 100 - - 2025/08/08 13:25:52.319 + 2025/09/04 21:35:58.671 - - 2025/08/08 13:25:52.319 + 2025/09/04 21:35:58.671 - Create job execution record - Table input + Change job status on error + Abort Y - Table input - Identify last row in a stream + Create job execution record + foerign_diploma_input Y @@ -35,36 +35,36 @@ Change job status on success Y - - Detect empty stream - Change job status on success - Y - - - Change job status on error - Abort - Y - Identify last row in a stream - Insert / update - Y - - - Insert / update - Filter rows - Y - - - Insert / update Detect empty stream Y - Insert / update + foerign_diploma_input + Identify last row in a stream + Y + + + Identify last row in a stream + foreign_diploma_upsert + Y + + + foreign_diploma_upsert + Filter rows + Y + + + foreign_diploma_upsert Change job status on error Y + + Detect empty stream + Change job status on success + Y + Abort @@ -77,13 +77,13 @@ none - ABORT_WITH_ERROR + ABORT Y 0 1248 - 400 + 624 @@ -117,8 +117,8 @@ and recruitment_id = '${IDM_ID}'; - 960 - 400 + 1024 + 624 @@ -148,8 +148,8 @@ and recruitment_id = '${IDM_ID}'; - 1440 - 48 + 1248 + 272 @@ -171,32 +171,17 @@ and recruitment_id = '${IDM_ID}'; Y N N - INSERT INTO etl.job_execution ( - id, - job_name, - status, - execution_datetime, - error_description, - recruitment_id -) -VALUES ( - DEFAULT, - '${JOB_NAME}', - 'PROCESSING', - DEFAULT, - NULL, - '${IDM_ID}' -) -ON CONFLICT (job_name, recruitment_id) -DO UPDATE SET + UPDATE etl.job_execution +SET status = 'PROCESSING', - execution_datetime = DEFAULT, + execution_datetime = current_timestamp, error_description = NULL - +where job_name = '${JOB_NAME}' +and recruitment_id = '${IDM_ID}'; - 272 - 208 + 320 + 448 @@ -212,8 +197,8 @@ DO UPDATE SET - 1440 - 208 + 736 + 272 @@ -248,15 +233,15 @@ DO UPDATE SET Change job status on success - 960 - 48 + 1248 + 448 Identify last row in a stream DetectLastRow - Y + N 1 @@ -266,151 +251,166 @@ DO UPDATE SET last_row - 720 - 208 + 736 + 448 - Insert / update - InsertUpdate + foerign_diploma_input + TableInput - N + Y 1 none - 100 + ervu-dashboard + N + + WITH filteredData AS (SELECT ri.recruit_id, + ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr + FROM recruits_info ri + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr') = 'array' + and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' +) +SELECT fd.recruit_id, + diploma_elem ->> 'id' AS source_id, + to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, + diploma_elem ->> 'kodUrObr' as education_level_code, + diploma_elem ->> 'naimUchZaved' as educational_institution_name, + diploma_elem ->> 'naimInoDok' as foreign_doc_name, + diploma_elem ->> 'serNomDok' as ser_num_doc, + diploma_elem ->> 'regNomDok' as reg_num, + diploma_elem ->> 'nomSved' as recognition_certificate_number, + to_date(diploma_elem ->> 'dataPriz', 'YYYY-MM-DD') as recognition_date, + diploma_elem -> 'svedSpecz' ->> 'naimSpecz' as speciality_name, + diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, + diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, + diploma_elem ->> 'kodOKSM' as oksm_code, + coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec +FROM filteredData fd + CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem; + + Y + + + 544 + 448 + + + + foreign_diploma_upsert + InsertUpdate + + Y + + 1 + + none + + + 10000 ervu-dashboard = - id - id - - - - = - created_at - created_at - + source_id + source_id ervu_dashboard - restriction_document
+ foreign_diploma
- id - id + recruit_id + recruit_id N - subpoena_id - subpoena_id - Y - - - created_at - created_at + source_id + source_id N - updated_at - updated_at + source_update_date + source_update_date Y - vk_id - vk_id + education_level_code + education_level_code Y - user_id - user_id + educational_institution_name + educational_institution_name Y - user_name - user_name + foreign_doc_name + foreign_doc_name Y - status - status + ser_num_doc + ser_num_doc Y - decision_number - decision_number + reg_num + reg_num Y - decision_date - decision_date + recognition_certificate_number + recognition_certificate_number Y - decision_reason - decision_reason + recognition_date + recognition_date Y - extra_info - extra_info + speciality_name + speciality_name Y - type - type + minobr_speciality_code + minobr_speciality_code Y - recruitment_name - recruitment_name + minprosv_speciality_code + minprosv_speciality_code + Y + + + oksm_code + oksm_code + Y + + + vus_spec + vus_spec Y N - 960 - 208 - - - - Table input - TableInput - - N - - 1 - - none - - - postgres.subpoena - N - SELECT -* -FROM public.restriction_document -WHERE - '${IDM_ID}' != '' -- Проверка на пустую строку - AND vk_id = '${IDM_ID}' - AND '${M_R_CR_DATE}' >= created_at -${LIMIT_FW} - Y - - - 480 - 208 + 1024 + 448 - Insert / update + foreign_diploma_upsert Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/prosecution/recruitments_five_flow.hpl b/mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow.hpl similarity index 81% rename from mappings/info_recruits/citizen_tables/prosecution/recruitments_five_flow.hpl rename to mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow.hpl index 2d2c4ac..06fe914 100644 --- a/mappings/info_recruits/citizen_tables/prosecution/recruitments_five_flow.hpl +++ b/mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow.hpl @@ -1,7 +1,7 @@ - recruitments_five_flow + recruitment_five_flow Y @@ -13,41 +13,41 @@ 1000 100 - - 2025/04/18 09:48:01.970 + 2025/09/04 20:54:08.982 - - 2025/04/18 09:48:01.970 + 2025/09/04 20:54:08.982 - Get all recruitments ordered by created_date - prosecution_flow.hpl + Get all recruitments + foreign_diploma_flow.hpl Y - Get all recruitments ordered by created_date - prosecution_flow.hpl 2 + Get all recruitments + foreign_diploma_flow.hpl 2 Y - Get all recruitments ordered by created_date - prosecution_flow.hpl 3 + Get all recruitments + foreign_diploma_flow.hpl 3 Y - Get all recruitments ordered by created_date - prosecution_flow.hpl 4 + Get all recruitments + foreign_diploma_flow.hpl 4 Y - Get all recruitments ordered by created_date - prosecution_flow.hpl 5 + Get all recruitments + foreign_diploma_flow.hpl 5 Y - Get all recruitments ordered by created_date + Get all recruitments TableInput Y @@ -57,21 +57,71 @@ none - ervu-dashboard + ervu-dashboard-test N - SELECT -idm_id AS recruitment +idm_id FROM ervu_dashboard.recruitment; N - 432 + 448 + 464 + + + + foreign_diploma_flow.hpl + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 720 304 - prosecution_flow.hpl + foreign_diploma_flow.hpl 2 PipelineExecutor Y @@ -82,7 +132,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl N 1 @@ -91,14 +141,9 @@ FROM ervu_dashboard.recruitment; IDM_ID - recruitment + idm_id - - JOB_NAME - - prosecution_job - Y @@ -122,180 +167,12 @@ FROM ervu_dashboard.recruitment; - 704 - 144 - - - - prosecution_flow.hpl 2 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl - N - - 1 - - - - - IDM_ID - recruitment - - - - JOB_NAME - - prosecution_job - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 704 - 224 - - - - prosecution_flow.hpl 3 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl - N - - 1 - - - - - IDM_ID - recruitment - - - - JOB_NAME - - prosecution_job - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 704 - 304 - - - - prosecution_flow.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl - N - - 1 - - - - - IDM_ID - recruitment - - - - JOB_NAME - - prosecution_job - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 704 + 720 384 - prosecution_flow.hpl 5 + foreign_diploma_flow.hpl 3 PipelineExecutor Y @@ -306,7 +183,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl N 1 @@ -315,14 +192,9 @@ FROM ervu_dashboard.recruitment; IDM_ID - recruitment + idm_id - - JOB_NAME - - prosecution_job - Y @@ -346,10 +218,112 @@ FROM ervu_dashboard.recruitment; - 704 + 720 464 + + foreign_diploma_flow.hpl 4 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 720 + 544 + + + + foreign_diploma_flow.hpl 5 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 720 + 624 + + diff --git a/mappings/info_recruits/citizen_tables/deferment_liberation_permission/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_delta.hpl similarity index 77% rename from mappings/info_recruits/citizen_tables/deferment_liberation_permission/recruitment_five_flow_delta.hpl rename to mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_delta.hpl index e55fa8d..ea8c53a 100644 --- a/mappings/info_recruits/citizen_tables/deferment_liberation_permission/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_delta.hpl @@ -13,36 +13,41 @@ 1000 100 - - 2025/08/11 13:16:11.204 + 2025/08/28 21:23:43.331 - - 2025/08/11 13:16:11.204 + 2025/08/28 21:23:43.331 Table input - deferment_liberation_permission_flow_delta.hpl + foreign_diploma_flow_delta.hpl + Y + + + get_max_source_update_date + Table input Y Table input - deferment_liberation_permission_flow_delta.hpl 2 + foreign_diploma_flow_delta.hpl 2 Y Table input - deferment_liberation_permission_flow_delta.hpl 3 + foreign_diploma_flow_delta.hpl 3 Y Table input - deferment_liberation_permission_flow_delta.hpl 4 + foreign_diploma_flow_delta.hpl 4 Y Table input - deferment_liberation_permission_flow_delta.hpl 5 + foreign_diploma_flow_delta.hpl 5 Y @@ -57,249 +62,35 @@ none - ervu-dashboard + ervu-dashboard-test N 0 + get_max_source_update_date SELECT - r.idm_id as recruitment_id + r.idm_id, + ? max_source_update_date FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - and job_name = 'deferment_liberation_permission_job' + and job_name = '${JOB_NAME}' + JOIN recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > ( + SELECT MAX(execution_datetime) + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND recruitment_id = r.idm_id + ) where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); N - 352 - 288 - - - - deferment_liberation_permission_flow_delta.hpl - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_delta.hpl - N - - 1 - - - - - IDM_ID - recruitment_id - - - - JOB_NAME - - deferment_liberation_permission_job - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 640 - 128 - - - - deferment_liberation_permission_flow_delta.hpl 2 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_delta.hpl - N - - 1 - - - - - IDM_ID - recruitment_id - - - - JOB_NAME - - deferment_liberation_permission_job - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 640 - 208 - - - - deferment_liberation_permission_flow_delta.hpl 3 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_delta.hpl - N - - 1 - - - - - IDM_ID - recruitment_id - - - - JOB_NAME - - deferment_liberation_permission_job - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 640 - 288 - - - - deferment_liberation_permission_flow_delta.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_delta.hpl - N - - 1 - - - - - IDM_ID - recruitment_id - - - - JOB_NAME - - deferment_liberation_permission_job - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 640 + 608 384 - deferment_liberation_permission_flow_delta.hpl 5 + foreign_diploma_flow_delta.hpl PipelineExecutor Y @@ -310,7 +101,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING local - ${PROJECT_HOME}/info_recruits/citizen_tables/deferment_liberation_permission/parallel/deferment_liberation_permission_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl N 1 @@ -319,13 +110,13 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id - JOB_NAME - - deferment_liberation_permission_job + MAX_SOURCE_UPDATE_DATE + max_source_update_date + Y @@ -350,8 +141,255 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 640 - 480 + 800 + 224 + + + + foreign_diploma_flow_delta.hpl 2 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + + MAX_SOURCE_UPDATE_DATE + max_source_update_date + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 800 + 304 + + + + foreign_diploma_flow_delta.hpl 3 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + + MAX_SOURCE_UPDATE_DATE + max_source_update_date + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 800 + 384 + + + + foreign_diploma_flow_delta.hpl 4 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + + MAX_SOURCE_UPDATE_DATE + max_source_update_date + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 800 + 464 + + + + foreign_diploma_flow_delta.hpl 5 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + + MAX_SOURCE_UPDATE_DATE + max_source_update_date + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 800 + 544 + + + + get_max_source_update_date + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + select max(source_update_date) +from foreign_diploma; + N + + + 432 + 384 diff --git a/mappings/info_recruits/citizen_tables/child/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_repeat.hpl similarity index 84% rename from mappings/info_recruits/citizen_tables/child/recruitment_five_flow_delta.hpl rename to mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_repeat.hpl index 71e650a..818600f 100644 --- a/mappings/info_recruits/citizen_tables/child/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_repeat.hpl @@ -1,7 +1,7 @@ - recruitment_five_flow_delta + recruitment_five_flow_repeat Y @@ -13,36 +13,36 @@ 1000 100 - - 2025/08/11 13:16:11.204 + 2025/09/05 08:56:57.814 - - 2025/08/11 13:16:11.204 + 2025/09/05 08:56:57.814 Table input - child_flow_delta.hpl + foreign_diploma_flow_repeat.hpl Y Table input - child_flow_delta.hpl 2 + foreign_diploma_flow_repeat.hpl 2 Y Table input - child_flow_delta.hpl 3 + foreign_diploma_flow_repeat.hpl 3 Y Table input - child_flow_delta.hpl 4 + foreign_diploma_flow_repeat.hpl 4 Y Table input - child_flow_delta.hpl 5 + foreign_diploma_flow_repeat.hpl 5 Y @@ -57,25 +57,25 @@ none - ervu-dashboard + ervu-dashboard-test N 0 - SELECT - r.idm_id as recruitment_id + SELECT r.idm_id AS recruitment_id FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - and job_name = 'child_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); + AND je.job_name = 'foreign_diploma_job' +WHERE je.id IS NULL + OR je.status IN ('ERROR', 'PROCESSING'); N - 352 - 288 + 624 + 512 - child_flow_delta.hpl + foreign_diploma_flow_repeat.hpl PipelineExecutor Y @@ -86,7 +86,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING local - ${PROJECT_HOME}/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl N 1 @@ -98,11 +98,6 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING recruitment_id - - JOB_NAME - - child_job - Y @@ -126,12 +121,12 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 544 - 128 + 832 + 352 - child_flow_delta.hpl 2 + foreign_diploma_flow_repeat.hpl 2 PipelineExecutor Y @@ -142,7 +137,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING local - ${PROJECT_HOME}/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl N 1 @@ -154,11 +149,6 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING recruitment_id - - JOB_NAME - - child_job - Y @@ -182,12 +172,12 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 544 - 208 + 832 + 432 - child_flow_delta.hpl 3 + foreign_diploma_flow_repeat.hpl 3 PipelineExecutor Y @@ -198,7 +188,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING local - ${PROJECT_HOME}/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl N 1 @@ -210,11 +200,6 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING recruitment_id - - JOB_NAME - - child_job - Y @@ -238,12 +223,12 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 544 - 288 + 832 + 512 - child_flow_delta.hpl 4 + foreign_diploma_flow_repeat.hpl 4 PipelineExecutor Y @@ -254,7 +239,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING local - ${PROJECT_HOME}/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl N 1 @@ -266,11 +251,6 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING recruitment_id - - JOB_NAME - - child_job - Y @@ -294,12 +274,12 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 544 - 368 + 832 + 592 - child_flow_delta.hpl 5 + foreign_diploma_flow_repeat.hpl 5 PipelineExecutor Y @@ -310,7 +290,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING local - ${PROJECT_HOME}/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl N 1 @@ -322,11 +302,6 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING recruitment_id - - JOB_NAME - - child_job - Y @@ -350,8 +325,8 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 544 - 448 + 832 + 672 diff --git a/mappings/info_recruits/citizen_tables/education/russian_diploma/constants/russian_diploma_constants_flow.hpl b/mappings/info_recruits/citizen_tables/education/russian_diploma/constants/russian_diploma_constants_flow.hpl new file mode 100644 index 0000000..aee3c4e --- /dev/null +++ b/mappings/info_recruits/citizen_tables/education/russian_diploma/constants/russian_diploma_constants_flow.hpl @@ -0,0 +1,199 @@ + + + + russian_diploma_constants_flow + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/09/22 08:32:44.247 + - + 2025/09/22 08:32:44.247 + + + + + + minobr_doc_type_input + minobr_doc_type_output + Y + + + minprosv_doc_type_input + minprosv_doc_type_output + Y + + + + minobr_doc_type_input + TableInput + + Y + + 1 + + none + + + nsi + N + 0 + SELECT + key as code, + value ->> 'value' as value, + NOT hidden as actual, + updated_at as update_date +FROM classifier_records +WHERE code = 'educationNaimVidDocMinobr'; + N + + + 384 + 272 + + + + minprosv_doc_type_input + TableInput + + Y + + 1 + + none + + + nsi + N + 0 + SELECT + key as code, + value ->> 'value' as value, + NOT hidden as actual, + updated_at as update_date +FROM classifier_records +WHERE code = 'educationNaimVidDocMinprosv'; + N + + + 384 + 384 + + + + minobr_doc_type_output + TableOutput + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + code + code + + + value + value + + + actual + actual + + + update_date + update_date + + + N + N + N + N + + Y + + N + ervu_dashboard + Y + minobr_doc_type
+ + N + Y + N + Y + + + 624 + 272 + + + + minprosv_doc_type_output + TableOutput + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + code + code + + + value + value + + + actual + actual + + + update_date + update_date + + + N + N + N + N + + Y + + N + ervu_dashboard + Y + minprosv_doc_type
+ + N + Y + N + Y + + + 624 + 384 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/education/russian_diploma/constants/russian_diploma_constants_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/education/russian_diploma/constants/russian_diploma_constants_flow_repeat.hpl new file mode 100644 index 0000000..54dedda --- /dev/null +++ b/mappings/info_recruits/citizen_tables/education/russian_diploma/constants/russian_diploma_constants_flow_repeat.hpl @@ -0,0 +1,255 @@ + + + + foreign_diploma_constants_flow + Y + + + + Normal + + + N + 1000 + 100 + - + 2025/09/22 08:32:45.452 + - + 2025/09/22 08:32:45.452 + + + + + + minobr_doc_type_input + minobr_doc_type_upsert + Y + + + minprosv_doc_type_input + minprosv_doc_type_upsert + Y + + + get_max_minobr_doc_type_update_date + minobr_doc_type_input + Y + + + get_max_minprosv_doc_type_date + minprosv_doc_type_input + Y + + + + minobr_doc_type_input + TableInput + + Y + + 1 + + none + + + nsi + N + 0 + get_max_minobr_doc_type_update_date + SELECT + key as code, + value ->> 'value' as value, + NOT hidden as actual +FROM classifier_records +WHERE code = 'educationNaimVidDocMinobr' +AND updated_at > ?; + N + + + 720 + 256 + + + + minobr_doc_type_upsert + InsertUpdate + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + = + code + code + + + ervu_dashboard + minobr_doc_type
+ + code + code + N + + + value + value + Y + + + actual + actual + Y + + + update_date + update_date + Y + + + N + + + 992 + 256 + + + + minprosv_doc_type_input + TableInput + + Y + + 1 + + none + + + nsi + N + 0 + get_max_minprosv_doc_type_date + SELECT + key as code, + value ->> 'value' as value, + NOT hidden as actual +FROM classifier_records +WHERE code = 'educationNaimVidDocMinprosv' +AND updated_at > ?; + N + + + 720 + 416 + + + + minprosv_doc_type_upsert + InsertUpdate + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + = + code + code + + + ervu_dashboard + minprosv_doc_type
+ + code + code + N + + + value + value + Y + + + actual + actual + Y + + + update_date + update_date + Y + + + N + + + 992 + 416 + + + + get_max_minobr_doc_type_update_date + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT + max(update_date) as max_update_date +FROM minobr_doc_type; + + N + + + 464 + 256 + + + + get_max_minprosv_doc_type_date + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT + max(update_date) as max_update_date +FROM minprosv_doc_type; + + N + + + 464 + 416 + + + + + + diff --git a/mappings/info_recruits/citizen_tables/work/self_employed/support/check_if_self_employed_job_execution_exists.hpl b/mappings/info_recruits/citizen_tables/education/russian_diploma/constants/support/check_if_constants_exists.hpl similarity index 75% rename from mappings/info_recruits/citizen_tables/work/self_employed/support/check_if_self_employed_job_execution_exists.hpl rename to mappings/info_recruits/citizen_tables/education/russian_diploma/constants/support/check_if_constants_exists.hpl index 494e70f..fd54f04 100644 --- a/mappings/info_recruits/citizen_tables/work/self_employed/support/check_if_self_employed_job_execution_exists.hpl +++ b/mappings/info_recruits/citizen_tables/education/russian_diploma/constants/support/check_if_constants_exists.hpl @@ -1,7 +1,7 @@ - check_if_self_employed_job_execution_exists + check_if_constants_exists Y @@ -13,9 +13,9 @@ 1000 100 - - 2025/08/18 20:34:56.618 + 2025/09/22 08:47:26.114 - - 2025/08/18 20:34:56.618 + 2025/09/22 08:47:26.114 @@ -39,16 +39,17 @@ - job_executed_flag - JOB_EXECUTED_FLAG + + constants_exists + CONSTANTS_EXISTS PARENT_WORKFLOW Y - 592 - 192 + 480 + 208 @@ -66,16 +67,15 @@ N 0 SELECT EXISTS ( - SELECT 1 - FROM etl.job_execution - WHERE job_name = 'self_employed_job' - limit 1 - ) AS job_executed_flag; - N + SELECT 1 FROM minobr_doc_type + UNION ALL + SELECT 1 FROM minprosv_doc_type +) AS constants_exists + Y - 304 - 192 + 192 + 208 diff --git a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl b/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl similarity index 64% rename from mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl rename to mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl index ebfe4cd..fda88dd 100644 --- a/mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl +++ b/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl @@ -1,7 +1,7 @@ - citizen_spouse_flow + russian_diploma_flow Y @@ -13,21 +13,21 @@ 1000 100 - - 2025/08/05 14:31:22.799 + 2025/09/01 16:13:06.345 - - 2025/08/05 14:31:22.799 + 2025/09/01 16:13:06.345 - Create job execution record - Table input + Change job status on error + Abort Y - Table output - Change job status on error + Create job execution record + education_input Y @@ -36,35 +36,35 @@ Y - Table output + Identify last row in a stream Detect empty stream Y - Detect empty stream - Change job status on success + citizen_education_diploma_output + Change job status on error Y - Change job status on error - Abort - Y - - - Table input + education_input Identify last row in a stream Y Identify last row in a stream - Table output + citizen_education_diploma_output Y - Table output + citizen_education_diploma_output Filter rows Y + + Detect empty stream + Change job status on success + Y + Abort @@ -77,13 +77,13 @@ none - ABORT_WITH_ERROR + ABORT Y 0 - 1136 - 528 + 1152 + 560 @@ -117,8 +117,8 @@ and recruitment_id = '${IDM_ID}'; - 720 - 528 + 928 + 560 @@ -143,14 +143,13 @@ and recruitment_id = '${IDM_ID}'; UPDATE etl.job_execution SET status = 'SUCCESS' WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'PROCESSING'; +and recruitment_id = '${IDM_ID}'; - 1120 - 64 + 1152 + 224 @@ -178,8 +177,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 80 - 320 + 224 + 400 @@ -195,8 +194,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 1120 - 320 + 640 + 224 @@ -231,15 +230,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 720 - 64 + 1152 + 400 Identify last row in a stream DetectLastRow - Y + N 1 @@ -249,12 +248,113 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); last_row - 576 - 320 + 640 + 400 - Table input + citizen_education_diploma_output + TableOutput + + Y + + 1 + + none + + + 10000 + ervu-dashboard + + + recruit_id + recruit_id + + + source_id + source_id + + + source_update_date + source_update_date + + + education_level_code + education_level_code + + + document_type + document_type + + + minobr_document_type_code + minobr_document_type_code + + + minprosv_document_type_code + minprosv_document_type_code + + + seria + seria + + + number + number + + + reg_number + reg_number + + + issue_date + issue_date + + + educational_institution_name + educational_institution_name + + + speciality_name + speciality_name + + + minobr_speciality_code + minobr_speciality_code + + + minprosv_speciality_code + minprosv_speciality_code + + + vus_spec + vus_spec + + + N + N + N + N + + Y + + N + ervu_dashboard + Y + russian_diploma
+ + N + Y + N + Y + + + 928 + 400 + + + + education_input TableInput Y @@ -267,159 +367,46 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - SELECT - ri.recruit_id AS recruit_id, - (supr->>'id')::uuid AS spouse_external_id, - NULLIF(supr->>'idERN', '') AS spouse_id_ern, - NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type, - supr->'svedFLBS'->'fio'->>'familiya' AS last_name, - supr->'svedFLBS'->'fio'->>'imya' AS first_name, - supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name, - CONCAT_WS(' ', - supr->'svedFLBS'->'fio'->>'familiya', - supr->'svedFLBS'->'fio'->>'imya', - supr->'svedFLBS'->'fio'->>'otchestvo' - ) AS full_name, - MAKE_DATE( - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int, - NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int - ) AS birth_date, - (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date, - supr->'svedSmert'->>'nomerZapis' AS death_az_number, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date, - ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number, - (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date, - CASE - WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true - ELSE false - END AS information_excluded -FROM ervu_dashboard.recruits_info ri -JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id -CROSS JOIN LATERAL ( - SELECT supri - FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri - WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array' -) AS supr(supr) -WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND '${M_R_CR_DATE}'::timestamp >= r.recruit_create_date; + WITH filteredData AS (SELECT ri.recruit_id, + ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr + FROM recruits_info ri + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedRObr' -> 'robr') = 'array' + and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' +) +SELECT fd.recruit_id, + diploma_elem ->> 'id' AS source_id, + to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, + diploma_elem ->> 'kodUrObr' as education_level_code, + diploma_elem -> 'vidDok' ->> 'naimVidDok' as document_type, + diploma_elem -> 'vidDokMinobr' ->> 'kodVidDok' as minobr_document_type_code, + diploma_elem -> 'vidDokMinprosv' ->> 'kodVidDok' as minprosv_document_type_code, + diploma_elem ->> 'seriyaBlank' as seria, + diploma_elem ->> 'nomerBlank' as number, + diploma_elem ->> 'regNomer' as reg_number, + to_date(diploma_elem ->> 'dataVyd', 'YYYY-MM-DD') as issue_date, + diploma_elem ->> 'naimOrg' as educational_institution_name, + diploma_elem -> 'svedSpecz' ->> 'naimSpecz' as speciality_name, + diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, + diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, + coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec +FROM filteredData fd + CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem; Y - 368 - 320 - - - - Table output - TableOutput - - N - - 1 - - none - - - 1000 - ervu-dashboard - - - recruit_id - recruit_id - - - spouse_external_id - spouse_external_id - - - spouse_id_ern - spouse_id_ern - - - kinship_type - kinship_type - - - last_name - last_name - - - first_name - first_name - - - middle_name - middle_name - - - full_name - full_name - - - birth_date - birth_date - - - death_date - death_date - - - death_az_number - death_az_number - - - marriage_az_number - marriage_az_number - - - marriage_date - marriage_date - - - divorce_az_number - divorce_az_number - - - divorce_date - divorce_date - - - information_excluded - information_excluded - - - N - N - N - N - Y - N - ervu_dashboard - Y - citizen_spouse
- N - Y - N - Y - - - 720 - 320 + 448 + 400 - Table output + citizen_education_diploma_output Change job status on error Y error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl b/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl similarity index 65% rename from mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl rename to mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl index 262fee7..61e43e6 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl @@ -1,7 +1,7 @@ - citizen_address_flow_delta + russian_diploma_flow_delta Y @@ -13,26 +13,21 @@ 1000 100 - - 2025/08/11 13:22:56.048 + 2025/09/01 17:09:06.132 - - 2025/08/11 13:22:56.048 + 2025/09/01 17:09:06.132 + + Change job status on error + Abort + Y + Create job execution record - Table input - Y - - - Insert / update - Change job status on error - Y - - - Detect empty stream - Change job status on success + education_input Y @@ -41,28 +36,33 @@ Y - Change job status on error - Abort - Y - - - Table input - Insert / update - Y - - - Insert / update - Identify last row in a stream + Identify last row in a stream + Detect empty stream Y Identify last row in a stream - Filter rows + Insert / update + Y + + + education_input + Identify last row in a stream Y Insert / update - Detect empty stream + Change job status on error + Y + + + Insert / update + Filter rows + Y + + + Detect empty stream + Change job status on success Y @@ -77,13 +77,13 @@ none - ABORT_WITH_ERROR + ABORT Y 0 - 1008 - 464 + 1392 + 656 @@ -117,8 +117,8 @@ and recruitment_id = '${IDM_ID}'; - 752 - 464 + 1168 + 656 @@ -143,14 +143,13 @@ and recruitment_id = '${IDM_ID}'; UPDATE etl.job_execution SET status = 'DELTA_SUCCESS' WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'DELTA_PROCESSING'; +and recruitment_id = '${IDM_ID}'; - 1120 - 16 + 1392 + 288 @@ -175,14 +174,14 @@ and status = 'DELTA_PROCESSING'; UPDATE etl.job_execution SET status = 'DELTA_PROCESSING', - execution_datetime = DEFAULT, + execution_datetime = current_timestamp, error_description = NULL where job_name = '${JOB_NAME}' and recruitment_id = '${IDM_ID}'; - 128 - 224 + 464 + 464 @@ -198,8 +197,8 @@ and recruitment_id = '${IDM_ID}'; - 752 - 16 + 880 + 288 @@ -234,15 +233,15 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1120 - 224 + 1392 + 464 Identify last row in a stream DetectLastRow - Y + N 1 @@ -252,96 +251,121 @@ and recruitment_id = '${IDM_ID}'; last_row - 944 - 224 + 880 + 464 Insert / update InsertUpdate - N + Y 1 none - 1000 + 10000 ervu-dashboard = - recruit_id - recruit_id + source_id + source_id ervu_dashboard - citizen_address
+ russian_diploma
recruit_id recruit_id N - residence_address - residence_address + source_id + source_id + N + + + source_update_date + source_update_date Y - residence_registration_date - residence_registration_date + education_level_code + education_level_code Y - residence_deregistration_date - residence_deregistration_date + document_type + document_type Y - place_stay_address - place_stay_address + minobr_document_type_code + minobr_document_type_code Y - place_stay_registration_date - place_stay_registration_date + minprosv_document_type_code + minprosv_document_type_code Y - place_stay_deregistration_date - place_stay_deregistration_date + seria + seria Y - place_stay_registration_type - place_stay_registration_type + number + number Y - factual_address - factual_address + reg_number + reg_number Y - factual_info_received_date - factual_info_received_date + issue_date + issue_date Y - factual_info_expiry_date - factual_info_expiry_date + educational_institution_name + educational_institution_name + Y + + + speciality_name + speciality_name + Y + + + minobr_speciality_code + minobr_speciality_code + Y + + + minprosv_speciality_code + minprosv_speciality_code + Y + + + vus_spec + vus_spec Y N - 752 - 224 + 1168 + 464 - Table input + education_input TableInput Y @@ -354,37 +378,35 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - SELECT - ri.recruit_id::uuid, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, - - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, - CASE - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' - ELSE NULL - END AS place_stay_registration_type, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date -FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr -WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date - AND c.update_date >= '${M_R_UP_DATE}'::timestamp -GROUP BY ri.recruit_id; + WITH filteredData AS (SELECT ri.recruit_id, + ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr + FROM recruits_info ri + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedRObr' -> 'robr') = 'array' + and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}') +SELECT fd.recruit_id, + diploma_elem ->> 'id' AS source_id, + to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, + diploma_elem ->> 'kodUrObr' as education_level_code, + diploma_elem -> 'vidDok' ->> 'naimVidDok' as document_type, + diploma_elem -> 'vidDokMinobr' ->> 'kodVidDok' as minobr_document_type_code, + diploma_elem -> 'vidDokMinprosv' ->> 'kodVidDok' as minprosv_document_type_code, + diploma_elem ->> 'seriyaBlank' as seria, + diploma_elem ->> 'nomerBlank' as number, + diploma_elem ->> 'regNomer' as reg_number, + to_date(diploma_elem ->> 'dataVyd', 'YYYY-MM-DD') as issue_date, + diploma_elem ->> 'naimOrg' as educational_institution_name, + diploma_elem -> 'svedSpecz' ->> 'naimSpecz' as speciality_name, + diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, + diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, + coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec +FROM filteredData fd + CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem +where to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'; Y - 400 - 224 + 688 + 464 @@ -395,7 +417,7 @@ GROUP BY ri.recruit_id; error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl similarity index 64% rename from mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl rename to mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl index 6722b66..0dbeced 100644 --- a/mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl @@ -1,7 +1,7 @@ - citizen_address_flow_repeat + russian_diploma_flow_repeat Y @@ -13,16 +13,41 @@ 1000 100 - - 2025/08/08 13:25:52.319 + 2025/09/01 17:03:13.520 - - 2025/08/08 13:25:52.319 + 2025/09/01 17:03:13.520 + + Change job status on error + Abort + Y + Create job execution record - Table input + education_input + Y + + + Filter rows + Change job status on success + Y + + + Identify last row in a stream + Detect empty stream + Y + + + education_input + Identify last row in a stream + Y + + + Identify last row in a stream + Insert / update Y @@ -32,7 +57,7 @@ Insert / update - Detect empty stream + Filter rows Y @@ -40,31 +65,6 @@ Change job status on success Y - - Filter rows - Change job status on success - Y - - - Change job status on error - Abort - Y - - - Insert / update - Identify last row in a stream - Y - - - Identify last row in a stream - Filter rows - Y - - - Table input - Insert / update - Y - Abort @@ -77,12 +77,12 @@ none - ABORT_WITH_ERROR + ABORT Y 0 - 1232 + 1136 544 @@ -117,7 +117,7 @@ and recruitment_id = '${IDM_ID}'; - 768 + 912 544 @@ -143,14 +143,13 @@ and recruitment_id = '${IDM_ID}'; UPDATE etl.job_execution SET status = 'SUCCESS' WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}' -and status = 'PROCESSING'; +and recruitment_id = '${IDM_ID}'; - 1120 - 16 + 1136 + 208 @@ -172,31 +171,17 @@ and status = 'PROCESSING'; Y N N - INSERT INTO etl.job_execution ( - id, - job_name, - status, - execution_datetime, - error_description, - recruitment_id -) -VALUES ( - DEFAULT, - '${JOB_NAME}', - 'PROCESSING', - DEFAULT, - NULL, - '${IDM_ID}' -) + INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id) +VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}') ON CONFLICT (job_name, recruitment_id) -DO UPDATE SET - status = 'PROCESSING', - execution_datetime = DEFAULT, - error_description = NULL; +DO UPDATE +SET status = 'PROCESSING', + error_description = null, + execution_datetime = current_timestamp; - 176 - 272 + 208 + 384 @@ -212,8 +197,8 @@ DO UPDATE SET - 768 - 16 + 624 + 208 @@ -248,15 +233,15 @@ DO UPDATE SET Change job status on success - 1120 - 272 + 1136 + 384 Identify last row in a stream DetectLastRow - Y + N 1 @@ -266,96 +251,121 @@ DO UPDATE SET last_row - 960 - 272 + 624 + 384 Insert / update InsertUpdate - N + Y 1 none - 1000 + 10000 ervu-dashboard = - recruit_id - recruit_id + source_id + source_id ervu_dashboard - citizen_address
+ russian_diploma
recruit_id recruit_id N - residence_address - residence_address + source_id + source_id + N + + + source_update_date + source_update_date Y - residence_registration_date - residence_registration_date + education_level_code + education_level_code Y - residence_deregistration_date - residence_deregistration_date + document_type + document_type Y - place_stay_address - place_stay_address + minobr_document_type_code + minobr_document_type_code Y - place_stay_registration_date - place_stay_registration_date + minprosv_document_type_code + minprosv_document_type_code Y - place_stay_deregistration_date - place_stay_deregistration_date + seria + seria Y - place_stay_registration_type - place_stay_registration_type + number + number Y - factual_address - factual_address + reg_number + reg_number Y - factual_info_received_date - factual_info_received_date + issue_date + issue_date Y - factual_info_expiry_date - factual_info_expiry_date + educational_institution_name + educational_institution_name + Y + + + speciality_name + speciality_name + Y + + + minobr_speciality_code + minobr_speciality_code + Y + + + minprosv_speciality_code + minprosv_speciality_code + Y + + + vus_spec + vus_spec Y N - 768 - 272 + 912 + 384 - Table input + education_input TableInput Y @@ -368,36 +378,35 @@ DO UPDATE SET ervu-dashboard N - SELECT - ri.recruit_id::uuid, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date, - MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date, - - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date, - MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date, - CASE - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица' - WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы' - ELSE NULL - END AS place_stay_registration_type, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date, - MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date -FROM ervu_dashboard.recruits_info ri - JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id - CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr -WHERE - '${IDM_ID}' != '' - AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' - AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date -GROUP BY ri.recruit_id; + WITH filteredData AS (SELECT ri.recruit_id, + ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr + FROM recruits_info ri + WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedRObr' -> 'robr') = 'array' + and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' +) +SELECT fd.recruit_id, + diploma_elem ->> 'id' AS source_id, + to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, + diploma_elem ->> 'kodUrObr' as education_level_code, + diploma_elem -> 'vidDok' ->> 'naimVidDok' as document_type, + diploma_elem -> 'vidDokMinobr' ->> 'kodVidDok' as minobr_document_type_code, + diploma_elem -> 'vidDokMinprosv' ->> 'kodVidDok' as minprosv_document_type_code, + diploma_elem ->> 'seriyaBlank' as seria, + diploma_elem ->> 'nomerBlank' as number, + diploma_elem ->> 'regNomer' as reg_number, + to_date(diploma_elem ->> 'dataVyd', 'YYYY-MM-DD') as issue_date, + diploma_elem ->> 'naimOrg' as educational_institution_name, + diploma_elem -> 'svedSpecz' ->> 'naimSpecz' as speciality_name, + diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, + diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, + coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec +FROM filteredData fd + CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem; Y - 480 - 272 + 432 + 384 @@ -408,7 +417,7 @@ GROUP BY ri.recruit_id; error_description - error_code + diff --git a/mappings/info_recruits/citizen_tables/punishment/recruitments_five_flow.hpl b/mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow.hpl similarity index 81% rename from mappings/info_recruits/citizen_tables/punishment/recruitments_five_flow.hpl rename to mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow.hpl index 33ee4da..3860444 100644 --- a/mappings/info_recruits/citizen_tables/punishment/recruitments_five_flow.hpl +++ b/mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow.hpl @@ -1,7 +1,7 @@ - recruitments_five_flow + recruitment_five_flow Y @@ -13,41 +13,41 @@ 1000 100 - - 2025/04/18 09:48:01.970 + 2025/09/01 16:12:33.364 - - 2025/04/18 09:48:01.970 + 2025/09/01 16:12:33.364 - Get all recruitments ordered by created_date - punishment_flow.hpl + Get all recruitments + russian_diploma_flow.hpl Y - Get all recruitments ordered by created_date - punishment_flow.hpl 2 + Get all recruitments + russian_diploma_flow.hpl 2 Y - Get all recruitments ordered by created_date - punishment_flow.hpl 3 + Get all recruitments + russian_diploma_flow.hpl 3 Y - Get all recruitments ordered by created_date - punishment_flow.hpl 4 + Get all recruitments + russian_diploma_flow.hpl 4 Y - Get all recruitments ordered by created_date - punishment_flow.hpl 5 + Get all recruitments + russian_diploma_flow.hpl 5 Y - Get all recruitments ordered by created_date + Get all recruitments TableInput Y @@ -57,21 +57,20 @@ none - ervu-dashboard + ervu-dashboard-test N - SELECT -idm_id AS recruitment +idm_id FROM ervu_dashboard.recruitment; N 432 - 304 + 512 - punishment_flow.hpl + russian_diploma_flow.hpl PipelineExecutor Y @@ -82,7 +81,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl N 1 @@ -91,14 +90,9 @@ FROM ervu_dashboard.recruitment; IDM_ID - recruitment + idm_id - - JOB_NAME - - punishment_job - Y @@ -123,11 +117,11 @@ FROM ervu_dashboard.recruitment; 704 - 144 + 352 - punishment_flow.hpl 2 + russian_diploma_flow.hpl 2 PipelineExecutor Y @@ -138,7 +132,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl N 1 @@ -147,14 +141,9 @@ FROM ervu_dashboard.recruitment; IDM_ID - recruitment + idm_id - - JOB_NAME - - punishment_job - Y @@ -179,11 +168,11 @@ FROM ervu_dashboard.recruitment; 704 - 224 + 432 - punishment_flow.hpl 3 + russian_diploma_flow.hpl 3 PipelineExecutor Y @@ -194,7 +183,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl N 1 @@ -203,14 +192,9 @@ FROM ervu_dashboard.recruitment; IDM_ID - recruitment + idm_id - - JOB_NAME - - punishment_job - Y @@ -235,11 +219,11 @@ FROM ervu_dashboard.recruitment; 704 - 304 + 512 - punishment_flow.hpl 4 + russian_diploma_flow.hpl 4 PipelineExecutor Y @@ -250,7 +234,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl N 1 @@ -259,14 +243,9 @@ FROM ervu_dashboard.recruitment; IDM_ID - recruitment + idm_id - - JOB_NAME - - punishment_job - Y @@ -291,11 +270,11 @@ FROM ervu_dashboard.recruitment; 704 - 384 + 592 - punishment_flow.hpl 5 + russian_diploma_flow.hpl 5 PipelineExecutor Y @@ -306,7 +285,7 @@ FROM ervu_dashboard.recruitment; local - ${PROJECT_HOME}/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl N 1 @@ -315,14 +294,9 @@ FROM ervu_dashboard.recruitment; IDM_ID - recruitment + idm_id - - JOB_NAME - - punishment_job - Y @@ -347,7 +321,7 @@ FROM ervu_dashboard.recruitment; 704 - 464 + 672 diff --git a/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow_delta.hpl similarity index 76% rename from mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_delta.hpl rename to mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow_delta.hpl index 864793c..38479ff 100644 --- a/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow_delta.hpl @@ -13,36 +13,41 @@ 1000 100 - - 2025/08/11 13:16:11.204 + 2025/09/01 17:07:38.569 - - 2025/08/11 13:16:11.204 + 2025/09/01 17:07:38.569 Table input - citizenship_foreign_flow_delta.hpl + russian_diploma_flow_delta.hpl + Y + + + get_max_source_update_date + Table input Y Table input - citizenship_foreign_flow_delta.hpl 2 + russian_diploma_flow_delta.hpl 2 Y Table input - citizenship_foreign_flow_delta.hpl 3 + russian_diploma_flow_delta.hpl 3 Y Table input - citizenship_foreign_flow_delta.hpl 4 + russian_diploma_flow_delta.hpl 4 Y Table input - citizenship_foreign_flow_delta.hpl 5 + russian_diploma_flow_delta.hpl 5 Y @@ -57,249 +62,58 @@ none - ervu-dashboard + ervu-dashboard-test N 0 + get_max_source_update_date SELECT - r.idm_id as recruitment_id + r.idm_id, + ? max_source_update_date FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - and job_name = 'citizenship_foreign_job' + and job_name = '${JOB_NAME}' + JOIN recruits_info ri + ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id + AND ri.updated_at > ( + SELECT MAX(execution_datetime) + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND recruitment_id = r.idm_id + ) where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); N - 352 - 288 - - - - citizenship_foreign_flow_delta.hpl - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl - N - - 1 - - - - - IDM_ID - recruitment_id - - - - JOB_NAME - - citizenship_foreign_job - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 640 - 128 - - - - citizenship_foreign_flow_delta.hpl 2 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl - N - - 1 - - - - - IDM_ID - recruitment_id - - - - JOB_NAME - - citizenship_foreign_job - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 656 - 208 - - - - citizenship_foreign_flow_delta.hpl 3 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl - N - - 1 - - - - - IDM_ID - recruitment_id - - - - JOB_NAME - - citizenship_foreign_job - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 656 - 288 - - - - citizenship_foreign_flow_delta.hpl 4 - PipelineExecutor - - Y - - 1 - - none - - - local - ${PROJECT_HOME}/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl - N - - 1 - - - - - IDM_ID - recruitment_id - - - - JOB_NAME - - citizenship_foreign_job - - Y - - - ExecutionTime - ExecutionResult - ExecutionNrErrors - ExecutionLinesRead - ExecutionLinesWritten - ExecutionLinesInput - ExecutionLinesOutput - ExecutionLinesRejected - ExecutionLinesUpdated - ExecutionLinesDeleted - ExecutionFilesRetrieved - ExecutionExitStatus - ExecutionLogText - ExecutionLogChannelId - - - FileName - - - - 656 + 688 368 - citizenship_foreign_flow_delta.hpl 5 + get_max_source_update_date + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + select max(source_update_date) +from russian_diploma; + N + + + 512 + 368 + + + + russian_diploma_flow_delta.hpl PipelineExecutor Y @@ -310,7 +124,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING local - ${PROJECT_HOME}/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl N 1 @@ -319,13 +133,18 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING IDM_ID - recruitment_id + idm_id JOB_NAME - citizenship_foreign_job + russian_diploma_job + + + MAX_SOURCE_UPDATE_DATE + max_source_update_date + Y @@ -350,10 +169,254 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 656 + 880 + 208 + + + + russian_diploma_flow_delta.hpl 2 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + + JOB_NAME + + russian_diploma_job + + + MAX_SOURCE_UPDATE_DATE + max_source_update_date + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 880 + 288 + + + + russian_diploma_flow_delta.hpl 3 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + + JOB_NAME + + russian_diploma_job + + + MAX_SOURCE_UPDATE_DATE + max_source_update_date + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 880 + 368 + + + + russian_diploma_flow_delta.hpl 4 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + + JOB_NAME + + russian_diploma_job + + + MAX_SOURCE_UPDATE_DATE + max_source_update_date + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 880 448 + + russian_diploma_flow_delta.hpl 5 + PipelineExecutor + + Y + + 1 + + none + + + local + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl + N + + 1 + + + + + IDM_ID + idm_id + + + + JOB_NAME + + russian_diploma_job + + + MAX_SOURCE_UPDATE_DATE + max_source_update_date + + + Y + + + ExecutionTime + ExecutionResult + ExecutionNrErrors + ExecutionLinesRead + ExecutionLinesWritten + ExecutionLinesInput + ExecutionLinesOutput + ExecutionLinesRejected + ExecutionLinesUpdated + ExecutionLinesDeleted + ExecutionFilesRetrieved + ExecutionExitStatus + ExecutionLogText + ExecutionLogChannelId + + + FileName + + + + 880 + 528 + + diff --git a/mappings/info_recruits/citizen_tables/border_crossing/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow_repeat.hpl similarity index 81% rename from mappings/info_recruits/citizen_tables/border_crossing/recruitment_five_flow_delta.hpl rename to mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow_repeat.hpl index 01ec515..fca1560 100644 --- a/mappings/info_recruits/citizen_tables/border_crossing/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow_repeat.hpl @@ -1,7 +1,7 @@ - recruitment_five_flow_delta + recruitment_five_flow_repeat Y @@ -13,36 +13,36 @@ 1000 100 - - 2025/08/11 13:16:11.204 + 2025/09/01 17:02:32.905 - - 2025/08/11 13:16:11.204 + 2025/09/01 17:02:32.905 Table input - border_crossing_flow_delta.hpl + russian_diploma_flow_repeat.hpl Y Table input - border_crossing_flow_delta.hpl 2 + russian_diploma_flow_repeat.hpl 2 Y Table input - border_crossing_flow_delta.hpl 3 + russian_diploma_flow_repeat.hpl 3 Y Table input - border_crossing_flow_delta.hpl 4 + russian_diploma_flow_repeat.hpl 4 Y Table input - border_crossing_flow_delta.hpl 5 + russian_diploma_flow_repeat.hpl 5 Y @@ -57,25 +57,25 @@ none - ervu-dashboard + ervu-dashboard-test N 0 - SELECT - r.idm_id as recruitment_id + SELECT r.idm_id AS recruitment_id FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - and job_name = 'border_crossing_job' -where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING'); + AND je.job_name = 'russian_diploma_job' +WHERE je.id IS NULL + OR je.status IN ('ERROR', 'PROCESSING'); N - 352 - 288 + 368 + 352 - border_crossing_flow_delta.hpl + russian_diploma_flow_repeat.hpl PipelineExecutor Y @@ -86,7 +86,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING local - ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl N 1 @@ -94,15 +94,10 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - IDM_ID + recruitment_id - - JOB_NAME - - border_crossing_job - Y @@ -126,12 +121,12 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 640 - 128 + 576 + 192 - border_crossing_flow_delta.hpl 2 + russian_diploma_flow_repeat.hpl 2 PipelineExecutor Y @@ -142,7 +137,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING local - ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl N 1 @@ -150,15 +145,10 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - IDM_ID + recruitment_id - - JOB_NAME - - border_crossing_job - Y @@ -182,12 +172,12 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 656 - 224 + 576 + 272 - border_crossing_flow_delta.hpl 3 + russian_diploma_flow_repeat.hpl 3 PipelineExecutor Y @@ -198,7 +188,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING local - ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl N 1 @@ -206,15 +196,10 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - IDM_ID + recruitment_id - - JOB_NAME - - border_crossing_job - Y @@ -238,12 +223,12 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 656 - 304 + 576 + 352 - border_crossing_flow_delta.hpl 4 + russian_diploma_flow_repeat.hpl 4 PipelineExecutor Y @@ -254,7 +239,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING local - ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl N 1 @@ -262,15 +247,10 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - IDM_ID + recruitment_id - - JOB_NAME - - border_crossing_job - Y @@ -294,12 +274,12 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 656 - 384 + 576 + 432 - border_crossing_flow_delta.hpl 5 + russian_diploma_flow_repeat.hpl 5 PipelineExecutor Y @@ -310,7 +290,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING local - ${PROJECT_HOME}/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow_delta.hpl + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl N 1 @@ -318,15 +298,10 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - IDM_ID + recruitment_id - - JOB_NAME - - border_crossing_job - Y @@ -350,8 +325,8 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING - 656 - 480 + 576 + 512 diff --git a/mappings/info_recruits/citizen_tables/education/russian_diploma/russian_diploma_job.hwf b/mappings/info_recruits/citizen_tables/education/russian_diploma/russian_diploma_job.hwf new file mode 100644 index 0000000..cc7726c --- /dev/null +++ b/mappings/info_recruits/citizen_tables/education/russian_diploma/russian_diploma_job.hwf @@ -0,0 +1,419 @@ + + + russian_diploma_job + Y + + + + - + 2025/09/01 15:59:15.492 + - + 2025/09/01 15:59:15.492 + + + + + Start + + SPECIAL + + 1 + 12 + 60 + 0 + 0 + N + 0 + 1 + N + 272 + 208 + + + + check_if_job_execution_exists.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_job_execution_exists.hpl + + + Basic + + Y + + N + local + N + N + Y + N + 1200 + 224 + + + + employer_job_execution_exists_check + + SIMPLE_EVAL + + boolean + false + equal + equal + N + variable + JOB_EXECUTED_FLAG + N + 1504 + 224 + + + + recruitment_five_flow.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow.hpl + Basic + + Y + + N + local + N + N + Y + Y + 1744 + 224 + + + + recruitment_five_flow_repeat.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow_repeat.hpl + Basic + + Y + + N + local + N + N + Y + N + 1728 + 528 + + + + check_if_need_to_repeat.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_need_to_repeat.hpl + + + Basic + + Y + + N + local + N + N + Y + N + 1504 + 368 + + + + Simple evaluation + + SIMPLE_EVAL + + boolean + true + equal + equal + N + variable + NEED_TO_REPEAT_JOB + N + 1504 + 528 + + + + recruitment_five_flow_delta.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow_delta.hpl + Basic + + Y + + N + local + N + N + Y + N + 1504 + 672 + + + + check_if_constants_exists.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/constants/support/check_if_constants_exists.hpl + + + Basic + + Y + + N + local + N + N + Y + N + 432 + 208 + + + + constants_exists + + SIMPLE_EVAL + + boolean + true + equal + equal + N + variable + CONSTANTS_EXISTS + N + 608 + 208 + + + + russian_diploma_constants_flow.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/constants/russian_diploma_constants_flow.hpl + + + Basic + + Y + + N + local + N + N + Y + N + 800 + 128 + + + + russian_diploma_constants_flow_repeat.hpl + + PIPELINE + + N + N + N + N + N + N + ${PROJECT_HOME}/info_recruits/citizen_tables/education/russian_diploma/constants/russian_diploma_constants_flow_repeat.hpl + + + Basic + + Y + + N + local + N + N + Y + N + 800 + 304 + + + + init_job_name + + SET_VARIABLES + + + + JOB_NAME + CURRENT_WORKFLOW + russian_diploma_job + + + CURRENT_WORKFLOW + + N + N + 992 + 224 + + + + + + check_if_job_execution_exists.hpl + employer_job_execution_exists_check + Y + Y + N + + + employer_job_execution_exists_check + check_if_need_to_repeat.hpl + Y + N + N + + + check_if_need_to_repeat.hpl + Simple evaluation + Y + Y + N + + + check_if_constants_exists.hpl + constants_exists + Y + Y + N + + + constants_exists + russian_diploma_constants_flow.hpl + Y + N + N + + + constants_exists + russian_diploma_constants_flow_repeat.hpl + Y + Y + N + + + russian_diploma_constants_flow.hpl + init_job_name + Y + Y + N + + + russian_diploma_constants_flow_repeat.hpl + init_job_name + Y + Y + N + + + Start + check_if_constants_exists.hpl + Y + Y + Y + + + init_job_name + check_if_job_execution_exists.hpl + Y + Y + N + + + employer_job_execution_exists_check + recruitment_five_flow.hpl + Y + Y + N + + + Simple evaluation + recruitment_five_flow_repeat.hpl + Y + Y + N + + + Simple evaluation + recruitment_five_flow_delta.hpl + Y + N + N + + + + + +