diff --git a/mappings/info_recruits/citizen_tables/deputy/parallel/deputy_flow.hpl b/mappings/info_recruits/citizen_tables/deputy/parallel/deputy_flow.hpl index 3e3d3ab..a3fb132 100644 --- a/mappings/info_recruits/citizen_tables/deputy/parallel/deputy_flow.hpl +++ b/mappings/info_recruits/citizen_tables/deputy/parallel/deputy_flow.hpl @@ -35,49 +35,39 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - deputy_output - Y - deputy_output Change job status on error Y - - deputy_input - Identify last row in a stream - Y - Detect empty stream Change job status on success Y - - deputy_flag_update - Filter rows - Y - deputy_flag_update Change job status on error Y - Identify last row in a stream - Block until transforms finish + deputy_input + deputy_flag_update Y - Block until transforms finish - deputy_flag_update + deputy_input + Detect empty stream + Y + + + deputy_input + deputy_output + Y + + + deputy_output + Filter rows Y @@ -97,30 +87,8 @@ 0 - 1504 - 592 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - deputy_output - - - - - 976 - 416 + 1072 + 608 @@ -155,8 +123,8 @@ and recruitment_id = '${IDM_ID}'; - 1216 - 592 + 784 + 608 @@ -187,7 +155,7 @@ and recruitment_id = '${IDM_ID}'; - 1504 + 1072 240 @@ -233,7 +201,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 704 + 512 240 @@ -269,25 +237,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1504 - 416 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 704 + 1072 416 @@ -322,15 +272,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Y - 1216 - 416 + 512 + 608 deputy_input TableInput - Y + N 1 @@ -341,7 +291,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); N WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedDeputat' -> 'deputat' as deputy_elem + ri.info -> 'svedFL' -> 'svedDeputat' -> 'deputat' as deputy_elem, + ctid FROM ervu_dashboard.recruits_info ri WHERE ri.info -> 'svedFL' -> 'svedDeputat' ->> 'deputat' <> 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -355,7 +306,8 @@ SELECT fd.recruit_id, to_date(deputy_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS end_date, deputy_elem ->> 'prichinaDosrochKod' AS termination_reason_code, to_date(deputy_elem ->> 'dataDosroch', 'YYYY-MM-DD') AS termination_date, - true as has_deputy_info + true as has_deputy_info, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd; Y @@ -430,8 +382,8 @@ FROM filteredData fd; Y - 976 - 592 + 784 + 416 diff --git a/mappings/info_recruits/citizen_tables/deputy/parallel/deputy_flow_delta.hpl b/mappings/info_recruits/citizen_tables/deputy/parallel/deputy_flow_delta.hpl index 0ebc6b3..8cd0ff3 100644 --- a/mappings/info_recruits/citizen_tables/deputy/parallel/deputy_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/deputy/parallel/deputy_flow_delta.hpl @@ -40,43 +40,33 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - deputy_flag_update Change job status on error Y - - deputy_input - Identify last row in a stream - Y - - - Identify last row in a stream - deputy_upsert - Y - deputy_upsert Change job status on error Y - Identify last row in a stream + deputy_input + Detect empty stream + Y + + + deputy_input deputy_flag_update Y - deputy_upsert - Block until transforms finish + deputy_input + deputy_upsert Y - Block until transforms finish + deputy_upsert Filter rows Y @@ -97,32 +87,10 @@ 0 - 1360 + 928 688 - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - deputy_flag_update - - - - - 1152 - 416 - - Change job status on error ExecSql @@ -155,7 +123,7 @@ and recruitment_id = '${IDM_ID}'; - 944 + 688 688 @@ -187,7 +155,7 @@ and recruitment_id = '${IDM_ID}'; - 1360 + 928 240 @@ -236,7 +204,7 @@ and recruitment_id = '${IDM_ID}'; - 624 + 432 240 @@ -272,25 +240,7 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1360 - 416 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 624 + 928 416 @@ -325,7 +275,7 @@ and recruitment_id = '${IDM_ID}'; Y - 624 + 432 688 @@ -333,7 +283,7 @@ and recruitment_id = '${IDM_ID}'; deputy_input TableInput - Y + N 1 @@ -344,7 +294,8 @@ and recruitment_id = '${IDM_ID}'; N WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedDeputat' -> 'deputat' as deputy_elem + ri.info -> 'svedFL' -> 'svedDeputat' -> 'deputat' as deputy_elem, + ctid FROM ervu_dashboard.recruits_info ri WHERE ri.info -> 'svedFL' -> 'svedDeputat' ->> 'deputat' <> 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -362,7 +313,8 @@ SELECT fd.recruit_id, to_date(deputy_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS end_date, deputy_elem ->> 'prichinaDosrochKod' AS termination_reason_code, to_date(deputy_elem ->> 'dataDosroch', 'YYYY-MM-DD') AS termination_date, - true as has_deputy_info + true as has_deputy_info, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd; Y @@ -442,7 +394,7 @@ FROM filteredData fd; N - 944 + 688 416 diff --git a/mappings/info_recruits/citizen_tables/deputy/parallel/deputy_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/deputy/parallel/deputy_flow_repeat.hpl index c43b1a8..fc03463 100644 --- a/mappings/info_recruits/citizen_tables/deputy/parallel/deputy_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/deputy/parallel/deputy_flow_repeat.hpl @@ -51,46 +51,36 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - deputy_flag_update Change job status on error Y - - deputy_input - Identify last row in a stream - Y - - - Identify last row in a stream - deputy_upsert - Y - deputy_upsert Change job status on error Y - deputy_upsert - Block until transforms finish + deputy_input + Detect empty stream Y - Block until transforms finish - Filter rows - Y - - - Identify last row in a stream + deputy_input deputy_flag_update Y + + deputy_input + deputy_upsert + Y + + + deputy_upsert + Filter rows + Y + Abort @@ -108,30 +98,8 @@ 0 - 1440 - 784 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - deputy_flag_update - - - - - 1248 - 512 + 1072 + 768 @@ -166,8 +134,8 @@ and recruitment_id = '${IDM_ID}'; - 1072 - 784 + 816 + 768 @@ -198,7 +166,7 @@ and recruitment_id = '${IDM_ID}'; - 1440 + 1072 336 @@ -253,7 +221,7 @@ ON CONFLICT (job_name, recruitment_id) - 784 + 576 336 @@ -289,25 +257,7 @@ ON CONFLICT (job_name, recruitment_id) Change job status on success - 1440 - 512 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 784 + 1072 512 @@ -342,15 +292,15 @@ ON CONFLICT (job_name, recruitment_id) Y - 784 - 784 + 576 + 768 deputy_input TableInput - Y + N 1 @@ -359,9 +309,9 @@ ON CONFLICT (job_name, recruitment_id) ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedDeputat' -> 'deputat' as deputy_elem + ri.info -> 'svedFL' -> 'svedDeputat' -> 'deputat' as deputy_elem, + ctid FROM ervu_dashboard.recruits_info ri WHERE ri.info -> 'svedFL' -> 'svedDeputat' ->> 'deputat' <> 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -375,7 +325,8 @@ SELECT fd.recruit_id, to_date(deputy_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS end_date, deputy_elem ->> 'prichinaDosrochKod' AS termination_reason_code, to_date(deputy_elem ->> 'dataDosroch', 'YYYY-MM-DD') AS termination_date, - true as has_deputy_info + true as has_deputy_info, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd; Y @@ -454,7 +405,7 @@ FROM filteredData fd; N - 1072 + 816 512 diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl index f057260..d920f29 100644 --- a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl +++ b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl @@ -61,21 +61,6 @@ Change job status on success Y - - citizen_education_output - block_until_all_complete - Y - - - block_until_all_complete - Filter rows - Y - - - change_job_status_on_error_upsert - abort_2 - Y - sort_by_recruit_id Update @@ -96,29 +81,34 @@ sort_by_source_id Y - - sort_by_source_id - Identify last row in a stream - Y - - - Identify last row in a stream - citizen_education_output - Y - citizen_education_output change_job_status_on_error_main_upsert Y - education_leave_output - change_job_status_on_error_upsert + citizen_education_output + Filter rows + Y + + + sort_by_source_id + citizen_education_output Y Update - change_job_status_on_error_upsert + change_job_status_on_error_main_upsert + Y + + + education_leave_output + change_job_status_on_error_main_upsert + Y + + + education_input + Detect empty stream Y @@ -150,8 +140,8 @@ and recruitment_id = '${IDM_ID}'; - 1824 - 256 + 1696 + 208 @@ -196,8 +186,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 624 - 256 + 432 + 208 @@ -232,8 +222,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1824 - 432 + 1696 + 320 @@ -260,25 +250,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); block_until_main_finished - 848 - 576 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 912 + 704 432 @@ -313,8 +285,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Y - 1264 - 768 + 1120 + 544 @@ -333,53 +305,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 0 - 1824 - 560 - - - - abort_2 - Abort - - Y - - 1 - - none - - - ABORT - Y - 0 - - - 1824 - 656 - - - - block_until_all_complete - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - education_leave_output - - - Update - - - - - 1600 - 432 + 1696 + 448 @@ -400,8 +327,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 1056 - 576 + 928 + 432 @@ -435,44 +362,8 @@ and recruitment_id = '${IDM_ID}'; - 1600 - 560 - - - - change_job_status_on_error_upsert - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 1600 - 656 + 1456 + 448 @@ -689,8 +580,8 @@ and recruitment_id = '${IDM_ID}'; Y - 1264 - 432 + 1120 + 320 @@ -709,7 +600,8 @@ and recruitment_id = '${IDM_ID}'; WITH filteredData AS (SELECT ri.recruit_id, education_elem, academ_elem, - 'Образовательная организация высшего образования' as educational_institution_type + 'Образовательная организация высшего образования' as educational_institution_type, + ri.ctid FROM ervu_dashboard.recruits_info ri CROSS JOIN LATERAL jsonb_array_elements( ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') AS education_elem @@ -728,7 +620,8 @@ and recruitment_id = '${IDM_ID}'; SELECT ri.recruit_id, education_elem, academ_elem, - 'Профессиональная образовательная организация' as educational_institution_type + 'Профессиональная образовательная организация' as educational_institution_type, + ri.ctid FROM ervu_dashboard.recruits_info ri CROSS JOIN LATERAL jsonb_array_elements( ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') AS education_elem @@ -747,7 +640,8 @@ and recruitment_id = '${IDM_ID}'; SELECT ri.recruit_id, education_elem, academ_elem, - 'Общеобразовательная организация' as educational_institution_type + 'Общеобразовательная организация' as educational_institution_type, + ri.ctid FROM ervu_dashboard.recruits_info ri CROSS JOIN LATERAL jsonb_array_elements( ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') AS education_elem @@ -773,7 +667,7 @@ SELECT education_elem ->> 'id' education_elem -> 'svedOrg' ->> 'naimOrg' as educational_institution_name, education_elem -> 'svedOrg' -> 'extend' ->> 'statusOrg' as educational_institution_status_code, education_elem -> 'svedOrg' -> 'extend' ->> 'vidPodrazdOrg' as educational_organization_separate_unit_type, - education_elem -> 'svedOrg' -> 'extend' ->> 'adresPodrazdOrg' as educational_organization_separate_unit_address, + education_elem -> 'svedOrg' -> 'extend' ->> 'adresPodrazdOrg' as educational_organization_separate_unit_address, education_elem ->> 'naimStatus' as student_status, case when educational_institution_type = 'Образовательная организация высшего образования' @@ -825,7 +719,8 @@ SELECT education_elem ->> 'id' to_date(academ_elem ->> 'dataNach', 'YYYY-MM-DD') as academ_start_date, to_date(academ_elem ->> 'dataOkonch', 'YYYY-MM-DD') as academ_end_date, academ_elem -> 'naimPrichOtpusk' as academ_reason, - true as has_educational_institution_info + true as has_educational_institution_info, + fd.ctid = (SELECT max(ctid) FROM filteredData) AS last_row FROM filteredData fd; Y @@ -869,22 +764,19 @@ FROM filteredData fd; N N N - Y - N ervu_dashboard Y educational_leave
- N Y N Y - 1264 - 576 + 1120 + 432 @@ -917,8 +809,8 @@ FROM filteredData fd; - 848 - 768 + 704 + 544 @@ -951,14 +843,14 @@ FROM filteredData fd; - 672 - 432 + 704 + 320 Update - change_job_status_on_error_upsert + change_job_status_on_error_main_upsert Y @@ -982,7 +874,7 @@ FROM filteredData fd; education_leave_output - change_job_status_on_error_upsert + change_job_status_on_error_main_upsert Y error_description diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl index 7367fcb..50c9116 100644 --- a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl @@ -540,13 +540,11 @@ and recruitment_id = '${IDM_ID}'; = educational_institution_learning_source_id source_id - = start_date academ_start_date - ervu_dashboard educational_leave
@@ -850,9 +848,11 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N + WITH filteredData AS (SELECT ri.recruit_id, ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz' as education_arr, - 'Образовательная организация высшего образования' as educational_institution_type + 'Образовательная организация высшего образования' as educational_institution_typ, + ri.ctid FROM ervu_dashboard.recruits_info ri WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') = 'array' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -861,7 +861,8 @@ and recruitment_id = '${IDM_ID}'; SELECT ri.recruit_id, ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh' as education_arr, - 'Профессиональная образовательная организация' as educational_institution_type + 'Профессиональная образовательная организация' as educational_institution_type, + ri.ctid FROM ervu_dashboard.recruits_info ri WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') = 'array' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -870,7 +871,8 @@ and recruitment_id = '${IDM_ID}'; SELECT ri.recruit_id, ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola' as education_arr, - 'Общеобразовательная организация' as educational_institution_type + 'Общеобразовательная организация' as educational_institution_type, + ri.ctid FROM ervu_dashboard.recruits_info ri WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') = 'array' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -940,7 +942,8 @@ SELECT education_elem ->> 'id' to_date(academ_elem ->> 'dataNach', 'YYYY-MM-DD') as academ_start_date, to_date(academ_elem ->> 'dataOkonch', 'YYYY-MM-DD') as academ_end_date, academ_elem -> 'naimPrichOtpusk' as academ_reason, - true as has_educational_institution_info + true as has_educational_institution_info, + fd.ctid = (SELECT max(ctid) FROM filteredData) AS last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(education_arr) AS education_elem LEFT JOIN LATERAL jsonb_array_elements( diff --git a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl index 712b968..b1e84d8 100644 --- a/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl @@ -51,36 +51,6 @@ block_until_main_finished Y - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - Filter_null_parental_leave_dates - Y - - - Identify last row in a stream - sort_by_recruit_id - Y - - - Identify last row in a stream - sort_by_source_id - Y - - - block_until_all_complete - Filter rows - Y - - - education_input - Identify last row in a stream - Y - sort_by_source_id Group by @@ -91,11 +61,6 @@ upsert_edu_inst_info Y - - upsert_edu_inst_info - block_until_all_complete - Y - block_until_main_finished upsert_educational_leave @@ -106,24 +71,39 @@ Change job status on error Y - - Change job status on error 2 - Abort 2 - Y - sort_by_recruit_id Update Y + + upsert_edu_inst_info + Filter rows + Y + + + education_input + sort_by_source_id + Y + + + education_input + Filter_null_parental_leave_dates + Y + + + education_input + sort_by_recruit_id + Y + upsert_educational_leave - Change job status on error 2 + Change job status on error Y Update - Change job status on error 2 + Change job status on error Y @@ -143,28 +123,8 @@ 0 - 1888 - 800 - - - - Abort 2 - Abort - - Y - - 1 - - none - - - ABORT - Y - 0 - - - 2048 - 1024 + 1936 + 640 @@ -199,44 +159,8 @@ and recruitment_id = '${IDM_ID}'; - 1664 - 800 - - - - Change job status on error 2 - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 1824 - 1024 + 1728 + 640 @@ -267,8 +191,8 @@ and recruitment_id = '${IDM_ID}'; - 2032 - 464 + 1936 + 368 @@ -322,8 +246,8 @@ ON CONFLICT (job_name, recruitment_id) - 1056 - 464 + 864 + 368 @@ -358,8 +282,8 @@ ON CONFLICT (job_name, recruitment_id) Change job status on success - 2032 - 640 + 1936 + 512 @@ -386,8 +310,8 @@ ON CONFLICT (job_name, recruitment_id) block_until_main_finished - 1152 - 912 + 1120 + 640 @@ -646,26 +570,8 @@ ON CONFLICT (job_name, recruitment_id) grp - 1472 - 640 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 1056 - 640 + 1312 + 512 @@ -699,33 +605,8 @@ ON CONFLICT (job_name, recruitment_id) Y - 1600 - 1152 - - - - block_until_all_complete - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - upsert_educational_leave - - - Update - - - - - 1888 - 640 + 1504 + 784 @@ -746,15 +627,15 @@ ON CONFLICT (job_name, recruitment_id) - 1408 - 912 + 1312 + 640 education_input TableInput - Y + N 1 @@ -766,7 +647,8 @@ ON CONFLICT (job_name, recruitment_id) WITH filteredData AS (SELECT ri.recruit_id, education_elem, academ_elem, - 'Образовательная организация высшего образования' as educational_institution_type + 'Образовательная организация высшего образования' as educational_institution_type, + ri.ctid FROM ervu_dashboard.recruits_info ri CROSS JOIN LATERAL jsonb_array_elements( ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') AS education_elem @@ -785,7 +667,8 @@ ON CONFLICT (job_name, recruitment_id) SELECT ri.recruit_id, education_elem, academ_elem, - 'Профессиональная образовательная организация' as educational_institution_type + 'Профессиональная образовательная организация' as educational_institution_type, + ri.ctid FROM ervu_dashboard.recruits_info ri CROSS JOIN LATERAL jsonb_array_elements( ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') AS education_elem @@ -804,7 +687,8 @@ ON CONFLICT (job_name, recruitment_id) SELECT ri.recruit_id, education_elem, academ_elem, - 'Общеобразовательная организация' as educational_institution_type + 'Общеобразовательная организация' as educational_institution_type, + ri.ctid FROM ervu_dashboard.recruits_info ri CROSS JOIN LATERAL jsonb_array_elements( ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') AS education_elem @@ -817,9 +701,8 @@ ON CONFLICT (job_name, recruitment_id) ) AS academ_elem ON true WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') = 'array' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))) -SELECT fd.recruit_id, - true as has_educational_institution_info, - education_elem ->> 'id' AS source_id, +SELECT education_elem ->> 'id' AS source_id, + fd.recruit_id, to_date(education_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, educational_institution_type, education_elem -> 'svedOrg' ->> 'naimTipOrg' as org_type, @@ -828,6 +711,7 @@ SELECT fd.recruit_id, coalesce(education_elem -> 'svedOrg' ->> 'ogrn', education_elem -> 'svedOrg' ->> 'ogrnip') as ogrn, education_elem -> 'svedOrg' ->> 'adresOrg' as educational_institution_address, + education_elem -> 'svedOrg' ->> 'naimOrg' as educational_institution_name, education_elem -> 'svedOrg' -> 'extend' ->> 'statusOrg' as educational_institution_status_code, education_elem -> 'svedOrg' -> 'extend' ->> 'vidPodrazdOrg' as educational_organization_separate_unit_type, education_elem -> 'svedOrg' -> 'extend' ->> 'adresPodrazdOrg' as educational_organization_separate_unit_address, @@ -881,7 +765,9 @@ SELECT fd.recruit_id, 'YYYY-MM-DD') as military_department_end_date, to_date(academ_elem ->> 'dataNach', 'YYYY-MM-DD') as academ_start_date, to_date(academ_elem ->> 'dataOkonch', 'YYYY-MM-DD') as academ_end_date, - academ_elem -> 'naimPrichOtpusk' as academ_reason + academ_elem -> 'naimPrichOtpusk' as academ_reason, + true as has_educational_institution_info, + fd.ctid = (SELECT max(ctid) FROM filteredData) AS last_row FROM filteredData fd; Y @@ -920,8 +806,8 @@ FROM filteredData fd; - 1152 - 1152 + 1120 + 784 @@ -954,8 +840,8 @@ FROM filteredData fd; - 1280 - 640 + 1120 + 512 @@ -1203,8 +1089,8 @@ FROM filteredData fd; N - 1664 - 640 + 1504 + 512 @@ -1225,7 +1111,6 @@ FROM filteredData fd; = educational_institution_learning_source_id source_id - ervu_dashboard educational_leave
@@ -1253,14 +1138,14 @@ FROM filteredData fd; N - 1600 - 912 + 1504 + 640 Update - Change job status on error 2 + Change job status on error Y error_description @@ -1284,7 +1169,7 @@ FROM filteredData fd; upsert_educational_leave - Change job status on error 2 + Change job status on error Y error_description diff --git a/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl b/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl index df91373..b5064e6 100644 --- a/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl +++ b/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl @@ -35,36 +35,16 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - citizen_education_diploma_output - Y - citizen_education_diploma_output Change job status on error Y - - foerign_diploma_input - Identify last row in a stream - Y - Detect empty stream Change job status on success Y - - Identify last row in a stream - Sort rows - Y - Sort rows Update @@ -76,12 +56,22 @@ Y - citizen_education_diploma_output - Block until transforms finish + foerign_diploma_input + citizen_education_diploma_output Y - Block until transforms finish + foerign_diploma_input + Detect empty stream + Y + + + foerign_diploma_input + Sort rows + Y + + + citizen_education_diploma_output Filter rows Y @@ -102,30 +92,8 @@ 0 - 1488 - 608 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - Update - - - - - 1280 - 432 + 1248 + 528 @@ -160,8 +128,8 @@ and recruitment_id = '${IDM_ID}'; - 1264 - 608 + 1024 + 528 @@ -192,7 +160,7 @@ and recruitment_id = '${IDM_ID}'; - 1488 + 1024 256 @@ -238,7 +206,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 736 + 544 256 @@ -274,25 +242,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1488 - 432 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 736 + 1024 432 @@ -307,24 +257,27 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); none - N ${java.io.tmpdir} + + 1000000 + + N + + Y + recruit_id Y N N 0 - recruit_id N - 1000000 - Y - 736 - 608 + 544 + 624 @@ -358,8 +311,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Y - 1024 - 608 + 768 + 624 @@ -452,7 +405,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Y - 1024 + 768 432 @@ -460,7 +413,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); foerign_diploma_input TableInput - Y + N 1 @@ -471,7 +424,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); N WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr + ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr, + ctid FROM ervu_dashboard.recruits_info ri WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr') = 'array' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -491,7 +445,8 @@ SELECT fd.recruit_id, diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, diploma_elem ->> 'kodOKSM' as oksm_code, coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec, - true as has_foreign_diploma + true as has_foreign_diploma, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem; diff --git a/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl b/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl index 445a39d..841a100 100644 --- a/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl @@ -35,21 +35,6 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - foreign_diploma_upsert - Y - - - foreign_diploma_input - Identify last row in a stream - Y - foreign_diploma_upsert Change job status on error @@ -71,20 +56,20 @@ Y - Identify last row in a stream + foreign_diploma_input + Detect empty stream + Y + + + foreign_diploma_input + foreign_diploma_upsert + Y + + + foreign_diploma_input Sort rows Y - - foreign_diploma_upsert - Block until transforms finish - Y - - - Block until transforms finish - Filter rows - Y - Abort @@ -102,32 +87,10 @@ 0 - 1472 + 1328 688 - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - Update - - - - - 1232 - 496 - - Change job status on error ExecSql @@ -160,7 +123,7 @@ and recruitment_id = '${IDM_ID}'; - 1280 + 1136 688 @@ -192,7 +155,7 @@ and recruitment_id = '${IDM_ID}'; - 1472 + 1136 320 @@ -241,7 +204,7 @@ and recruitment_id = '${IDM_ID}'; - 768 + 576 320 @@ -277,25 +240,7 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1472 - 496 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 768 + 1136 496 @@ -310,23 +255,26 @@ and recruitment_id = '${IDM_ID}'; none - N ${java.io.tmpdir} + + 1000000 + + N + + Y + recruit_id Y N N 0 - recruit_id N - 1000000 - Y - 768 + 576 688 @@ -361,7 +309,7 @@ and recruitment_id = '${IDM_ID}'; Y - 992 + 832 688 @@ -369,7 +317,7 @@ and recruitment_id = '${IDM_ID}'; foreign_diploma_input TableInput - Y + N 1 @@ -378,12 +326,13 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr + ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr, + ctid FROM ervu_dashboard.recruits_info ri WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr') = 'array' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) +) SELECT fd.recruit_id, diploma_elem ->> 'id' AS source_id, to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, @@ -399,7 +348,8 @@ SELECT fd.recruit_id, diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, diploma_elem ->> 'kodOKSM' as oksm_code, coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec, - true as has_foreign_diploma + true as has_foreign_diploma, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem where case @@ -514,7 +464,7 @@ where case N - 992 + 832 496 diff --git a/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl index 3aa0f3a..28b2db2 100644 --- a/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl @@ -35,21 +35,6 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - - - foerign_diploma_input - Identify last row in a stream - Y - - - Identify last row in a stream - foreign_diploma_upsert - Y - foreign_diploma_upsert Change job status on error @@ -65,23 +50,28 @@ Update Y - - Identify last row in a stream - Sort rows - Y - Update Change job status on error Y - foreign_diploma_upsert - Block until transforms finish + foerign_diploma_input + Sort rows Y - Block until transforms finish + foerign_diploma_input + Detect empty stream + Y + + + foerign_diploma_input + foreign_diploma_upsert + Y + + + foreign_diploma_upsert Filter rows Y @@ -102,32 +92,10 @@ 0 - 1488 + 1216 608 - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - Update - - - - - 1248 - 448 - - Change job status on error ExecSql @@ -160,7 +128,7 @@ and recruitment_id = '${IDM_ID}'; - 1232 + 960 608 @@ -192,7 +160,7 @@ and recruitment_id = '${IDM_ID}'; - 1488 + 960 272 @@ -247,7 +215,7 @@ ON CONFLICT (job_name, recruitment_id) - 736 + 544 272 @@ -283,25 +251,7 @@ ON CONFLICT (job_name, recruitment_id) Change job status on success - 1488 - 448 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 736 + 960 448 @@ -316,23 +266,26 @@ ON CONFLICT (job_name, recruitment_id) none - N ${java.io.tmpdir} + + 1000000 + + N + + Y + recruit_id Y N N 0 - recruit_id N - 1000000 - Y - 736 + 544 608 @@ -367,7 +320,7 @@ ON CONFLICT (job_name, recruitment_id) Y - 960 + 720 608 @@ -375,7 +328,7 @@ ON CONFLICT (job_name, recruitment_id) foerign_diploma_input TableInput - Y + N 1 @@ -384,9 +337,9 @@ ON CONFLICT (job_name, recruitment_id) ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr + ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr, + ctid FROM ervu_dashboard.recruits_info ri WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr') = 'array' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -396,8 +349,8 @@ SELECT fd.recruit_id, to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, diploma_elem ->> 'kodUrObr' as education_level_code, diploma_elem ->> 'naimUchZaved' as educational_institution_name, - diploma_elem ->> 'naimInoDok' as foreign_doc_name, - diploma_elem ->> 'serNomDok' as ser_num_doc, + diploma_elem ->> 'naimInoDok' as foreign_doc_name, + diploma_elem ->> 'serNomDok' as ser_num_doc, diploma_elem ->> 'regNomDok' as reg_num, diploma_elem ->> 'nomSved' as recognition_certificate_number, to_date(diploma_elem ->> 'dataPriz', 'YYYY-MM-DD') as recognition_date, @@ -406,7 +359,8 @@ SELECT fd.recruit_id, diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, diploma_elem ->> 'kodOKSM' as oksm_code, coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec, - true as has_foreign_diploma + true as has_foreign_diploma, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem; @@ -517,7 +471,7 @@ FROM filteredData fd N - 960 + 736 448 diff --git a/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl b/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl index 0fcbc6d..40b9aee 100644 --- a/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl +++ b/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl @@ -35,26 +35,11 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - citizen_education_diploma_output Change job status on error Y - - education_input - Identify last row in a stream - Y - - - Identify last row in a stream - citizen_education_diploma_output - Y - Detect empty stream Change job status on success @@ -65,23 +50,28 @@ Update Y - - Identify last row in a stream - Sort rows - Y - Update Change job status on error Y - citizen_education_diploma_output - Block until transforms finish + education_input + Detect empty stream Y - Block until transforms finish + education_input + citizen_education_diploma_output + Y + + + education_input + Sort rows + Y + + + citizen_education_diploma_output Filter rows Y @@ -102,32 +92,10 @@ 0 - 1328 + 1200 560 - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - Update - - - - - 1152 - 400 - - Change job status on error ExecSql @@ -160,7 +128,7 @@ and recruitment_id = '${IDM_ID}'; - 1088 + 960 560 @@ -192,7 +160,7 @@ and recruitment_id = '${IDM_ID}'; - 1328 + 960 224 @@ -238,7 +206,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 640 + 448 224 @@ -274,25 +242,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1328 - 400 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 640 + 960 400 @@ -307,23 +257,26 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); none - N ${java.io.tmpdir} + + 1000000 + + N + + Y + recruit_id Y N N 0 - recruit_id N - 1000000 - Y - 640 + 448 560 @@ -358,7 +311,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Y - 832 + 720 560 @@ -456,7 +409,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Y - 928 + 720 400 @@ -464,7 +417,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); education_input TableInput - Y + N 1 @@ -475,7 +428,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); N WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr + ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr, + ctid FROM ervu_dashboard.recruits_info ri WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedRObr' -> 'robr') = 'array' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -496,7 +450,8 @@ SELECT fd.recruit_id, diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec, - true as has_russian_diploma + true as has_russian_diploma, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem; Y diff --git a/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl b/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl index f25b941..a39f17c 100644 --- a/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl @@ -51,21 +51,6 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - Insert / update - Y - - - education_input - Identify last row in a stream - Y - Insert / update Change job status on error @@ -81,11 +66,6 @@ Update Y - - Identify last row in a stream - Sort rows - Y - Update Change job status on error @@ -96,6 +76,21 @@ Filter rows Y + + education_input + Detect empty stream + Y + + + education_input + Insert / update + Y + + + education_input + Sort rows + Y + Abort @@ -113,7 +108,7 @@ 0 - 1472 + 1328 656 @@ -149,7 +144,7 @@ and recruitment_id = '${IDM_ID}'; - 1248 + 1120 656 @@ -181,7 +176,7 @@ and recruitment_id = '${IDM_ID}'; - 1472 + 1120 288 @@ -230,7 +225,7 @@ and recruitment_id = '${IDM_ID}'; - 880 + 688 288 @@ -266,25 +261,7 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1472 - 464 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 880 + 1120 464 @@ -393,7 +370,7 @@ and recruitment_id = '${IDM_ID}'; N - 1248 + 912 464 @@ -427,7 +404,7 @@ and recruitment_id = '${IDM_ID}'; - 880 + 688 656 @@ -462,7 +439,7 @@ and recruitment_id = '${IDM_ID}'; Y - 1072 + 912 656 @@ -470,7 +447,7 @@ and recruitment_id = '${IDM_ID}'; education_input TableInput - Y + N 1 @@ -479,13 +456,13 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - -WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' as diploma_arr + + WITH filteredData AS (SELECT ri.recruit_id, + ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr, + ctid FROM ervu_dashboard.recruits_info ri WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedRObr' -> 'robr') = 'array' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - ) SELECT fd.recruit_id, diploma_elem ->> 'id' AS source_id, @@ -503,7 +480,8 @@ SELECT fd.recruit_id, diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec, - true as has_russian_diploma + true as has_russian_diploma, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem where case diff --git a/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl index 753a286..64eec28 100644 --- a/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl @@ -41,21 +41,6 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - - - education_input - Identify last row in a stream - Y - - - Identify last row in a stream - Insert / update - Y - Insert / update Change job status on error @@ -71,23 +56,28 @@ Update Y - - Identify last row in a stream - Sort rows - Y - Update Change job status on error Y - Insert / update - Block until transforms finish + education_input + Detect empty stream Y - Block until transforms finish + education_input + Insert / update + Y + + + education_input + Sort rows + Y + + + Insert / update Filter rows Y @@ -108,32 +98,10 @@ 0 - 1216 + 1184 560 - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - Update - - - - - 1040 - 384 - - Change job status on error ExecSql @@ -166,7 +134,7 @@ and recruitment_id = '${IDM_ID}'; - 992 + 944 560 @@ -198,7 +166,7 @@ and recruitment_id = '${IDM_ID}'; - 1216 + 944 208 @@ -253,7 +221,7 @@ ON CONFLICT (job_name, recruitment_id) - 624 + 432 208 @@ -289,25 +257,7 @@ ON CONFLICT (job_name, recruitment_id) Change job status on success - 1216 - 384 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 624 + 944 384 @@ -416,7 +366,7 @@ ON CONFLICT (job_name, recruitment_id) N - 864 + 672 384 @@ -450,7 +400,7 @@ ON CONFLICT (job_name, recruitment_id) - 624 + 432 560 @@ -468,13 +418,11 @@ ON CONFLICT (job_name, recruitment_id) 10000 ervu-dashboard N - = recruit_id recruit_id - ervu_dashboard citizen
@@ -487,7 +435,7 @@ ON CONFLICT (job_name, recruitment_id) Y - 816 + 672 560 @@ -495,7 +443,7 @@ ON CONFLICT (job_name, recruitment_id) education_input TableInput - Y + N 1 @@ -504,8 +452,10 @@ ON CONFLICT (job_name, recruitment_id) ervu-dashboard N + WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr + ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr, + ctid FROM ervu_dashboard.recruits_info ri WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedRObr' -> 'robr') = 'array' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -526,7 +476,8 @@ SELECT fd.recruit_id, diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code, diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code, coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec, - true as has_russian_diploma + true as has_russian_diploma, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem; Y diff --git a/mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_delta.hpl b/mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_delta.hpl index 11b1f39..8ea9313 100644 --- a/mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_delta.hpl +++ b/mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_delta.hpl @@ -41,43 +41,33 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - election_candidate_flag_update Change job status on error Y - - election_candidate_input - Identify last row in a stream - Y - - - Identify last row in a stream - election_candidate_upsert - Y - election_candidate_upsert Change job status on error Y - Identify last row in a stream + election_candidate_input + Detect empty stream + Y + + + election_candidate_input + election_candidate_upsert + Y + + + election_candidate_input election_candidate_flag_update Y election_candidate_upsert - Block until transforms finish - Y - - - Block until transforms finish Filter rows Y @@ -98,30 +88,8 @@ 0 - 1296 - 688 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - election_candidate_upsert - - - - - 1104 - 416 + 976 + 672 @@ -156,8 +124,8 @@ and recruitment_id = '${IDM_ID}'; - 896 - 688 + 704 + 672 @@ -188,7 +156,7 @@ and recruitment_id = '${IDM_ID}'; - 1296 + 976 240 @@ -237,7 +205,7 @@ and recruitment_id = '${IDM_ID}'; - 624 + 432 240 @@ -273,25 +241,7 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1296 - 416 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 624 + 976 416 @@ -331,15 +281,15 @@ and recruitment_id = '${IDM_ID}'; Y - 624 - 688 + 432 + 672 election_candidate_input TableInput - Y + N 1 @@ -350,7 +300,8 @@ and recruitment_id = '${IDM_ID}'; N WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem + ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem, + ctid FROM ervu_dashboard.recruits_info ri WHERE ri.info -> 'svedFL' -> 'svedKandidat' ->> 'kandidat' <> 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -367,7 +318,8 @@ SELECT fd.recruit_id, to_date(deputy_elem ->> 'dataReg', 'YYYY-MM-DD') AS registration_date, deputy_elem ->> 'prichinaSnyatNaim' AS termination_reason, to_date(deputy_elem ->> 'dataSnyatKandidat', 'YYYY-MM-DD') AS termination_date, - true as has_election_candidate_info + true as has_election_candidate_info, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd; Y @@ -441,7 +393,7 @@ FROM filteredData fd; N - 896 + 704 416 diff --git a/mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_flow.hpl b/mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_flow.hpl index 038ec8e..3b42fa0 100644 --- a/mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_flow.hpl +++ b/mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_flow.hpl @@ -46,16 +46,6 @@ Change job status on error Y - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - election_candidate_output - Y - Detect empty stream Change job status on success @@ -68,24 +58,24 @@ election_candidate_input - Identify last row in a stream + Detect empty stream + Y + + + election_candidate_input + election_candidate_output + Y + + + election_candidate_input + election_candidate_flag_update Y election_candidate_output - Block until transforms finish - Y - - - Block until transforms finish Filter rows Y - - Identify last row in a stream - election_candidate_flag_update - Y - Abort @@ -103,32 +93,10 @@ 0 - 1248 + 1072 592 - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - election_candidate_flag_update - - - - - 1248 - 416 - - Change job status on error ExecSql @@ -161,7 +129,7 @@ and recruitment_id = '${IDM_ID}'; - 976 + 832 592 @@ -193,7 +161,7 @@ and recruitment_id = '${IDM_ID}'; - 1504 + 1072 240 @@ -239,7 +207,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 704 + 512 240 @@ -275,25 +243,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1504 - 416 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 704 + 1072 416 @@ -328,7 +278,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Y - 704 + 512 592 @@ -336,7 +286,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); election_candidate_input TableInput - Y + N 1 @@ -347,7 +297,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); N WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem + ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem, + ctid FROM ervu_dashboard.recruits_info ri WHERE ri.info -> 'svedFL' -> 'svedKandidat' ->> 'kandidat' <> 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -360,7 +311,8 @@ SELECT fd.recruit_id, to_date(deputy_elem ->> 'dataReg', 'YYYY-MM-DD') AS registration_date, deputy_elem ->> 'prichinaSnyatNaim' AS termination_reason, to_date(deputy_elem ->> 'dataSnyatKandidat', 'YYYY-MM-DD') AS termination_date, - true as has_election_candidate_info + true as has_election_candidate_info, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd; Y @@ -431,7 +383,7 @@ FROM filteredData fd; Y - 976 + 832 416 diff --git a/mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_repeat.hpl b/mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_repeat.hpl index 0762ce4..e1a34e8 100644 --- a/mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_repeat.hpl @@ -46,21 +46,11 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - election_candidate_flag_update Change job status on error Y - - Identify last row in a stream - election_candidate_upsert - Y - election_candidate_upsert Change job status on error @@ -73,22 +63,22 @@ election_candidate_input - Identify last row in a stream + Detect empty stream Y - Identify last row in a stream - election_candidate_flag_update + election_candidate_input + election_candidate_upsert Y election_candidate_upsert - Block until transforms finish + Filter rows Y - Block until transforms finish - Filter rows + election_candidate_input + election_candidate_flag_update Y @@ -108,32 +98,10 @@ 0 - 1584 + 1056 688 - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - election_candidate_flag_update - - - - - 1408 - 512 - - Change job status on error ExecSql @@ -166,7 +134,7 @@ and recruitment_id = '${IDM_ID}'; - 1152 + 848 688 @@ -198,7 +166,7 @@ and recruitment_id = '${IDM_ID}'; - 1584 + 1056 336 @@ -247,7 +215,7 @@ and recruitment_id = '${IDM_ID}'; - 784 + 576 336 @@ -283,25 +251,7 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1584 - 512 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 784 + 1056 512 @@ -336,7 +286,7 @@ and recruitment_id = '${IDM_ID}'; Y - 784 + 576 688 @@ -344,7 +294,7 @@ and recruitment_id = '${IDM_ID}'; election_candidate_input TableInput - Y + N 1 @@ -355,7 +305,8 @@ and recruitment_id = '${IDM_ID}'; N WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem + ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem, + ctid FROM ervu_dashboard.recruits_info ri WHERE ri.info -> 'svedFL' -> 'svedKandidat' ->> 'kandidat' <> 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -368,7 +319,8 @@ SELECT fd.recruit_id, to_date(deputy_elem ->> 'dataReg', 'YYYY-MM-DD') AS registration_date, deputy_elem ->> 'prichinaSnyatNaim' AS termination_reason, to_date(deputy_elem ->> 'dataSnyatKandidat', 'YYYY-MM-DD') AS termination_date, - true as has_election_candidate_info + true as has_election_candidate_info, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd; Y @@ -442,7 +394,7 @@ FROM filteredData fd; N - 1152 + 848 512 diff --git a/mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow.hpl b/mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow.hpl index 3230af7..51799db 100644 --- a/mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow.hpl +++ b/mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow.hpl @@ -46,44 +46,39 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - disability_output - Y - disability_output Change job status on error Y - - disability_input - Identify last row in a stream - Y - Detect empty stream Change job status on success Y - disability_output + has_disability_update + Change job status on error + Y + + + disability_input has_disability_update Y - has_disability_update - Filter rows + disability_input + disability_output Y - has_disability_update - Change job status on error + disability_input + Detect empty stream + Y + + + disability_output + Filter rows Y @@ -103,7 +98,7 @@ 0 - 1504 + 960 576 @@ -139,7 +134,7 @@ and recruitment_id = '${IDM_ID}'; - 1216 + 736 576 @@ -171,7 +166,7 @@ and recruitment_id = '${IDM_ID}'; - 1504 + 960 240 @@ -217,7 +212,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 704 + 512 240 @@ -253,25 +248,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1504 - 416 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 704 + 960 416 @@ -279,7 +256,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); disability_input TableInput - Y + N 1 @@ -290,7 +267,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); N WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' AS invalid_elem + ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' AS invalid_elem, + ctid FROM ervu_dashboard.recruits_info ri WHERE ri.info -> 'svedFL' -> 'svedInvalid' ->> 'invalid' <> 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -304,7 +282,8 @@ SELECT fd.recruit_id, to_date(invalid_elem -> 'ustanInvalid' ->> 'dataPodtverzhdPlan', 'YYYY-MM-DD') as confirmation_date, to_date(invalid_elem ->> 'dataSnyat', 'YYYY-MM-DD') as deregistration_date, coalesce(invalid_elem -> 'ustanInvalid' ->> 'prBessrInvl' = '1', false) as permanent, - true as has_disability + true as has_disability, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd; Y @@ -379,7 +358,7 @@ FROM filteredData fd; Y - 992 + 736 416 @@ -414,8 +393,8 @@ FROM filteredData fd; Y - 1216 - 416 + 512 + 576 diff --git a/mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_delta.hpl b/mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_delta.hpl index 794da48..1a1d901 100644 --- a/mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_delta.hpl @@ -41,16 +41,6 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - disability_upsert - Y - disability_upsert Change job status on error @@ -61,16 +51,6 @@ Change job status on success Y - - disability_upsert - has_disability_update - Y - - - has_disability_update - Filter rows - Y - has_disability_update Change job status on error @@ -78,7 +58,22 @@ disability_input - Identify last row in a stream + has_disability_update + Y + + + disability_input + Detect empty stream + Y + + + disability_input + disability_upsert + Y + + + disability_upsert + Filter rows Y @@ -98,7 +93,7 @@ 0 - 1968 + 1472 560 @@ -134,7 +129,7 @@ and recruitment_id = '${IDM_ID}'; - 1744 + 1280 560 @@ -166,7 +161,7 @@ and recruitment_id = '${IDM_ID}'; - 1968 + 1472 208 @@ -215,7 +210,7 @@ and recruitment_id = '${IDM_ID}'; - 1264 + 1040 208 @@ -251,25 +246,7 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1968 - 384 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 1264 + 1472 384 @@ -277,7 +254,7 @@ and recruitment_id = '${IDM_ID}'; disability_input TableInput - Y + N 1 @@ -288,7 +265,8 @@ and recruitment_id = '${IDM_ID}'; N WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' AS invalid_elem + ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' AS invalid_elem, + ctid FROM ervu_dashboard.recruits_info ri WHERE ri.info -> 'svedFL' -> 'svedInvalid' ->> 'invalid' <> 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -308,7 +286,8 @@ SELECT fd.recruit_id, to_date(invalid_elem -> 'ustanInvalid' ->> 'dataPodtverzhdPlan', 'YYYY-MM-DD') as confirmation_date, to_date(invalid_elem ->> 'dataSnyat', 'YYYY-MM-DD') as deregistration_date, coalesce(invalid_elem -> 'ustanInvalid' ->> 'prBessrInvl' = '1', false) as permanent, - true as has_disability + true as has_disability, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd; Y @@ -387,7 +366,7 @@ FROM filteredData fd; N - 1552 + 1280 384 @@ -422,8 +401,8 @@ FROM filteredData fd; Y - 1744 - 384 + 1040 + 560 diff --git a/mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_repeat.hpl index 59150de..0ae7ff4 100644 --- a/mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_repeat.hpl @@ -35,21 +35,6 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - - - disability_input - Identify last row in a stream - Y - - - Identify last row in a stream - disability_upsert - Y - disability_upsert Change job status on error @@ -61,18 +46,28 @@ Y - disability_upsert + has_disability_update + Change job status on error + Y + + + disability_input has_disability_update Y - has_disability_update - Filter rows + disability_input + Detect empty stream Y - has_disability_update - Change job status on error + disability_input + disability_upsert + Y + + + disability_upsert + Filter rows Y @@ -92,7 +87,7 @@ 0 - 1504 + 1104 592 @@ -128,7 +123,7 @@ and recruitment_id = '${IDM_ID}'; - 1120 + 896 592 @@ -160,7 +155,7 @@ and recruitment_id = '${IDM_ID}'; - 1504 + 1104 256 @@ -209,7 +204,7 @@ and recruitment_id = '${IDM_ID}'; - 832 + 640 256 @@ -245,25 +240,7 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1504 - 432 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 832 + 1104 432 @@ -271,7 +248,7 @@ and recruitment_id = '${IDM_ID}'; disability_input TableInput - Y + N 1 @@ -282,9 +259,10 @@ and recruitment_id = '${IDM_ID}'; N WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' AS invalid_elem + ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' AS invalid_elem, + ctid FROM ervu_dashboard.recruits_info ri - WHERE ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' <> 'null' + WHERE ri.info -> 'svedFL' -> 'svedInvalid' ->> 'invalid' <> 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) ) SELECT fd.recruit_id, @@ -295,7 +273,9 @@ SELECT fd.recruit_id, to_date(invalid_elem -> 'ustanInvalid' ->> 'dataUstan', 'YYYY-MM-DD') as register_date, to_date(invalid_elem -> 'ustanInvalid' ->> 'dataPodtverzhdPlan', 'YYYY-MM-DD') as confirmation_date, to_date(invalid_elem ->> 'dataSnyat', 'YYYY-MM-DD') as deregistration_date, - coalesce(invalid_elem -> 'ustanInvalid' ->> 'prBessrInvl' = '1', false) as permanent + coalesce(invalid_elem -> 'ustanInvalid' ->> 'prBessrInvl' = '1', false) as permanent, + true as has_disability, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd; Y @@ -374,7 +354,7 @@ FROM filteredData fd; N - 1120 + 896 432 @@ -409,8 +389,8 @@ FROM filteredData fd; Y - 1312 - 432 + 640 + 592 diff --git a/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow.hpl b/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow.hpl index e6f4a05..993ffa5 100644 --- a/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow.hpl +++ b/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow.hpl @@ -35,16 +35,6 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - health_state_output - Y - health_state_output Change job status on error @@ -61,13 +51,13 @@ Y - Update - Change job status on error 2 + health_state_input + Filter rows 2 Y - Update 2 - Block until transforms finish + Filter rows 2 + health_state_output Y @@ -76,23 +66,18 @@ Y - Block until transforms finish + Update + Change job status on error + Y + + + Update 2 Filter rows Y health_state_input - Filter rows 2 - Y - - - Filter rows 2 - Identify last row in a stream - Y - - - Change job status on error 2 - Abort 2 + Detect empty stream Y @@ -112,52 +97,10 @@ 0 - 1472 + 1440 672 - - Abort 2 - Abort - - Y - - 1 - - none - - - ABORT - Y - 0 - - - 928 - 768 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - Update - - - - - 1584 - 464 - - Change job status on error ExecSql @@ -190,46 +133,10 @@ and recruitment_id = '${IDM_ID}'; - 1248 + 1104 672 - - Change job status on error 2 - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'DELTA_ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 752 - 768 - - Change job status on success ExecSql @@ -258,8 +165,8 @@ and recruitment_id = '${IDM_ID}'; - 1744 - 288 + 1424 + 304 @@ -304,8 +211,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 960 - 288 + 560 + 304 @@ -340,7 +247,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1744 + 1424 464 @@ -374,31 +281,13 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Update - Identify last row in a stream + health_state_output 752 464 - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 960 - 464 - - Update Update @@ -431,7 +320,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 752 - 624 + 672 @@ -465,7 +354,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Y - 1392 + 1248 464 @@ -473,7 +362,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); health_state_input TableInput - Y + N 1 @@ -482,12 +371,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved' as source_update_date, ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov' as zdorov_arr, - coalesce(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'priznakNalichSved' = - '1', false) as has_health_state_info + coalesce(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'priznakNalichSved' = '1', false) as has_health_state_info, + ctid FROM ervu_dashboard.recruits_info ri WHERE ri.info -> 'svedFL' -> 'svedSostZdorov' ->> 'svedZdorov' <> 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -498,7 +386,8 @@ SELECT fd.recruit_id, source_update_date, zdorov_elem ->> 'kodZabol' AS disease_code, to_date(zdorov_elem ->> 'dataDiagn', 'YYYY-MM-DD') AS diagnosis_date, - coalesce(zdorov_elem ->> 'kodZabol' <> 'null', false) as has_disease + coalesce(zdorov_elem ->> 'kodZabol' <> 'null', false) as has_disease, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(COALESCE( NULLIF(zdorov_arr, 'null'::jsonb), @@ -560,14 +449,14 @@ FROM filteredData fd Y - 1248 + 1104 464 Update - Change job status on error 2 + Change job status on error Y error_description diff --git a/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_delta.hpl b/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_delta.hpl index 4f30cf6..5d21209 100644 --- a/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_delta.hpl @@ -505,8 +505,8 @@ and recruitment_id = '${IDM_ID}'; WITH filteredData AS (SELECT ri.recruit_id, ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved' as source_update_date, ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov' as zdorov_arr, - coalesce(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'priznakNalichSved' = - '1', false) as has_health_state_info + coalesce(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'priznakNalichSved' = '1', false) as has_health_state_info, + ctid FROM ervu_dashboard.recruits_info ri WHERE ri.info -> 'svedFL' -> 'svedSostZdorov' ->> 'svedZdorov' <> 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -523,7 +523,8 @@ SELECT fd.recruit_id, source_update_date, zdorov_elem ->> 'kodZabol' AS disease_code, to_date(zdorov_elem ->> 'dataDiagn', 'YYYY-MM-DD') AS diagnosis_date, - coalesce(zdorov_elem ->> 'kodZabol' <> 'null', false) as has_disease + coalesce(zdorov_elem ->> 'kodZabol' <> 'null', false) as has_disease, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(COALESCE( NULLIF(zdorov_arr, 'null'::jsonb), diff --git a/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_repeat.hpl index 0935353..5cf4ad9 100644 --- a/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_repeat.hpl @@ -46,16 +46,6 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - health_state_upsert - Y - health_state_upsert Change job status on error @@ -66,31 +56,11 @@ Change job status on success Y - - Change job status on error 2 - Abort 2 - Y - - - Update - Change job status on error 2 - Y - - - Update 2 - Block until transforms finish - Y - health_state_upsert Update 2 Y - - Block until transforms finish - Filter rows - Y - health_state_input Filter rows 2 @@ -98,12 +68,27 @@ Filter rows 2 - Identify last row in a stream + Update + Y + + + Update 2 + Filter rows Y Filter rows 2 - Update + health_state_upsert + Y + + + Update + Change job status on error + Y + + + health_state_input + Detect empty stream Y @@ -123,50 +108,8 @@ 0 - 1696 - 640 - - - - Abort 2 - Abort - - Y - - 1 - - none - - - ABORT - Y - 0 - - - 848 - 752 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - Update - - - - - 1488 - 448 + 1184 + 608 @@ -201,44 +144,8 @@ and recruitment_id = '${IDM_ID}'; - 1136 - 640 - - - - Change job status on error 2 - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'DELTA_ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 640 - 752 + 832 + 608 @@ -269,8 +176,8 @@ and recruitment_id = '${IDM_ID}'; - 1696 - 272 + 1184 + 288 @@ -318,8 +225,8 @@ and recruitment_id = '${IDM_ID}'; - 848 - 272 + 464 + 288 @@ -354,7 +261,7 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1696 + 1184 448 @@ -388,31 +295,13 @@ and recruitment_id = '${IDM_ID}'; Update - Identify last row in a stream + health_state_upsert 640 448 - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 848 - 448 - - Update Update @@ -483,7 +372,7 @@ and recruitment_id = '${IDM_ID}'; Y - 1296 + 992 448 @@ -491,7 +380,7 @@ and recruitment_id = '${IDM_ID}'; health_state_input TableInput - Y + N 1 @@ -502,10 +391,10 @@ and recruitment_id = '${IDM_ID}'; N WITH filteredData AS (SELECT ri.recruit_id, - to_date(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, + ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved' as source_update_date, ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov' as zdorov_arr, - coalesce(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'priznakNalichSved' = - '1', false) as has_health_state_info + coalesce(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'priznakNalichSved' = '1', false) as has_health_state_info, + ctid FROM ervu_dashboard.recruits_info ri WHERE ri.info -> 'svedFL' -> 'svedSostZdorov' ->> 'svedZdorov' <> 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -516,7 +405,8 @@ SELECT fd.recruit_id, source_update_date, zdorov_elem ->> 'kodZabol' AS disease_code, to_date(zdorov_elem ->> 'dataDiagn', 'YYYY-MM-DD') AS diagnosis_date, - coalesce(zdorov_elem ->> 'kodZabol' <> 'null', false) as has_disease + coalesce(zdorov_elem ->> 'kodZabol' <> 'null', false) as has_disease, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(COALESCE( NULLIF(zdorov_arr, 'null'::jsonb), @@ -578,14 +468,14 @@ FROM filteredData fd N - 1136 + 832 448 Update - Change job status on error 2 + Change job status on error Y error_description diff --git a/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow.hpl b/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow.hpl index f85755a..0d562e4 100644 --- a/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow.hpl +++ b/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow.hpl @@ -35,54 +35,44 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - incapacity_output - Y - incapacity_output Change job status on error Y - - incapacity_input - Identify last row in a stream - Y - Detect empty stream Change job status on success Y - - incapacity_output - Block until transforms finish - Y - - - Block until transforms finish - Filter rows - Y - Update Change job status on error Y - Identify last row in a stream + Sort rows + Update + Y + + + incapacity_input + Detect empty stream + Y + + + incapacity_input + incapacity_output + Y + + + incapacity_input Sort rows Y - Sort rows - Update + incapacity_output + Filter rows Y @@ -102,30 +92,8 @@ 0 - 1424 - 560 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - Update - - - - - 1168 - 416 + 1152 + 576 @@ -160,8 +128,8 @@ and recruitment_id = '${IDM_ID}'; - 1136 - 560 + 960 + 576 @@ -192,7 +160,7 @@ and recruitment_id = '${IDM_ID}'; - 1424 + 1152 240 @@ -238,7 +206,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 704 + 512 240 @@ -274,25 +242,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1424 - 416 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 704 + 1152 416 @@ -326,8 +276,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 704 - 560 + 512 + 576 @@ -361,15 +311,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Y - 912 - 560 + 720 + 576 incapacity_input TableInput - Y + N 1 @@ -378,10 +328,10 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - WITH filteredData AS ( SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob' as nedeesposob_arr + ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob' as nedeesposob_arr, + ctid FROM ervu_dashboard.recruits_info ri WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob') = 'array' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -393,7 +343,8 @@ SELECT fd.recruit_id, nedeesposob_elem ->> 'naimOrg' AS court_name, to_date(nedeesposob_elem ->> 'dataPrisv', 'YYYY-MM-DD') AS register_date, to_date(nedeesposob_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS deregistration_date, - true as has_incapacity_info + true as has_incapacity_info, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(fd.nedeesposob_arr) AS nedeesposob_elem; Y @@ -461,7 +412,7 @@ FROM filteredData fd Y - 912 + 720 416 diff --git a/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_delta.hpl b/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_delta.hpl index f483f91..d012594 100644 --- a/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_delta.hpl @@ -36,21 +36,6 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - incapacity_upsert - Y - - - incapacity_input - Identify last row in a stream - Y - incapacity_upsert Change job status on error @@ -67,23 +52,28 @@ Y - incapacity_upsert - Block until transforms finish + Sort rows + Update Y - Block until transforms finish - Filter rows + incapacity_input + incapacity_upsert Y - Identify last row in a stream + incapacity_input + Detect empty stream + Y + + + incapacity_input Sort rows Y - Sort rows - Update + incapacity_upsert + Filter rows Y @@ -103,32 +93,10 @@ 0 - 1888 + 1696 560 - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - Update - - - - - 1744 - 384 - - Change job status on error ExecSql @@ -161,7 +129,7 @@ and recruitment_id = '${IDM_ID}'; - 1568 + 1440 560 @@ -193,7 +161,7 @@ and recruitment_id = '${IDM_ID}'; - 1888 + 1696 208 @@ -242,7 +210,7 @@ and recruitment_id = '${IDM_ID}'; - 1232 + 1040 208 @@ -278,25 +246,7 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1888 - 384 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 1232 + 1696 384 @@ -311,23 +261,26 @@ and recruitment_id = '${IDM_ID}'; none - N ${java.io.tmpdir} + + 1000000 + + N + + Y + recruit_id Y N N 0 - recruit_id N - 1000000 - Y - 1232 + 1040 560 @@ -362,7 +315,7 @@ and recruitment_id = '${IDM_ID}'; Y - 1392 + 1200 560 @@ -370,7 +323,7 @@ and recruitment_id = '${IDM_ID}'; incapacity_input TableInput - Y + N 1 @@ -381,19 +334,21 @@ and recruitment_id = '${IDM_ID}'; N WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob' as nedeesposob_arr + ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob' as nedeesposob_arr, + ctid FROM ervu_dashboard.recruits_info ri WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob') = 'array' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) ) SELECT fd.recruit_id, - nedeesposob_elem ->> 'id' AS source_id, + nedeesposob_elem ->> 'id' AS source_id, to_date(nedeesposob_elem ->> 'dataSved', 'YYYY-MM-DD') AS source_update_date, nedeesposob_elem ->> 'naimStatus' AS capacity_status, nedeesposob_elem ->> 'naimOrg' AS court_name, to_date(nedeesposob_elem ->> 'dataPrisv', 'YYYY-MM-DD') AS register_date, to_date(nedeesposob_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS deregistration_date, - true as has_incapacity_info + true as has_incapacity_info, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(nedeesposob_arr) AS nedeesposob_elem WHERE case when '${MAX_SOURCE_UPDATE_DATE}' <> '' @@ -467,7 +422,7 @@ WHERE case when '${MAX_SOURCE_UPDATE_DATE}' <> '' N - 1568 + 1440 384 diff --git a/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_repeat.hpl index f7c7971..fb9fd74 100644 --- a/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_repeat.hpl @@ -35,21 +35,6 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - - - incapacity_input - Identify last row in a stream - Y - - - Identify last row in a stream - incapacity_upsert - Y - incapacity_upsert Change job status on error @@ -66,23 +51,28 @@ Y - incapacity_upsert - Block until transforms finish + Sort rows + Update Y - Block until transforms finish - Filter rows - Y - - - Identify last row in a stream + incapacity_input Sort rows Y - Sort rows - Update + incapacity_input + Detect empty stream + Y + + + incapacity_input + incapacity_upsert + Y + + + incapacity_upsert + Filter rows Y @@ -101,31 +91,9 @@ Y 0 - - 1424 - 640 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - Update - - - 1280 - 432 + 592 @@ -160,8 +128,8 @@ and recruitment_id = '${IDM_ID}'; - 1120 - 640 + 1056 + 592 @@ -192,7 +160,7 @@ and recruitment_id = '${IDM_ID}'; - 1424 + 1280 256 @@ -241,7 +209,7 @@ and recruitment_id = '${IDM_ID}'; - 832 + 640 256 @@ -277,25 +245,7 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1424 - 432 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 832 + 1280 432 @@ -329,8 +279,8 @@ and recruitment_id = '${IDM_ID}'; - 832 - 640 + 640 + 592 @@ -364,15 +314,15 @@ and recruitment_id = '${IDM_ID}'; Y - 976 - 640 + 832 + 592 incapacity_input TableInput - Y + N 1 @@ -381,13 +331,13 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH filteredData AS ( SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob' as nedeesposob_arr + ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob' as nedeesposob_arr, + ctid FROM ervu_dashboard.recruits_info ri WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob') = 'array' - AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) + AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) ) SELECT fd.recruit_id, nedeesposob_elem ->> 'id' AS source_id, @@ -395,7 +345,9 @@ SELECT fd.recruit_id, nedeesposob_elem ->> 'naimStatus' AS capacity_status, nedeesposob_elem ->> 'naimOrg' AS court_name, to_date(nedeesposob_elem ->> 'dataPrisv', 'YYYY-MM-DD') AS register_date, - to_date(nedeesposob_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS deregistration_date + to_date(nedeesposob_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS deregistration_date, + true as has_incapacity_info, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(fd.nedeesposob_arr) AS nedeesposob_elem; Y @@ -465,7 +417,7 @@ FROM filteredData fd N - 1120 + 1056 432 diff --git a/mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow.hpl b/mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow.hpl index 2421f29..9fd77e6 100644 --- a/mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow.hpl +++ b/mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow.hpl @@ -46,16 +46,6 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - vich_output - Y - vich_output change_job_status_on_output_error @@ -86,16 +76,6 @@ Update Y - - change_job_status_on_update_error - abort_on_update_error - Y - - - Update - change_job_status_on_update_error - Y - vich_input Filter rows 2 @@ -103,7 +83,17 @@ Filter rows 2 - Identify last row in a stream + vich_output + Y + + + vich_input + Detect empty stream + Y + + + Update + change_job_status_on_output_error Y @@ -135,7 +125,7 @@ and recruitment_id = '${IDM_ID}'; - 1664 + 1360 240 @@ -181,7 +171,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 928 + 512 240 @@ -217,7 +207,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1664 + 1360 416 @@ -250,31 +240,13 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - Identify last row in a stream + vich_output 720 416 - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 928 - 416 - - Sort rows SortRows @@ -286,24 +258,27 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); none - N ${java.io.tmpdir} + + 1000000 + + N + + Y + recruit_id Y N N 0 - recruit_id N - 1000000 - Y 512 - 576 + 592 @@ -341,8 +316,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Y - 512 - 704 + 720 + 592 @@ -361,30 +336,10 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 0 - 1440 + 1136 592 - - abort_on_update_error - Abort - - Y - - 1 - - none - - - ABORT - Y - 0 - - - 720 - 864 - - block_until_update_finished BlockUntilTransformsFinish @@ -403,7 +358,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 1440 + 1136 416 @@ -439,46 +394,10 @@ and recruitment_id = '${IDM_ID}'; - 1216 + 912 592 - - change_job_status_on_update_error - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 512 - 864 - - vich_input TableInput @@ -492,7 +411,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, to_date(ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, @@ -500,7 +418,8 @@ and recruitment_id = '${IDM_ID}'; coalesce( ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' ->> 'priznakNalichSved' = '1', false) as vich_registered, - true as has_vich_info + true as has_vich_info, + ctid FROM ervu_dashboard.recruits_info ri WHERE ri.info -> 'svedFL' -> 'svedUchetVICH' ->> 'prOtsSvedUchetVICH' IS DISTINCT FROM '1' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -511,7 +430,8 @@ SELECT fd.recruit_id, to_date(vich_elem ->> 'dataSnyat', 'YYYY-MM-DD') AS register_date, to_date(vich_elem ->> 'dataPostUchet', 'YYYY-MM-DD') AS deregistration_date, vich_registered, - has_vich_info + has_vich_info, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(COALESCE( NULLIF(vich_arr, 'null'::jsonb), @@ -573,14 +493,14 @@ FROM filteredData fd Y - 1216 + 912 416 Update - change_job_status_on_update_error + change_job_status_on_output_error Y error_description diff --git a/mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_delta.hpl b/mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_delta.hpl index 829788e..e86c0d3 100644 --- a/mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_delta.hpl @@ -46,16 +46,6 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - vich_upsert - Y - vich_upsert Change job status on error @@ -71,16 +61,6 @@ Update Y - - Update - change_job_status_on_update_error - Y - - - change_job_status_on_update_error - abort_on_update_error - Y - vich_input Sort rows @@ -103,7 +83,17 @@ Filter rows 2 - Identify last row in a stream + vich_upsert + Y + + + Update + Change job status on error + Y + + + vich_input + Detect empty stream Y @@ -123,7 +113,7 @@ 0 - 2160 + 1856 560 @@ -159,7 +149,7 @@ and recruitment_id = '${IDM_ID}'; - 1776 + 1472 560 @@ -191,8 +181,8 @@ and recruitment_id = '${IDM_ID}'; - 2160 - 208 + 1856 + 224 @@ -240,8 +230,8 @@ and recruitment_id = '${IDM_ID}'; - 1488 - 208 + 1040 + 224 @@ -276,7 +266,7 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 2160 + 1856 384 @@ -309,31 +299,13 @@ and recruitment_id = '${IDM_ID}'; - Identify last row in a stream + vich_upsert 1248 384 - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 1488 - 384 - - Sort rows SortRows @@ -345,24 +317,27 @@ and recruitment_id = '${IDM_ID}'; none - N ${java.io.tmpdir} + + 1000000 + + N + + Y + recruit_id Y N N 0 - recruit_id N - 1000000 - Y 1040 - 528 + 560 @@ -395,29 +370,9 @@ and recruitment_id = '${IDM_ID}'; N Y - - 1040 - 656 - - - - abort_on_update_error - Abort - - Y - - 1 - - none - - - ABORT - Y - 0 - 1248 - 816 + 560 @@ -438,46 +393,10 @@ and recruitment_id = '${IDM_ID}'; - 1968 + 1664 384 - - change_job_status_on_update_error - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 1040 - 816 - - vich_input TableInput @@ -499,7 +418,8 @@ and recruitment_id = '${IDM_ID}'; coalesce( ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' ->> 'priznakNalichSved' = '1', false) as vich_registered, - true as has_vich_info + true as has_vich_info, + ctid FROM ervu_dashboard.recruits_info ri WHERE ri.info -> 'svedFL' -> 'svedUchetVICH' ->> 'prOtsSvedUchetVICH' IS DISTINCT FROM '1' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -510,7 +430,8 @@ SELECT fd.recruit_id, to_date(vich_elem ->> 'dataSnyat', 'YYYY-MM-DD') AS register_date, to_date(vich_elem ->> 'dataPostUchet', 'YYYY-MM-DD') AS deregistration_date, vich_registered, - has_vich_info + has_vich_info, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(COALESCE( NULLIF(vich_arr, 'null'::jsonb), @@ -575,14 +496,14 @@ WHERE case when '${MAX_SOURCE_UPDATE_DATE}' <> '' N - 1776 + 1472 384 Update - change_job_status_on_update_error + Change job status on error Y error_description diff --git a/mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_repeat.hpl index 4184624..2f92477 100644 --- a/mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_repeat.hpl @@ -46,16 +46,6 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - - - Identify last row in a stream - vich_upsert - Y - vich_upsert Change job status on error @@ -71,41 +61,36 @@ Update Y - - Update - change_job_status_on_update_error - Y - - - change_job_status_on_update_error - abort_on_update_error - Y - vich_input Filter rows 2 Y - - Filter rows 2 - Identify last row in a stream - Y - vich_input Sort rows Y - vich_upsert - block_until_update_finished + Filter rows 2 + vich_upsert Y - block_until_update_finished + vich_upsert Filter rows Y + + vich_input + Detect empty stream + Y + + + Update + Change job status on error + Y + Abort @@ -123,7 +108,7 @@ 0 - 1744 + 1424 608 @@ -159,7 +144,7 @@ and recruitment_id = '${IDM_ID}'; - 1328 + 1008 608 @@ -191,8 +176,8 @@ and recruitment_id = '${IDM_ID}'; - 1744 - 256 + 1424 + 288 @@ -240,8 +225,8 @@ and recruitment_id = '${IDM_ID}'; - 1040 - 256 + 640 + 288 @@ -276,7 +261,7 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1744 + 1424 432 @@ -309,31 +294,13 @@ and recruitment_id = '${IDM_ID}'; - Identify last row in a stream + vich_upsert 816 432 - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 1040 - 432 - - Sort rows SortRows @@ -345,24 +312,27 @@ and recruitment_id = '${IDM_ID}'; none - N ${java.io.tmpdir} + + 1000000 + + N + + Y + recruit_id Y N N 0 - recruit_id N - 1000000 - Y 640 - 576 + 608 @@ -400,86 +370,8 @@ and recruitment_id = '${IDM_ID}'; Y - 640 - 704 - - - - abort_on_update_error - Abort - - Y - - 1 - - none - - - ABORT - Y - 0 - - - 848 - 864 - - - - block_until_update_finished - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - Update - - - - - 1536 - 432 - - - - change_job_status_on_update_error - ExecSql - - Y - - 1 - - none - - - - - error_description - - - ervu-dashboard - Y - N - Y - Y - N - UPDATE etl.job_execution -SET status = 'ERROR', - error_description = ?, - execution_end_datetime = current_timestamp -WHERE job_name = '${JOB_NAME}' -and recruitment_id = '${IDM_ID}'; - - - - - 640 - 864 + 816 + 608 @@ -495,7 +387,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, to_date(ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, @@ -503,7 +394,8 @@ and recruitment_id = '${IDM_ID}'; coalesce( ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' ->> 'priznakNalichSved' = '1', false) as vich_registered, - true as has_vich_info + true as has_vich_info, + ctid FROM ervu_dashboard.recruits_info ri WHERE ri.info -> 'svedFL' -> 'svedUchetVICH' ->> 'prOtsSvedUchetVICH' IS DISTINCT FROM '1' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -514,7 +406,8 @@ SELECT fd.recruit_id, to_date(vich_elem ->> 'dataSnyat', 'YYYY-MM-DD') AS register_date, to_date(vich_elem ->> 'dataPostUchet', 'YYYY-MM-DD') AS deregistration_date, vich_registered, - has_vich_info + has_vich_info, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(COALESCE( NULLIF(vich_arr, 'null'::jsonb), @@ -576,14 +469,14 @@ FROM filteredData fd N - 1328 + 1008 432 Update - change_job_status_on_update_error + Change job status on error Y error_description diff --git a/mappings/info_recruits/citizen_tables/sport/parallel/sport_flow.hpl b/mappings/info_recruits/citizen_tables/sport/parallel/sport_flow.hpl index c6d7156..d9a4a11 100644 --- a/mappings/info_recruits/citizen_tables/sport/parallel/sport_flow.hpl +++ b/mappings/info_recruits/citizen_tables/sport/parallel/sport_flow.hpl @@ -46,21 +46,11 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - sport_rank_output Change job status on error Y - - sport_input - Identify last row in a stream - Y - Detect empty stream Change job status on success @@ -76,31 +66,6 @@ Change job status on error Y - - Identify last row in a stream - sport_flags_update - Y - - - sport_title_output - Block until transforms finish - Y - - - sport_rank_output - Block until transforms finish - Y - - - Block until transforms finish - Filter rows - Y - - - Identify last row in a stream - Filter rows 2 - Y - Filter rows 2 sport_title_output @@ -111,6 +76,31 @@ sport_rank_output Y + + sport_input + Detect empty stream + Y + + + sport_input + sport_flags_update + Y + + + sport_input + Filter rows 2 + Y + + + sport_title_output + Filter rows + Y + + + sport_rank_output + Filter rows + Y + Abort @@ -128,30 +118,8 @@ 0 - 1504 - 672 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - sport_flags_update - - - - - 1264 - 432 + 1296 + 576 @@ -185,8 +153,8 @@ and recruitment_id = '${IDM_ID}'; - 1216 - 672 + 1120 + 576 @@ -217,8 +185,8 @@ and recruitment_id = '${IDM_ID}'; - 1504 - 240 + 1296 + 272 @@ -263,8 +231,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 704 - 240 + 512 + 272 @@ -299,7 +267,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1504 + 1296 432 @@ -336,25 +304,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); sport_title_output - 880 - 416 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 704 + 688 416 @@ -393,15 +343,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Y - 704 - 672 + 512 + 576 sport_input TableInput - Y + N 1 @@ -412,7 +362,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); N WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedSport' -> 'sport' AS sport_arr + ri.info -> 'svedFL' -> 'svedSport' -> 'sport' AS sport_arr, + ctid FROM ervu_dashboard.recruits_info ri WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSport' -> 'sport') = 'array' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -431,7 +382,8 @@ SELECT fd.recruit_id, sport_elem -> 'sportZvan' ->> 'sbornaya' as title_national_team, sport_elem -> 'sportZvan' ->> 'zvanieNaim' as title, to_date(sport_elem -> 'sportZvan' ->> 'dataPrisv', 'YYYY-MM-DD') as title_assigment_date, - coalesce(sport_elem ->> 'sportZvan' <> 'null', false) as has_sport_title + coalesce(sport_elem ->> 'sportZvan' <> 'null', false) as has_sport_title, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(sport_arr) as sport_elem; Y @@ -507,8 +459,8 @@ FROM filteredData fd Y - 1056 - 480 + 928 + 464 @@ -569,7 +521,7 @@ FROM filteredData fd Y - 1056 + 928 368 diff --git a/mappings/info_recruits/citizen_tables/sport/parallel/sport_flow_delta.hpl b/mappings/info_recruits/citizen_tables/sport/parallel/sport_flow_delta.hpl index e6ccd91..de2d08f 100644 --- a/mappings/info_recruits/citizen_tables/sport/parallel/sport_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/sport/parallel/sport_flow_delta.hpl @@ -40,21 +40,11 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - sport_rank_flag_update Change job status on error Y - - sport_input - Identify last row in a stream - Y - sport_rank_upsert Change job status on error @@ -65,31 +55,6 @@ Change job status on error Y - - Identify last row in a stream - sport_rank_flag_update - Y - - - sport_rank_upsert - Block until transforms finish - Y - - - sports_title_upsert - Block until transforms finish - Y - - - Block until transforms finish - Filter rows - Y - - - Identify last row in a stream - Filter rows 2 - Y - Filter rows 2 sport_rank_upsert @@ -100,6 +65,31 @@ sports_title_upsert Y + + sport_input + Filter rows 2 + Y + + + sport_input + Detect empty stream + Y + + + sport_input + sport_rank_flag_update + Y + + + sport_rank_upsert + Filter rows + Y + + + sports_title_upsert + Filter rows + Y + Abort @@ -117,30 +107,8 @@ 0 - 1552 - 688 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - sport_rank_flag_update - - - - - 1264 - 416 + 1120 + 640 @@ -175,8 +143,8 @@ and recruitment_id = '${IDM_ID}'; - 1136 - 688 + 896 + 640 @@ -207,8 +175,8 @@ and recruitment_id = '${IDM_ID}'; - 1552 - 240 + 1120 + 256 @@ -256,8 +224,8 @@ and recruitment_id = '${IDM_ID}'; - 624 - 240 + 432 + 256 @@ -292,7 +260,7 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1552 + 1120 416 @@ -328,24 +296,6 @@ and recruitment_id = '${IDM_ID}'; sports_title_upsert sport_rank_upsert - - 784 - 416 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - 624 416 @@ -355,7 +305,7 @@ and recruitment_id = '${IDM_ID}'; sport_input TableInput - Y + N 1 @@ -364,9 +314,9 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedSport' -> 'sport' AS sport_arr + ri.info -> 'svedFL' -> 'svedSport' -> 'sport' AS sport_arr, + ctid FROM ervu_dashboard.recruits_info ri WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSport' -> 'sport') = 'array' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -385,7 +335,8 @@ SELECT fd.recruit_id, sport_elem -> 'sportZvan' ->> 'sbornaya' as title_national_team, sport_elem -> 'sportZvan' ->> 'zvanieNaim' as title, to_date(sport_elem -> 'sportZvan' ->> 'dataPrisv', 'YYYY-MM-DD') as title_assigment_date, - coalesce(sport_elem ->> 'sportZvan' <> 'null', false) as has_sport_title + coalesce(sport_elem ->> 'sportZvan' <> 'null', false) as has_sport_title, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(sport_arr) as sport_elem where case @@ -436,8 +387,8 @@ where case Y - 624 - 688 + 432 + 640 @@ -510,7 +461,7 @@ where case N - 944 + 784 368 @@ -574,7 +525,7 @@ where case N - 944 + 784 512 diff --git a/mappings/info_recruits/citizen_tables/sport/parallel/sport_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/sport/parallel/sport_flow_repeat.hpl index 888cb47..eb0670f 100644 --- a/mappings/info_recruits/citizen_tables/sport/parallel/sport_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/sport/parallel/sport_flow_repeat.hpl @@ -40,21 +40,11 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - sport_flags_update Change job status on error Y - - sport_input - Identify last row in a stream - Y - sports_title_upsert Change job status on error @@ -65,31 +55,6 @@ Change job status on error Y - - Identify last row in a stream - sport_flags_update - Y - - - sports_title_upsert - Block until transforms finish - Y - - - sport_rank_upsert - Block until transforms finish - Y - - - Block until transforms finish - Filter rows - Y - - - Identify last row in a stream - Filter rows 2 - Y - Filter rows 2 sport_rank_upsert @@ -100,6 +65,31 @@ sports_title_upsert Y + + sport_rank_upsert + Filter rows + Y + + + sports_title_upsert + Filter rows + Y + + + sport_input + Filter rows 2 + Y + + + sport_input + Detect empty stream + Y + + + sport_input + sport_flags_update + Y + Abort @@ -117,30 +107,8 @@ 0 - 1584 - 784 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - sport_flags_update - - - - - 1392 - 512 + 1280 + 688 @@ -175,8 +143,8 @@ and recruitment_id = '${IDM_ID}'; - 1296 - 784 + 1072 + 688 @@ -207,7 +175,7 @@ and recruitment_id = '${IDM_ID}'; - 1584 + 1280 336 @@ -256,7 +224,7 @@ and recruitment_id = '${IDM_ID}'; - 784 + 592 336 @@ -292,7 +260,7 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1584 + 1280 512 @@ -329,25 +297,7 @@ and recruitment_id = '${IDM_ID}'; sport_rank_upsert - 944 - 512 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 784 + 752 512 @@ -386,15 +336,15 @@ and recruitment_id = '${IDM_ID}'; Y - 784 - 784 + 592 + 688 sport_input TableInput - Y + N 1 @@ -403,9 +353,9 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedSport' -> 'sport' AS sport_arr + ri.info -> 'svedFL' -> 'svedSport' -> 'sport' AS sport_arr, + ctid FROM ervu_dashboard.recruits_info ri WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSport' -> 'sport') = 'array' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -424,7 +374,8 @@ SELECT fd.recruit_id, sport_elem -> 'sportZvan' ->> 'sbornaya' as title_national_team, sport_elem -> 'sportZvan' ->> 'zvanieNaim' as title, to_date(sport_elem -> 'sportZvan' ->> 'dataPrisv', 'YYYY-MM-DD') as title_assigment_date, - coalesce(sport_elem ->> 'sportZvan' <> 'null', false) as has_sport_title + coalesce(sport_elem ->> 'sportZvan' <> 'null', false) as has_sport_title, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(sport_arr) as sport_elem; Y @@ -504,8 +455,8 @@ FROM filteredData fd N - 1104 - 448 + 912 + 464 @@ -568,8 +519,8 @@ FROM filteredData fd N - 1104 - 592 + 912 + 560 diff --git a/mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow.hpl b/mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow.hpl index ef6089d..df35f3a 100644 --- a/mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow.hpl +++ b/mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow.hpl @@ -35,21 +35,6 @@ Change job status on success Y - - Identify last row in a stream - individual_entrepreneur_output - Y - - - individual_entrepreneur_input - Identify last row in a stream - Y - - - Identify last row in a stream - Detect empty stream - Y - Detect empty stream Change job status on success @@ -75,16 +60,6 @@ Change job status on error Y - - Identify last row in a stream - filter_null_npd_dates - Y - - - Identify last row in a stream - filter_non_actual_ip - Y - filter_non_actual_ip Update @@ -96,12 +71,27 @@ Y - individual_entrepreneur_output - Block until transforms finish + individual_entrepreneur_input + filter_null_npd_dates Y - Block until transforms finish + individual_entrepreneur_input + filter_non_actual_ip + Y + + + individual_entrepreneur_input + Detect empty stream + Y + + + individual_entrepreneur_input + individual_entrepreneur_output + Y + + + individual_entrepreneur_output Filter rows Y @@ -121,34 +111,9 @@ Y 0 - - 1632 - 704 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - Update - - - individual_entrepreneur_npd_output - - - 1424 - 448 + 560 @@ -183,8 +148,8 @@ and recruitment_id = '${IDM_ID}'; - 1424 - 704 + 1232 + 560 @@ -215,7 +180,7 @@ and recruitment_id = '${IDM_ID}'; - 1600 + 1232 288 @@ -261,7 +226,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 800 + 592 288 @@ -297,25 +262,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1600 - 448 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 800 + 1232 448 @@ -350,8 +297,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Y - 1184 - 576 + 992 + 544 @@ -386,8 +333,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Update - 976 - 576 + 784 + 544 @@ -414,15 +361,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); unique_by_rec_id - 800 - 704 + 592 + 672 individual_entrepreneur_input TableInput - Y + N 1 @@ -439,7 +386,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 'YYYY-MM-DD') as reg_npd_date, to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataSnUchNPD', 'YYYY-MM-DD') as dereg_npd_date, - true as actual + true as actual, + ri.ctid from ervu_dashboard.recruits_info ri where ri.info -> 'svedFL' -> 'svedRegIP' ->> 'regIP' != 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -452,7 +400,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 'YYYY-MM-DD') as reg_npd_date, to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataSnUchNPD', 'YYYY-MM-DD') as dereg_npd_date, - false as actual + false as actual, + ri.ctid from ervu_dashboard.recruits_info ri where ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'predRegIP' != 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -462,10 +411,11 @@ select fd.recruit_id, ip_elem ->> 'ogrnip' as ogrnip, to_date(ip_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, to_date(ip_elem ->> 'dataRegIP', 'YYYY-MM-DD') as registration_date, - to_date(ip_elem ->> 'dataINNNed', 'YYYY-MM-DD') as deregistration_date, + to_date(ip_elem ->> 'dataINNNed', 'YYYY-MM-DD') as deregistration_date, reg_npd_date, dereg_npd_date, - fd.actual as actual + fd.actual as actual, + fd.ctid = (select max(ctid) from filteredData) as last_row from filtered_data fd; Y @@ -520,8 +470,8 @@ from filtered_data fd; Y - 1184 - 704 + 976 + 672 @@ -582,7 +532,7 @@ from filtered_data fd; Y - 1184 + 992 448 @@ -607,8 +557,8 @@ from filtered_data fd; - 976 - 704 + 768 + 672 diff --git a/mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow_delta.hpl b/mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow_delta.hpl index 8549743..30f272b 100644 --- a/mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow_delta.hpl @@ -20,11 +20,6 @@ - - Block until transforms finish 2 - Change job status on success - Y - Change job status on error Abort @@ -50,11 +45,6 @@ upsert_pred Y - - Filter rows - Block until transforms finish 2 - Y - delete_akt_become_null Change job status on error @@ -155,6 +145,11 @@ Change job status on error Y + + Filter rows + Change job status on success + Y + Abort @@ -172,36 +167,8 @@ 0 - 1872 - 576 - - - - Block until transforms finish 2 - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - update_akt_flag_on_delete - - - update_akt_flag - - - upsert_pred - - - - - 1872 - 400 + 1856 + 656 @@ -236,8 +203,8 @@ and recruitment_id = '${IDM_ID}'; - 1696 - 576 + 1648 + 656 @@ -268,7 +235,7 @@ and recruitment_id = '${IDM_ID}'; - 2096 + 1920 400 @@ -384,7 +351,7 @@ and recruitment_id = '${IDM_ID}'; - Block until transforms finish 2 + Change job status on success 1696 @@ -630,7 +597,8 @@ and recruitment_id = '${IDM_ID}'; to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataUchNPD', 'YYYY-MM-DD') as reg_npd_date, to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataSnUchNPD', - 'YYYY-MM-DD') as dereg_npd_date + 'YYYY-MM-DD') as dereg_npd_date, + ctid from ervu_dashboard.recruits_info ri where ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'aktRegIP' != 'null' and (to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'aktRegIP' ->> 'dataSved', @@ -652,7 +620,8 @@ select fd.recruit_id, to_date(pred_ip_elem ->> 'dataSved', 'YYYY-MM-DD') as pred_source_update_date, to_date(pred_ip_elem ->> 'dataRegIP', 'YYYY-MM-DD') as pred_registration_date, to_date(pred_ip_elem ->> 'dataINNNed', 'YYYY-MM-DD') as pred_deregistration_date, - to_date(fd.pred_ip_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' as pred_need_update + to_date(fd.pred_ip_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' as pred_need_update, + fd.ctid = (select max(ctid) from filteredData) as last_row from filtered_data fd; Y diff --git a/mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow_repeat.hpl index 44026bf..1d602eb 100644 --- a/mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow_repeat.hpl @@ -36,31 +36,16 @@ Change job status on success Y - - individual_entrepreneur_input - Identify last row in a stream - Y - filter_null_npd_dates unique_by_rec_id Y - - Identify last row in a stream - Detect empty stream - Y - Detect empty stream Change job status on success Y - - Identify last row in a stream - individual_entrepreneur_upsert - Y - individual_entrepreneur_upsert Change job status on error @@ -71,11 +56,6 @@ individual_entrepreneur_npd_upsert Y - - Identify last row in a stream - filter_null_npd_dates - Y - individual_entrepreneur_npd_upsert Change job status on error @@ -86,23 +66,33 @@ Update Y - - Identify last row in a stream - filter_non_actual_ip - Y - Update Change job status on error Y - individual_entrepreneur_upsert - Block until transforms finish + individual_entrepreneur_input + filter_null_npd_dates Y - Block until transforms finish + individual_entrepreneur_input + filter_non_actual_ip + Y + + + individual_entrepreneur_input + Detect empty stream + Y + + + individual_entrepreneur_input + individual_entrepreneur_upsert + Y + + + individual_entrepreneur_upsert Filter rows Y @@ -123,33 +113,8 @@ 0 - 2032 - 736 - - - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - Update - - - individual_entrepreneur_npd_upsert - - - - - 1808 - 464 + 1632 + 624 @@ -184,8 +149,8 @@ and recruitment_id = '${IDM_ID}'; - 1568 - 736 + 1456 + 624 @@ -216,8 +181,8 @@ and recruitment_id = '${IDM_ID}'; - 2032 - 304 + 1440 + 288 @@ -265,8 +230,8 @@ SET status = 'PROCESSING', - 976 - 304 + 736 + 288 @@ -301,25 +266,7 @@ SET status = 'PROCESSING', Change job status on success - 2032 - 464 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 976 + 1440 464 @@ -354,8 +301,8 @@ SET status = 'PROCESSING', Y - 1328 - 592 + 1088 + 576 @@ -390,8 +337,8 @@ SET status = 'PROCESSING', Update - 1136 - 592 + 896 + 576 @@ -418,15 +365,15 @@ SET status = 'PROCESSING', unique_by_rec_id - 976 - 736 + 736 + 720 individual_entrepreneur_input TableInput - Y + N 1 @@ -443,9 +390,10 @@ SET status = 'PROCESSING', 'YYYY-MM-DD') as reg_npd_date, to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataSnUchNPD', 'YYYY-MM-DD') as dereg_npd_date, - true as actual + true as actual, + ri.ctid from ervu_dashboard.recruits_info ri - where ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'aktRegIP' != 'null' + where ri.info -> 'svedFL' -> 'svedRegIP' ->> 'regIP' != 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) union all @@ -456,10 +404,11 @@ SET status = 'PROCESSING', 'YYYY-MM-DD') as reg_npd_date, to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataSnUchNPD', 'YYYY-MM-DD') as dereg_npd_date, - false as actual + false as actual, + ri.ctid from ervu_dashboard.recruits_info ri where ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'predRegIP' != 'null' - AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) + AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) ) select fd.recruit_id, fd.source_id, @@ -469,7 +418,8 @@ select fd.recruit_id, to_date(ip_elem ->> 'dataINNNed', 'YYYY-MM-DD') as deregistration_date, reg_npd_date, dereg_npd_date, - fd.actual as actual + fd.actual as actual, + fd.ctid = (select max(ctid) from filteredData) as last_row from filtered_data fd; Y @@ -523,8 +473,8 @@ from filtered_data fd; N - 1328 - 736 + 1088 + 720 @@ -587,7 +537,7 @@ from filtered_data fd; N - 1568 + 1088 464 @@ -612,8 +562,8 @@ from filtered_data fd; - 1136 - 736 + 896 + 720 diff --git a/mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow.hpl b/mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow.hpl index 3b8d8a9..42515b7 100644 --- a/mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow.hpl +++ b/mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow.hpl @@ -30,16 +30,6 @@ self_employed_input Y - - self_employed_input - Identify last row in a stream - Y - - - Identify last row in a stream - self_employed_output - Y - Filter rows Change job status on success @@ -50,21 +40,11 @@ Change job status on error Y - - Identify last row in a stream - Detect empty stream - Y - Detect empty stream Change job status on success Y - - Identify last row in a stream - filter_non_actual_self_employed - Y - filter_non_actual_self_employed Update @@ -76,15 +56,25 @@ Y - self_employed_output - Block until transforms finish + self_employed_input + self_employed_output Y - Block until transforms finish + self_employed_output Filter rows Y + + self_employed_input + filter_non_actual_self_employed + Y + + + self_employed_input + Detect empty stream + Y + Abort @@ -102,32 +92,10 @@ 0 - 1264 + 832 496 - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - Update - - - - - 1088 - 224 - - Change job status on error ExecSql @@ -160,7 +128,7 @@ and recruitment_id = '${IDM_ID}'; - 896 + 656 496 @@ -192,7 +160,7 @@ and recruitment_id = '${IDM_ID}'; - 1264 + 832 48 @@ -238,7 +206,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 624 + 432 48 @@ -274,25 +242,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1264 - 224 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 624 + 832 224 @@ -327,7 +277,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Y - 624 + 432 496 @@ -363,7 +313,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Update - 624 + 432 368 @@ -371,7 +321,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); self_employed_input TableInput - Y + N 1 @@ -383,7 +333,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 0 with filtered_data as (select ri.recruit_id, ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' as npd_elem, - true as actual + true as actual, + ctid from ervu_dashboard.recruits_info ri where ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' ->> 'aktNPD' != 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -392,7 +343,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); select ri.recruit_id, ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' as npd_elem, - false as actual + false as actual, + ctid from ervu_dashboard.recruits_info ri where ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' ->> 'predNPD' != 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -402,7 +354,8 @@ select fd.recruit_id, to_date(npd_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, to_date(npd_elem ->> 'dataUchNPD', 'YYYY-MM-DD') as register_date, to_date(npd_elem ->> 'dataSnUchNPD', 'YYYY-MM-DD') as deregistration_date, - actual + actual, + fd.ctid = (select max(ctid) from filteredData) as last_row from filtered_data fd; Y @@ -465,7 +418,7 @@ from filtered_data fd; Y - 896 + 656 224 diff --git a/mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_delta.hpl b/mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_delta.hpl index 25b2bad..634f45b 100644 --- a/mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_delta.hpl @@ -20,11 +20,6 @@ - - Block until transforms finish 2 - Change job status on success - Y - Change job status on error Abort @@ -50,11 +45,6 @@ upsert_pred Y - - Filter rows - Block until transforms finish 2 - Y - delete_akt_become_null Change job status on error @@ -135,6 +125,11 @@ Filter rows Y + + Filter rows + Change job status on success + Y + Abort @@ -156,34 +151,6 @@ 736 - - Block until transforms finish 2 - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - update_akt_flag_on_delete - - - update_akt_flag - - - upsert_pred - - - - - 1872 - 384 - - Change job status on error ExecSql @@ -246,7 +213,7 @@ and recruitment_id = '${IDM_ID}'; - 2096 + 1856 384 @@ -362,7 +329,7 @@ and recruitment_id = '${IDM_ID}'; - Block until transforms finish 2 + Change job status on success 1696 @@ -526,10 +493,11 @@ and recruitment_id = '${IDM_ID}'; with filtered_data as (select ri.recruit_id, ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' as akt_npd_elem, - ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' as pred_npd_elem + ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' as pred_npd_elem, + ctid from ervu_dashboard.recruits_info ri where ri.info -> 'svedFL' -> 'svedNPD' ->> 'npd' != 'null' - and (to_date(ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' + and (to_date(ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' or to_date(ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}') AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) ) @@ -545,7 +513,8 @@ select fd.recruit_id, to_date(pred_npd_elem ->> 'dataSved', 'YYYY-MM-DD') as pred_source_update_date, to_date(pred_npd_elem ->> 'dataUchNPD', 'YYYY-MM-DD') as pred_register_date, to_date(pred_npd_elem ->> 'dataSnUchNPD', 'YYYY-MM-DD') as pred_deregistration_date, - to_date(pred_npd_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' as pred_need_update + to_date(pred_npd_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' as pred_need_update, + fd.ctid = (select max(ctid) from filteredData) as last_row from filtered_data fd; Y diff --git a/mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_repeat.hpl index 2587135..01be147 100644 --- a/mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_repeat.hpl @@ -35,26 +35,11 @@ Change job status on success Y - - self_employed_input - Identify last row in a stream - Y - - - Identify last row in a stream - self_employed_output - Y - self_employed_output Change job status on error Y - - Identify last row in a stream - Detect empty stream - Y - Detect empty stream Change job status on success @@ -65,23 +50,28 @@ Update Y - - Identify last row in a stream - filter_non_actual_self_employed - Y - Update Change job status on error Y - self_employed_output - Block until transforms finish + self_employed_input + filter_non_actual_self_employed Y - Block until transforms finish + self_employed_input + self_employed_output + Y + + + self_employed_input + Detect empty stream + Y + + + self_employed_output Filter rows Y @@ -102,32 +92,10 @@ 0 - 1296 + 880 656 - - Block until transforms finish - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - Update - - - - - 1072 - 416 - - Change job status on error ExecSql @@ -159,7 +127,7 @@ and recruitment_id = '${IDM_ID}'; - 864 + 688 656 @@ -190,7 +158,7 @@ and recruitment_id = '${IDM_ID}'; - 1296 + 880 240 @@ -239,7 +207,7 @@ SET status = 'PROCESSING', - 656 + 464 240 @@ -275,25 +243,7 @@ SET status = 'PROCESSING', Change job status on success - 1296 - 416 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 656 + 880 416 @@ -328,7 +278,7 @@ SET status = 'PROCESSING', Y - 656 + 464 656 @@ -364,7 +314,7 @@ SET status = 'PROCESSING', Update - 656 + 464 544 @@ -372,7 +322,7 @@ SET status = 'PROCESSING', self_employed_input TableInput - Y + N 1 @@ -384,7 +334,8 @@ SET status = 'PROCESSING', 0 with filtered_data as (select ri.recruit_id, ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' as npd_elem, - true as actual + true as actual, + ctid from ervu_dashboard.recruits_info ri where ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' ->> 'aktNPD' != 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -393,7 +344,8 @@ SET status = 'PROCESSING', select ri.recruit_id, ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' as npd_elem, - false as actual + false as actual, + ctid from ervu_dashboard.recruits_info ri where ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' ->> 'predNPD' != 'null' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -403,7 +355,8 @@ select fd.recruit_id, to_date(npd_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, to_date(npd_elem ->> 'dataUchNPD', 'YYYY-MM-DD') as register_date, to_date(npd_elem ->> 'dataSnUchNPD', 'YYYY-MM-DD') as deregistration_date, - actual + actual, + fd.ctid = (select max(ctid) from filteredData) as last_row from filtered_data fd; Y @@ -462,7 +415,7 @@ from filtered_data fd; N - 864 + 688 416 diff --git a/mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow.hpl b/mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow.hpl index 7a083d8..8736b17 100644 --- a/mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow.hpl +++ b/mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow.hpl @@ -41,11 +41,6 @@ Change job status on success Y - - Identify last row in a stream - Detect empty stream - Y - employee_parental_leave_output Change job status on error @@ -71,39 +66,34 @@ Filter_null_parental_leave_dates Y - - citizen_work_activity_input - Group by - Y - - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - work_activity_output - Y - work_activity_output Change job status on error Y - work_activity_output - Block until transforms finish 2 + Create job execution record + citizen_work_activity_input Y - Block until transforms finish 2 + work_activity_output Filter rows Y - Create job execution record - citizen_work_activity_input + citizen_work_activity_input + Detect empty stream + Y + + + citizen_work_activity_input + Sort rows + Y + + + Sort rows + work_activity_output Y @@ -123,8 +113,8 @@ 0 - 1728 - 624 + 1456 + 560 @@ -145,30 +135,8 @@ - 928 - 624 - - - - Block until transforms finish 2 - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - employee_parental_leave_output - - - - - 1392 - 320 + 720 + 560 @@ -203,8 +171,8 @@ and recruitment_id = '${IDM_ID}'; - 1504 - 624 + 1152 + 560 @@ -235,7 +203,7 @@ and recruitment_id = '${IDM_ID}'; - 1584 + 1152 144 @@ -281,7 +249,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 928 + 496 144 @@ -317,7 +285,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1584 + 1152 320 @@ -346,101 +314,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Block until transforms finish - 704 - 624 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - recruit_id - recruit_id - FIRST_INCL_NULL - - - employer_name - employer_name - FIRST_INCL_NULL - - - position - position - FIRST_INCL_NULL - - - personnel_event_date - personnel_event_date - FIRST_INCL_NULL - - - personnel_event_type - personnel_event_type - FIRST_INCL_NULL - - - source_update_date - source_update_date - FIRST_INCL_NULL - - - actual_employer - actual_employer - FIRST_INCL_NULL - - - parental_leave_start_date - parental_leave_start_date - FIRST_INCL_NULL - - - parental_leave_end_date - parental_leave_end_date - FIRST_INCL_NULL - - - N - - - source_id - - - N - grp - - - 736 - 320 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 928 - 320 + 496 + 560 @@ -456,9 +331,9 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' as trud_arr + ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' as trud_arr, + ctid FROM ervu_dashboard.recruits_info ri WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat') = 'array' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -472,7 +347,8 @@ SELECT fd.recruit_id, to_date(trud_elem ->> 'dataSved', 'YYYY-MM-DD') AS source_update_date, coalesce((trud_elem ->> 'prAktMestRab') = '1', false) AS actual_employer, to_date(uhod_elem ->> 'dataNachUhodReb', 'YYYY-MM-DD') AS parental_leave_start_date, - to_date(uhod_elem ->> 'dataKonUhodReb', 'YYYY-MM-DD') AS parental_leave_end_date + to_date(uhod_elem ->> 'dataKonUhodReb', 'YYYY-MM-DD') AS parental_leave_end_date, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(trud_arr) AS trud_elem LEFT JOIN LATERAL jsonb_array_elements( @@ -531,8 +407,8 @@ FROM filteredData fd Y - 1200 - 624 + 944 + 560 @@ -597,7 +473,41 @@ FROM filteredData fd Y - 1200 + 896 + 320 + + + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + source_id + Y + N + N + 0 + N + + + + + 704 320 diff --git a/mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow_delta.hpl b/mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow_delta.hpl index 3e62548..2d0bc0a 100644 --- a/mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow_delta.hpl @@ -45,11 +45,6 @@ Block until transforms finish Y - - Identify last row in a stream - Detect empty stream - Y - Detect empty stream Change job status on success @@ -67,30 +62,50 @@ citizen_work_activity_input - Group by - Y - - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - citizen_work_activity_upsert + Detect empty stream Y citizen_work_activity_upsert - Block until transforms finish 2 - Y - - - Block until transforms finish 2 Filter rows Y + + Change job status on error + Abort + Y + + + citizen_work_activity_input + Sort rows + Y + + + Sort rows + citizen_work_activity_upsert + Y + + + Abort + Abort + + Y + + 1 + + none + + + ABORT_WITH_ERROR + Y + 0 + + + 1344 + 432 + + Block until transforms finish BlockUntilTransformsFinish @@ -109,32 +124,10 @@ - 880 + 672 432 - - Block until transforms finish 2 - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - parental_leave_upsert - - - - - 1360 - 224 - - Change job status on error ExecSql @@ -170,7 +163,7 @@ and recruitment_id = '${IDM_ID}'; - 1552 + 1168 432 @@ -202,7 +195,7 @@ and recruitment_id = '${IDM_ID}'; - 1552 + 1184 48 @@ -251,7 +244,7 @@ and recruitment_id = '${IDM_ID}'; - 832 + 432 48 @@ -287,7 +280,7 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1552 + 1184 224 @@ -316,103 +309,10 @@ and recruitment_id = '${IDM_ID}'; Block until transforms finish - 640 + 432 432 - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - recruit_id - recruit_id - FIRST_INCL_NULL - - - employer_name - employer_name - FIRST_INCL_NULL - - - position - position - FIRST_INCL_NULL - - - personnel_event_date - personnel_event_date - FIRST_INCL_NULL - - - personnel_event_type - personnel_event_type - FIRST_INCL_NULL - - - source_update_date - source_update_date - FIRST_INCL_NULL - - - actual_employer - actual_employer - FIRST_INCL_NULL - - - parental_leave_start_date - parental_leave_start_date - FIRST_INCL_NULL - - - parental_leave_end_date - parental_leave_end_date - FIRST_INCL_NULL - - - N - - - source_id - - - N - grp - - - 640 - 224 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 832 - 224 - - citizen_work_activity_input TableInput @@ -426,9 +326,9 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' as trud_arr + ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' as trud_arr, + ctid FROM ervu_dashboard.recruits_info ri WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat') = 'array' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -442,7 +342,8 @@ SELECT fd.recruit_id, to_date(trud_elem ->> 'dataSved', 'YYYY-MM-DD') AS source_update_date, coalesce((trud_elem ->> 'prAktMestRab') = '1', false) AS actual_employer, to_date(uhod_elem ->> 'dataNachUhodReb', 'YYYY-MM-DD') AS parental_leave_start_date, - to_date(uhod_elem ->> 'dataKonUhodReb', 'YYYY-MM-DD') AS parental_leave_end_date + to_date(uhod_elem ->> 'dataKonUhodReb', 'YYYY-MM-DD') AS parental_leave_end_date, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(trud_arr) AS trud_elem LEFT JOIN LATERAL jsonb_array_elements( @@ -526,7 +427,7 @@ where to_date(trud_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DAT N - 1136 + 928 224 @@ -570,10 +471,44 @@ where to_date(trud_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DAT N - 1136 + 928 432 + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + source_id + Y + N + N + 0 + N + + + + + 672 + 224 + + citizen_work_activity_upsert diff --git a/mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow_repeat.hpl index bf6720a..3618bcb 100644 --- a/mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow_repeat.hpl @@ -36,11 +36,6 @@ Block until transforms finish Y - - Identify last row in a stream - Detect empty stream - Y - Block until transforms finish parental_leave_upsert @@ -56,21 +51,6 @@ Filter_null_parental_leave_dates Y - - citizen_work_activity_input - Group by - Y - - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - citizen_work_activity_upsert - Y - citizen_work_activity_upsert Change job status on error @@ -82,18 +62,28 @@ Y - citizen_work_activity_upsert - Block until transforms finish 2 + Change job status on error + Abort Y - Block until transforms finish 2 + citizen_work_activity_input + Detect empty stream + Y + + + citizen_work_activity_upsert Filter rows Y - Change job status on error - Abort + citizen_work_activity_input + Sort rows + Y + + + Sort rows + citizen_work_activity_upsert Y @@ -113,8 +103,8 @@ 0 - 1968 - 704 + 1552 + 720 @@ -135,30 +125,8 @@ - 1088 - 704 - - - - Block until transforms finish 2 - BlockUntilTransformsFinish - - Y - - 1 - - none - - - - - parental_leave_upsert - - - - - 1600 - 496 + 896 + 720 @@ -193,8 +161,8 @@ and recruitment_id = '${IDM_ID}'; - 1776 - 704 + 1344 + 720 @@ -225,8 +193,8 @@ and recruitment_id = '${IDM_ID}'; - 1776 - 320 + 1344 + 336 @@ -274,8 +242,8 @@ and recruitment_id = '${IDM_ID}'; - 1056 - 320 + 640 + 336 @@ -310,7 +278,7 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1776 + 1344 496 @@ -339,101 +307,8 @@ and recruitment_id = '${IDM_ID}'; Block until transforms finish - 848 - 704 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - recruit_id - recruit_id - FIRST_INCL_NULL - - - employer_name - employer_name - FIRST_INCL_NULL - - - position - position - FIRST_INCL_NULL - - - personnel_event_date - personnel_event_date - FIRST_INCL_NULL - - - personnel_event_type - personnel_event_type - FIRST_INCL_NULL - - - source_update_date - source_update_date - FIRST_INCL_NULL - - - actual_employer - actual_employer - FIRST_INCL_NULL - - - parental_leave_start_date - parental_leave_start_date - FIRST_INCL_NULL - - - parental_leave_end_date - parental_leave_end_date - FIRST_INCL_NULL - - - N - - - source_id - - - N - grp - - - 832 - 496 - - - - Identify last row in a stream - DetectLastRow - - N - - 1 - - none - - - last_row - - - 1056 - 496 + 640 + 720 @@ -450,7 +325,8 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N WITH filteredData AS (SELECT ri.recruit_id, - ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' as trud_arr + ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' as trud_arr, + ctid FROM ervu_dashboard.recruits_info ri WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat') = 'array' AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -464,7 +340,8 @@ SELECT fd.recruit_id, to_date(trud_elem ->> 'dataSved', 'YYYY-MM-DD') AS source_update_date, coalesce((trud_elem ->> 'prAktMestRab') = '1', false) AS actual_employer, to_date(uhod_elem ->> 'dataNachUhodReb', 'YYYY-MM-DD') AS parental_leave_start_date, - to_date(uhod_elem ->> 'dataKonUhodReb', 'YYYY-MM-DD') AS parental_leave_end_date + to_date(uhod_elem ->> 'dataKonUhodReb', 'YYYY-MM-DD') AS parental_leave_end_date, + fd.ctid = (select max(ctid) from filteredData) as last_row FROM filteredData fd CROSS JOIN LATERAL jsonb_array_elements(trud_arr) AS trud_elem LEFT JOIN LATERAL jsonb_array_elements( @@ -546,7 +423,7 @@ FROM filteredData fd N - 1360 + 1136 496 @@ -568,7 +445,6 @@ FROM filteredData fd = citizen_work_activity_source_id source_id - ervu_dashboard educational_leave
@@ -596,8 +472,42 @@ FROM filteredData fd N - 1360 - 704 + 1136 + 720 + + + + Sort rows + SortRows + + Y + + 1 + + none + + + ${java.io.tmpdir} + out + 1000000 + + N + + Y + + + source_id + Y + N + N + 0 + N + + + + + 896 + 496