diff --git a/mappings/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow.hpl b/mappings/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow.hpl index 95257fe..dbebd8b 100644 --- a/mappings/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow.hpl @@ -45,46 +45,46 @@ Change job status on error Y - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - User defined Java expression - Y - - - User defined Java expression - Update - Y - - - Table input - Table output - Y - - - Table output - Group by - Y - Update Change job status on error Y - Update + Table input + Identify last row in a stream + Y + + + Identify last row in a stream + Table output + Y + + + Identify last row in a stream Detect empty stream Y - Update + Table output + Block until transforms finish + Y + + + Block until transforms finish Filter rows Y + + Identify last row in a stream + sort_by_recruit_id + Y + + + sort_by_recruit_id + Update + Y + Abort @@ -102,10 +102,32 @@ 0 - 1328 + 1504 496 + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + Update + + + + + 1312 + 320 + + Change job status on error ExecSql @@ -138,8 +160,8 @@ and recruitment_id = '${IDM_ID}'; - 1328 - 320 + 1280 + 496 @@ -170,8 +192,8 @@ and recruitment_id = '${IDM_ID}'; - 1856 - 176 + 1536 + 160 @@ -216,8 +238,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 1536 - 176 + 688 + 160 @@ -253,49 +275,14 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); 1536 - 304 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - acquired_citizenship_raw - is_acquired_citizenship - MAX - - - N - - - recruit_id - - - N - grp - - - 688 - 176 + 320 Identify last row in a stream DetectLastRow - Y + N 1 @@ -305,8 +292,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); last_row - 880 - 176 + 688 + 320 @@ -322,7 +309,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); ervu-dashboard N - select ri.recruit_id, to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataSved', 'YYYY-MM-DD') AS source_update_date, -- дата обновления сведений @@ -335,13 +321,12 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshVyh', 'YYYY-MM-DD') renunciation_date, -- дата решения о выходе из гражданства РФ + ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'organReshVyh' renunciation_authority, -- наименование органа, принявшего решение о выходе из гражданства РФ + CASE - WHEN ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' is not null THEN 1 - ELSE 0 - END AS is_acquired_citizenship + WHEN ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' is not null THEN true + ELSE false + END AS has_acquired_citizenship FROM ervu_dashboard.recruits_info ri join ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and (ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' is not null or ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'osnVyh' is not null) Y @@ -421,7 +406,7 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and Y - 688 + 1056 320 @@ -456,13 +441,13 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and Y - 1328 - 176 + 1056 + 496 - User defined Java expression - Janino + sort_by_recruit_id + SortRows Y @@ -471,18 +456,25 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and none - - acquired_citizenship - acquired_citizenship_raw == 1 - Boolean - -1 - -1 - - + N + ${java.io.tmpdir} + + + Y + N + N + 0 + recruit_id + N + + + srt + 1000000 + Y - 1104 - 176 + 688 + 496 @@ -493,7 +485,7 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and error_description - error_code + @@ -503,7 +495,7 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow_delta.hpl index 2d30e1d..b5efb3d 100644 --- a/mappings/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow_delta.hpl @@ -40,49 +40,49 @@ Abort Y - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - User defined Java expression - Y - - - User defined Java expression - Update - Y - Update - Detect empty stream + Change job status on error Y - Update - Filter rows - Y - - - Update + Insert / update Change job status on error Y Table input + Identify last row in a stream + Y + + + Identify last row in a stream Insert / update Y - Insert / update - Group by + Identify last row in a stream + Detect empty stream Y Insert / update - Change job status on error + Block until transforms finish + Y + + + Block until transforms finish + Filter rows + Y + + + Identify last row in a stream + sort_by_recruit_id + Y + + + sort_by_recruit_id + Update Y @@ -106,6 +106,28 @@ 528 + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + Update + + + + + 1200 + 352 + + Change job status on error ExecSql @@ -138,8 +160,8 @@ and recruitment_id = '${IDM_ID}'; - 1392 - 352 + 1216 + 528 @@ -170,7 +192,7 @@ and recruitment_id = '${IDM_ID}'; - 1872 + 1408 208 @@ -219,7 +241,7 @@ and recruitment_id = '${IDM_ID}'; - 1632 + 768 208 @@ -255,50 +277,15 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1632 - 320 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - acquired_citizenship_raw - is_acquired_citizenship - MAX - - - N - - - recruit_id - - - N - grp - - - 752 - 208 + 1408 + 352 Identify last row in a stream DetectLastRow - Y + N 1 @@ -308,8 +295,8 @@ and recruitment_id = '${IDM_ID}'; last_row - 944 - 208 + 768 + 352 @@ -397,7 +384,7 @@ and recruitment_id = '${IDM_ID}'; N - 752 + 960 352 @@ -427,9 +414,9 @@ and recruitment_id = '${IDM_ID}'; to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshVyh', 'YYYY-MM-DD') renunciation_date, -- дата решения о выходе из гражданства РФ + ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'organReshVyh' renunciation_authority, -- наименование органа, принявшего решение о выходе из гражданства РФ + CASE - WHEN ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' is not null THEN 1 - ELSE 0 - END AS is_acquired_citizenship + WHEN ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' is not null THEN true + ELSE false + END AS has_acquired_citizenship FROM ervu_dashboard.recruits_info ri join ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) @@ -480,13 +467,13 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and Y - 1392 - 208 + 960 + 528 - User defined Java expression - Janino + sort_by_recruit_id + SortRows Y @@ -495,18 +482,25 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and none - - acquired_citizenship - acquired_citizenship_raw == 1 - Boolean - -1 - -1 - - + N + ${java.io.tmpdir} + + + Y + N + N + 0 + recruit_id + N + + + srt + 1000000 + Y - 1168 - 208 + 768 + 528 @@ -515,7 +509,7 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and Change job status on error Y - + error_description @@ -527,7 +521,7 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow_repeat.hpl index fbc11dd..7099292 100644 --- a/mappings/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow_repeat.hpl @@ -46,43 +46,43 @@ Y - Table input - Insert / update + Update + Change job status on error Y - Group by + Table input Identify last row in a stream Y Identify last row in a stream - User defined Java expression - Y - - - User defined Java expression - Update + Insert / update Y Insert / update - Group by + Block until transforms finish Y - Update - Detect empty stream - Y - - - Update + Block until transforms finish Filter rows Y - Update - Change job status on error + Identify last row in a stream + Detect empty stream + Y + + + Identify last row in a stream + sort_by_recruit_id + Y + + + sort_by_recruit_id + Update Y @@ -106,6 +106,28 @@ 400 + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + Update + + + + + 1168 + 208 + + Change job status on error ExecSql @@ -138,8 +160,8 @@ and recruitment_id = '${IDM_ID}'; - 1312 - 208 + 1104 + 400 @@ -170,7 +192,7 @@ and recruitment_id = '${IDM_ID}'; - 1776 + 1424 64 @@ -234,7 +256,7 @@ DO UPDATE SET - 1568 + 672 64 @@ -270,50 +292,15 @@ DO UPDATE SET Change job status on success - 1568 - 176 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - acquired_citizenship_raw - is_acquired_citizenship - MAX - - - N - - - recruit_id - - - N - grp - - - 672 - 64 + 1424 + 208 Identify last row in a stream DetectLastRow - Y + N 1 @@ -323,8 +310,8 @@ DO UPDATE SET last_row - 864 - 64 + 672 + 208 @@ -412,7 +399,7 @@ DO UPDATE SET N - 672 + 912 208 @@ -442,13 +429,12 @@ DO UPDATE SET to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshVyh', 'YYYY-MM-DD') renunciation_date, -- дата решения о выходе из гражданства РФ + ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'organReshVyh' renunciation_authority, -- наименование органа, принявшего решение о выходе из гражданства РФ + CASE - WHEN ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' is not null THEN 1 - ELSE 0 - END AS is_acquired_citizenship + WHEN ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' is not null THEN true + ELSE false + END AS has_acquired_citizenship FROM ervu_dashboard.recruits_info ri join ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and (ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' is not null or ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'osnVyh' is not null) Y @@ -494,13 +480,13 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and Y - 1312 - 64 + 912 + 400 - User defined Java expression - Janino + sort_by_recruit_id + SortRows Y @@ -509,18 +495,25 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and none - - acquired_citizenship - acquired_citizenship_raw == 1 - Boolean - -1 - -1 - - + N + ${java.io.tmpdir} + + + Y + N + N + 0 + recruit_id + N + + + srt + 1000000 + Y - 1088 - 64 + 672 + 400 @@ -529,7 +522,7 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and Change job status on error Y - + error_description @@ -541,7 +534,7 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/citizenship/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizenship/recruitment_five_flow_delta.hpl index 8dc2a87..61f0b8c 100644 --- a/mappings/info_recruits/citizen_tables/citizenship/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship/recruitment_five_flow_delta.hpl @@ -65,23 +65,28 @@ ervu-dashboard N 0 - WITH mud AS ( - SELECT - recruitment_id, - MAX(execution_datetime) AS max_upd_date - FROM etl.job_execution - WHERE job_name = '${JOB_NAME}' - AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING') - GROUP BY recruitment_id -) -SELECT - r.idm_id + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r JOIN mud ON mud.recruitment_id = r.idm_id JOIN recruits_info ri ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id - AND ri.updated_at > mud.max_upd_date; - N + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; + Y 352 @@ -144,29 +149,6 @@ FROM ervu_dashboard.recruitment r 128 - - get_max_source_update_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT max(source_update_date) AS max_source_update_date -FROM ervu_dashboard.citizenship; - N - - - 144 - 288 - - citizenship_flow_delta.hpl 2 PipelineExecutor @@ -391,6 +373,29 @@ FROM ervu_dashboard.citizenship; 448 + + get_max_source_update_date + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT max(source_update_date) AS max_source_update_date +FROM ervu_dashboard.citizenship; + N + + + 144 + 288 + + diff --git a/mappings/info_recruits/citizen_tables/citizenship/recruitment_five_flow_on_error.hpl b/mappings/info_recruits/citizen_tables/citizenship/recruitment_five_flow_on_error.hpl index c68cd8e..4cc707f 100644 --- a/mappings/info_recruits/citizen_tables/citizenship/recruitment_five_flow_on_error.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship/recruitment_five_flow_on_error.hpl @@ -65,9 +65,9 @@ FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - and job_name = 'citizenship_job' + and job_name = '${JOB_NAME}' where je.status is null or je.status in('ERROR', 'PROCESSING'); - N + Y 272 @@ -121,7 +121,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING'); - 544 + 560 208 diff --git a/mappings/info_recruits/citizen_tables/citizenship/recruitments_five_flow.hpl b/mappings/info_recruits/citizen_tables/citizenship/recruitments_five_flow.hpl index b9099b2..af3f2f8 100644 --- a/mappings/info_recruits/citizen_tables/citizenship/recruitments_five_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship/recruitments_five_flow.hpl @@ -63,24 +63,8 @@ SELECT idm_id AS recruitment FROM ervu_dashboard.recruitment -${COMMENT}/* -where idm_id in ('110686b2-4a01-4976-885b-d26a2980f53a', -'cff66da8-2be0-4839-bd8e-332b1a150031', -'34c8685e-f08a-4877-afb1-a0589800a21c', -'0e10678c-f272-41c3-94dd-c15830321f9a', -'9703f108-761a-467a-aad3-89617e5539ed', -'55eae374-9bb5-4295-9e3a-5fbcdaba870d', -'dad6c101-34be-45c3-8173-57190cb69dfb', -'fc10523f-af31-4f6b-b0e2-00799758f634', -'61807cc2-9913-4060-8802-0672877c28c8', -'eba7fd7d-e3da-49a7-8e65-8fc85e47190e', -'f59afe6d-c166-4196-a489-6d17f02fc05e', -'22c64b79-f8e4-41e7-a2f3-1aa9df424fd9', -'0fe2bdf0-afbb-402d-93da-67e511799afa', -'6139de3b-2378-496f-9fca-7df7dab90832', -'50ba73dd-1726-48f1-8b7c-7a77ff965299') -${COMMENT}*/ - Y + + N 432 diff --git a/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow.hpl b/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow.hpl index b396dc8..14164c6 100644 --- a/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow.hpl @@ -45,46 +45,46 @@ Change job status on error Y - - Group by - Identify last row in a stream - Y - - - Identify last row in a stream - User defined Java expression - Y - - - User defined Java expression - Update - Y - - - Table input - Table output - Y - - - Table output - Group by - Y - Update Change job status on error Y - Update + Table input + Identify last row in a stream + Y + + + Identify last row in a stream + Table output + Y + + + Identify last row in a stream Detect empty stream Y - Update + Table output + Block until transforms finish + Y + + + Block until transforms finish Filter rows Y + + Identify last row in a stream + sort_by_recruit_id + Y + + + sort_by_recruit_id + Update + Y + Abort @@ -102,8 +102,30 @@ 0 - 1328 - 496 + 1344 + 480 + + + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + Update + + + + + 1120 + 320 @@ -138,8 +160,8 @@ and recruitment_id = '${IDM_ID}'; - 1328 - 320 + 1104 + 480 @@ -170,7 +192,7 @@ and recruitment_id = '${IDM_ID}'; - 1856 + 1328 176 @@ -216,7 +238,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); - 1536 + 688 176 @@ -252,55 +274,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); Change job status on success - 1536 - 304 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - has_foreign_residence_raw - is_foreign_residence - MAX - - - has_foreign_citizenship_raw - is_foreign_citizenship - MAX - - - N - - - recruit_id - - - N - grp - - - 688 - 176 + 1328 + 320 Identify last row in a stream DetectLastRow - Y + N 1 @@ -310,8 +292,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); last_row - 880 - 176 + 688 + 320 @@ -335,7 +317,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at WHERE jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedIG') = 'array' OR jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedPP') = 'array' ), @@ -373,13 +354,13 @@ SELECT END::date AS citizenship_date, type, CASE - WHEN type = 'residense' THEN 1 - ELSE 0 - END AS is_foreign_residence, + WHEN type = 'residense' THEN true + ELSE false + END AS has_foreign_residence, CASE - WHEN type = 'citizenship' THEN 1 - ELSE 0 - END AS is_foreign_citizenship + WHEN type = 'citizenship' THEN true + ELSE false + END AS has_foreign_citizenship FROM citizen Y @@ -450,7 +431,7 @@ FROM citizen Y - 688 + 896 320 @@ -489,13 +470,13 @@ FROM citizen Y - 1328 - 176 + 896 + 480 - User defined Java expression - Janino + sort_by_recruit_id + SortRows Y @@ -504,26 +485,25 @@ FROM citizen none - - has_foreign_citizenship - has_foreign_citizenship_raw == 1 - Boolean - -1 - -1 - - - - has_foreign_residence - has_foreign_residence_raw == 1 - None - -1 - -1 - - + N + ${java.io.tmpdir} + + + Y + N + N + 0 + recruit_id + N + + + srt + 1000000 + Y - 1104 - 176 + 688 + 480 @@ -534,7 +514,7 @@ FROM citizen error_description - error_code + @@ -544,7 +524,7 @@ FROM citizen Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl index 71579e2..cd6b03c 100644 --- a/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl @@ -41,48 +41,48 @@ Y - Group by + Update + Change job status on error + Y + + + Insert / update + Change job status on error + Y + + + Table input Identify last row in a stream Y Identify last row in a stream - User defined Java expression - Y - - - User defined Java expression - Update - Y - - - Table input Insert / update Y - Insert / update - Group by + Identify last row in a stream + sort_by_recruit_id Y - Update + Identify last row in a stream Detect empty stream Y - Update + Insert / update + Block until transforms finish + Y + + + Block until transforms finish Filter rows Y - Update - Change job status on error - Y - - - Insert / update - Change job status on error + sort_by_recruit_id + Update Y @@ -106,6 +106,28 @@ 528 + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + Update + + + + + 1232 + 352 + + Change job status on error ExecSql @@ -138,8 +160,8 @@ and recruitment_id = '${IDM_ID}'; - 1392 - 352 + 1168 + 528 @@ -170,8 +192,8 @@ and recruitment_id = '${IDM_ID}'; - 1872 - 208 + 1408 + 192 @@ -219,8 +241,8 @@ and recruitment_id = '${IDM_ID}'; - 1632 - 208 + 784 + 192 @@ -255,55 +277,15 @@ and recruitment_id = '${IDM_ID}'; Change job status on success - 1632 - 320 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - has_foreign_residence_raw - is_foreign_residence - MAX - - - has_foreign_citizenship_raw - is_foreign_citizenship - MAX - - - N - - - recruit_id - - - N - grp - - - 752 - 208 + 1408 + 352 Identify last row in a stream DetectLastRow - Y + N 1 @@ -313,8 +295,8 @@ and recruitment_id = '${IDM_ID}'; last_row - 944 - 208 + 784 + 352 @@ -387,7 +369,7 @@ and recruitment_id = '${IDM_ID}'; N - 752 + 976 352 @@ -404,7 +386,6 @@ and recruitment_id = '${IDM_ID}'; ervu-dashboard N - WITH base AS ( SELECT ri.recruit_id, @@ -457,13 +438,13 @@ SELECT END::date AS citizenship_date, type, CASE - WHEN type = 'residense' THEN 1 - ELSE 0 - END AS is_foreign_residence, + WHEN type = 'residense' THEN true + ELSE false + END AS has_foreign_residence, CASE - WHEN type = 'citizenship' THEN 1 - ELSE 0 - END AS is_foreign_citizenship + WHEN type = 'citizenship' THEN true + ELSE false + END AS has_foreign_citizenship FROM citizen Y @@ -507,13 +488,13 @@ FROM citizen Y - 1392 - 208 + 992 + 528 - User defined Java expression - Janino + sort_by_recruit_id + SortRows Y @@ -522,26 +503,25 @@ FROM citizen none - - has_foreign_citizenship - has_foreign_citizenship_raw == 1 - Boolean - -1 - -1 - - - - has_foreign_residence - has_foreign_residence_raw == 1 - None - -1 - -1 - - + N + ${java.io.tmpdir} + + + Y + N + N + 0 + recruit_id + N + + + srt + 1000000 + Y - 1168 - 208 + 784 + 528 @@ -550,7 +530,7 @@ FROM citizen Change job status on error Y - + error_description @@ -562,7 +542,7 @@ FROM citizen Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_repeat.hpl b/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_repeat.hpl index bb7db32..0d00ff5 100644 --- a/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_repeat.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_repeat.hpl @@ -46,45 +46,45 @@ Y - Table input - Insert / update + Update + Change job status on error Y - Group by + Table input Identify last row in a stream Y Identify last row in a stream - User defined Java expression + Insert / update Y - User defined Java expression + Identify last row in a stream + Detect empty stream + Y + + + Identify last row in a stream + sort_by_recruit_id + Y + + + sort_by_recruit_id Update Y Insert / update - Group by + Block until transforms finish Y - Update - Detect empty stream - Y - - - Update + Block until transforms finish Filter rows Y - - Update - Change job status on error - Y - Abort @@ -106,6 +106,28 @@ 400 + + Block until transforms finish + BlockUntilTransformsFinish + + Y + + 1 + + none + + + + + Update + + + + + 1168 + 208 + + Change job status on error ExecSql @@ -138,8 +160,8 @@ and recruitment_id = '${IDM_ID}'; - 1312 - 208 + 1120 + 400 @@ -170,7 +192,7 @@ and recruitment_id = '${IDM_ID}'; - 1776 + 1360 64 @@ -234,7 +256,7 @@ DO UPDATE SET - 1568 + 704 64 @@ -270,55 +292,15 @@ DO UPDATE SET Change job status on success - 1568 - 176 - - - - Group by - GroupBy - - Y - - 1 - - none - - - N - N - ${java.io.tmpdir} - - - has_foreign_residence_raw - is_foreign_residence - MAX - - - has_foreign_citizenship_raw - is_foreign_citizenship - MAX - - - N - - - recruit_id - - - N - grp - - - 672 - 64 + 1360 + 208 Identify last row in a stream DetectLastRow - Y + N 1 @@ -328,8 +310,8 @@ DO UPDATE SET last_row - 864 - 64 + 704 + 208 @@ -402,7 +384,7 @@ DO UPDATE SET N - 672 + 912 208 @@ -419,7 +401,6 @@ DO UPDATE SET ervu-dashboard N - WITH base AS ( SELECT ri.recruit_id, @@ -427,7 +408,6 @@ DO UPDATE SET FROM ervu_dashboard.recruits_info ri JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) - AND '${M_R_CR_DATE}'::timestamp >= ri.created_at WHERE jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedIG') = 'array' OR jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedPP') = 'array' ), @@ -465,13 +445,13 @@ SELECT END::date AS citizenship_date, type, CASE - WHEN type = 'residense' THEN 1 - ELSE 0 - END AS is_foreign_residence, + WHEN type = 'residense' THEN true + ELSE false + END AS has_foreign_residence, CASE - WHEN type = 'citizenship' THEN 1 - ELSE 0 - END AS is_foreign_citizenship + WHEN type = 'citizenship' THEN true + ELSE false + END AS has_foreign_citizenship FROM citizen Y @@ -515,13 +495,13 @@ FROM citizen Y - 1312 - 64 + 912 + 400 - User defined Java expression - Janino + sort_by_recruit_id + SortRows Y @@ -530,26 +510,25 @@ FROM citizen none - - has_foreign_citizenship - has_foreign_citizenship_raw == 1 - Boolean - -1 - -1 - - - - has_foreign_residence - has_foreign_residence_raw == 1 - None - -1 - -1 - - + N + ${java.io.tmpdir} + + + Y + N + N + 0 + recruit_id + N + + + srt + 1000000 + Y - 1088 - 64 + 704 + 400 @@ -558,7 +537,7 @@ FROM citizen Change job status on error Y - + error_description @@ -570,7 +549,7 @@ FROM citizen Change job status on error Y - + error_description diff --git a/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_delta.hpl b/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_delta.hpl index 66991ad..dbf2231 100644 --- a/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_delta.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_delta.hpl @@ -65,22 +65,27 @@ ervu-dashboard N 0 - WITH mud AS ( - SELECT - recruitment_id, - MAX(execution_datetime) AS max_upd_date - FROM etl.job_execution - WHERE job_name = '${JOB_NAME}' - AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING') - GROUP BY recruitment_id -) -SELECT - r.idm_id + WITH mud AS (SELECT recruitment_id, + MAX(execution_datetime) AS max_upd_date + FROM etl.job_execution + WHERE job_name = '${JOB_NAME}' + AND status IN ('SUCCESS', 'DELTA_SUCCESS') + GROUP BY recruitment_id) +SELECT r.idm_id FROM ervu_dashboard.recruitment r JOIN mud ON mud.recruitment_id = r.idm_id JOIN recruits_info ri ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id - AND ri.updated_at > mud.max_upd_date; + AND ri.updated_at > mud.max_upd_date + +union + +select r2.idm_id +from recruitment r2 + join etl.job_execution je + on r2.idm_id = je.recruitment_id +where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') + and je.job_name = '${JOB_NAME}'; Y @@ -144,29 +149,6 @@ FROM ervu_dashboard.recruitment r 128 - - get_max_source_update_date - TableInput - - Y - - 1 - - none - - - ervu-dashboard - N - 0 - SELECT max(source_update_date) AS max_source_update_date -FROM ervu_dashboard.citizenship_foreign; - N - - - 160 - 288 - - citizenship_foreign_flow_delta.hpl 2 PipelineExecutor @@ -391,6 +373,29 @@ FROM ervu_dashboard.citizenship_foreign; 480 + + get_max_source_update_date + TableInput + + Y + + 1 + + none + + + ervu-dashboard + N + 0 + SELECT max(source_update_date) AS max_source_update_date +FROM ervu_dashboard.citizenship_foreign; + N + + + 160 + 288 + + diff --git a/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_on_error.hpl b/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_on_error.hpl index 8e216e8..a0de7f8 100644 --- a/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_on_error.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_on_error.hpl @@ -65,9 +65,9 @@ FROM ervu_dashboard.recruitment r LEFT JOIN etl.job_execution je ON r.idm_id = je.recruitment_id - and job_name = 'citizenship_foreign_job' + and job_name = '${JOB_NAME}' where je.status is null or je.status in('ERROR', 'PROCESSING'); - N + Y 272 diff --git a/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitments_five_flow.hpl b/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitments_five_flow.hpl index ba9e2e3..6ea0db8 100644 --- a/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitments_five_flow.hpl +++ b/mappings/info_recruits/citizen_tables/citizenship_foreign/recruitments_five_flow.hpl @@ -63,24 +63,8 @@ SELECT idm_id AS recruitment FROM ervu_dashboard.recruitment -${COMMENT}/* -where idm_id in ('110686b2-4a01-4976-885b-d26a2980f53a', -'cff66da8-2be0-4839-bd8e-332b1a150031', -'34c8685e-f08a-4877-afb1-a0589800a21c', -'0e10678c-f272-41c3-94dd-c15830321f9a', -'9703f108-761a-467a-aad3-89617e5539ed', -'55eae374-9bb5-4295-9e3a-5fbcdaba870d', -'dad6c101-34be-45c3-8173-57190cb69dfb', -'fc10523f-af31-4f6b-b0e2-00799758f634', -'61807cc2-9913-4060-8802-0672877c28c8', -'eba7fd7d-e3da-49a7-8e65-8fc85e47190e', -'f59afe6d-c166-4196-a489-6d17f02fc05e', -'22c64b79-f8e4-41e7-a2f3-1aa9df424fd9', -'0fe2bdf0-afbb-402d-93da-67e511799afa', -'6139de3b-2378-496f-9fca-7df7dab90832', -'50ba73dd-1726-48f1-8b7c-7a77ff965299') -${COMMENT}*/ - Y + + N 432 diff --git a/mappings/info_recruits/job_parallel_subpoena_decision_archive_person.hwf b/mappings/info_recruits/job_parallel_subpoena_decision_archive_person.hwf index 6354dab..68d9007 100644 --- a/mappings/info_recruits/job_parallel_subpoena_decision_archive_person.hwf +++ b/mappings/info_recruits/job_parallel_subpoena_decision_archive_person.hwf @@ -666,7 +666,7 @@ Start job_last_update_date_subpoena.hwf - Y + N Y Y @@ -729,14 +729,14 @@ Start job_last_update_date_deferment_decision.hwf - Y + N Y Y Start job_last_update_date_recruit_archive.hwf - Y + N Y Y @@ -750,7 +750,7 @@ Start job_last_update_date_restriction_document.hwf - Y + N Y Y @@ -764,7 +764,7 @@ Start job_last_update_date_restriction_document_item.hwf - Y + N Y Y @@ -785,7 +785,7 @@ Start job_last_update_date_subpoena_appearance.hwf - Y + N Y Y diff --git a/mappings/info_recruits/supportive_jobs/job2_info_from_json.hwf b/mappings/info_recruits/supportive_jobs/job2_info_from_json.hwf index 44b13dc..f8e19f1 100644 --- a/mappings/info_recruits/supportive_jobs/job2_info_from_json.hwf +++ b/mappings/info_recruits/supportive_jobs/job2_info_from_json.hwf @@ -376,7 +376,7 @@ row_last_recruit_update_date.hpl drivers_licence_job.hwf - Y + N Y N @@ -390,7 +390,7 @@ row_last_recruit_update_date.hpl property_job.hwf - Y + N Y N diff --git a/mappings/info_recruits/supportive_jobs/job3_info_from_json.hwf b/mappings/info_recruits/supportive_jobs/job3_info_from_json.hwf index 43a5e44..ae61e85 100644 --- a/mappings/info_recruits/supportive_jobs/job3_info_from_json.hwf +++ b/mappings/info_recruits/supportive_jobs/job3_info_from_json.hwf @@ -40,18 +40,14 @@ N N ${PROJECT_HOME}/info_recruits/citizen_tables/snils/snils_job.hwf - - Nothing M_R_UP_DATE - ${M_R_UP_DATE} M_R_CR_DATE - ${M_R_CR_DATE} Y @@ -322,18 +318,14 @@ N N ${PROJECT_HOME}/info_recruits/citizen_tables/inn/inn_job.hwf - - Nothing M_R_UP_DATE - ${M_R_UP_DATE} M_R_CR_DATE - ${M_R_CR_DATE} Y @@ -419,7 +411,7 @@ row_last_recruit_update_date.hpl snils_job.hwf - Y + N Y N @@ -461,7 +453,7 @@ row_last_recruit_update_date.hpl inn_job.hwf - Y + N Y N