micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

recruit archive + CL_fz53 refatoring

Browse source
This commit is contained in:
Fusionshh 2025-11-10 14:11:49 +03:00
parent 5700985794
commit 2e28e88dae
7 changed files with 542 additions and 330 deletions
Download patch file Download diff file Expand all files Collapse all files

4
mappings/info_recruits/citizen_tables/citizen_liability_fz53/parallel/citizen_liability_fz53_flow_delta.hpl
View file

@ -404,9 +404,9 @@ and recruitment_id = '${IDM_ID}';</sql>
NOT i.hidden AS has_liability_fz53,
ctid
FROM public.infringement i
JOIN public.recruit r ON r.id = i.recruit_id -- Проверка на пустую строку
JOIN public.recruit r ON r.id = i.recruit_id
AND (r.current_recruitment = '${IDM_ID}' or (r.current_recruitment is null and r.target_recruitment = '${IDM_ID}'))
AND i.updated_at >= '${M_INFR_UP_DATE}'::timestamp
AND i.updated_at > '${MAX_UPDATE_DATE}'::timestamp
WHERE type = 'ADMINISTRATIVE')
select fd.*,
fd.ctid = (select max(ctid) from filtered_data) as last_row

218
mappings/info_recruits/citizen_tables/citizen_liability_fz53/recruitment_five_flow_delta.hpl
View file

@ -21,34 +21,59 @@
</notepads>
<order>
<hop>
<from>Table input</from>
<from>failed_deferment_liberation_permission_idm_ids_on_delta</from>
<to>Append streams</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>max_update_date_from_deferment_liberation</from>
<to>failed_deferment_liberation_permission_idm_ids_on_delta</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>max_update_date_from_deferment_liberation</from>
<to>updated_idm_ids_fromrecruits_deferment</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>updated_idm_ids_fromrecruits_deferment</from>
<to>Append streams</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Append streams</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>citizen_liability_fz53_flow_delta.hpl</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>Sort rows</from>
<to>citizen_liability_fz53_flow_delta.hpl 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>Sort rows</from>
<to>citizen_liability_fz53_flow_delta.hpl 3</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>Sort rows</from>
<to>citizen_liability_fz53_flow_delta.hpl 4</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>Sort rows</from>
<to>citizen_liability_fz53_flow_delta.hpl 5</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Table input</name>
<type>TableInput</type>
<name>Append streams</name>
<type>Append</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -57,34 +82,9 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>WITH mud AS (SELECT recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS', 'DELTA_SUCCESS')
GROUP BY recruitment_id)
SELECT r.idm_id
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN ervu_dashboard.recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date
union
select r2.idm_id
from ervu_dashboard.recruitment r2
join etl.job_execution je
on r2.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>352</xloc>
<xloc>864</xloc>
<yloc>288</yloc>
</GUI>
</transform>
@ -112,6 +112,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -135,10 +140,125 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/>
<attributes/>
<GUI>
<xloc>592</xloc>
<xloc>1312</xloc>
<yloc>128</yloc>
</GUI>
</transform>
<transform>
<name>failed_deferment_liberation_permission_idm_ids_on_delta</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>max_update_date_from_deferment_liberation</lookup>
<sql>select r.idm_id as recruitment_id,
? as max_update_date
from ervu_dashboard.recruitment r
join etl.job_execution je
on r.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>656</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>max_update_date_from_deferment_liberation</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(updated_at) as max_update_date
from citizen_liability_fz53;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>464</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform>
<name>updated_idm_ids_fromrecruits_deferment</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu_person_registry</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>max_update_date_from_deferment_liberation</lookup>
<sql>WITH max_update_date(val) AS (SELECT CAST(? AS timestamp))
SELECT DISTINCT COALESCE(r.current_recruitment, r.target_recruitment) AS recruitment_id,
? as max_update_date
FROM recruit r
JOIN infringement i ON i.recruit_id = r.id
JOIN max_update_date mud ON TRUE
WHERE (mud.val IS NULL OR i.updated_at > mud.val);</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>656</xloc>
<yloc>192</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruitment_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform>
<name>citizen_liability_fz53_flow_delta.hpl 2</name>
<type>PipelineExecutor</type>
@ -163,6 +283,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -186,8 +311,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/>
<attributes/>
<GUI>
<xloc>608</xloc>
<yloc>224</yloc>
<xloc>1312</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
@ -214,6 +339,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -237,7 +367,7 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/>
<attributes/>
<GUI>
<xloc>624</xloc>
<xloc>1312</xloc>
<yloc>288</yloc>
</GUI>
</transform>
@ -265,6 +395,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -288,7 +423,7 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/>
<attributes/>
<GUI>
<xloc>640</xloc>
<xloc>1312</xloc>
<yloc>368</yloc>
</GUI>
</transform>
@ -316,6 +451,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -339,7 +479,7 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/>
<attributes/>
<GUI>
<xloc>624</xloc>
<xloc>1312</xloc>
<yloc>448</yloc>
</GUI>
</transform>

174
mappings/info_recruits/citizen_tables/deferment_liberation_permission/recruitment_five_flow_delta.hpl
View file

@ -21,12 +21,12 @@
</notepads>
<order>
<hop>
<from>Table input 2</from>
<from>updated_idm_ids_fromrecruits_deferment</from>
<to>Append streams</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input 3</from>
<from>failed_deferment_liberation_permission_idm_ids_on_delta</from>
<to>Append streams</to>
<enabled>Y</enabled>
</hop>
@ -36,8 +36,8 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Table input 2</to>
<from>max_update_date_from_deferment_liberation</from>
<to>updated_idm_ids_fromrecruits_deferment</to>
<enabled>Y</enabled>
</hop>
<hop>
@ -46,8 +46,8 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Table input 3</to>
<from>max_update_date_from_deferment_liberation</from>
<to>failed_deferment_liberation_permission_idm_ids_on_delta</to>
<enabled>Y</enabled>
</hop>
<hop>
@ -122,87 +122,6 @@
<yloc>288</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(updated_at) as max_update_date
from deferment_liberation;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>192</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform>
<name>Table input 2</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu_person_registry</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>Table input</lookup>
<sql>WITH max_update_date(val) AS (SELECT CAST(? AS timestamp))
SELECT DISTINCT COALESCE(r.current_recruitment_id, r.target_recruitment_id) AS recruitment_id,
? as max_update_date
FROM recruits r
JOIN recruits_deferment rd ON rd.recruit_id = r.id
JOIN max_update_date mud ON TRUE
WHERE (mud.val IS NULL OR rd.system_update_date > mud.val);</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>384</xloc>
<yloc>192</yloc>
</GUI>
</transform>
<transform>
<name>Table input 3</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>Table input</lookup>
<sql>select r.idm_id as recruitment_id,
? as max_update_date
from ervu_dashboard.recruitment r
join etl.job_execution je
on r.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>384</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>deferment_liberation_permission_flow_delta.hpl</name>
<type>PipelineExecutor</type>
@ -483,6 +402,87 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>failed_deferment_liberation_permission_idm_ids_on_delta</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>max_update_date_from_deferment_liberation</lookup>
<sql>select r.idm_id as recruitment_id,
? as max_update_date
from ervu_dashboard.recruitment r
join etl.job_execution je
on r.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>384</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>max_update_date_from_deferment_liberation</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(updated_at) as max_update_date
from deferment_liberation;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>192</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform>
<name>updated_idm_ids_fromrecruits_deferment</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu_person_registry</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>max_update_date_from_deferment_liberation</lookup>
<sql>WITH max_update_date(val) AS (SELECT CAST(? AS timestamp))
SELECT DISTINCT COALESCE(r.current_recruitment_id, r.target_recruitment_id) AS recruitment_id,
? as max_update_date
FROM recruits r
JOIN recruits_deferment rd ON rd.recruit_id = r.id
JOIN max_update_date mud ON TRUE
WHERE (mud.val IS NULL OR rd.system_update_date > mud.val);</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>384</xloc>
<yloc>192</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>

87
mappings/info_recruits/citizen_tables/recruit_archive/parallel/recruit_archive_flow.hpl
View file

@ -50,26 +50,6 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Table output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>sort_by_recruit_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sort_by_recruit_id</from>
<to>Update</to>
@ -80,6 +60,21 @@
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Table output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>sort_by_recruit_id</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -97,7 +92,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1344</xloc>
<xloc>1168</xloc>
<yloc>480</yloc>
</GUI>
</transform>
@ -133,7 +128,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1136</xloc>
<xloc>960</xloc>
<yloc>480</yloc>
</GUI>
</transform>
@ -165,7 +160,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1360</xloc>
<xloc>960</xloc>
<yloc>176</yloc>
</GUI>
</transform>
@ -211,7 +206,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>720</xloc>
<xloc>496</xloc>
<yloc>176</yloc>
</GUI>
</transform>
@ -247,13 +242,13 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1360</xloc>
<xloc>960</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
@ -262,27 +257,10 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>720</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu_person_archive</connection>
<execute_each_row>N</execute_each_row>
<sql>SELECT
<limit/>
<sql>with filtered_data as (SELECT
id,
recruit_id,
archivation_reason_id,
@ -290,13 +268,12 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
updated_at,
hidden,
recruit_extract_enable,
CASE
WHEN recruit_extract_enable IS true THEN true
ELSE false
END AS card_in_archive
ctid
FROM public.recruit_archive
WHERE '${IDM_ID}' != '' -- Проверка на пустую строку
AND vk_id = '${IDM_ID}'
WHERE vk_id = '${IDM_ID}')
select fd.*,
fd.ctid = (select max(ctid) from filtered_data) as last_row
from filtered_data fd;
</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -363,7 +340,7 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>912</xloc>
<xloc>736</xloc>
<yloc>320</yloc>
</GUI>
</transform>
@ -398,7 +375,7 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>912</xloc>
<xloc>736</xloc>
<yloc>480</yloc>
</GUI>
</transform>
@ -432,7 +409,7 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку
</fields>
<attributes/>
<GUI>
<xloc>720</xloc>
<xloc>496</xloc>
<yloc>480</yloc>
</GUI>
</transform>

92
mappings/info_recruits/citizen_tables/recruit_archive/parallel/recruit_archive_flow_delta.hpl
View file

@ -45,26 +45,6 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Insert / update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>sort_by_recruit_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sort_by_recruit_id</from>
<to>Update</to>
@ -80,6 +60,21 @@
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>sort_by_recruit_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Insert / update</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -97,7 +92,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1392</xloc>
<xloc>1136</xloc>
<yloc>496</yloc>
</GUI>
</transform>
@ -133,7 +128,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1152</xloc>
<xloc>896</xloc>
<yloc>496</yloc>
</GUI>
</transform>
@ -165,7 +160,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1376</xloc>
<xloc>896</xloc>
<yloc>224</yloc>
</GUI>
</transform>
@ -214,7 +209,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>704</xloc>
<xloc>544</xloc>
<yloc>224</yloc>
</GUI>
</transform>
@ -250,25 +245,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1376</xloc>
<yloc>352</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>704</xloc>
<xloc>896</xloc>
<yloc>352</yloc>
</GUI>
</transform>
@ -332,7 +309,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>944</xloc>
<xloc>736</xloc>
<yloc>352</yloc>
</GUI>
</transform>
@ -340,7 +317,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -349,7 +326,8 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu_person_archive</connection>
<execute_each_row>N</execute_each_row>
<sql>SELECT
<limit/>
<sql>with filtered_data as (SELECT
id,
recruit_id,
archivation_reason_id,
@ -357,14 +335,16 @@ and recruitment_id = '${IDM_ID}';</sql>
updated_at,
hidden,
recruit_extract_enable,
CASE
WHEN recruit_extract_enable IS true THEN true
ELSE false
END AS card_in_archive
FROM public.recruit_archive
WHERE '${IDM_ID}' != '' -- Проверка на пустую строку
AND vk_id = '${IDM_ID}'
AND updated_at >= '${M_RECARCH_UP_DATE}'::timestamp</sql>
ctid
FROM public.recruit_archive ra
WHERE vk_id = '${IDM_ID}'
and case when '${MAX_UPDATE_DATE}' &lt;&gt; ''
then ra.updated_at > to_date('${MAX_UPDATE_DATE}', 'YYYY-MM-DD')
else true end)
select fd.*,
fd.ctid = (select max(ctid) from filtered_data) as last_row
from filtered_data fd;
</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -408,7 +388,7 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>944</xloc>
<xloc>736</xloc>
<yloc>496</yloc>
</GUI>
</transform>
@ -442,7 +422,7 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку
</fields>
<attributes/>
<GUI>
<xloc>704</xloc>
<xloc>544</xloc>
<yloc>496</yloc>
</GUI>
</transform>

86
mappings/info_recruits/citizen_tables/recruit_archive/parallel/recruit_archive_flow_repeat.hpl
View file

@ -50,26 +50,6 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Insert / update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>sort_by_recruit_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sort_by_recruit_id</from>
<to>Update</to>
@ -80,6 +60,21 @@
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>sort_by_recruit_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Insert / update</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -97,7 +92,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1120</xloc>
<xloc>912</xloc>
<yloc>352</yloc>
</GUI>
</transform>
@ -133,7 +128,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>928</xloc>
<xloc>720</xloc>
<yloc>352</yloc>
</GUI>
</transform>
@ -165,7 +160,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1152</xloc>
<xloc>720</xloc>
<yloc>80</yloc>
</GUI>
</transform>
@ -229,7 +224,7 @@ DO UPDATE SET
</partitioning>
<attributes/>
<GUI>
<xloc>528</xloc>
<xloc>320</xloc>
<yloc>80</yloc>
</GUI>
</transform>
@ -265,25 +260,7 @@ DO UPDATE SET
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1152</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>528</xloc>
<xloc>720</xloc>
<yloc>208</yloc>
</GUI>
</transform>
@ -305,7 +282,6 @@ DO UPDATE SET
<condition>=</condition>
<field>id</field>
<name>id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>recruit_archive</table>
@ -348,7 +324,7 @@ DO UPDATE SET
<update_bypassed>Y</update_bypassed>
<attributes/>
<GUI>
<xloc>720</xloc>
<xloc>528</xloc>
<yloc>208</yloc>
</GUI>
</transform>
@ -356,7 +332,7 @@ DO UPDATE SET
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -365,7 +341,8 @@ DO UPDATE SET
</partitioning>
<connection>ervu_person_archive</connection>
<execute_each_row>N</execute_each_row>
<sql>SELECT
<limit/>
<sql>with filtered_data as (SELECT
id,
recruit_id,
archivation_reason_id,
@ -373,13 +350,12 @@ DO UPDATE SET
updated_at,
hidden,
recruit_extract_enable,
CASE
WHEN recruit_extract_enable IS true THEN true
ELSE false
END AS card_in_archive
ctid
FROM public.recruit_archive
WHERE '${IDM_ID}' != '' -- Проверка на пустую строку
AND vk_id = '${IDM_ID}'
WHERE vk_id = '${IDM_ID}')
select fd.*,
fd.ctid = (select max(ctid) from filtered_data) as last_row
from filtered_data fd;
</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -424,7 +400,7 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>720</xloc>
<xloc>528</xloc>
<yloc>352</yloc>
</GUI>
</transform>
@ -458,7 +434,7 @@ WHERE '${IDM_ID}' != '' -- Проверка на пустую строку
</fields>
<attributes/>
<GUI>
<xloc>528</xloc>
<xloc>320</xloc>
<yloc>352</yloc>
</GUI>
</transform>

211
mappings/info_recruits/citizen_tables/recruit_archive/recruitment_five_flow_delta.hpl
View file

@ -21,33 +21,109 @@
</notepads>
<order>
<hop>
<from>Table input</from>
<from>Append streams</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>max_update_date_from_recruit_archive</from>
<to>updated_idm_ids_from_source_recruit_archive</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>max_update_date_from_recruit_archive</from>
<to>failed_recruit_archive_idm_ids_on_delta</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>updated_idm_ids_from_source_recruit_archive</from>
<to>Append streams</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>failed_recruit_archive_idm_ids_on_delta</from>
<to>Append streams</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>recruit_archive_flow_delta.hpl</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>Sort rows</from>
<to>recruit_archive_flow_delta.hpl 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>Sort rows</from>
<to>recruit_archive_flow_delta.hpl 3</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>Sort rows</from>
<to>recruit_archive_flow_delta.hpl 4</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>Sort rows</from>
<to>recruit_archive_flow_delta.hpl 5</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Table input</name>
<name>Append streams</name>
<type>Append</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>336</xloc>
<yloc>304</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruitment_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>304</yloc>
</GUI>
</transform>
<transform>
<name>failed_recruit_archive_idm_ids_on_delta</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
@ -60,32 +136,42 @@
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>WITH mud AS (SELECT recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS', 'DELTA_SUCCESS')
GROUP BY recruitment_id)
SELECT r.idm_id
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN ervu_dashboard.recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date
union
select r2.idm_id
from ervu_dashboard.recruitment r2
<lookup>max_update_date_from_recruit_archive</lookup>
<sql>select r.idm_id as recruitment_id,
? as max_update_date
from ervu_dashboard.recruitment r
join etl.job_execution je
on r2.idm_id = je.recruitment_id
on r.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>352</xloc>
<yloc>288</yloc>
<xloc>176</xloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>max_update_date_from_recruit_archive</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(updated_at) as max_update_date
from recruit_archive;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>0</xloc>
<yloc>304</yloc>
</GUI>
</transform>
<transform>
@ -112,6 +198,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -135,8 +226,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>128</yloc>
<xloc>752</xloc>
<yloc>144</yloc>
</GUI>
</transform>
<transform>
@ -163,6 +254,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -186,8 +282,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>208</yloc>
<xloc>752</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
@ -214,6 +310,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -237,8 +338,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>288</yloc>
<xloc>752</xloc>
<yloc>304</yloc>
</GUI>
</transform>
<transform>
@ -265,6 +366,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -288,8 +394,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>368</yloc>
<xloc>752</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
@ -316,6 +422,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -339,8 +450,36 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/>
<attributes/>
<GUI>
<xloc>656</xloc>
<yloc>448</yloc>
<xloc>752</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>updated_idm_ids_from_source_recruit_archive</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu_person_archive</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>max_update_date_from_recruit_archive</lookup>
<sql>WITH max_update_date(val) AS (SELECT CAST(? AS timestamp))
SELECT DISTINCT ra.vk_id as recruitment_id,
? as max_update_date
FROM public.recruit_archive ra
JOIN max_update_date mud ON TRUE
WHERE (mud.val IS NULL OR ra.updated_at > mud.val);</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>176</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform_error_handling>