micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

фикс rdi и rd

Browse source
This commit is contained in:
adel.ka 2025-11-14 14:05:46 +03:00
parent ff4b6c1ecc
commit 6fd41db91b
8 changed files with 911 additions and 644 deletions
Download patch file Download diff file Expand all files Collapse all files

194
mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow.hpl
View file

@ -21,53 +21,82 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>Create job execution record</from> <from>Create_job_execution_record_RD</from>
<to>Table input</to> <to>input_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>filter_rows_RD</from>
<to>Change job status on success</to> <to>сhange_status_on_success_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>detect_empty_stream_RD</from>
<to>Change job status on success</to> <to>сhange_status_on_success_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Change job status on error</from> <from>change_job_status_on_error_RD</from>
<to>Abort</to> <to>abort_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>output_RD</from>
<to>Identify last row in a stream</to> <to>change_job_status_on_error_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>output_RD</from>
<to>Table output</to> <to>detect_empty_stream_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table output</from> <from>output_RD</from>
<to>Filter rows</to> <to>identify_last_row_in_a_stream_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table output</from> <from>identify_last_row_in_a_stream_RD</from>
<to>Change job status on error</to> <to>filter_rows_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table output</from> <from>input_RD</from>
<to>Detect empty stream</to> <to>output_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
<transform> <transform>
<name>Abort</name> <name>Create_job_execution_record_RD</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<attributes/>
<GUI>
<xloc>288</xloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>abort_RD</name>
<type>Abort</type> <type>Abort</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -83,11 +112,11 @@
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1376</xloc> <xloc>1376</xloc>
<yloc>512</yloc> <yloc>592</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Change job status on error</name> <name>change_job_status_on_error_RD</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -118,71 +147,11 @@ and recruitment_id = '${IDM_ID}';
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1024</xloc> <xloc>1024</xloc>
<yloc>512</yloc> <yloc>592</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Change job status on success</name> <name>detect_empty_stream_RD</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS'
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1632</xloc>
<yloc>160</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<attributes/>
<GUI>
<xloc>288</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>Detect empty stream</name>
<type>DetectEmptyStream</type> <type>DetectEmptyStream</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -195,11 +164,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1632</xloc> <xloc>1632</xloc>
<yloc>320</yloc> <yloc>400</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Filter rows</name> <name>filter_rows_RD</name>
<type>FilterRows</type> <type>FilterRows</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -227,15 +196,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value> </value>
</condition> </condition>
</compare> </compare>
<send_true_to>Change job status on success</send_true_to> <send_true_to>сhange_status_on_success_RD</send_true_to>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1024</xloc> <xloc>1024</xloc>
<yloc>160</yloc> <yloc>64</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Identify last row in a stream</name> <name>identify_last_row_in_a_stream_RD</name>
<type>DetectLastRow</type> <type>DetectLastRow</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -248,12 +217,12 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<resultfieldname>last_row</resultfieldname> <resultfieldname>last_row</resultfieldname>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>752</xloc> <xloc>1024</xloc>
<yloc>320</yloc> <yloc>224</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Table input</name> <name>input_RD</name>
<type>TableInput</type> <type>TableInput</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -275,12 +244,12 @@ WHERE
<variables_active>Y</variables_active> <variables_active>Y</variables_active>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>496</xloc> <xloc>640</xloc>
<yloc>320</yloc> <yloc>400</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Table output</name> <name>output_RD</name>
<type>TableOutput</type> <type>TableOutput</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -366,13 +335,44 @@ WHERE
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1024</xloc> <xloc>1024</xloc>
<yloc>320</yloc> <yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>сhange_status_on_success_RD</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS'
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1632</xloc>
<yloc>64</yloc>
</GUI> </GUI>
</transform> </transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>Table output</source_transform> <source_transform>output_RD</source_transform>
<target_transform>Change job status on error</target_transform> <target_transform>change_job_status_on_error_RD</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>

90
mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_delta.hpl
View file

@ -21,53 +21,53 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>Create job execution record</from> <from>create_job_execution_record_RD</from>
<to>Table input</to> <to>table_input_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>detect_empty_stream_RD</from>
<to>Identify last row in a stream</to> <to>change_job_status_on_success_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>filter_rows_RD</from>
<to>Change job status on success</to> <to>change_job_status_on_success_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>change_job_status_on_error_RD</from>
<to>Change job status on success</to> <to>abort_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Change job status on error</from> <from>insert_or_update_RD</from>
<to>Abort</to> <to>detect_empty_stream_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>insert_or_update_RD</from>
<to>Insert / update</to> <to>change_job_status_on_error_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update</from> <from>insert_or_update_RD</from>
<to>Filter rows</to> <to>identify_last_row_in_a_stream_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update</from> <from>identify_last_row_in_a_stream_RD</from>
<to>Detect empty stream</to> <to>filter_rows_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update</from> <from>table_input_RD</from>
<to>Change job status on error</to> <to>insert_or_update_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
<transform> <transform>
<name>Abort</name> <name>abort_RD</name>
<type>Abort</type> <type>Abort</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -83,11 +83,11 @@
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1168</xloc> <xloc>1168</xloc>
<yloc>400</yloc> <yloc>544</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Change job status on error</name> <name>change_job_status_on_error_RD</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -118,11 +118,11 @@ and recruitment_id = '${IDM_ID}';
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>944</xloc> <xloc>944</xloc>
<yloc>400</yloc> <yloc>544</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Change job status on success</name> <name>change_job_status_on_success_RD</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -153,7 +153,7 @@ and recruitment_id = '${IDM_ID}';
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Create job execution record</name> <name>create_job_execution_record_RD</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -181,11 +181,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>336</xloc> <xloc>336</xloc>
<yloc>224</yloc> <yloc>368</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Detect empty stream</name> <name>detect_empty_stream_RD</name>
<type>DetectEmptyStream</type> <type>DetectEmptyStream</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -198,11 +198,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1248</xloc> <xloc>1248</xloc>
<yloc>224</yloc> <yloc>368</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Filter rows</name> <name>filter_rows_RD</name>
<type>FilterRows</type> <type>FilterRows</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -230,7 +230,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</value> </value>
</condition> </condition>
</compare> </compare>
<send_true_to>Change job status on success</send_true_to> <send_true_to>change_job_status_on_success_RD</send_true_to>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>944</xloc> <xloc>944</xloc>
@ -238,7 +238,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Identify last row in a stream</name> <name>identify_last_row_in_a_stream_RD</name>
<type>DetectLastRow</type> <type>DetectLastRow</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -251,12 +251,12 @@ and recruitment_id = '${IDM_ID}';</sql>
<resultfieldname>last_row</resultfieldname> <resultfieldname>last_row</resultfieldname>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>736</xloc> <xloc>944</xloc>
<yloc>224</yloc> <yloc>208</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Insert / update</name> <name>insert_or_update_RD</name>
<type>InsertUpdate</type> <type>InsertUpdate</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -274,11 +274,6 @@ and recruitment_id = '${IDM_ID}';</sql>
<field>id</field> <field>id</field>
<name>id</name> <name>id</name>
</key> </key>
<key>
<condition>=</condition>
<field>created_at</field>
<name>created_at</name>
</key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>restriction_document</table> <table>restriction_document</table>
<value> <value>
@ -289,12 +284,12 @@ and recruitment_id = '${IDM_ID}';</sql>
<value> <value>
<name>subpoena_id</name> <name>subpoena_id</name>
<rename>subpoena_id</rename> <rename>subpoena_id</rename>
<update>Y</update> <update>N</update>
</value> </value>
<value> <value>
<name>created_at</name> <name>created_at</name>
<rename>created_at</rename> <rename>created_at</rename>
<update>N</update> <update>Y</update>
</value> </value>
<value> <value>
<name>updated_at</name> <name>updated_at</name>
@ -356,11 +351,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>944</xloc> <xloc>944</xloc>
<yloc>224</yloc> <yloc>368</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Table input</name> <name>table_input_RD</name>
<type>TableInput</type> <type>TableInput</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -372,26 +367,25 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning> </partitioning>
<connection>postgres.subpoena</connection> <connection>postgres.subpoena</connection>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<limit/>
<sql>SELECT <sql>SELECT
* *
FROM public.restriction_document FROM public.restriction_document
WHERE WHERE
'${IDM_ID}' != '' -- Проверка на пустую строку '${IDM_ID}' != '' -- Проверка на пустую строку
AND vk_id = '${IDM_ID}' AND vk_id = '${IDM_ID}'
AND updated_at >= '${M_RESTRDOC_UP_DATE}'::timestamp AND updated_at >= '${MAX_UPDATE_DATE}'::timestamp
</sql> </sql>
<variables_active>Y</variables_active> <variables_active>Y</variables_active>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>544</xloc> <xloc>640</xloc>
<yloc>224</yloc> <yloc>368</yloc>
</GUI> </GUI>
</transform> </transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>Insert / update</source_transform> <source_transform>insert_or_update_RD</source_transform>
<target_transform>Change job status on error</target_transform> <target_transform>change_job_status_on_error_RD</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>

138
mappings/info_recruits/raw_data/restriction_document/parallel/restriction_document_flow_repeat.hpl
View file

@ -21,53 +21,53 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>Create job execution record</from> <from>create_job_execution_record_RD</from>
<to>Table input</to> <to>input_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>filter_rows_RD</from>
<to>Identify last row in a stream</to> <to>change_job_status_on_success_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>detect_empty_stream_RD</from>
<to>Change job status on success</to> <to>change_job_status_on_success_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>change_job_status_on_error_RD</from>
<to>Change job status on success</to> <to>abort_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Change job status on error</from> <from>insert_or_update_RD</from>
<to>Abort</to> <to>detect_empty_stream_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>insert_or_update_RD</from>
<to>Insert / update</to> <to>change_job_status_on_error_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update</from> <from>insert_or_update_RD</from>
<to>Filter rows</to> <to>identify_last_row_in_a_stream_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update</from> <from>identify_last_row_in_a_stream_RD</from>
<to>Detect empty stream</to> <to>filter_rows_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update</from> <from>input_RD</from>
<to>Change job status on error</to> <to>insert_or_update_RD</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
<transform> <transform>
<name>Abort</name> <name>abort_RD</name>
<type>Abort</type> <type>Abort</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -83,11 +83,11 @@
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1248</xloc> <xloc>1248</xloc>
<yloc>400</yloc> <yloc>656</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Change job status on error</name> <name>change_job_status_on_error_RD</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -118,11 +118,11 @@ and recruitment_id = '${IDM_ID}';
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>960</xloc> <xloc>960</xloc>
<yloc>400</yloc> <yloc>656</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Change job status on success</name> <name>change_job_status_on_success_RD</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -153,7 +153,7 @@ and recruitment_id = '${IDM_ID}';
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Create job execution record</name> <name>create_job_execution_record_RD</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -196,11 +196,11 @@ DO UPDATE SET
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>272</xloc> <xloc>272</xloc>
<yloc>208</yloc> <yloc>416</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Detect empty stream</name> <name>detect_empty_stream_RD</name>
<type>DetectEmptyStream</type> <type>DetectEmptyStream</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -213,11 +213,11 @@ DO UPDATE SET
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1440</xloc> <xloc>1440</xloc>
<yloc>208</yloc> <yloc>416</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Filter rows</name> <name>filter_rows_RD</name>
<type>FilterRows</type> <type>FilterRows</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -245,7 +245,7 @@ DO UPDATE SET
</value> </value>
</condition> </condition>
</compare> </compare>
<send_true_to>Change job status on success</send_true_to> <send_true_to>change_job_status_on_success_RD</send_true_to>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>960</xloc> <xloc>960</xloc>
@ -253,7 +253,7 @@ DO UPDATE SET
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Identify last row in a stream</name> <name>identify_last_row_in_a_stream_RD</name>
<type>DetectLastRow</type> <type>DetectLastRow</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -266,12 +266,39 @@ DO UPDATE SET
<resultfieldname>last_row</resultfieldname> <resultfieldname>last_row</resultfieldname>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>720</xloc> <xloc>960</xloc>
<yloc>208</yloc> <yloc>224</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Insert / update</name> <name>input_RD</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>postgres.subpoena</connection>
<execute_each_row>N</execute_each_row>
<sql>SELECT
*
FROM public.restriction_document
WHERE
'${IDM_ID}' != '' -- Проверка на пустую строку
AND vk_id = '${IDM_ID}'
</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>608</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>insert_or_update_RD</name>
<type>InsertUpdate</type> <type>InsertUpdate</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -288,11 +315,7 @@ DO UPDATE SET
<condition>=</condition> <condition>=</condition>
<field>id</field> <field>id</field>
<name>id</name> <name>id</name>
</key> <name2/>
<key>
<condition>=</condition>
<field>created_at</field>
<name>created_at</name>
</key> </key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>restriction_document</table> <table>restriction_document</table>
@ -304,12 +327,12 @@ DO UPDATE SET
<value> <value>
<name>subpoena_id</name> <name>subpoena_id</name>
<rename>subpoena_id</rename> <rename>subpoena_id</rename>
<update>Y</update> <update>N</update>
</value> </value>
<value> <value>
<name>created_at</name> <name>created_at</name>
<rename>created_at</rename> <rename>created_at</rename>
<update>N</update> <update>Y</update>
</value> </value>
<value> <value>
<name>updated_at</name> <name>updated_at</name>
@ -367,44 +390,17 @@ DO UPDATE SET
<update>Y</update> <update>Y</update>
</value> </value>
</lookup> </lookup>
<update_bypassed>N</update_bypassed> <update_bypassed>Y</update_bypassed>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>960</xloc> <xloc>960</xloc>
<yloc>208</yloc> <yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>postgres.subpoena</connection>
<execute_each_row>N</execute_each_row>
<sql>SELECT
*
FROM public.restriction_document
WHERE
'${IDM_ID}' != '' -- Проверка на пустую строку
AND vk_id = '${IDM_ID}'
</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>480</xloc>
<yloc>208</yloc>
</GUI> </GUI>
</transform> </transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>Insert / update</source_transform> <source_transform>insert_or_update_RD</source_transform>
<target_transform>Change job status on error</target_transform> <target_transform>change_job_status_on_error_RD</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>

225
mappings/info_recruits/raw_data/restriction_document/recruitment_five_flow_delta.hpl
View file

@ -21,33 +21,111 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>Table input</from> <from>max_update_date_from_restriction_document</from>
<to>restriction_document_flow_delta.hpl</to> <to>failed_restriction_document_idm_ids_on_delta</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>max_update_date_from_restriction_document</from>
<to>updated_idm_ids_from_source_restriction_document</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>restriction_document_flow_delta.hpl 1</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>restriction_document_flow_delta.hpl 2</to> <to>restriction_document_flow_delta.hpl 2</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>Sort rows</from>
<to>restriction_document_flow_delta.hpl 3</to> <to>restriction_document_flow_delta.hpl 3</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>Sort rows</from>
<to>restriction_document_flow_delta.hpl 4</to> <to>restriction_document_flow_delta.hpl 4</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>Sort rows</from>
<to>restriction_document_flow_delta.hpl 5</to> <to>restriction_document_flow_delta.hpl 5</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop>
<from>updated_idm_ids_from_source_restriction_document</from>
<to>Append streams</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>failed_restriction_document_idm_ids_on_delta</from>
<to>Append streams</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Append streams</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
</order> </order>
<transform> <transform>
<name>Table input</name> <name>Append streams</name>
<type>Append</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<head_name>updated_idm_ids_from_source_restriction_document</head_name>
<tail_name>failed_restriction_document_idm_ids_on_delta</tail_name>
<attributes/>
<GUI>
<xloc>592</xloc>
<yloc>800</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruitment_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>768</xloc>
<yloc>800</yloc>
</GUI>
</transform>
<transform>
<name>failed_restriction_document_idm_ids_on_delta</name>
<type>TableInput</type> <type>TableInput</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -60,36 +138,46 @@
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<limit>0</limit> <limit>0</limit>
<sql>WITH mud AS (SELECT recruitment_id, <lookup>max_update_date_from_restriction_document</lookup>
MAX(execution_datetime) AS max_upd_date <sql>select r.idm_id as recruitment_id,
FROM etl.job_execution ? as max_update_date
WHERE job_name = '${JOB_NAME}' from ervu_dashboard.recruitment r
AND status IN ('SUCCESS', 'DELTA_SUCCESS')
GROUP BY recruitment_id)
SELECT r.idm_id
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date
union
select r2.idm_id
from ervu_dashboard.recruitment r2
join etl.job_execution je join etl.job_execution je
on r2.idm_id = je.recruitment_id on r.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql> and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active> <variables_active>Y</variables_active>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>352</xloc> <xloc>352</xloc>
<yloc>288</yloc> <yloc>1008</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>restriction_document_flow_delta.hpl</name> <name>max_update_date_from_restriction_document</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(updated_at) as max_update_date
from ervu_dashboard.restriction_document;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>352</xloc>
<yloc>800</yloc>
</GUI>
</transform>
<transform>
<name>restriction_document_flow_delta.hpl 1</name>
<type>PipelineExecutor</type> <type>PipelineExecutor</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -109,7 +197,12 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<parameters> <parameters>
<variable_mapping> <variable_mapping>
<variable>IDM_ID</variable> <variable>IDM_ID</variable>
<field>idm_id</field> <field>recruitment_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/> <input/>
</variable_mapping> </variable_mapping>
<inherit_all_vars>Y</inherit_all_vars> <inherit_all_vars>Y</inherit_all_vars>
@ -135,8 +228,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/> <executors_output_transform/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>656</xloc> <xloc>928</xloc>
<yloc>128</yloc> <yloc>672</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -160,7 +253,12 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<parameters> <parameters>
<variable_mapping> <variable_mapping>
<variable>IDM_ID</variable> <variable>IDM_ID</variable>
<field>idm_id</field> <field>recruitment_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/> <input/>
</variable_mapping> </variable_mapping>
<inherit_all_vars>Y</inherit_all_vars> <inherit_all_vars>Y</inherit_all_vars>
@ -186,8 +284,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/> <executors_output_transform/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>656</xloc> <xloc>1024</xloc>
<yloc>208</yloc> <yloc>752</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -211,7 +309,12 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<parameters> <parameters>
<variable_mapping> <variable_mapping>
<variable>IDM_ID</variable> <variable>IDM_ID</variable>
<field>idm_id</field> <field>recruitment_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/> <input/>
</variable_mapping> </variable_mapping>
<inherit_all_vars>Y</inherit_all_vars> <inherit_all_vars>Y</inherit_all_vars>
@ -237,8 +340,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/> <executors_output_transform/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>656</xloc> <xloc>1024</xloc>
<yloc>288</yloc> <yloc>848</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -262,7 +365,12 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<parameters> <parameters>
<variable_mapping> <variable_mapping>
<variable>IDM_ID</variable> <variable>IDM_ID</variable>
<field>idm_id</field> <field>recruitment_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/> <input/>
</variable_mapping> </variable_mapping>
<inherit_all_vars>Y</inherit_all_vars> <inherit_all_vars>Y</inherit_all_vars>
@ -288,8 +396,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/> <executors_output_transform/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>656</xloc> <xloc>1024</xloc>
<yloc>368</yloc> <yloc>944</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -313,7 +421,12 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<parameters> <parameters>
<variable_mapping> <variable_mapping>
<variable>IDM_ID</variable> <variable>IDM_ID</variable>
<field>idm_id</field> <field>recruitment_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/> <input/>
</variable_mapping> </variable_mapping>
<inherit_all_vars>Y</inherit_all_vars> <inherit_all_vars>Y</inherit_all_vars>
@ -339,8 +452,36 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/> <executors_output_transform/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>656</xloc> <xloc>976</xloc>
<yloc>448</yloc> <yloc>1040</yloc>
</GUI>
</transform>
<transform>
<name>updated_idm_ids_from_source_restriction_document</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>postgres.subpoena</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>max_update_date_from_restriction_document</lookup>
<sql>WITH max_update_date(val) AS (SELECT CAST(? AS timestamp))
SELECT DISTINCT ra.vk_id as recruitment_id,
mud.val as max_update_date
FROM public.restriction_document ra
JOIN max_update_date mud ON TRUE
WHERE (mud.val IS NULL OR ra.updated_at > mud.val);</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>352</xloc>
<yloc>608</yloc>
</GUI> </GUI>
</transform> </transform>
<transform_error_handling> <transform_error_handling>

224
mappings/info_recruits/raw_data/restriction_document_item/parallel/restriction_document_item_flow.hpl
View file

@ -21,73 +21,68 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>Create job execution record</from> <from>create_job_execution_record_RDI</from>
<to>Table input</to> <to>table_input_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>filter_rows_RDI</from>
<to>Change job status on success</to> <to>change_job_status_on_success_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>detect_empty_stream_RDI</from>
<to>Change job status on success</to> <to>change_job_status_on_success_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Change job status on error</from> <from>change_job_status_on_error_RDI</from>
<to>Abort</to> <to>abort_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table output</from> <from>table_output_RDI</from>
<to>Change job status on error</to> <to>change_job_status_on_error_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Update</from> <from>update_flags_RDI</from>
<to>Change job status on error</to> <to>change_job_status_on_error_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>group_by_RDI</from>
<to>Identify last row in a stream</to> <to>update_flags_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>table_output_RDI</from>
<to>Table output</to> <to>identify_last_row_in_a_stream_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>identify_last_row_in_a_stream_RDI</from>
<to>Detect empty stream</to> <to>filter_rows_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>table_input_RDI</from>
<to>sort_by_recruit_id</to> <to>detect_empty_stream_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>sort_by_recruit_id</from> <from>table_input_RDI</from>
<to>Update</to> <to>group_by_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table output</from> <from>table_input_RDI</from>
<to>Block until transforms finish</to> <to>table_output_RDI</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
<transform> <transform>
<name>Abort</name> <name>abort_RDI</name>
<type>Abort</type> <type>Abort</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -99,37 +94,16 @@
</partitioning> </partitioning>
<abort_option>ABORT_WITH_ERROR</abort_option> <abort_option>ABORT_WITH_ERROR</abort_option>
<always_log_rows>Y</always_log_rows> <always_log_rows>Y</always_log_rows>
<message/>
<row_threshold>0</row_threshold> <row_threshold>0</row_threshold>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1360</xloc> <xloc>1360</xloc>
<yloc>496</yloc> <yloc>752</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Block until transforms finish</name> <name>change_job_status_on_error_RDI</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1152</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -145,8 +119,11 @@
</argument> </argument>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row> <execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>Y</set_params> <set_params>Y</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -157,14 +134,15 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'; and recruitment_id = '${IDM_ID}';
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1168</xloc> <xloc>1008</xloc>
<yloc>496</yloc> <yloc>752</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Change job status on success</name> <name>change_job_status_on_success_RDI</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -177,8 +155,11 @@ and recruitment_id = '${IDM_ID}';
<arguments> <arguments>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row> <execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>N</set_params> <set_params>N</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -188,14 +169,15 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'; and recruitment_id = '${IDM_ID}';
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1344</xloc> <xloc>1424</xloc>
<yloc>176</yloc> <yloc>176</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Create job execution record</name> <name>create_job_execution_record_RDI</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -208,8 +190,11 @@ and recruitment_id = '${IDM_ID}';
<arguments> <arguments>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>N</set_params> <set_params>N</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -217,14 +202,15 @@ and recruitment_id = '${IDM_ID}';
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>288</xloc> <xloc>288</xloc>
<yloc>320</yloc> <yloc>400</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Detect empty stream</name> <name>detect_empty_stream_RDI</name>
<type>DetectEmptyStream</type> <type>DetectEmptyStream</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -241,7 +227,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Filter rows</name> <name>filter_rows_RDI</name>
<type>FilterRows</type> <type>FilterRows</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -269,15 +255,52 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value> </value>
</condition> </condition>
</compare> </compare>
<send_true_to>Change job status on success</send_true_to> <send_true_to>change_job_status_on_success_RDI</send_true_to>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1344</xloc> <xloc>1424</xloc>
<yloc>320</yloc> <yloc>400</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Identify last row in a stream</name> <name>group_by_RDI</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<aggregate>has_active_temporary_measure</aggregate>
<subject>has_active_temporary_measure</subject>
<type>MAX</type>
<valuefield/>
</field>
</fields>
<give_back_row>N</give_back_row>
<group>
<field>
<name>recruit_id</name>
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<linenr_fieldname/>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
<name>identify_last_row_in_a_stream_RDI</name>
<type>DetectLastRow</type> <type>DetectLastRow</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -290,15 +313,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<resultfieldname>last_row</resultfieldname> <resultfieldname>last_row</resultfieldname>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>704</xloc> <xloc>1200</xloc>
<yloc>320</yloc> <yloc>400</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Table input</name> <name>table_input_RDI</name>
<type>TableInput</type> <type>TableInput</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>N</distribute>
<custom_distribution/> <custom_distribution/>
<copies>1</copies> <copies>1</copies>
<partitioning> <partitioning>
@ -323,16 +346,16 @@ s.id = rd.subpoena_id
WHERE WHERE
'${IDM_ID}' != '' -- Проверка на пустую строку '${IDM_ID}' != '' -- Проверка на пустую строку
AND rd.vk_id = '${IDM_ID}' AND rd.vk_id = '${IDM_ID}'
</sql> ORDER BY s.recruit_id</sql>
<variables_active>Y</variables_active> <variables_active>Y</variables_active>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>496</xloc> <xloc>704</xloc>
<yloc>320</yloc> <yloc>400</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Table output</name> <name>table_output_RDI</name>
<type>TableOutput</type> <type>TableOutput</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -410,23 +433,26 @@ WHERE
<only_when_have_rows>N</only_when_have_rows> <only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily> <partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled> <partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly> <partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys> <return_keys>N</return_keys>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields> <specify_fields>Y</specify_fields>
<table>restriction_document_item</table> <table>restriction_document_item</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field> <tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table> <tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate> <truncate>N</truncate>
<use_batch>Y</use_batch> <use_batch>Y</use_batch>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>928</xloc> <xloc>1008</xloc>
<yloc>320</yloc> <yloc>400</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Update</name> <name>update_flags_RDI</name>
<type>Update</type> <type>Update</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -439,11 +465,13 @@ WHERE
<commit>10000</commit> <commit>10000</commit>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored> <error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup> <lookup>
<key> <key>
<condition>=</condition> <condition>=</condition>
<field>recruit_id</field> <field>recruit_id</field>
<name>recruit_id</name> <name>recruit_id</name>
<name2/>
</key> </key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>citizen</table> <table>citizen</table>
@ -455,47 +483,15 @@ WHERE
<skip_lookup>N</skip_lookup> <skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch> <use_batch>Y</use_batch>
<attributes/> <attributes/>
<GUI>
<xloc>928</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>sort_by_recruit_id</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compress>N</compress>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<name>recruit_id</name>
<presorted>N</presorted>
</field>
</fields>
<sort_prefix>srt</sort_prefix>
<sort_size>1000000</sort_size>
<unique_rows>Y</unique_rows>
<attributes/>
<GUI> <GUI>
<xloc>704</xloc> <xloc>704</xloc>
<yloc>496</yloc> <yloc>752</yloc>
</GUI> </GUI>
</transform> </transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>Table output</source_transform> <source_transform>table_output_RDI</source_transform>
<target_transform>Change job status on error</target_transform> <target_transform>change_job_status_on_error_RDI</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>
@ -506,8 +502,8 @@ WHERE
<min_pct_rows/> <min_pct_rows/>
</error> </error>
<error> <error>
<source_transform>Update</source_transform> <source_transform>update_flags_RDI</source_transform>
<target_transform>Change job status on error</target_transform> <target_transform>change_job_status_on_error_RDI</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>

240
mappings/info_recruits/raw_data/restriction_document_item/parallel/restriction_document_item_flow_delta.hpl
View file

@ -21,73 +21,68 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>Create job execution record</from> <from>create_job_execution_record_RDI</from>
<to>Table input</to> <to>table_input_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>detect_empty_stream_RDI</from>
<to>Change job status on success</to> <to>change_job_status_on_success_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>filter_rows_RDI</from>
<to>Change job status on success</to> <to>change_job_status_on_success_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Change job status on error</from> <from>change_job_status_on_error_RDI</from>
<to>Abort</to> <to>abort_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update</from> <from>insert_or_update_RDI</from>
<to>Change job status on error</to> <to>change_job_status_on_error_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Update</from> <from>update_flags_RDI</from>
<to>Change job status on error</to> <to>change_job_status_on_error_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>insert_or_update_RDI</from>
<to>Identify last row in a stream</to> <to>identify_last_row_in_a_stream_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>identify_last_row_in_a_stream_RDI</from>
<to>Insert / update</to> <to>filter_rows_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>table_input_RDI</from>
<to>Detect empty stream</to> <to>insert_or_update_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update</from> <from>table_input_RDI</from>
<to>Block until transforms finish</to> <to>detect_empty_stream_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Block until transforms finish</from> <from>table_input_RDI</from>
<to>Filter rows</to> <to>group_by_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>group_by_RDI</from>
<to>sort_by_recruit_id</to> <to>update_flags_RDI</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sort_by_recruit_id</from>
<to>Update</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
<transform> <transform>
<name>Abort</name> <name>abort_RDI</name>
<type>Abort</type> <type>Abort</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -99,37 +94,16 @@
</partitioning> </partitioning>
<abort_option>ABORT_WITH_ERROR</abort_option> <abort_option>ABORT_WITH_ERROR</abort_option>
<always_log_rows>Y</always_log_rows> <always_log_rows>Y</always_log_rows>
<message/>
<row_threshold>0</row_threshold> <row_threshold>0</row_threshold>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1408</xloc> <xloc>1488</xloc>
<yloc>368</yloc> <yloc>784</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Block until transforms finish</name> <name>change_job_status_on_error_RDI</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1264</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -145,8 +119,11 @@
</argument> </argument>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row> <execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>Y</set_params> <set_params>Y</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -157,14 +134,15 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'; and recruitment_id = '${IDM_ID}';
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1232</xloc> <xloc>1152</xloc>
<yloc>368</yloc> <yloc>784</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Change job status on success</name> <name>change_job_status_on_success_RDI</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -177,8 +155,11 @@ and recruitment_id = '${IDM_ID}';
<arguments> <arguments>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row> <execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>N</set_params> <set_params>N</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -188,14 +169,15 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'; and recruitment_id = '${IDM_ID}';
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1472</xloc> <xloc>1552</xloc>
<yloc>80</yloc> <yloc>304</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Create job execution record</name> <name>create_job_execution_record_RDI</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -208,8 +190,11 @@ and recruitment_id = '${IDM_ID}';
<arguments> <arguments>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>N</set_params> <set_params>N</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -220,14 +205,15 @@ SET
error_description = NULL error_description = NULL
where job_name = '${JOB_NAME}' where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql> and recruitment_id = '${IDM_ID}';</sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>336</xloc> <xloc>320</xloc>
<yloc>224</yloc> <yloc>576</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Detect empty stream</name> <name>detect_empty_stream_RDI</name>
<type>DetectEmptyStream</type> <type>DetectEmptyStream</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -239,12 +225,12 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning> </partitioning>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>768</xloc> <xloc>752</xloc>
<yloc>80</yloc> <yloc>304</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Filter rows</name> <name>filter_rows_RDI</name>
<type>FilterRows</type> <type>FilterRows</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -272,15 +258,52 @@ and recruitment_id = '${IDM_ID}';</sql>
</value> </value>
</condition> </condition>
</compare> </compare>
<send_true_to>Change job status on success</send_true_to> <send_true_to>change_job_status_on_success_RDI</send_true_to>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1472</xloc> <xloc>1552</xloc>
<yloc>224</yloc> <yloc>576</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Identify last row in a stream</name> <name>group_by_RDI</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<aggregate>has_active_temporary_measure</aggregate>
<subject>has_active_temporary_measure</subject>
<type>MAX</type>
<valuefield/>
</field>
</fields>
<give_back_row>N</give_back_row>
<group>
<field>
<name>recruit_id</name>
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<linenr_fieldname/>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>784</yloc>
</GUI>
</transform>
<transform>
<name>identify_last_row_in_a_stream_RDI</name>
<type>DetectLastRow</type> <type>DetectLastRow</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -293,12 +316,12 @@ and recruitment_id = '${IDM_ID}';</sql>
<resultfieldname>last_row</resultfieldname> <resultfieldname>last_row</resultfieldname>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>768</xloc> <xloc>1344</xloc>
<yloc>224</yloc> <yloc>576</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Insert / update</name> <name>insert_or_update_RDI</name>
<type>InsertUpdate</type> <type>InsertUpdate</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -315,11 +338,13 @@ and recruitment_id = '${IDM_ID}';</sql>
<condition>=</condition> <condition>=</condition>
<field>id</field> <field>id</field>
<name>id</name> <name>id</name>
<name2/>
</key> </key>
<key> <key>
<condition>=</condition> <condition>=</condition>
<field>created_at</field> <field>created_at</field>
<name>created_at</name> <name>created_at</name>
<name2/>
</key> </key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>restriction_document_item</table> <table>restriction_document_item</table>
@ -402,23 +427,24 @@ and recruitment_id = '${IDM_ID}';</sql>
<update_bypassed>N</update_bypassed> <update_bypassed>N</update_bypassed>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>992</xloc> <xloc>1152</xloc>
<yloc>224</yloc> <yloc>576</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Table input</name> <name>table_input_RDI</name>
<type>TableInput</type> <type>TableInput</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>N</distribute>
<custom_distribution/> <custom_distribution/>
<copies>1</copies> <copies>1</copies>
<partitioning> <partitioning>
<method>none</method> <method>none</method>
<schema_name/> <schema_name/>
</partitioning> </partitioning>
<connection>ervu_person_registry</connection> <connection>postgres.subpoena</connection>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<limit/>
<sql>select <sql>select
s.recruit_id, s.recruit_id,
CASE CASE
@ -434,16 +460,17 @@ s.id = rd.subpoena_id
WHERE WHERE
'${IDM_ID}' != '' -- Проверка на пустую строку '${IDM_ID}' != '' -- Проверка на пустую строку
AND rd.vk_id = '${IDM_ID}' AND rd.vk_id = '${IDM_ID}'
AND rdi.updated_at >= '${M_RDI_UP_DATE}'::timestamp</sql> AND rdi.updated_at >= '${MAX_UPDATE_DATE}'::timestamp
ORDER BY s.recruit_id</sql>
<variables_active>Y</variables_active> <variables_active>Y</variables_active>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>544</xloc> <xloc>752</xloc>
<yloc>224</yloc> <yloc>576</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Update</name> <name>update_flags_RDI</name>
<type>Update</type> <type>Update</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -456,16 +483,19 @@ WHERE
<commit>10000</commit> <commit>10000</commit>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored> <error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup> <lookup>
<key> <key>
<condition>=</condition> <condition>=</condition>
<field>recruit_id</field> <field>recruit_id</field>
<name>recruit_id</name> <name>recruit_id</name>
<name2/>
</key> </key>
<key> <key>
<condition>&lt;></condition> <condition>&lt;></condition>
<field>has_active_temporary_measure</field> <field>has_active_temporary_measure</field>
<name>has_active_temporary_measure</name> <name>has_active_temporary_measure</name>
<name2/>
</key> </key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>citizen</table> <table>citizen</table>
@ -478,46 +508,14 @@ WHERE
<use_batch>Y</use_batch> <use_batch>Y</use_batch>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>992</xloc> <xloc>960</xloc>
<yloc>368</yloc> <yloc>784</yloc>
</GUI>
</transform>
<transform>
<name>sort_by_recruit_id</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compress>N</compress>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<name>recruit_id</name>
<presorted>N</presorted>
</field>
</fields>
<sort_prefix>srt</sort_prefix>
<sort_size>1000000</sort_size>
<unique_rows>Y</unique_rows>
<attributes/>
<GUI>
<xloc>768</xloc>
<yloc>368</yloc>
</GUI> </GUI>
</transform> </transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>Insert / update</source_transform> <source_transform>insert_or_update_RDI</source_transform>
<target_transform>Change job status on error</target_transform> <target_transform>change_job_status_on_error_RDI</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>
@ -528,8 +526,8 @@ WHERE
<min_pct_rows/> <min_pct_rows/>
</error> </error>
<error> <error>
<source_transform>Update</source_transform> <source_transform>update_flags_RDI</source_transform>
<target_transform>Change job status on error</target_transform> <target_transform>change_job_status_on_error_RDI</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>

227
mappings/info_recruits/raw_data/restriction_document_item/parallel/restriction_document_item_flow_repeat.hpl
View file

@ -21,73 +21,68 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>Create job execution record</from> <from>create_job_execution_record_RDI</from>
<to>Table input</to> <to>table_input_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>filter_rows_RDI</from>
<to>Change job status on success</to> <to>change_job_status_on_success_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>detect_empty_stream_RDI</from>
<to>Change job status on success</to> <to>change_job_status_on_success_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Change job status on error</from> <from>change_job_status_on_error_RDI</from>
<to>Abort</to> <to>abort_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update</from> <from>insert_or_update_RDI</from>
<to>Change job status on error</to> <to>change_job_status_on_error_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Update</from> <from>update_flags_RDI</from>
<to>Change job status on error</to> <to>change_job_status_on_error_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>table_input_RDI</from>
<to>Identify last row in a stream</to> <to>group_by_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>group_by_RDI</from>
<to>Insert / update</to> <to>update_flags_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update</from> <from>identify_last_row_in_a_stream_RDI</from>
<to>Block until transforms finish</to> <to>filter_rows_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>detect_empty_stream_RDI</from>
<to>sort_by_recruit_id</to> <to>table_input_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>sort_by_recruit_id</from> <from>table_input_RDI</from>
<to>Update</to> <to>insert_or_update_RDI</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>insert_or_update_RDI</from>
<to>Detect empty stream</to> <to>identify_last_row_in_a_stream_RDI</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
<transform> <transform>
<name>Abort</name> <name>abort_RDI</name>
<type>Abort</type> <type>Abort</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -99,37 +94,16 @@
</partitioning> </partitioning>
<abort_option>ABORT_WITH_ERROR</abort_option> <abort_option>ABORT_WITH_ERROR</abort_option>
<always_log_rows>Y</always_log_rows> <always_log_rows>Y</always_log_rows>
<message/>
<row_threshold>0</row_threshold> <row_threshold>0</row_threshold>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1312</xloc> <xloc>1440</xloc>
<yloc>368</yloc> <yloc>448</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Block until transforms finish</name> <name>change_job_status_on_error_RDI</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1184</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -145,8 +119,11 @@
</argument> </argument>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row> <execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>Y</set_params> <set_params>Y</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -157,14 +134,15 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'; and recruitment_id = '${IDM_ID}';
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1136</xloc> <xloc>1024</xloc>
<yloc>368</yloc> <yloc>448</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Change job status on success</name> <name>change_job_status_on_success_RDI</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -177,8 +155,11 @@ and recruitment_id = '${IDM_ID}';
<arguments> <arguments>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row> <execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>N</set_params> <set_params>N</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -188,6 +169,7 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'; and recruitment_id = '${IDM_ID}';
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1440</xloc> <xloc>1440</xloc>
@ -195,7 +177,7 @@ and recruitment_id = '${IDM_ID}';
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Create job execution record</name> <name>create_job_execution_record_RDI</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -208,8 +190,11 @@ and recruitment_id = '${IDM_ID}';
<arguments> <arguments>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>N</set_params> <set_params>N</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -235,17 +220,18 @@ DO UPDATE SET
execution_datetime = DEFAULT, execution_datetime = DEFAULT,
error_description = NULL error_description = NULL
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>272</xloc> <xloc>272</xloc>
<yloc>208</yloc> <yloc>256</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Detect empty stream</name> <name>detect_empty_stream_RDI</name>
<type>DetectEmptyStream</type> <type>DetectEmptyStream</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>N</distribute>
<custom_distribution/> <custom_distribution/>
<copies>1</copies> <copies>1</copies>
<partitioning> <partitioning>
@ -254,12 +240,12 @@ DO UPDATE SET
</partitioning> </partitioning>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>672</xloc> <xloc>656</xloc>
<yloc>64</yloc> <yloc>64</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Filter rows</name> <name>filter_rows_RDI</name>
<type>FilterRows</type> <type>FilterRows</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -287,15 +273,52 @@ DO UPDATE SET
</value> </value>
</condition> </condition>
</compare> </compare>
<send_true_to>Change job status on success</send_true_to> <send_true_to>change_job_status_on_success_RDI</send_true_to>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1440</xloc> <xloc>1440</xloc>
<yloc>208</yloc> <yloc>256</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Identify last row in a stream</name> <name>group_by_RDI</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<aggregate>has_active_temporary_measure</aggregate>
<subject>has_active_temporary_measure</subject>
<type>MAX</type>
<valuefield/>
</field>
</fields>
<give_back_row>N</give_back_row>
<group>
<field>
<name>recruit_id</name>
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<linenr_fieldname/>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>656</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>identify_last_row_in_a_stream_RDI</name>
<type>DetectLastRow</type> <type>DetectLastRow</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -308,12 +331,12 @@ DO UPDATE SET
<resultfieldname>last_row</resultfieldname> <resultfieldname>last_row</resultfieldname>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>672</xloc> <xloc>1264</xloc>
<yloc>208</yloc> <yloc>256</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Insert / update</name> <name>insert_or_update_RDI</name>
<type>InsertUpdate</type> <type>InsertUpdate</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -330,11 +353,13 @@ DO UPDATE SET
<condition>=</condition> <condition>=</condition>
<field>id</field> <field>id</field>
<name>id</name> <name>id</name>
<name2/>
</key> </key>
<key> <key>
<condition>=</condition> <condition>=</condition>
<field>created_at</field> <field>created_at</field>
<name>created_at</name> <name>created_at</name>
<name2/>
</key> </key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>restriction_document_item</table> <table>restriction_document_item</table>
@ -417,15 +442,15 @@ DO UPDATE SET
<update_bypassed>N</update_bypassed> <update_bypassed>N</update_bypassed>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>912</xloc> <xloc>1024</xloc>
<yloc>208</yloc> <yloc>256</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Table input</name> <name>table_input_RDI</name>
<type>TableInput</type> <type>TableInput</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>N</distribute>
<custom_distribution/> <custom_distribution/>
<copies>1</copies> <copies>1</copies>
<partitioning> <partitioning>
@ -450,16 +475,17 @@ s.id = rd.subpoena_id
WHERE WHERE
'${IDM_ID}' != '' -- Проверка на пустую строку '${IDM_ID}' != '' -- Проверка на пустую строку
AND rd.vk_id = '${IDM_ID}' AND rd.vk_id = '${IDM_ID}'
ORDER BY s.recruit_id
</sql> </sql>
<variables_active>Y</variables_active> <variables_active>Y</variables_active>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>480</xloc> <xloc>656</xloc>
<yloc>208</yloc> <yloc>256</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Update</name> <name>update_flags_RDI</name>
<type>Update</type> <type>Update</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -472,16 +498,19 @@ WHERE
<commit>10000</commit> <commit>10000</commit>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored> <error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup> <lookup>
<key> <key>
<condition>=</condition> <condition>=</condition>
<field>recruit_id</field> <field>recruit_id</field>
<name>recruit_id</name> <name>recruit_id</name>
<name2/>
</key> </key>
<key> <key>
<condition>&lt;></condition> <condition>&lt;></condition>
<field>has_active_temporary_measure</field> <field>has_active_temporary_measure</field>
<name>has_active_temporary_measure</name> <name>has_active_temporary_measure</name>
<name2/>
</key> </key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>citizen</table> <table>citizen</table>
@ -494,46 +523,14 @@ WHERE
<use_batch>Y</use_batch> <use_batch>Y</use_batch>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>912</xloc> <xloc>832</xloc>
<yloc>368</yloc> <yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>sort_by_recruit_id</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compress>N</compress>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<name>recruit_id</name>
<presorted>N</presorted>
</field>
</fields>
<sort_prefix>srt</sort_prefix>
<sort_size>1000000</sort_size>
<unique_rows>Y</unique_rows>
<attributes/>
<GUI>
<xloc>672</xloc>
<yloc>368</yloc>
</GUI> </GUI>
</transform> </transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>Insert / update</source_transform> <source_transform>insert_or_update_RDI</source_transform>
<target_transform>Change job status on error</target_transform> <target_transform>change_job_status_on_error_RDI</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>
@ -544,8 +541,8 @@ WHERE
<min_pct_rows/> <min_pct_rows/>
</error> </error>
<error> <error>
<source_transform>Update</source_transform> <source_transform>update_flags_RDI</source_transform>
<target_transform>Change job status on error</target_transform> <target_transform>change_job_status_on_error_RDI</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>

217
mappings/info_recruits/raw_data/restriction_document_item/recruitment_five_flow_delta.hpl
View file

@ -21,33 +21,111 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>Table input</from> <from>Append streams</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>failed_restriction_document_item_idm_ids_on_delta</from>
<to>Append streams</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>max_update_date_from_restriction_document_item</from>
<to>failed_restriction_document_item_idm_ids_on_delta</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>max_update_date_from_restriction_document_item</from>
<to>updated_idm_ids_from_source_restriction_document_item</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>updated_idm_ids_from_source_restriction_document_item</from>
<to>Append streams</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>restriction_document_item_flow_delta.hpl</to> <to>restriction_document_item_flow_delta.hpl</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>Sort rows</from>
<to>restriction_document_item_flow_delta.hpl 2</to> <to>restriction_document_item_flow_delta.hpl 2</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>Sort rows</from>
<to>restriction_document_item_flow_delta.hpl 3</to> <to>restriction_document_item_flow_delta.hpl 3</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>Sort rows</from>
<to>restriction_document_item_flow_delta.hpl 4</to> <to>restriction_document_item_flow_delta.hpl 4</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>Sort rows</from>
<to>restriction_document_item_flow_delta.hpl 5</to> <to>restriction_document_item_flow_delta.hpl 5</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
<transform> <transform>
<name>Table input</name> <name>Append streams</name>
<type>Append</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<head_name>updated_idm_ids_from_source_restriction_document_item</head_name>
<tail_name>failed_restriction_document_item_idm_ids_on_delta</tail_name>
<attributes/>
<GUI>
<xloc>272</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruitment_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>448</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>failed_restriction_document_item_idm_ids_on_delta</name>
<type>TableInput</type> <type>TableInput</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -60,32 +138,42 @@
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<limit>0</limit> <limit>0</limit>
<sql>WITH mud AS (SELECT recruitment_id, <lookup>max_update_date_from_restriction_document_item</lookup>
MAX(execution_datetime) AS max_upd_date <sql>select r.idm_id as recruitment_id,
FROM etl.job_execution ? as max_update_date
WHERE job_name = '${JOB_NAME}' from ervu_dashboard.recruitment r
AND status IN ('SUCCESS', 'DELTA_SUCCESS')
GROUP BY recruitment_id)
SELECT r.idm_id
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date
union
select r2.idm_id
from ervu_dashboard.recruitment r2
join etl.job_execution je join etl.job_execution je
on r2.idm_id = je.recruitment_id on r.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING') where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql> and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active> <variables_active>Y</variables_active>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>352</xloc> <xloc>0</xloc>
<yloc>288</yloc> <yloc>784</yloc>
</GUI>
</transform>
<transform>
<name>max_update_date_from_restriction_document_item</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(updated_at) as max_update_date
from ervu_dashboard.restriction_document_item;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>0</xloc>
<yloc>576</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -112,6 +200,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<field>recruitment_id</field> <field>recruitment_id</field>
<input/> <input/>
</variable_mapping> </variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars> <inherit_all_vars>Y</inherit_all_vars>
</parameters> </parameters>
<execution_result_target_transform/> <execution_result_target_transform/>
@ -135,8 +228,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/> <executors_output_transform/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>656</xloc> <xloc>688</xloc>
<yloc>128</yloc> <yloc>432</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -163,6 +256,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<field>recruitment_id</field> <field>recruitment_id</field>
<input/> <input/>
</variable_mapping> </variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars> <inherit_all_vars>Y</inherit_all_vars>
</parameters> </parameters>
<execution_result_target_transform/> <execution_result_target_transform/>
@ -186,8 +284,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/> <executors_output_transform/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>656</xloc> <xloc>688</xloc>
<yloc>208</yloc> <yloc>512</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -214,6 +312,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<field>recruitment_id</field> <field>recruitment_id</field>
<input/> <input/>
</variable_mapping> </variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars> <inherit_all_vars>Y</inherit_all_vars>
</parameters> </parameters>
<execution_result_target_transform/> <execution_result_target_transform/>
@ -237,8 +340,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/> <executors_output_transform/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>656</xloc> <xloc>688</xloc>
<yloc>288</yloc> <yloc>592</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -265,6 +368,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<field>recruitment_id</field> <field>recruitment_id</field>
<input/> <input/>
</variable_mapping> </variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars> <inherit_all_vars>Y</inherit_all_vars>
</parameters> </parameters>
<execution_result_target_transform/> <execution_result_target_transform/>
@ -288,8 +396,8 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/> <executors_output_transform/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>656</xloc> <xloc>688</xloc>
<yloc>368</yloc> <yloc>672</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -316,6 +424,11 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<field>recruitment_id</field> <field>recruitment_id</field>
<input/> <input/>
</variable_mapping> </variable_mapping>
<variable_mapping>
<variable>MAX_UPDATE_DATE</variable>
<field>max_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars> <inherit_all_vars>Y</inherit_all_vars>
</parameters> </parameters>
<execution_result_target_transform/> <execution_result_target_transform/>
@ -339,8 +452,40 @@ where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
<executors_output_transform/> <executors_output_transform/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>656</xloc> <xloc>688</xloc>
<yloc>448</yloc> <yloc>752</yloc>
</GUI>
</transform>
<transform>
<name>updated_idm_ids_from_source_restriction_document_item</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>postgres.subpoena</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>max_update_date_from_restriction_document_item</lookup>
<sql>WITH max_update_date(val) AS (SELECT CAST(? AS timestamp))
SELECT DISTINCT rd.vk_id as recruitment_id,
mud.val as max_update_date
FROM public.restriction_document_item rdi
join public.restriction_document rd on
rd.id = coalesce(rdi.restriction_document_create_id,rdi.restriction_document_cancel_id)
join public.subpoena s on
s.id = rd.subpoena_id
JOIN max_update_date mud ON TRUE
WHERE (mud.val IS NULL OR rdi.updated_at > mud.val);</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>0</xloc>
<yloc>384</yloc>
</GUI> </GUI>
</transform> </transform>
<transform_error_handling> <transform_error_handling>