micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

rename

Browse source
This commit is contained in:
adel.ka 2025-11-14 16:17:39 +03:00
parent 6fd41db91b
commit 66840c2c28
6 changed files with 926 additions and 876 deletions
Download patch file Download diff file Expand all files Collapse all files

416
mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl
View file

@ -21,68 +21,68 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>Create job execution record</from> <from>create_job_execution_record_punishment</from>
<to>punishment_input</to> <to>table_input_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_output</from> <from>table_output_punishment</from>
<to>punishment_error_job_status</to> <to>change_job_status_on_error_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_error_job_status</from> <from>change_job_status_on_error_punishment</from>
<to>Abort</to> <to>abort_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>detect_empty_stream_punishment</from>
<to>punishment_success_job_status</to> <to>change_job_status_on_success_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_input</from> <from>table_input_punishment</from>
<to>punishment_output</to> <to>table_output_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_update_flags</from> <from>update_flags_punishment</from>
<to>punishment_error_job_status</to> <to>change_job_status_on_error_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_sort_rows</from> <from>sort_rows_punishment</from>
<to>punishment_update_flags</to> <to>update_flags_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_input</from> <from>table_input_punishment</from>
<to>punishment_sort_rows</to> <to>sort_rows_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_input</from> <from>table_input_punishment</from>
<to>Detect empty stream</to> <to>detect_empty_stream_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_output</from> <from>table_output_punishment</from>
<to>Identify last row in a stream</to> <to>identify_last_row_in_a_stream_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>identify_last_row_in_a_stream_punishment</from>
<to>Filter rows</to> <to>filter_rows_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>filter_rows_punishment</from>
<to>punishment_success_job_status</to> <to>change_job_status_on_success_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
<transform> <transform>
<name>Abort</name> <name>abort_punishment</name>
<type>Abort</type> <type>Abort</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -94,44 +94,16 @@
</partitioning> </partitioning>
<abort_option>ABORT_WITH_ERROR</abort_option> <abort_option>ABORT_WITH_ERROR</abort_option>
<always_log_rows>Y</always_log_rows> <always_log_rows>Y</always_log_rows>
<message/>
<row_threshold>0</row_threshold> <row_threshold>0</row_threshold>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1168</xloc> <xloc>1472</xloc>
<yloc>448</yloc> <yloc>544</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Create job execution record</name> <name>detect_empty_stream_punishment</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<attributes/>
<GUI>
<xloc>80</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>Detect empty stream</name>
<type>DetectEmptyStream</type> <type>DetectEmptyStream</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -148,7 +120,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Filter rows</name> <name>filter_rows_punishment</name>
<type>FilterRows</type> <type>FilterRows</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -176,15 +148,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value> </value>
</condition> </condition>
</compare> </compare>
<send_true_to>punishment_success_job_status</send_true_to> <send_true_to>change_job_status_on_success_punishment</send_true_to>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>720</xloc> <xloc>928</xloc>
<yloc>144</yloc> <yloc>144</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Identify last row in a stream</name> <name>identify_last_row_in_a_stream_punishment</name>
<type>DetectLastRow</type> <type>DetectLastRow</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -197,12 +169,41 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<resultfieldname>last_row</resultfieldname> <resultfieldname>last_row</resultfieldname>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>720</xloc> <xloc>928</xloc>
<yloc>224</yloc> <yloc>272</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>punishment_error_job_status</name> <name>create_job_execution_record_punishment</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<attributes/>
<GUI>
<xloc>80</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>change_job_status_on_error_punishment</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -218,8 +219,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</argument> </argument>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row> <execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>Y</set_params> <set_params>Y</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -231,14 +235,139 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'; and recruitment_id = '${IDM_ID}';
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1008</xloc> <xloc>1216</xloc>
<yloc>448</yloc> <yloc>544</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>punishment_input</name> <name>sort_rows_punishment</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>336</xloc>
<yloc>720</yloc>
</GUI>
</transform>
<transform>
<name>change_job_status_on_success_punishment</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>688</xloc>
<yloc>144</yloc>
</GUI>
</transform>
<transform>
<name>update_flags_punishment</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_punishment_execution_info</field>
<name>has_punishment_execution_info</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_sentence_info</field>
<name>has_sentence_info</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_punishment_execution_info</name>
<rename>has_punishment_execution_info</rename>
</value>
<value>
<name>has_sentence_info</name>
<rename>has_sentence_info</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>720</yloc>
</GUI>
</transform>
<transform>
<name>table_input_punishment</name>
<type>TableInput</type> <type>TableInput</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -250,7 +379,6 @@ and recruitment_id = '${IDM_ID}';
</partitioning> </partitioning>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH punishment_data AS ( <sql>WITH punishment_data AS (
SELECT SELECT
ri.recruit_id, ri.recruit_id,
@ -299,11 +427,11 @@ GROUP BY recruit_id, punishment_info;</sql>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>336</xloc> <xloc>336</xloc>
<yloc>320</yloc> <yloc>416</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>punishment_output</name> <name>table_output_punishment</name>
<type>TableOutput</type> <type>TableOutput</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -408,142 +536,14 @@ GROUP BY recruit_id, punishment_info;</sql>
<use_batch>Y</use_batch> <use_batch>Y</use_batch>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>720</xloc> <xloc>928</xloc>
<yloc>320</yloc> <yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>punishment_sort_rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>336</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
<name>punishment_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>512</xloc>
<yloc>144</yloc>
</GUI>
</transform>
<transform>
<name>punishment_update_flags</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<key>
<condition>&lt;></condition>
<field>has_punishment_execution_info</field>
<name>has_punishment_execution_info</name>
</key>
<key>
<condition>&lt;></condition>
<field>has_sentence_info</field>
<name>has_sentence_info</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_punishment_execution_info</name>
<rename>has_punishment_execution_info</rename>
</value>
<value>
<name>has_sentence_info</name>
<rename>has_sentence_info</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>624</yloc>
</GUI> </GUI>
</transform> </transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>punishment_output</source_transform> <source_transform>update_flags_punishment</source_transform>
<target_transform>punishment_error_job_status</target_transform> <target_transform>change_job_status_on_error_punishment</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>punishment_update_flags</source_transform>
<target_transform>punishment_error_job_status</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_decription</descriptions_valuename> <descriptions_valuename>error_decription</descriptions_valuename>
@ -553,6 +553,18 @@ and status = 'PROCESSING';
<max_pct_errors/> <max_pct_errors/>
<min_pct_rows/> <min_pct_rows/>
</error> </error>
<error>
<source_transform>table_output_punishment</source_transform>
<target_transform>change_job_status_on_error_punishment</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
</transform_error_handling> </transform_error_handling>
<attributes/> <attributes/>
</pipeline> </pipeline>

103
mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl
View file

@ -40,68 +40,68 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>Create job execution record</from> <from>create_job_execution_record_punishment</from>
<to>punishment_input</to> <to>punishment_input</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_insert_or_update</from> <from>insert_or_update_punishment</from>
<to>punishment_error_job_status</to> <to>change_job_status_on_error_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>detect_empty_stream_punishment</from>
<to>punishment_success_job_status</to> <to>change_job_status_on_success_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_error_job_status</from> <from>change_job_status_on_error_punishment</from>
<to>Abort</to> <to>abort_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_input</from> <from>punishment_input</from>
<to>punishment_insert_or_update</to> <to>insert_or_update_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_update_flags</from> <from>update_flags_punishment</from>
<to>punishment_error_job_status</to> <to>change_job_status_on_error_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_input</from> <from>punishment_input</from>
<to>punishment_sort_rows</to> <to>sort_rows_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_input</from> <from>punishment_input</from>
<to>Detect empty stream</to> <to>detect_empty_stream_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_sort_rows</from> <from>sort_rows_punishment</from>
<to>punishment_update_flags</to> <to>update_flags_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_insert_or_update</from> <from>insert_or_update_punishment</from>
<to>Identify last row in a stream</to> <to>Identify last row in a stream</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>Identify last row in a stream</from>
<to>Filter rows</to> <to>filter_rows_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>filter_rows_punishment</from>
<to>punishment_success_job_status</to> <to>change_job_status_on_success_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
<transform> <transform>
<name>Abort</name> <name>abort_punishment</name>
<type>Abort</type> <type>Abort</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -113,15 +113,16 @@
</partitioning> </partitioning>
<abort_option>ABORT_WITH_ERROR</abort_option> <abort_option>ABORT_WITH_ERROR</abort_option>
<always_log_rows>Y</always_log_rows> <always_log_rows>Y</always_log_rows>
<message/>
<row_threshold>0</row_threshold> <row_threshold>0</row_threshold>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1120</xloc> <xloc>1392</xloc>
<yloc>384</yloc> <yloc>368</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Create job execution record</name> <name>create_job_execution_record_punishment</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -134,8 +135,11 @@
<arguments> <arguments>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>N</set_params> <set_params>N</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -146,6 +150,7 @@ SET
error_description = NULL error_description = NULL
where job_name = '${JOB_NAME}' where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql> and recruitment_id = '${IDM_ID}';</sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>128</xloc> <xloc>128</xloc>
@ -153,7 +158,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Detect empty stream</name> <name>detect_empty_stream_punishment</name>
<type>DetectEmptyStream</type> <type>DetectEmptyStream</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -170,7 +175,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Filter rows</name> <name>filter_rows_punishment</name>
<type>FilterRows</type> <type>FilterRows</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -198,10 +203,10 @@ and recruitment_id = '${IDM_ID}';</sql>
</value> </value>
</condition> </condition>
</compare> </compare>
<send_true_to>punishment_success_job_status</send_true_to> <send_true_to>change_job_status_on_success_punishment</send_true_to>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>784</xloc> <xloc>944</xloc>
<yloc>0</yloc> <yloc>0</yloc>
</GUI> </GUI>
</transform> </transform>
@ -219,12 +224,12 @@ and recruitment_id = '${IDM_ID}';</sql>
<resultfieldname>last_row</resultfieldname> <resultfieldname>last_row</resultfieldname>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>784</xloc> <xloc>944</xloc>
<yloc>96</yloc> <yloc>80</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>punishment_error_job_status</name> <name>change_job_status_on_error_punishment</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -240,8 +245,11 @@ and recruitment_id = '${IDM_ID}';</sql>
</argument> </argument>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row> <execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>Y</set_params> <set_params>Y</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -253,10 +261,11 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'; and recruitment_id = '${IDM_ID}';
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>960</xloc> <xloc>1120</xloc>
<yloc>384</yloc> <yloc>368</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -325,7 +334,7 @@ GROUP BY recruit_id, punishment_info;</sql>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>punishment_insert_or_update</name> <name>insert_or_update_punishment</name>
<type>InsertUpdate</type> <type>InsertUpdate</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -342,11 +351,13 @@ GROUP BY recruit_id, punishment_info;</sql>
<condition>=</condition> <condition>=</condition>
<field>recruit_id</field> <field>recruit_id</field>
<name>recruit_id</name> <name>recruit_id</name>
<name2/>
</key> </key>
<key> <key>
<condition>=</condition> <condition>=</condition>
<field>punishment_external_id</field> <field>punishment_external_id</field>
<name>punishment_external_id</name> <name>punishment_external_id</name>
<name2/>
</key> </key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>punishment</table> <table>punishment</table>
@ -449,12 +460,12 @@ GROUP BY recruit_id, punishment_info;</sql>
<update_bypassed>N</update_bypassed> <update_bypassed>N</update_bypassed>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>784</xloc> <xloc>944</xloc>
<yloc>224</yloc> <yloc>224</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>punishment_sort_rows</name> <name>sort_rows_punishment</name>
<type>SortRows</type> <type>SortRows</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -488,7 +499,7 @@ GROUP BY recruit_id, punishment_info;</sql>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>punishment_success_job_status</name> <name>change_job_status_on_success_punishment</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -501,8 +512,11 @@ GROUP BY recruit_id, punishment_info;</sql>
<arguments> <arguments>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row> <execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>N</set_params> <set_params>N</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -514,14 +528,15 @@ and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING'; and status = 'DELTA_PROCESSING';
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>576</xloc> <xloc>688</xloc>
<yloc>0</yloc> <yloc>0</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>punishment_update_flags</name> <name>update_flags_punishment</name>
<type>Update</type> <type>Update</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -534,21 +549,25 @@ and status = 'DELTA_PROCESSING';
<commit>10000</commit> <commit>10000</commit>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored> <error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup> <lookup>
<key> <key>
<condition>=</condition> <condition>=</condition>
<field>recruit_id</field> <field>recruit_id</field>
<name>recruit_id</name> <name>recruit_id</name>
<name2/>
</key> </key>
<key> <key>
<condition>&lt;></condition> <condition>&lt;></condition>
<field>has_punishment_execution_info</field> <field>has_punishment_execution_info</field>
<name>has_punishment_execution_info</name> <name>has_punishment_execution_info</name>
<name2/>
</key> </key>
<key> <key>
<condition>&lt;></condition> <condition>&lt;></condition>
<field>has_sentence_info</field> <field>has_sentence_info</field>
<name>has_sentence_info</name> <name>has_sentence_info</name>
<name2/>
</key> </key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>citizen</table> <table>citizen</table>
@ -565,14 +584,14 @@ and status = 'DELTA_PROCESSING';
<use_batch>Y</use_batch> <use_batch>Y</use_batch>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>960</xloc> <xloc>1120</xloc>
<yloc>608</yloc> <yloc>608</yloc>
</GUI> </GUI>
</transform> </transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>punishment_insert_or_update</source_transform> <source_transform>insert_or_update_punishment</source_transform>
<target_transform>punishment_error_job_status</target_transform> <target_transform>change_job_status_on_error_punishment</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>
@ -583,8 +602,8 @@ and status = 'DELTA_PROCESSING';
<min_pct_rows/> <min_pct_rows/>
</error> </error>
<error> <error>
<source_transform>punishment_update_flags</source_transform> <source_transform>update_flags_punishment</source_transform>
<target_transform>punishment_error_job_status</target_transform> <target_transform>change_job_status_on_error_punishment</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>

413
mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl
View file

@ -21,52 +21,52 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>Create job execution record</from> <from>create_job_execution_record_punishment</from>
<to>punishment_input</to> <to>table_input_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_error_job_status</from> <from>change_job_status_on_error_punishment</from>
<to>Abort</to> <to>abort_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>detect_empty_stream_punishment</from>
<to>punishment_success_job_status</to> <to>change_job_status_on_success_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_update_flags</from> <from>update_flags_punishment</from>
<to>punishment_error_job_status</to> <to>change_job_status_on_error_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_insert_or_update</from> <from>insert_or_update_punishment</from>
<to>punishment_error_job_status</to> <to>change_job_status_on_error_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_input</from> <from>table_input_punishment</from>
<to>punishment_insert_or_update</to> <to>insert_or_update_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_input</from> <from>table_input_punishment</from>
<to>punishment_sort_rows</to> <to>sort_rows_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_sort_rows</from> <from>sort_rows_punishment</from>
<to>punishment_update_flags</to> <to>update_flags_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_input</from> <from>table_input_punishment</from>
<to>Detect empty stream</to> <to>detect_empty_stream_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>punishment_insert_or_update</from> <from>insert_or_update_punishment</from>
<to>Identify last row in a stream</to> <to>Identify last row in a stream</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
@ -77,12 +77,66 @@
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>Filter rows</from>
<to>punishment_success_job_status</to> <to>change_job_status_on_success_punishment</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
<transform> <transform>
<name>Abort</name> <name>Filter rows</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>change_job_status_on_success_punishment</send_true_to>
<attributes/>
<GUI>
<xloc>992</xloc>
<yloc>64</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>992</xloc>
<yloc>144</yloc>
</GUI>
</transform>
<transform>
<name>abort_punishment</name>
<type>Abort</type> <type>Abort</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -97,12 +151,81 @@
<row_threshold>0</row_threshold> <row_threshold>0</row_threshold>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1152</xloc> <xloc>1264</xloc>
<yloc>432</yloc> <yloc>432</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Create job execution record</name> <name>change_job_status_on_error_punishment</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>992</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>change_job_status_on_success_punishment</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>688</xloc>
<yloc>64</yloc>
</GUI>
</transform>
<transform>
<name>create_job_execution_record_punishment</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -148,7 +271,7 @@ DO UPDATE SET
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Detect empty stream</name> <name>detect_empty_stream_punishment</name>
<type>DetectEmptyStream</type> <type>DetectEmptyStream</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -165,162 +288,7 @@ DO UPDATE SET
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Filter rows</name> <name>insert_or_update_punishment</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>punishment_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>800</xloc>
<yloc>64</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>800</xloc>
<yloc>144</yloc>
</GUI>
</transform>
<transform>
<name>punishment_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>992</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>punishment_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH punishment_data AS (
SELECT
ri.recruit_id,
jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
AND '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null'
)
SELECT recruit_id,
(punishment_info ->> 'id')::uuid AS punishment_external_id,
-- Блок "Сведения о вынесенном приговоре"
punishment_info ->> 'vidNakaz' AS punishment_type, -- Вид наказания
punishment_info ->> 'kodVidNakaz' AS punishment_code, -- Код вида наказания
TO_DATE(punishment_info ->> 'dataOsuzhd', 'YYYY-MM-DD') AS sentense_date, -- Дата осуждения
punishment_info ->> 'naimSud' AS authority_name, -- Наименование суда
punishment_info ->> 'srokNakaz' AS sentence, -- Срок наказания
CASE
WHEN punishment_info ->> 'kodIzmSrokNakaz' = '1' THEN 'Изменений срока наказания не было'
WHEN punishment_info ->> 'kodIzmSrokNakaz' = '2' THEN 'Были изменения срока наказания'
ELSE punishment_info ->> 'kodIzmSrokNakaz'
END AS sentence_changed, -- Признак изменения срока
STRING_AGG(TRIM(BOTH ' "' FROM law_articles), ', ') AS law_point, -- Статьи УК
-- Блок "Сведения об отбывании наказания"
punishment_info ->> 'katUcheta' AS registration_category, -- Категория учета
punishment_info ->> 'naimSubekt' AS region_name, -- Наименование субъекта
punishment_info ->> 'naimOrg' AS facility_name, -- Название учреждения
punishment_info ->> 'adresOrg' AS facility_address, -- Адрес учреждения
punishment_info ->> 'nomerUD' AS criminal_case_number, -- Номер уголовного дела
TO_DATE(punishment_info ->> 'dataNachSrok', 'YYYY-MM-DD') AS start_date, -- Дата начала срока
TO_DATE(punishment_info ->> 'dataOkonchSrok', 'YYYY-MM-DD') AS planned_date, -- Планируемая дата освобождения
CASE
WHEN punishment_info ->> 'kodPerekvalif' = '1' THEN 'Изменений статьи не было'
WHEN punishment_info ->> 'kodPerekvalif' = '2' THEN 'Были изменения статьи'
ELSE punishment_info ->> 'kodPerekvalif'
END AS article_modification_flag, -- Признак переквалификации
punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения
TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения
true AS has_punishment_execution_info,
true AS has_sentence_info
FROM punishment_data
LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true
GROUP BY recruit_id, punishment_info;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>416</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform>
<name>punishment_insert_or_update</name>
<type>InsertUpdate</type> <type>InsertUpdate</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -444,12 +412,12 @@ GROUP BY recruit_id, punishment_info;</sql>
<update_bypassed>Y</update_bypassed> <update_bypassed>Y</update_bypassed>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>800</xloc> <xloc>992</xloc>
<yloc>272</yloc> <yloc>272</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>punishment_sort_rows</name> <name>sort_rows_punishment</name>
<type>SortRows</type> <type>SortRows</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -483,40 +451,71 @@ GROUP BY recruit_id, punishment_info;</sql>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>punishment_success_job_status</name> <name>table_input_punishment</name>
<type>ExecSql</type> <type>TableInput</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>N</distribute>
<custom_distribution/> <custom_distribution/>
<copies>1</copies> <copies>1</copies>
<partitioning> <partitioning>
<method>none</method> <method>none</method>
<schema_name/> <schema_name/>
</partitioning> </partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row> <execute_each_row>N</execute_each_row>
<quoteString>N</quoteString> <sql>WITH punishment_data AS (
<replace_variables>Y</replace_variables> SELECT
<set_params>N</set_params> ri.recruit_id,
<single_statement>N</single_statement> jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info
<sql>UPDATE etl.job_execution FROM ervu_dashboard.recruits_info ri
SET status = 'SUCCESS', JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
execution_end_datetime = current_timestamp AND '${IDM_ID}' != ''
WHERE job_name = '${JOB_NAME}' AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
and recruitment_id = '${IDM_ID}' WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null'
and status = 'PROCESSING'; )
SELECT recruit_id,
</sql> (punishment_info ->> 'id')::uuid AS punishment_external_id,
-- Блок "Сведения о вынесенном приговоре"
punishment_info ->> 'vidNakaz' AS punishment_type, -- Вид наказания
punishment_info ->> 'kodVidNakaz' AS punishment_code, -- Код вида наказания
TO_DATE(punishment_info ->> 'dataOsuzhd', 'YYYY-MM-DD') AS sentense_date, -- Дата осуждения
punishment_info ->> 'naimSud' AS authority_name, -- Наименование суда
punishment_info ->> 'srokNakaz' AS sentence, -- Срок наказания
CASE
WHEN punishment_info ->> 'kodIzmSrokNakaz' = '1' THEN 'Изменений срока наказания не было'
WHEN punishment_info ->> 'kodIzmSrokNakaz' = '2' THEN 'Были изменения срока наказания'
ELSE punishment_info ->> 'kodIzmSrokNakaz'
END AS sentence_changed, -- Признак изменения срока
STRING_AGG(TRIM(BOTH ' "' FROM law_articles), ', ') AS law_point, -- Статьи УК
-- Блок "Сведения об отбывании наказания"
punishment_info ->> 'katUcheta' AS registration_category, -- Категория учета
punishment_info ->> 'naimSubekt' AS region_name, -- Наименование субъекта
punishment_info ->> 'naimOrg' AS facility_name, -- Название учреждения
punishment_info ->> 'adresOrg' AS facility_address, -- Адрес учреждения
punishment_info ->> 'nomerUD' AS criminal_case_number, -- Номер уголовного дела
TO_DATE(punishment_info ->> 'dataNachSrok', 'YYYY-MM-DD') AS start_date, -- Дата начала срока
TO_DATE(punishment_info ->> 'dataOkonchSrok', 'YYYY-MM-DD') AS planned_date, -- Планируемая дата освобождения
CASE
WHEN punishment_info ->> 'kodPerekvalif' = '1' THEN 'Изменений статьи не было'
WHEN punishment_info ->> 'kodPerekvalif' = '2' THEN 'Были изменения статьи'
ELSE punishment_info ->> 'kodPerekvalif'
END AS article_modification_flag, -- Признак переквалификации
punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения
TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения
true AS has_punishment_execution_info,
true AS has_sentence_info
FROM punishment_data
LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true
GROUP BY recruit_id, punishment_info;</sql>
<variables_active>Y</variables_active>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>608</xloc> <xloc>416</xloc>
<yloc>64</yloc> <yloc>272</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>punishment_update_flags</name> <name>update_flags_punishment</name>
<type>Update</type> <type>Update</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -566,8 +565,8 @@ and status = 'PROCESSING';
</transform> </transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>punishment_insert_or_update</source_transform> <source_transform>insert_or_update_punishment</source_transform>
<target_transform>punishment_error_job_status</target_transform> <target_transform>change_job_status_on_error_punishment</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>
@ -578,8 +577,8 @@ and status = 'PROCESSING';
<min_pct_rows/> <min_pct_rows/>
</error> </error>
<error> <error>
<source_transform>punishment_update_flags</source_transform> <source_transform>update_flags_punishment</source_transform>
<target_transform>punishment_error_job_status</target_transform> <target_transform>change_job_status_on_error_punishment</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>

64
mappings/info_recruits/citizen_tables/snils/parallel/snils_flow.hpl
View file

@ -21,53 +21,53 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>snils_create_job</from> <from>create_job_execution_record_snils</from>
<to>snils_input</to> <to>table_input_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>filter_rows_snils</from>
<to>snils_success_job</to> <to>snils_success_job</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>detect_empty_stream_snils</from>
<to>snils_success_job</to> <to>snils_success_job</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>snils_failure_job</from> <from>change_job_status_on_error_snils</from>
<to>Abort</to> <to>abort_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>snils_output</from> <from>table_output_snils</from>
<to>snils_failure_job</to> <to>change_job_status_on_error_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>snils_input</from> <from>table_input_snils</from>
<to>Detect empty stream</to> <to>detect_empty_stream_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>snils_output</from> <from>table_output_snils</from>
<to>Identify last row in a stream </to> <to>identify_last_row_in_a_stream_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream </from> <from>identify_last_row_in_a_stream_snils</from>
<to>Filter rows</to> <to>filter_rows_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>snils_input</from> <from>table_input_snils</from>
<to>snils_output</to> <to>table_output_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
<transform> <transform>
<name>Abort</name> <name>abort_snils</name>
<type>Abort</type> <type>Abort</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -79,6 +79,7 @@
</partitioning> </partitioning>
<abort_option>ABORT_WITH_ERROR</abort_option> <abort_option>ABORT_WITH_ERROR</abort_option>
<always_log_rows>Y</always_log_rows> <always_log_rows>Y</always_log_rows>
<message/>
<row_threshold>0</row_threshold> <row_threshold>0</row_threshold>
<attributes/> <attributes/>
<GUI> <GUI>
@ -87,7 +88,7 @@
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Detect empty stream</name> <name>detect_empty_stream_snils</name>
<type>DetectEmptyStream</type> <type>DetectEmptyStream</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -104,7 +105,7 @@
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Filter rows</name> <name>filter_rows_snils</name>
<type>FilterRows</type> <type>FilterRows</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -140,7 +141,7 @@
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Identify last row in a stream </name> <name>identify_last_row_in_a_stream_snils</name>
<type>DetectLastRow</type> <type>DetectLastRow</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -158,7 +159,7 @@
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>snils_create_job</name> <name>create_job_execution_record_snils</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -171,8 +172,11 @@
<arguments> <arguments>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>N</set_params> <set_params>N</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -180,6 +184,7 @@
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>288</xloc> <xloc>288</xloc>
@ -187,7 +192,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>snils_failure_job</name> <name>change_job_status_on_error_snils</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -203,8 +208,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</argument> </argument>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row> <execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>Y</set_params> <set_params>Y</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -216,6 +224,7 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'; and recruitment_id = '${IDM_ID}';
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1024</xloc> <xloc>1024</xloc>
@ -223,7 +232,7 @@ and recruitment_id = '${IDM_ID}';
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>snils_input</name> <name>table_input_snils</name>
<type>TableInput</type> <type>TableInput</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -280,7 +289,7 @@ WHERE LENGTH(pred_snils_cleaned) = 11</sql>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>snils_output</name> <name>table_output_snils</name>
<type>TableOutput</type> <type>TableOutput</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -318,11 +327,14 @@ WHERE LENGTH(pred_snils_cleaned) = 11</sql>
<only_when_have_rows>N</only_when_have_rows> <only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily> <partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled> <partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly> <partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys> <return_keys>N</return_keys>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields> <specify_fields>Y</specify_fields>
<table>snils</table> <table>snils</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field> <tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table> <tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate> <truncate>N</truncate>
@ -367,8 +379,8 @@ and recruitment_id = '${IDM_ID}';
</transform> </transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>snils_output</source_transform> <source_transform>table_output_snils</source_transform>
<target_transform>snils_failure_job</target_transform> <target_transform>change_job_status_on_error_snils</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>

443
mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_delta.hpl
View file

@ -21,53 +21,53 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>snils_create_job</from> <from>create_job_execution_record_snils</from>
<to>snils_input</to> <to>table_input_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>detect_empty_stream_snils</from>
<to>snils_success_job</to> <to>change_job_status_on_success_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>filter_rows_snils</from>
<to>snils_success_job</to> <to>change_job_status_on_success_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>snils_failure_job</from> <from>change_job_status_on_error_snils</from>
<to>Abort</to> <to>abort_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>snils_insert_or_update</from> <from>insert_or_update_snils</from>
<to>snils_failure_job</to> <to>change_job_status_on_error_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>snils_insert_or_update</from> <from>insert_or_update_snils</from>
<to>Identify last row in a stream 2</to> <to>identify_last_row_in_a_stream_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream 2</from> <from>identify_last_row_in_a_stream_snils</from>
<to>Filter rows</to> <to>filter_rows_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>snils_input</from> <from>table_input_snils</from>
<to>snils_insert_or_update</to> <to>insert_or_update_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>snils_input</from> <from>table_input_snils</from>
<to>Detect empty stream</to> <to>detect_empty_stream_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
<transform> <transform>
<name>Abort</name> <name>abort_snils</name>
<type>Abort</type> <type>Abort</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -79,6 +79,7 @@
</partitioning> </partitioning>
<abort_option>ABORT_WITH_ERROR</abort_option> <abort_option>ABORT_WITH_ERROR</abort_option>
<always_log_rows>Y</always_log_rows> <always_log_rows>Y</always_log_rows>
<message/>
<row_threshold>0</row_threshold> <row_threshold>0</row_threshold>
<attributes/> <attributes/>
<GUI> <GUI>
@ -87,7 +88,114 @@
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>snils_failure_job</name> <name>detect_empty_stream_snils</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>544</xloc>
<yloc>240</yloc>
</GUI>
</transform>
<transform>
<name>filter_rows_snils</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>change_job_status_on_success_snils</send_true_to>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>64</yloc>
</GUI>
</transform>
<transform>
<name>identify_last_row_in_a_stream_snils</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>240</yloc>
</GUI>
</transform>
<transform>
<name>create_job_execution_record_snils</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET
status = 'DELTA_PROCESSING',
execution_datetime = DEFAULT,
error_description = NULL
where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>336</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>change_job_status_on_error_snils</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -127,205 +235,7 @@ and recruitment_id = '${IDM_ID}';
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>snils_success_job</name> <name>table_input_snils</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>544</xloc>
<yloc>64</yloc>
</GUI>
</transform>
<transform>
<name>snils_create_job</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET
status = 'DELTA_PROCESSING',
execution_datetime = DEFAULT,
error_description = NULL
where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>336</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Detect empty stream</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>544</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform>
<name>Filter rows</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>snils_success_job</send_true_to>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>64</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream 2</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>240</yloc>
</GUI>
</transform>
<transform>
<name>snils_insert_or_update</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>snils</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>snils</name>
<rename>snils</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>source_id</name>
<rename>source_id</rename>
<update>N</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>snils_input</name>
<type>TableInput</type> <type>TableInput</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -397,10 +307,101 @@ FROM snils_data;</sql>
<yloc>432</yloc> <yloc>432</yloc>
</GUI> </GUI>
</transform> </transform>
<transform>
<name>insert_or_update_snils</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>snils</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>snils</name>
<rename>snils</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>source_id</name>
<rename>source_id</rename>
<update>N</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>change_job_status_on_success_snils</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>544</xloc>
<yloc>64</yloc>
</GUI>
</transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>snils_insert_or_update</source_transform> <source_transform>insert_or_update_snils</source_transform>
<target_transform>snils_failure_job</target_transform> <target_transform>change_job_status_on_error_snils</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>

363
mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_repeat.hpl
View file

@ -21,53 +21,53 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>snils_create_job</from> <from>create_job_execution_record_snils</from>
<to>snils_input</to> <to>table_input_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>filter_rows_snils</from>
<to>snils_success_job</to> <to>change_job_status_on_success_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>detect_empty_stream_snils</from>
<to>snils_success_job</to> <to>change_job_status_on_success_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>snils_failure_job</from> <from>change_job_status_on_error_snils</from>
<to>Abort</to> <to>abort_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>snils_insert_or_update</from> <from>insert_or_update_snils</from>
<to>snils_failure_job</to> <to>change_job_status_on_error_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>snils_input</from> <from>table_input_snils</from>
<to>snils_insert_or_update</to> <to>insert_or_update_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>snils_input</from> <from>table_input_snils</from>
<to>Detect empty stream</to> <to>detect_empty_stream_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>snils_insert_or_update</from> <from>insert_or_update_snils</from>
<to>Identify last row in a stream </to> <to>identify_last_row_in_a_stream_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream </from> <from>identify_last_row_in_a_stream_snils</from>
<to>Filter rows</to> <to>filter_rows_snils</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
<transform> <transform>
<name>Abort</name> <name>abort_snils</name>
<type>Abort</type> <type>Abort</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -79,6 +79,7 @@
</partitioning> </partitioning>
<abort_option>ABORT_WITH_ERROR</abort_option> <abort_option>ABORT_WITH_ERROR</abort_option>
<always_log_rows>Y</always_log_rows> <always_log_rows>Y</always_log_rows>
<message/>
<row_threshold>0</row_threshold> <row_threshold>0</row_threshold>
<attributes/> <attributes/>
<GUI> <GUI>
@ -87,7 +88,129 @@
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>snils_failure_job</name> <name>detect_empty_stream_snils</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>480</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
<name>filter_rows_snils</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>change_job_status_on_success_snils</send_true_to>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>48</yloc>
</GUI>
</transform>
<transform>
<name>identify_last_row_in_a_stream_snils</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>create_job_execution_record_snils</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (
id,
job_name,
status,
execution_datetime,
error_description,
recruitment_id
)
VALUES (
DEFAULT,
'${JOB_NAME}',
'PROCESSING',
DEFAULT,
NULL,
'${IDM_ID}'
)
ON CONFLICT (job_name, recruitment_id)
DO UPDATE SET
status = 'PROCESSING',
execution_datetime = DEFAULT,
error_description = NULL
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>272</xloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>change_job_status_on_error_snils</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -127,161 +250,7 @@ and recruitment_id = '${IDM_ID}';
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>snils_success_job</name> <name>table_input_snils</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>480</xloc>
<yloc>48</yloc>
</GUI>
</transform>
<transform>
<name>Detect empty stream</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>480</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
<name>Filter rows</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>snils_success_job</send_true_to>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>48</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream </name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>snils_create_job</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (
id,
job_name,
status,
execution_datetime,
error_description,
recruitment_id
)
VALUES (
DEFAULT,
'${JOB_NAME}',
'PROCESSING',
DEFAULT,
NULL,
'${IDM_ID}'
)
ON CONFLICT (job_name, recruitment_id)
DO UPDATE SET
status = 'PROCESSING',
execution_datetime = DEFAULT,
error_description = NULL
</sql>
<attributes/>
<GUI>
<xloc>272</xloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>snils_input</name>
<type>TableInput</type> <type>TableInput</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -293,6 +262,7 @@ DO UPDATE SET
</partitioning> </partitioning>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH <sql>WITH
base as ( base as (
SELECT SELECT
@ -337,7 +307,7 @@ WHERE LENGTH(pred_snils_cleaned) = 11</sql>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>snils_insert_or_update</name> <name>insert_or_update_snils</name>
<type>InsertUpdate</type> <type>InsertUpdate</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -354,6 +324,7 @@ WHERE LENGTH(pred_snils_cleaned) = 11</sql>
<condition>=</condition> <condition>=</condition>
<field>source_id</field> <field>source_id</field>
<name>source_id</name> <name>source_id</name>
<name2/>
</key> </key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>snils</table> <table>snils</table>
@ -390,10 +361,46 @@ WHERE LENGTH(pred_snils_cleaned) = 11</sql>
<yloc>400</yloc> <yloc>400</yloc>
</GUI> </GUI>
</transform> </transform>
<transform>
<name>change_job_status_on_success_snils</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>480</xloc>
<yloc>48</yloc>
</GUI>
</transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>snils_insert_or_update</source_transform> <source_transform>insert_or_update_snils</source_transform>
<target_transform>snils_failure_job</target_transform> <target_transform>change_job_status_on_error_snils</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>