micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ERVU-528: Фиксики(Симка и Нолик)

Browse source
This commit is contained in:
adel.ka 2025-08-15 17:43:46 +03:00
parent 40b058ee91
commit 7e75f73116
4 changed files with 485 additions and 121 deletions
Download patch file Download diff file Expand all files Collapse all files

227
mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl
View file

@ -35,26 +35,11 @@
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Table output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<to>Detect empty stream</to>
@ -65,6 +50,41 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Table output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<to>Group by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Group by</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Add constants</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Add constants</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -82,8 +102,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1120</xloc>
<yloc>640</yloc>
<xloc>1216</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
@ -117,8 +137,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>640</yloc>
<xloc>1056</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
@ -135,11 +155,8 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
@ -150,11 +167,10 @@ and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1456</xloc>
<yloc>112</yloc>
<yloc>32</yloc>
</GUI>
</transform>
<transform>
@ -182,8 +198,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<attributes/>
<GUI>
<xloc>288</xloc>
<yloc>320</yloc>
<xloc>272</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
@ -200,7 +216,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<attributes/>
<GUI>
<xloc>1456</xloc>
<yloc>320</yloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
@ -235,26 +251,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>112</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>672</xloc>
<yloc>320</yloc>
<xloc>1200</xloc>
<yloc>32</yloc>
</GUI>
</transform>
<transform>
@ -318,8 +316,8 @@ GROUP BY r.recruit_id, up;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>320</yloc>
<xloc>480</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
@ -436,10 +434,127 @@ GROUP BY r.recruit_id, up;</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>848</xloc>
<xloc>832</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>Group by</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
</fields>
<give_back_row>N</give_back_row>
<group>
<field>
<name>recruit_id</name>
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<linenr_fieldname/>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>832</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>832</xloc>
<yloc>176</yloc>
</GUI>
</transform>
<transform>
<name>Add constants</name>
<type>Constant</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<fields>
<field>
<currency/>
<decimal/>
<format/>
<group/>
<length>-1</length>
<name>has_criminal_prosecution</name>
<nullif>true</nullif>
<precision>-1</precision>
<set_empty_string>N</set_empty_string>
<type>Boolean</type>
</field>
</fields>
<attributes/>
<GUI>
<xloc>832</xloc>
<yloc>32</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_criminal_prosecution</name>
<rename>has_criminal_prosecution</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1056</xloc>
<yloc>32</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Table output</source_transform>
@ -453,6 +568,18 @@ GROUP BY r.recruit_id, up;</sql>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename>error_code</codes_valuename>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
</transform_error_handling>
<attributes/>
</pipeline>

199
mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl
View file

@ -40,21 +40,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Insert / update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
@ -65,6 +50,41 @@
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream 2</from>
<to>Add constants</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Group by</from>
<to>Identify last row in a stream 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Group by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Add constants</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Insert / update</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -82,8 +102,35 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>464</yloc>
<xloc>992</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>Add constants</name>
<type>Constant</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<fields>
<field>
<length>-1</length>
<name>has_criminal_prosecution</name>
<nullif>true</nullif>
<precision>-1</precision>
<set_empty_string>N</set_empty_string>
<type>Boolean</type>
</field>
</fields>
<attributes/>
<GUI>
<xloc>736</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
@ -117,8 +164,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>464</yloc>
<xloc>880</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
@ -149,8 +196,8 @@ and status = 'DELTA_PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>1120</xloc>
<yloc>80</yloc>
<xloc>1152</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
@ -181,8 +228,8 @@ where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql>
<attributes/>
<GUI>
<xloc>128</xloc>
<yloc>224</yloc>
<xloc>112</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
@ -198,8 +245,8 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>80</yloc>
<xloc>1152</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
@ -234,12 +281,42 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1120</xloc>
<yloc>224</yloc>
<xloc>1024</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<name>Group by</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
</fields>
<give_back_row>N</give_back_row>
<group>
<field>
<name>recruit_id</name>
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>736</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream 2</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
@ -252,8 +329,8 @@ and recruitment_id = '${IDM_ID}';</sql>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>560</xloc>
<yloc>224</yloc>
<xloc>736</xloc>
<yloc>528</yloc>
</GUI>
</transform>
<transform>
@ -274,13 +351,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>prosecution_external_id</field>
<name>prosecution_external_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>prosecution</table>
@ -389,17 +464,12 @@ and recruitment_id = '${IDM_ID}';</sql>
<rename>type_info</rename>
<update>Y</update>
</value>
<value>
<name>last_row</name>
<rename>last_row</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>224</yloc>
<xloc>736</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
@ -466,8 +536,43 @@ GROUP BY r.recruit_id, up;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>224</yloc>
<xloc>384</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_criminal_prosecution</name>
<rename>has_criminal_prosecution</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>880</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform_error_handling>
@ -483,6 +588,18 @@ GROUP BY r.recruit_id, up;</sql>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename>error_code</codes_valuename>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
</transform_error_handling>
<attributes/>
</pipeline>

178
mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl
View file

@ -40,29 +40,49 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Insert / update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Group by</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Insert / update</to>
<to>Add constants</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<to>Abort</to>
<from>Insert / update</from>
<to>Group by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Add constants</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
</order>
@ -82,8 +102,35 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1232</xloc>
<yloc>544</yloc>
<xloc>1120</xloc>
<yloc>736</yloc>
</GUI>
</transform>
<transform>
<name>Add constants</name>
<type>Constant</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<fields>
<field>
<length>-1</length>
<name>has_criminal_prosecution</name>
<nullif>true</nullif>
<precision>-1</precision>
<set_empty_string>N</set_empty_string>
<type>Boolean</type>
</field>
</fields>
<attributes/>
<GUI>
<xloc>768</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -117,8 +164,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>768</xloc>
<yloc>544</yloc>
<xloc>944</xloc>
<yloc>736</yloc>
</GUI>
</transform>
<transform>
@ -149,8 +196,8 @@ and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>16</yloc>
<xloc>1296</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -196,7 +243,7 @@ DO UPDATE SET
<attributes/>
<GUI>
<xloc>176</xloc>
<yloc>272</yloc>
<yloc>944</yloc>
</GUI>
</transform>
<transform>
@ -212,8 +259,8 @@ DO UPDATE SET
</partitioning>
<attributes/>
<GUI>
<xloc>768</xloc>
<yloc>16</yloc>
<xloc>1296</xloc>
<yloc>944</yloc>
</GUI>
</transform>
<transform>
@ -248,8 +295,38 @@ DO UPDATE SET
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>272</yloc>
<xloc>1088</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
<name>Group by</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
</fields>
<give_back_row>N</give_back_row>
<group>
<field>
<name>recruit_id</name>
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>768</xloc>
<yloc>800</yloc>
</GUI>
</transform>
<transform>
@ -266,8 +343,8 @@ DO UPDATE SET
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>608</xloc>
<yloc>272</yloc>
<xloc>768</xloc>
<yloc>704</yloc>
</GUI>
</transform>
<transform>
@ -288,11 +365,13 @@ DO UPDATE SET
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>prosecution_external_id</field>
<name>prosecution_external_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>prosecution</table>
@ -401,17 +480,12 @@ DO UPDATE SET
<rename>type_info</rename>
<update>Y</update>
</value>
<value>
<name>last_row</name>
<rename>last_row</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>768</xloc>
<yloc>272</yloc>
<yloc>944</yloc>
</GUI>
</transform>
<transform>
@ -427,7 +501,6 @@ DO UPDATE SET
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH
ugolov_presl AS (
SELECT
@ -477,7 +550,42 @@ GROUP BY r.recruit_id, up;</sql>
<attributes/>
<GUI>
<xloc>416</xloc>
<yloc>272</yloc>
<yloc>944</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_criminal_prosecution</name>
<rename>has_criminal_prosecution</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform_error_handling>
@ -493,6 +601,18 @@ GROUP BY r.recruit_id, up;</sql>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename>error_code</codes_valuename>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
</transform_error_handling>
<attributes/>
</pipeline>

2
mappings/info_recruits/citizen_tables/prosecution/recruitment_five_flow_delta.hpl
View file

@ -66,7 +66,7 @@ FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = 'prosecution_job'
where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>