micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/develop' into develop

Browse source
This commit is contained in:
Fusionshh 2025-10-20 13:24:32 +03:00
commit b08a07aeb5
3 changed files with 134 additions and 208 deletions
Download patch file Download diff file Expand all files Collapse all files

106
mappings/info_recruits/citizen_tables/drivers_licence/parallel/drivers_licence_flow.hpl
View file

@ -50,41 +50,36 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Table output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>sort_by_recruit_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sort_by_recruit_id</from>
<to>Identify last row in a stream 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream 2</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Table output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -106,28 +101,6 @@
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -160,7 +133,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1104</xloc>
<xloc>1088</xloc>
<yloc>496</yloc>
</GUI>
</transform>
@ -238,7 +211,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>704</xloc>
<xloc>560</xloc>
<yloc>176</yloc>
</GUI>
</transform>
@ -279,7 +252,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<name>Identify last row in a stream 2</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
@ -292,7 +265,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>704</xloc>
<xloc>1104</xloc>
<yloc>320</yloc>
</GUI>
</transform>
@ -300,7 +273,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -391,7 +364,7 @@ FROM all_data</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>496</xloc>
<xloc>560</xloc>
<yloc>320</yloc>
</GUI>
</transform>
@ -458,18 +431,21 @@ FROM all_data</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>drivers_licence</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>928</xloc>
<xloc>736</xloc>
<yloc>320</yloc>
</GUI>
</transform>
@ -477,7 +453,7 @@ FROM all_data</sql>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -508,8 +484,8 @@ FROM all_data</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>928</xloc>
<yloc>496</yloc>
<xloc>1328</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
@ -523,25 +499,27 @@ FROM all_data</sql>
<method>none</method>
<schema_name/>
</partitioning>
<compress>N</compress>
<directory>${java.io.tmpdir}</directory>
<prefix/>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<name>recruit_id</name>
<presorted>N</presorted>
</field>
</fields>
<sort_prefix>srt</sort_prefix>
<sort_size>1000000</sort_size>
<unique_rows>Y</unique_rows>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>496</yloc>
<xloc>912</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform_error_handling>

123
mappings/info_recruits/citizen_tables/drivers_licence/parallel/drivers_licence_flow_delta.hpl
View file

@ -52,36 +52,31 @@
</hop>
<hop>
<from>Table input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Insert / update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Insert / update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>sort_by_recruit_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sort_by_recruit_id</from>
<to>Identify last row in a stream 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream 2</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
@ -106,28 +101,6 @@
<yloc>528</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1232</xloc>
<yloc>352</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -224,7 +197,7 @@ where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql>
<attributes/>
<GUI>
<xloc>352</xloc>
<xloc>272</xloc>
<yloc>352</yloc>
</GUI>
</transform>
@ -241,7 +214,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>752</xloc>
<xloc>544</xloc>
<yloc>192</yloc>
</GUI>
</transform>
@ -281,24 +254,6 @@ and recruitment_id = '${IDM_ID}';</sql>
<yloc>352</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>352</yloc>
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<type>InsertUpdate</type>
@ -384,7 +339,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>960</xloc>
<xloc>720</xloc>
<yloc>352</yloc>
</GUI>
</transform>
@ -392,7 +347,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -498,7 +453,7 @@ FROM all_data</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>560</xloc>
<xloc>544</xloc>
<yloc>352</yloc>
</GUI>
</transform>
@ -537,8 +492,8 @@ FROM all_data</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>528</yloc>
<xloc>1216</xloc>
<yloc>352</yloc>
</GUI>
</transform>
<transform>
@ -552,25 +507,45 @@ FROM all_data</sql>
<method>none</method>
<schema_name/>
</partitioning>
<compress>N</compress>
<directory>${java.io.tmpdir}</directory>
<prefix/>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<name>recruit_id</name>
<presorted>N</presorted>
</field>
</fields>
<sort_prefix>srt</sort_prefix>
<sort_size>1000000</sort_size>
<unique_rows>Y</unique_rows>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>528</yloc>
<xloc>896</xloc>
<yloc>352</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream 2</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1056</xloc>
<yloc>352</yloc>
</GUI>
</transform>
<transform_error_handling>

113
mappings/info_recruits/citizen_tables/drivers_licence/parallel/drivers_licence_flow_repeat.hpl
View file

@ -53,39 +53,34 @@
</hop>
<hop>
<from>Table input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Insert / update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>sort_by_recruit_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sort_by_recruit_id</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<from>Table input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<from>Insert / update</from>
<to>sort_by_recruit_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sort_by_recruit_id</from>
<to>Identify last row in a stream 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream 2</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -107,28 +102,6 @@
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -161,7 +134,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1152</xloc>
<xloc>1232</xloc>
<yloc>384</yloc>
</GUI>
</transform>
@ -257,7 +230,7 @@ DO UPDATE SET
</partitioning>
<attributes/>
<GUI>
<xloc>720</xloc>
<xloc>480</xloc>
<yloc>64</yloc>
</GUI>
</transform>
@ -297,24 +270,6 @@ DO UPDATE SET
<yloc>208</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>720</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<type>InsertUpdate</type>
@ -400,7 +355,7 @@ DO UPDATE SET
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>960</xloc>
<xloc>672</xloc>
<yloc>208</yloc>
</GUI>
</transform>
@ -408,7 +363,7 @@ DO UPDATE SET
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -537,8 +492,8 @@ FROM all_data</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>960</xloc>
<yloc>384</yloc>
<xloc>1232</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
@ -571,8 +526,26 @@ FROM all_data</sql>
</fields>
<attributes/>
<GUI>
<xloc>720</xloc>
<yloc>384</yloc>
<xloc>880</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream 2</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1056</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform_error_handling>