micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

/

Browse source
This commit is contained in:
Ruslan 2025-03-07 10:41:11 +03:00
parent 64c096b90f
commit f7186f2fa1
2 changed files with 354 additions and 306 deletions
Download patch file Download diff file Expand all files Collapse all files

2
v1_apache-hop dash mapping/job_recruitments_all.hwf
View file

@ -268,7 +268,7 @@
<hop>
<from>job_recruitments_region.hwf</from>
<to>job_ratings.hwf</to>
<enabled>N</enabled>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>

658
v1_apache-hop dash mapping/mil_com/parameters/add_schema.hpl
View file

@ -88,12 +88,12 @@
<hop>
<from>Table input main_dashboard.recruitment_campaign</from>
<to>Insert / update (main_dashboard.recruitment_campaign)</to>
<enabled>N</enabled>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input recruitment_campaign.subpoenas</from>
<to>Insert / update (recruitment_campaign.subpoenas)</to>
<enabled>N</enabled>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
@ -102,32 +102,32 @@
</hop>
<hop>
<from>Table input</from>
<to>Update</to>
<to>Update (total_registered.driver_license)</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Update 2</to>
<to>Update (total_registered.child_minor)</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Update 3</to>
<to>Update (total_registered.age)</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Update 4</to>
<to>Update (total_registered.marital_status)</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Update 5</to>
<to>Update (total_registered.busyness)</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Update 6</to>
<to>Update (total_registered.education_level)</to>
<enabled>Y</enabled>
</hop>
<hop>
@ -155,6 +155,11 @@
<to>Update (main_dashboard.total_registered)</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Update (recruitment_campaign.appeals)</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Dummy (do nothing)</name>
@ -1218,7 +1223,87 @@ WHERE schema in ('Department', 'Organization', 'Region') </sql>
</GUI>
</transform>
<transform>
<name>Update</name>
<name>Update (main_dashboard.total_registered)</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
</key>
<schema>main_dashboard</schema>
<table>total_registered</table>
<value>
<name>schema</name>
<rename>schema</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>480</yloc>
</GUI>
</transform>
<transform>
<name>Update (main_dashboard.waiting_registration)</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
</key>
<schema>main_dashboard</schema>
<table>waiting_registration</table>
<value>
<name>schema</name>
<rename>schema</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>Update (total_registered.driver_license)</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
@ -1254,293 +1339,7 @@ WHERE schema in ('Department', 'Organization', 'Region') </sql>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>928</yloc>
</GUI>
</transform>
<transform>
<name>Update 2</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
</key>
<schema>total_registered</schema>
<table>child_minor</table>
<value>
<name>schema</name>
<rename>schema</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>1008</yloc>
</GUI>
</transform>
<transform>
<name>Update 3</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
</key>
<schema>total_registered</schema>
<table>age</table>
<value>
<name>schema</name>
<rename>schema</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>1088</yloc>
</GUI>
</transform>
<transform>
<name>Update 4</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
</key>
<schema>total_registered</schema>
<table>marital_status</table>
<value>
<name>schema</name>
<rename>schema</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>1184</yloc>
</GUI>
</transform>
<transform>
<name>Update 5</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
</key>
<schema>total_registered</schema>
<table>busyness</table>
<value>
<name>schema</name>
<rename>schema</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1232</xloc>
<yloc>1264</yloc>
</GUI>
</transform>
<transform>
<name>Update 6</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
</key>
<schema>total_registered</schema>
<table>education_level</table>
<value>
<name>schema</name>
<rename>schema</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1232</xloc>
<yloc>1344</yloc>
</GUI>
</transform>
<transform>
<name>Update (main_dashboard.total_registered)</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
<name2/>
</key>
<schema>main_dashboard</schema>
<table>total_registered</table>
<value>
<name>schema</name>
<rename>schema</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>480</yloc>
</GUI>
</transform>
<transform>
<name>Update (main_dashboard.waiting_registration)</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
<name2/>
</key>
<schema>main_dashboard</schema>
<table>waiting_registration</table>
<value>
<name>schema</name>
<rename>schema</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>560</yloc>
<yloc>880</yloc>
</GUI>
</transform>
<transform>
@ -1557,19 +1356,16 @@ WHERE schema in ('Department', 'Organization', 'Region') </sql>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
<name2/>
</key>
<schema>total_registered</schema>
<table>reg_mil_cat</table>
@ -1600,19 +1396,16 @@ WHERE schema in ('Department', 'Organization', 'Region') </sql>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
<name2/>
</key>
<schema>total_registered</schema>
<table>removed_registry</table>
@ -1643,19 +1436,16 @@ WHERE schema in ('Department', 'Organization', 'Region') </sql>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
<name2/>
</key>
<schema>total_registered</schema>
<table>subpoenas</table>
@ -1672,6 +1462,264 @@ WHERE schema in ('Department', 'Organization', 'Region') </sql>
<yloc>800</yloc>
</GUI>
</transform>
<transform>
<name>Update (total_registered.child_minor)</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
<name2/>
</key>
<schema>total_registered</schema>
<table>child_minor</table>
<value>
<name>schema</name>
<rename>schema</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>960</yloc>
</GUI>
</transform>
<transform>
<name>Update (total_registered.age)</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
<name2/>
</key>
<schema>total_registered</schema>
<table>age</table>
<value>
<name>schema</name>
<rename>schema</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>1040</yloc>
</GUI>
</transform>
<transform>
<name>Update (total_registered.marital_status)</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
<name2/>
</key>
<schema>total_registered</schema>
<table>marital_status</table>
<value>
<name>schema</name>
<rename>schema</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>1120</yloc>
</GUI>
</transform>
<transform>
<name>Update (total_registered.busyness)</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
<name2/>
</key>
<schema>total_registered</schema>
<table>busyness</table>
<value>
<name>schema</name>
<rename>schema</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>1200</yloc>
</GUI>
</transform>
<transform>
<name>Update (total_registered.education_level)</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
<name2/>
</key>
<schema>total_registered</schema>
<table>education_level</table>
<value>
<name>schema</name>
<rename>schema</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>1280</yloc>
</GUI>
</transform>
<transform>
<name>Update (recruitment_campaign.appeals)</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
<name2/>
</key>
<schema>recruitment_campaign</schema>
<table>appeals</table>
<value>
<name>schema</name>
<rename>schema</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>1376</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>