micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

правки для копирования сведений о судимости, отсрочкам и заполнение флагов

Browse source
This commit is contained in:
Булат Хайруллин 2025-04-25 16:20:06 +03:00
parent cc58c51c7f
commit 66bf0b8255
8 changed files with 496 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

11
metadata/workflow-run-configuration/local.json Normal file
View file

@ -0,0 +1,11 @@
{
"engineRunConfiguration": {
"Local": {
"safe_mode": false,
"transactional": false
}
},
"defaultSelection": true,
"name": "local",
"description": "Runs your workflows locally with the standard local Hop workflow engine"
}

156
v1_apache-hop dash mapping/job_copy_def_pun_fz53.hwf Normal file
View file

@ -0,0 +1,156 @@
<?xml version="1.0" encoding="UTF-8"?>
<workflow>
<name>job_copy_def_pun_fz53</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
<workflow_version/>
<created_user>-</created_user>
<created_date>2024/08/21 17:23:33.592</created_date>
<modified_user>-</modified_user>
<modified_date>2024/08/21 17:23:33.592</modified_date>
<parameters>
</parameters>
<actions>
<action>
<name>Start</name>
<description/>
<type>SPECIAL</type>
<attributes/>
<DayOfMonth>1</DayOfMonth>
<hour>12</hour>
<intervalMinutes>60</intervalMinutes>
<intervalSeconds>0</intervalSeconds>
<minutes>0</minutes>
<repeat>N</repeat>
<schedulerType>0</schedulerType>
<weekDay>1</weekDay>
<parallel>N</parallel>
<xloc>288</xloc>
<yloc>240</yloc>
<attributes_hac/>
</action>
<action>
<name>Success</name>
<description/>
<type>SUCCESS</type>
<attributes/>
<parallel>N</parallel>
<xloc>1168</xloc>
<yloc>240</yloc>
<attributes_hac/>
</action>
<action>
<name>copy_punishment_and_deferment_once.hwf</name>
<description/>
<type>WORKFLOW</type>
<attributes/>
<add_date>N</add_date>
<add_time>N</add_time>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/jobs_once_a_day/copy_punishment_and_deferment_once.hwf</filename>
<loglevel>Nothing</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<params_from_previous>N</params_from_previous>
<run_configuration>local</run_configuration>
<set_append_logfile>N</set_append_logfile>
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>560</xloc>
<yloc>240</yloc>
<attributes_hac/>
</action>
<action>
<name>citizen_flags.hwf</name>
<description/>
<type>WORKFLOW</type>
<attributes/>
<add_date>N</add_date>
<add_time>N</add_time>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/jobs_once_a_day/citizen_flags.hwf</filename>
<loglevel>Nothing</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<params_from_previous>N</params_from_previous>
<run_configuration>local</run_configuration>
<set_append_logfile>N</set_append_logfile>
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>832</xloc>
<yloc>240</yloc>
<attributes_hac/>
</action>
</actions>
<hops>
<hop>
<from>copy_punishment_and_deferment_once.hwf</from>
<to>citizen_flags.hwf</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>citizen_flags.hwf</from>
<to>Success</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>Start</from>
<to>copy_punishment_and_deferment_once.hwf</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
</hops>
<notepads>
</notepads>
<attributes>
<group>
<name>METASTORE.pentaho</name>
<attribute>
<key>Default Run Configuration</key>
<value>{"namespace":"pentaho","id":"Default Run Configuration","name":"Default Run Configuration","description":"Defines a default run configuration","metaStoreName":null}</value>
</attribute>
</group>
<group>
<name>debug_plugin</name>
<attribute>
<key>Write to log : action_log_result</key>
<value>N</value>
</attribute>
<attribute>
<key>Write to log : action_log_result_files</key>
<value>N</value>
</attribute>
<attribute>
<key>Write to log : action_log_result_rows</key>
<value>Y</value>
</attribute>
<attribute>
<key>Write to log : action_log_level</key>
<value>Basic</value>
</attribute>
<attribute>
<key>Write to log : action_log_variables</key>
<value>N</value>
</attribute>
</group>
<group>
<name>{"_":"Embedded MetaStore Elements","namespace":"pentaho","type":"Default Run Configuration"}</name>
<attribute>
<key>Pentaho local</key>
<value>{"children":[{"children":[],"id":"server","value":null},{"children":[],"id":"clustered","value":"N"},{"children":[],"id":"name","value":"Pentaho local"},{"children":[],"id":"description","value":null},{"children":[],"id":"pentaho","value":"N"},{"children":[],"id":"readOnly","value":"Y"},{"children":[],"id":"sendResources","value":"N"},{"children":[],"id":"logRemoteExecutionLocally","value":"N"},{"children":[],"id":"remote","value":"N"},{"children":[],"id":"local","value":"Y"},{"children":[],"id":"showTransformations","value":"N"}],"id":"Pentaho local","value":null,"name":"Pentaho local","owner":null,"ownerPermissionsList":[]}</value>
</attribute>
</group>
</attributes>
</workflow>

3
v1_apache-hop dash mapping/job_general.hwf
View file

@ -5,6 +5,7 @@
<description/>
<extended_description/>
<workflow_version/>
<workflow_status>0</workflow_status>
<created_user>-</created_user>
<created_date>2024/08/21 17:23:33.592</created_date>
<modified_user>-</modified_user>
@ -50,6 +51,8 @@
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${Internal.Entry.Current.Folder}/job_pack.pub_recruitment.hwf</filename>
<logext/>
<logfile/>
<loglevel>Nothing</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>

7
v1_apache-hop dash mapping/jobs_once_a_day/citizen_appealing_violations_fz53_once.hpl
View file

@ -80,7 +80,7 @@ WHERE hidden IS FALSE</sql>
<method>none</method>
<schema_name/>
</partitioning>
<commit>1000</commit>
<commit>20000</commit>
<connection>ervu-dashboard</connection>
<fields>
<field>
@ -120,11 +120,14 @@ WHERE hidden IS FALSE</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>citizen_appealing_violations_fz53</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -149,13 +152,11 @@ WHERE hidden IS FALSE</sql>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

4
v1_apache-hop dash mapping/jobs_once_a_day/citizen_criminal_liability_fz53_once.hpl
View file

@ -83,7 +83,7 @@ AND type = 'CRIMINAL'</sql>
<method>none</method>
<schema_name/>
</partitioning>
<commit>1000</commit>
<commit>20000</commit>
<connection>ervu-dashboard</connection>
<fields>
<field>
@ -151,13 +151,11 @@ AND type = 'CRIMINAL'</sql>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

314
v1_apache-hop dash mapping/jobs_once_a_day/citizen_flags.hwf Normal file
View file

@ -0,0 +1,314 @@
<?xml version="1.0" encoding="UTF-8"?>
<workflow>
<name>citizen_flags</name>
<name_sync_with_filename>N</name_sync_with_filename>
<description/>
<extended_description/>
<workflow_version/>
<workflow_status>0</workflow_status>
<created_user>-</created_user>
<created_date>2025/04/25 13:38:56.431</created_date>
<modified_user>-</modified_user>
<modified_date>2025/04/25 13:38:56.431</modified_date>
<parameters>
</parameters>
<actions>
<action>
<name>Start</name>
<description/>
<type>SPECIAL</type>
<attributes/>
<DayOfMonth>1</DayOfMonth>
<hour>12</hour>
<intervalMinutes>60</intervalMinutes>
<intervalSeconds>0</intervalSeconds>
<minutes>0</minutes>
<repeat>N</repeat>
<schedulerType>0</schedulerType>
<weekDay>1</weekDay>
<parallel>N</parallel>
<xloc>304</xloc>
<yloc>224</yloc>
<attributes_hac/>
</action>
<action>
<name>Success sitizen_flags</name>
<description/>
<type>SUCCESS</type>
<attributes/>
<parallel>N</parallel>
<xloc>1728</xloc>
<yloc>224</yloc>
<attributes_hac/>
</action>
<action>
<name>children_flags.hpl</name>
<description/>
<type>PIPELINE</type>
<attributes/>
<add_date>N</add_date>
<add_time>N</add_time>
<clear_files>N</clear_files>
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}\jobs_once_a_day\children_flags.hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<params_from_previous>N</params_from_previous>
<run_configuration>local</run_configuration>
<set_append_logfile>N</set_append_logfile>
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>480</xloc>
<yloc>128</yloc>
<attributes_hac/>
</action>
<action>
<name>opekun_flags.hpl</name>
<description/>
<type>PIPELINE</type>
<attributes/>
<add_date>N</add_date>
<add_time>N</add_time>
<clear_files>N</clear_files>
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}\jobs_once_a_day\opekun_flags.hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<params_from_previous>N</params_from_previous>
<run_configuration>local</run_configuration>
<set_append_logfile>N</set_append_logfile>
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>624</xloc>
<yloc>128</yloc>
<attributes_hac/>
</action>
<action>
<name>driver_licenses_flags.hpl</name>
<description/>
<type>PIPELINE</type>
<attributes/>
<add_date>N</add_date>
<add_time>N</add_time>
<clear_files>N</clear_files>
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}\jobs_once_a_day\driver_licenses_flags.hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<params_from_previous>N</params_from_previous>
<run_configuration>local</run_configuration>
<set_append_logfile>N</set_append_logfile>
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>768</xloc>
<yloc>128</yloc>
<attributes_hac/>
</action>
<action>
<name>citizenship_flags.hpl</name>
<description/>
<type>PIPELINE</type>
<attributes/>
<add_date>N</add_date>
<add_time>N</add_time>
<clear_files>N</clear_files>
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}\jobs_once_a_day\citizenship_flags.hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<params_from_previous>N</params_from_previous>
<run_configuration>local</run_configuration>
<set_append_logfile>N</set_append_logfile>
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>928</xloc>
<yloc>128</yloc>
<attributes_hac/>
</action>
<action>
<name>property_flags.hpl</name>
<description/>
<type>PIPELINE</type>
<attributes/>
<add_date>N</add_date>
<add_time>N</add_time>
<clear_files>N</clear_files>
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}\jobs_once_a_day\property_flags.hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<params_from_previous>N</params_from_previous>
<run_configuration>local</run_configuration>
<set_append_logfile>N</set_append_logfile>
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>1072</xloc>
<yloc>128</yloc>
<attributes_hac/>
</action>
<action>
<name>deputy_flags.hpl</name>
<description/>
<type>PIPELINE</type>
<attributes/>
<add_date>N</add_date>
<add_time>N</add_time>
<clear_files>N</clear_files>
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}\jobs_once_a_day\deputy_flags.hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<params_from_previous>N</params_from_previous>
<run_configuration>local</run_configuration>
<set_append_logfile>N</set_append_logfile>
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>1200</xloc>
<yloc>128</yloc>
<attributes_hac/>
</action>
<action>
<name>criminal_prosecution_flags.hpl</name>
<description/>
<type>PIPELINE</type>
<attributes/>
<add_date>N</add_date>
<add_time>N</add_time>
<clear_files>N</clear_files>
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}\jobs_once_a_day\criminal_prosecution_flags.hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<params_from_previous>N</params_from_previous>
<run_configuration>local</run_configuration>
<set_append_logfile>N</set_append_logfile>
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>1360</xloc>
<yloc>224</yloc>
<attributes_hac/>
</action>
<action>
<name>criminal_record_flags.hpl</name>
<description/>
<type>PIPELINE</type>
<attributes/>
<add_date>N</add_date>
<add_time>N</add_time>
<clear_files>N</clear_files>
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}\jobs_once_a_day\criminal_record_flags.hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<params_from_previous>N</params_from_previous>
<run_configuration>local</run_configuration>
<set_append_logfile>N</set_append_logfile>
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>1552</xloc>
<yloc>224</yloc>
<attributes_hac/>
</action>
</actions>
<hops>
<hop>
<from>children_flags.hpl</from>
<to>opekun_flags.hpl</to>
<enabled>Y</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>opekun_flags.hpl</from>
<to>driver_licenses_flags.hpl</to>
<enabled>Y</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>driver_licenses_flags.hpl</from>
<to>citizenship_flags.hpl</to>
<enabled>Y</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>citizenship_flags.hpl</from>
<to>property_flags.hpl</to>
<enabled>Y</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>property_flags.hpl</from>
<to>deputy_flags.hpl</to>
<enabled>Y</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>criminal_prosecution_flags.hpl</from>
<to>criminal_record_flags.hpl</to>
<enabled>Y</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>criminal_record_flags.hpl</from>
<to>Success sitizen_flags</to>
<enabled>Y</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>Start</from>
<to>criminal_prosecution_flags.hpl</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
</hops>
<notepads>
</notepads>
<attributes/>
</workflow>

7
v1_apache-hop dash mapping/jobs_once_a_day/citizen_liability_fz53_once.hpl
View file

@ -90,7 +90,7 @@ AND type = 'ADMINISTRATIVE'</sql>
<method>none</method>
<schema_name/>
</partitioning>
<commit>1000</commit>
<commit>20000</commit>
<connection>ervu-dashboard</connection>
<fields>
<field>
@ -150,11 +150,14 @@ AND type = 'ADMINISTRATIVE'</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>citizen_liability_fz53</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -179,13 +182,11 @@ AND type = 'ADMINISTRATIVE'</sql>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

4
v1_apache-hop dash mapping/jobs_once_a_day/copy_punishment_and_deferment_once.hwf
View file

@ -376,7 +376,9 @@ delete from ervu_dashboard.citizen_deferment;
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>C:\WORK\ervu-dashboard-etl-RT\v1_apache-hop dash mapping\jobs_once_a_day\children_flags.hpl</filename>
<filename>${PROJECT_HOME}\jobs_once_a_day\children_flags.hpl</filename>
<logext/>
<logfile/>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>