micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

temp commit

Browse source
This commit is contained in:
Fusionshh 2025-10-02 10:53:25 +03:00
parent b25bf7bc95
commit 64131dd076
21 changed files with 2353 additions and 1087 deletions
Download patch file Download diff file Expand all files Collapse all files

4
mappings/connections/db.json
View file

@ -6,7 +6,7 @@
"description" : ""
}, {
"name" : "DB_NSI_PASSWORD",
"value" : "Encrypted 336ebbb2179beaa7340a4f1620f3af40",
"value" : "Encrypted 2be98afc86aa7f2e4b91caf74d19ca3c3",
"description" : ""
}, {
"name" : "DB_NSI_HOST",
@ -18,7 +18,7 @@
"description" : ""
}, {
"name" : "DB_NSI_NAME",
"value" : "ervu-nsi",
"value" : "ervu_nsi",
"description" : ""
}, {
"name" : "DB_PERSON_HOST",

5
mappings/info_recruits/citizen_tables/deputy/constants/deputy_constants_flow.hpl
View file

@ -37,7 +37,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>nsi</connection>
<connection>ervu_nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT
@ -89,14 +89,11 @@ WHERE code = 'deputatDosrochPrichina';</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>deputy_termination_reason</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>

2
mappings/info_recruits/citizen_tables/deputy/constants/deputy_constants_flow_repeat.hpl
View file

@ -67,7 +67,7 @@ FROM deputy_termination_reason;
<method>none</method>
<schema_name/>
</partitioning>
<connection>nsi</connection>
<connection>ervu_nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_termination_reason_update_date</lookup>

3
mappings/info_recruits/citizen_tables/deputy/recruitment_five_flow.hpl
View file

@ -57,8 +57,9 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>SELECT
idm_id
FROM ervu_dashboard.recruitment;</sql>

2
mappings/info_recruits/citizen_tables/deputy/recruitment_five_flow_repeat.hpl
View file

@ -57,7 +57,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT r.idm_id AS recruitment_id

8
mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow.hpl
View file

@ -52,7 +52,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>nsi</connection>
<connection>ervu_nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT
@ -130,7 +130,7 @@ WHERE code = 'educationLevel';</sql>
<method>none</method>
<schema_name/>
</partitioning>
<connection>nsi</connection>
<connection>ervu_nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT
@ -208,7 +208,7 @@ WHERE code = 'organizationStatus';</sql>
<method>none</method>
<schema_name/>
</partitioning>
<connection>nsi</connection>
<connection>ervu_nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT
@ -287,7 +287,7 @@ WHERE code = 'educationSpecialityMinobr'
<method>none</method>
<schema_name/>
</partitioning>
<connection>nsi</connection>
<connection>ervu_nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT

59
mappings/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow_repeat.hpl
View file

@ -25,21 +25,6 @@
<to>education_level_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>get_max_educational_institution_status_update_date</from>
<to>educational_institution_status_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>get_max_minobr_speciality_update_date</from>
<to>minobr_speciality_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>get_max_minprosv_speciality_update_date</from>
<to>minprosv_speciality_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_level_input</from>
<to>education_level_upsert</to>
@ -50,6 +35,11 @@
<to>educational_institution_status_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>get_max_educational_institution_status_update_date</from>
<to>educational_institution_status_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>minobr_speciality_input</from>
<to>minobr_speciality_upsert</to>
@ -60,6 +50,16 @@
<to>minprosv_speciality_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>get_max_minobr_speciality_update_date</from>
<to>minobr_speciality_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>get_max_minprosv_speciality_update_date</from>
<to>minprosv_speciality_input</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>education_level_input</name>
@ -72,7 +72,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>nsi</connection>
<connection>ervu_nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_education_level_update_date</lookup>
@ -111,12 +111,6 @@ AND updated_at > ?;</sql>
<name>code</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>actual</field>
<name>actual</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>education_level</table>
<value>
@ -158,7 +152,7 @@ AND updated_at > ?;</sql>
<method>none</method>
<schema_name/>
</partitioning>
<connection>nsi</connection>
<connection>ervu_nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_educational_institution_status_update_date</lookup>
@ -196,11 +190,6 @@ AND updated_at > ?;</sql>
<field>code</field>
<name>code</name>
</key>
<key>
<condition>&lt;></condition>
<field>actual</field>
<name>actual</name>
</key>
<schema>ervu_dashboard</schema>
<table>educational_institution_status</table>
<value>
@ -342,7 +331,7 @@ FROM minprosv_education_speciality
<method>none</method>
<schema_name/>
</partitioning>
<connection>nsi</connection>
<connection>ervu_nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_minobr_speciality_update_date</lookup>
@ -381,11 +370,6 @@ AND updated_at > ?;</sql>
<field>code</field>
<name>code</name>
</key>
<key>
<condition>&lt;></condition>
<field>actual</field>
<name>actual</name>
</key>
<schema>ervu_dashboard</schema>
<table>minobr_education_speciality</table>
<value>
@ -427,7 +411,7 @@ AND updated_at > ?;</sql>
<method>none</method>
<schema_name/>
</partitioning>
<connection>nsi</connection>
<connection>ervu_nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_minprosv_speciality_update_date</lookup>
@ -467,11 +451,6 @@ AND updated_at > ?;</sql>
<field>code</field>
<name>code</name>
</key>
<key>
<condition>&lt;></condition>
<field>actual</field>
<name>actual</name>
</key>
<schema>ervu_dashboard</schema>
<table>minprosv_education_speciality</table>
<value>

9
mappings/info_recruits/citizen_tables/education/educational_institution_learning/educational_institution_learning_job.hwf
View file

@ -5,6 +5,7 @@
<description/>
<extended_description/>
<workflow_version/>
<workflow_status>0</workflow_status>
<created_user>-</created_user>
<created_date>2025/08/27 21:14:40.330</created_date>
<modified_user>-</modified_user>
@ -206,8 +207,6 @@
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/constants/support/check_if_constants_exists.hpl</filename>
<logext/>
<logfile/>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -406,5 +405,9 @@
</hops>
<notepads>
</notepads>
<attributes/>
<attributes>
<group>
<name>debug_plugin</name>
</group>
</attributes>
</workflow>

534
mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl
View file

@ -7,7 +7,18 @@
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<pipeline_status>0</pipeline_status>
<parameters>
<parameter>
<name>IDM_ID</name>
<default_value>0e0535ab-5303-43ea-b36a-8c0a2a8d0ed5</default_value>
<description/>
</parameter>
<parameter>
<name>JOB_NAME</name>
<default_value>education_test_job</default_value>
<description/>
</parameter>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
@ -21,8 +32,8 @@
</notepads>
<order>
<hop>
<from>Change job status on error</from>
<to>Abort</to>
<from>change_job_status_on_error_main_upsert</from>
<to>abort</to>
<enabled>Y</enabled>
</hop>
<hop>
@ -50,36 +61,16 @@
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_education_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_education_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_leave_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter_null_parental_leave_dates</from>
<to>Block until transforms finish</to>
<to>block_until_main_finished</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<from>block_until_main_finished</from>
<to>education_leave_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Group by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Group by</from>
<to>citizen_education_output</to>
@ -90,9 +81,64 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>sort_by_recruit_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>sort_by_source_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sort_by_source_id</from>
<to>Group by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_education_output</from>
<to>block_until_all_complete</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>block_until_all_complete</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_education_output</from>
<to>change_job_status_on_error_main_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>change_job_status_on_error_upsert</from>
<to>abort_2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_leave_output</from>
<to>change_job_status_on_error_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>change_job_status_on_error_update</from>
<to>abort_3</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>change_job_status_on_error_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sort_by_recruit_id</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
<name>abort</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
@ -104,37 +150,16 @@
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<message/>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1488</xloc>
<yloc>640</yloc>
<xloc>1824</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>citizen_education_output</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>880</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<name>change_job_status_on_error_main_upsert</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -150,8 +175,11 @@
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
@ -162,10 +190,11 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1264</xloc>
<yloc>640</yloc>
<xloc>1600</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
@ -195,7 +224,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1488</xloc>
<xloc>1824</xloc>
<yloc>256</yloc>
</GUI>
</transform>
@ -277,7 +306,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1488</xloc>
<xloc>1824</xloc>
<yloc>432</yloc>
</GUI>
</transform>
@ -302,11 +331,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<operator>-</operator>
</condition>
</compare>
<send_true_to>Block until transforms finish</send_true_to>
<send_true_to>block_until_main_finished</send_true_to>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>640</yloc>
<xloc>848</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -321,243 +350,284 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<all_rows>Y</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<aggregate>recruit_id</aggregate>
<subject>recruit_id</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>source_update_date</aggregate>
<subject>source_update_date</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>educational_institution_type</aggregate>
<subject>educational_institution_type</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>org_type</aggregate>
<subject>org_type</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>inn</aggregate>
<subject>inn</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>kpp</aggregate>
<subject>kpp</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>ogrn</aggregate>
<subject>ogrn</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>educational_institution_address</aggregate>
<subject>educational_institution_address</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>educational_institution_name</aggregate>
<subject>educational_institution_name</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>educational_institution_status_code</aggregate>
<subject>educational_institution_status_code</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>educational_organization_separate_unit_type</aggregate>
<subject>educational_organization_separate_unit_type</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>educational_organization_separate_unit_address</aggregate>
<subject>educational_organization_separate_unit_address</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>student_status</aggregate>
<subject>student_status</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>grade_number</aggregate>
<subject>grade_number</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>education_level_code</aggregate>
<subject>education_level_code</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>admission_date</aggregate>
<subject>admission_date</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>enrollment_date</aggregate>
<subject>enrollment_date</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>enrollment_order_number</aggregate>
<subject>enrollment_order_number</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>enrollment_order_date</aggregate>
<subject>enrollment_order_date</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>completion_date</aggregate>
<subject>completion_date</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>education_form</aggregate>
<subject>education_form</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>education_program_start_date</aggregate>
<subject>education_program_start_date</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>education_start_date</aggregate>
<subject>education_start_date</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>planned_completion_date</aggregate>
<subject>planned_completion_date</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>minobr_speciality_code</aggregate>
<subject>minobr_speciality_code</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>minprosv_speciality_code</aggregate>
<subject>minprosv_speciality_code</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>recovery_info</aggregate>
<subject>recovery_info</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>recovery_date</aggregate>
<subject>recovery_date</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>recovery_order_date</aggregate>
<subject>recovery_order_date</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>recovery_order_number</aggregate>
<subject>recovery_order_number</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>expulsion_info</aggregate>
<subject>expulsion_info</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>expulsion_date</aggregate>
<subject>expulsion_date</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>expulsion_order_date</aggregate>
<subject>expulsion_order_date</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>expulsion_order_number</aggregate>
<subject>expulsion_order_number</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>academical_leave_info</aggregate>
<subject>academical_leave_info</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>first_education</aggregate>
<subject>first_education</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>vus_spec</aggregate>
<subject>vus_spec</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>military_center_education_info</aggregate>
<subject>military_center_education_info</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>military_center_education_start_date</aggregate>
<subject>military_center_education_start_date</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>military_center_education_end_date</aggregate>
<subject>military_center_education_end_date</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>military_center_education_period</aggregate>
<subject>military_center_education_period</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>military_center_completion</aggregate>
<subject>military_center_completion</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>military_department_education_info</aggregate>
<subject>military_department_education_info</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>military_department_start_date</aggregate>
<subject>military_department_start_date</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>military_department_end_date</aggregate>
<subject>military_department_end_date</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
<field>
<aggregate>last_row</aggregate>
<subject>last_row</subject>
<type>LAST_INCL_NULL</type>
</field>
<field>
<aggregate>academ_start_date</aggregate>
<subject>academ_start_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>academ_end_date</aggregate>
<subject>academ_end_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>academ_reason</aggregate>
<subject>academ_reason</subject>
<type>FIRST_INCL_NULL</type>
<valuefield/>
</field>
</fields>
<give_back_row>N</give_back_row>
@ -567,10 +637,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<linenr_fieldname/>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>944</xloc>
<xloc>1056</xloc>
<yloc>432</yloc>
</GUI>
</transform>
@ -592,6 +663,90 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_educational_institution_info</name>
<rename>has_educational_institution_info</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1264</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
<name>block_until_all_complete</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>education_leave_output</name>
</transform>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1600</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>block_until_main_finished</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>citizen_education_output</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1056</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
<name>citizen_education_output</name>
<type>TableOutput</type>
@ -782,6 +937,14 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<column_name>military_department_end_date</column_name>
<stream_name>military_department_end_date</stream_name>
</field>
<field>
<column_name>educational_institution_name</column_name>
<stream_name>educational_institution_name</stream_name>
</field>
<field>
<column_name>educational_institution_status_code</column_name>
<stream_name>educational_institution_status_code</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
@ -869,8 +1032,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
) AS academ_elem ON true
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') = 'array'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
SELECT fd.recruit_id,
education_elem ->> 'id' AS source_id,
SELECT education_elem ->> 'id' AS source_id,
fd.recruit_id,
to_date(education_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
educational_institution_type,
education_elem -> 'svedOrg' ->> 'naimTipOrg' as org_type,
@ -879,7 +1042,8 @@ SELECT fd.recruit_id,
coalesce(education_elem -> 'svedOrg' ->> 'ogrn', education_elem -> 'svedOrg' ->>
'ogrnip') as ogrn,
education_elem -> 'svedOrg' ->> 'adresOrg' as educational_institution_address,
education_elem -> 'svedOrg' -> 'extend' ->> 'statusOrg' as educational_institution_address,
education_elem -> 'svedOrg' ->> 'naimOrg' as educational_institution_name,
education_elem -> 'svedOrg' -> 'extend' ->> 'statusOrg' as educational_institution_status_code,
education_elem -> 'svedOrg' -> 'extend' ->> 'vidPodrazdOrg' as educational_organization_separate_unit_type,
education_elem -> 'svedOrg' -> 'extend' ->> 'adresPodrazdOrg' as educational_organization_separate_unit_address,
education_elem ->> 'naimStatus' as student_status,
@ -901,7 +1065,7 @@ SELECT fd.recruit_id,
education_elem -> 'svedObrProg' ->> 'naimFormaObuch' as education_form,
education_elem -> 'svedObrProg' ->> 'dataNachObuch' as education_program_start_date,
to_date(education_elem -> 'extend' ->> 'dataNachObuch', 'YYYY-MM-DD') as education_start_date,
to_date(coalesce(education_elem -> 'svedObrProg' ->> 'planDataOkonch', education_elem -&gt;&gt; 'dataOzhid'),
to_date(coalesce(education_elem -> 'svedObrProg' ->> 'planDataOkonch', education_elem ->> 'dataOzhid'),
'YYYY-MM-DD') as planned_completion_date,
education_elem -> 'svedObrProg' -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code,
education_elem -> 'svedObrProg' -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code,
@ -932,9 +1096,9 @@ SELECT fd.recruit_id,
'YYYY-MM-DD') as military_department_end_date,
to_date(academ_elem ->> 'dataNach', 'YYYY-MM-DD') as academ_start_date,
to_date(academ_elem ->> 'dataOkonch', 'YYYY-MM-DD') as academ_end_date,
academ_elem -> 'naimPrichOtpusk' as academ_reason
FROM filteredData fd
ORDER BY source_id;</sql>
academ_elem -> 'naimPrichOtpusk' as academ_reason,
true as has_educational_institution_info
FROM filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -988,14 +1152,214 @@ ORDER BY source_id;</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1088</xloc>
<yloc>640</yloc>
<xloc>1264</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
<name>sort_by_recruit_id</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
<name>sort_by_source_id</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>N</unique_rows>
<fields>
<field>
<name>source_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>abort_2</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<message/>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1824</xloc>
<yloc>656</yloc>
</GUI>
</transform>
<transform>
<name>change_job_status_on_error_upsert</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1600</xloc>
<yloc>656</yloc>
</GUI>
</transform>
<transform>
<name>abort_3</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<message/>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1824</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
<name>change_job_status_on_error_update</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1600</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>change_job_status_on_error_update</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>citizen_education_output</source_transform>
<target_transform>Change job status on error</target_transform>
<target_transform>change_job_status_on_error_main_upsert</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -1007,7 +1371,7 @@ ORDER BY source_id;</sql>
</error>
<error>
<source_transform>education_leave_output</source_transform>
<target_transform>Change job status on error</target_transform>
<target_transform>change_job_status_on_error_upsert</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

1120
mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl
View file

File diff suppressed because it is too large Load diff

969
mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl
View file

File diff suppressed because it is too large Load diff

3
mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow.hpl
View file

@ -57,8 +57,9 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>SELECT
idm_id
FROM ervu_dashboard.recruitment;</sql>

2
mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_delta.hpl
View file

@ -62,7 +62,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_source_update_date</lookup>

2
mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_repeat.hpl
View file

@ -57,7 +57,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT r.idm_id AS recruitment_id

5
mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/foreign_diploma_constants_flow.hpl
View file

@ -37,7 +37,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>nsi</connection>
<connection>ervu_nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT
@ -90,14 +90,11 @@ WHERE code = 'countriesOKSM';
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>country_oksm</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>

108
mappings/info_recruits/citizen_tables/education/foreign_diploma/constants/foreign_diploma_constants_flow_repeat.hpl
View file

@ -31,6 +31,56 @@
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>country_oksm</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>code</field>
<name>code</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>country_oksm</table>
<value>
<name>code</name>
<rename>code</rename>
<update>N</update>
</value>
<value>
<name>value</name>
<rename>value</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>update_date</name>
<rename>update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>688</xloc>
<yloc>240</yloc>
</GUI>
</transform>
<transform>
<name>country_oksm_input</name>
<type>TableInput</type>
@ -42,7 +92,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>nsi</connection>
<connection>ervu_nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_country_oksm_update_date</lookup>
@ -86,62 +136,6 @@ FROM country_oksm;
<yloc>240</yloc>
</GUI>
</transform>
<transform>
<name>country_oksm</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>code</field>
<name>code</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>actual</field>
<name>actual</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>country_oksm</table>
<value>
<name>code</name>
<rename>code</rename>
<update>N</update>
</value>
<value>
<name>value</name>
<rename>value</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>update_date</name>
<rename>update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>688</xloc>
<yloc>240</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>

2
mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl
View file

@ -268,7 +268,6 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
diploma_elem
FROM recruits_info ri
@ -319,6 +318,7 @@ FROM filteredData fd;</sql>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>foreign_diploma</table>

3
mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow.hpl
View file

@ -57,8 +57,9 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>SELECT
idm_id
FROM ervu_dashboard.recruitment;</sql>

2
mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_delta.hpl
View file

@ -62,7 +62,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_source_update_date</lookup>

2
mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_repeat.hpl
View file

@ -57,7 +57,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT r.idm_id AS recruitment_id

596
mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_delta.hpl
View file

@ -21,13 +21,8 @@
</notepads>
<order>
<hop>
<from>Create job execution record</from>
<to>self_employed_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>self_employed_upsert</from>
<to>Change job status on error</to>
<from>Block until transforms finish 2</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
@ -36,52 +31,107 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>self_employed_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>self_employed_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<from>Create job execution record</from>
<to>self_employed_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>delete_akt_become_null</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<from>Detect empty stream 2</from>
<to>upsert_akt</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>filter_non_actual_self_employed</from>
<to>Update</to>
<from>Detect empty stream 3</from>
<to>upsert_pred</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>filter_non_actual_self_employed</to>
<from>Filter rows</from>
<to>Block until transforms finish 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<from>delete_akt_become_null</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>self_employed_upsert</from>
<to>Block until transforms finish</to>
<from>delete_akt_become_null</from>
<to>update_akt_flag_on_delete</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<from>filter_akt_rows_for_delete</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>filter_akt_rows_for_update</from>
<to>Detect empty stream 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>filter_pred_rows_for_update</from>
<to>Detect empty stream 3</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>self_employed_input</from>
<to>filter_akt_rows_for_delete</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>self_employed_input</from>
<to>filter_akt_rows_for_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>self_employed_input</from>
<to>filter_pred_rows_for_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>update_akt_flag</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>update_akt_flag</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>update_akt_flag_on_delete</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>update_akt_flag_on_delete</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>upsert_akt</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>upsert_akt</from>
<to>update_akt_flag</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>upsert_pred</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>upsert_pred</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
@ -97,13 +147,41 @@
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT_WITH_ERROR</abort_option>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>464</yloc>
<yloc>736</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish 2</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>update_akt_flag_on_delete</name>
</transform>
<transform>
<name>update_akt_flag</name>
</transform>
<transform>
<name>upsert_pred</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1872</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
@ -129,7 +207,7 @@
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
SET status = 'ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
@ -137,8 +215,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>464</yloc>
<xloc>1024</xloc>
<yloc>736</yloc>
</GUI>
</transform>
<transform>
@ -155,7 +233,7 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
@ -168,8 +246,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>16</yloc>
<xloc>2096</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
@ -200,8 +278,8 @@ where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql>
<attributes/>
<GUI>
<xloc>272</xloc>
<yloc>224</yloc>
<xloc>384</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
@ -217,8 +295,42 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>16</yloc>
<xloc>1024</xloc>
<yloc>304</yloc>
</GUI>
</transform>
<transform>
<name>Detect empty stream 2</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>1024</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Detect empty stream 3</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>1024</xloc>
<yloc>528</yloc>
</GUI>
</transform>
<transform>
@ -250,34 +362,16 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Block until transforms finish 2</send_true_to>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>224</yloc>
<xloc>1696</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<name>delete_akt_become_null</name>
<type>Delete</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -288,35 +382,28 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<field>source_id</field>
<name>source_id</name>
</key>
<key>
<condition>&lt;></condition>
<field>self_employed</field>
<name>actual</name>
<condition>=</condition>
<field>actual</field>
<name>delete_akt</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>self_employed</name>
<rename>actual</rename>
</value>
<table>self_employed</table>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>464</yloc>
<xloc>1200</xloc>
<yloc>304</yloc>
</GUI>
</transform>
<transform>
<name>filter_non_actual_self_employed</name>
<name>filter_akt_rows_for_delete</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
@ -331,7 +418,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<conditions>
</conditions>
<function>=</function>
<leftvalue>actual</leftvalue>
<leftvalue>delete_akt</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
@ -344,16 +431,16 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Update</send_true_to>
<send_true_to>Detect empty stream</send_true_to>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>336</yloc>
<xloc>848</xloc>
<yloc>304</yloc>
</GUI>
</transform>
<transform>
<name>self_employed_input</name>
<type>TableInput</type>
<name>filter_akt_rows_for_update</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -362,43 +449,254 @@ and recruitment_id = '${IDM_ID}';</sql>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>akt_need_update</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Detect empty stream 2</send_true_to>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>filter_pred_rows_for_update</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>pred_need_update</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Detect empty stream 3</send_true_to>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>528</yloc>
</GUI>
</transform>
<transform>
<name>self_employed_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<limit>1</limit>
<sql>with filtered_data as (select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' as npd_elem,
true as actual
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' as akt_npd_elem,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' as pred_npd_elem
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedNPD' ->> 'aktNPD' != 'null'
and
to_date(ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
union all
select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' as npd_elem,
false as actual
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedNPD' ->> 'predNPD' != 'null'
and to_date(ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'
and (to_date(ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'
or to_date(ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}')
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
select fd.recruit_id,
npd_elem ->> 'id' as source_id,
to_date(npd_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
to_date(npd_elem ->> 'dataUchNPD', 'YYYY-MM-DD') as register_date,
to_date(npd_elem ->> 'dataSnUchNPD', 'YYYY-MM-DD') as deregistration_date,
actual
akt_npd_elem ->> 'id' as akt_source_id,
to_date(akt_npd_elem ->> 'dataSved', 'YYYY-MM-DD') as akt_source_update_date,
to_date(akt_npd_elem ->> 'dataUchNPD', 'YYYY-MM-DD') as akt_register_date,
to_date(akt_npd_elem ->> 'dataSnUchNPD', 'YYYY-MM-DD') as akt_deregistration_date,
jsonb_typeof(akt_npd_elem) IS NULL as delete_akt,
to_date(fd.akt_npd_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' as akt_need_update,
pred_npd_elem ->> 'id' as pred_source_id,
to_date(pred_npd_elem ->> 'dataSved', 'YYYY-MM-DD') as pred_source_update_date,
to_date(pred_npd_elem ->> 'dataUchNPD', 'YYYY-MM-DD') as pred_register_date,
to_date(pred_npd_elem ->> 'dataSnUchNPD', 'YYYY-MM-DD') as pred_deregistration_date,
to_date(pred_npd_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' as pred_need_update
from filtered_data fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>448</xloc>
<yloc>224</yloc>
<xloc>592</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>self_employed_upsert</name>
<name>update_akt_flag</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>&lt;></condition>
<field>self_employed</field>
<name>akt_need_update</name>
</key>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>self_employed</name>
<rename>akt_need_update</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1392</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>update_akt_flag_on_delete</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>self_employed</field>
<name>delete_akt</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>self_employed</name>
<rename>akt_need_update</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1392</xloc>
<yloc>304</yloc>
</GUI>
</transform>
<transform>
<name>upsert_akt</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>self_employed</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>akt_source_id</name>
<rename>akt_source_id</rename>
<update>N</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>register_date</name>
<rename>register_date</rename>
<update>Y</update>
</value>
<value>
<name>deregistration_date</name>
<rename>deregistration_date</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>akt_need_update</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>upsert_pred</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
@ -425,61 +723,39 @@ from filtered_data fd;</sql>
</value>
<value>
<name>source_id</name>
<rename>source_id</rename>
<rename>pred_source_id</rename>
<update>N</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<rename>pred_source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>register_date</name>
<rename>register_date</rename>
<rename>pred_register_date</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>actual</rename>
<name>deregistration_date</name>
<rename>pred_deregistration_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1024</xloc>
<yloc>224</yloc>
<xloc>1392</xloc>
<yloc>528</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<source_transform>delete_akt_become_null</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
@ -487,7 +763,43 @@ from filtered_data fd;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>self_employed_upsert</source_transform>
<source_transform>update_akt_flag</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>update_akt_flag_on_delete</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>upsert_akt</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>upsert_pred</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>