micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Revert "ERVU-480 : work_activity flow changes"

Browse source 
This reverts commit a95b839917.
This commit is contained in:
Fusionshh 2025-09-23 15:59:01 +03:00
parent ad488c763b
commit 1a86b0cfb8
26 changed files with 1220 additions and 1744 deletions
Download patch file Download diff file Expand all files Collapse all files

57
mappings/info_recruits/citizen_tables/work/employer/employer_job.hwf
View file

@ -26,12 +26,12 @@
<schedulerType>0</schedulerType>
<weekDay>1</weekDay>
<parallel>N</parallel>
<xloc>0</xloc>
<xloc>48</xloc>
<yloc>48</yloc>
<attributes_hac/>
</action>
<action>
<name>check_if_job_execution_exists.hpl</name>
<name>check_if_employer_job_execution_exists.hpl</name>
<description/>
<type>PIPELINE</type>
<attributes/>
@ -41,9 +41,7 @@
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_job_execution_exists.hpl</filename>
<logext/>
<logfile/>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/employer/support/check_if_employer_job_execution_exists.hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -54,7 +52,7 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>352</xloc>
<xloc>272</xloc>
<yloc>48</yloc>
<attributes_hac/>
</action>
@ -71,7 +69,7 @@
<valuetype>variable</valuetype>
<variablename>JOB_EXECUTED_FLAG</variablename>
<parallel>N</parallel>
<xloc>656</xloc>
<xloc>576</xloc>
<yloc>48</yloc>
<attributes_hac/>
</action>
@ -97,7 +95,7 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>Y</parallel>
<xloc>976</xloc>
<xloc>896</xloc>
<yloc>48</yloc>
<attributes_hac/>
</action>
@ -123,7 +121,7 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>896</xloc>
<xloc>816</xloc>
<yloc>352</yloc>
<attributes_hac/>
</action>
@ -138,9 +136,7 @@
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_need_to_repeat.hpl</filename>
<logext/>
<logfile/>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/employer/support/check_if_need_to_repeat.hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -151,7 +147,7 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>656</xloc>
<xloc>576</xloc>
<yloc>192</yloc>
<attributes_hac/>
</action>
@ -168,7 +164,7 @@
<valuetype>variable</valuetype>
<variablename>NEED_TO_REPEAT_JOB</variablename>
<parallel>N</parallel>
<xloc>656</xloc>
<xloc>576</xloc>
<yloc>352</yloc>
<attributes_hac/>
</action>
@ -194,34 +190,14 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>656</xloc>
<xloc>576</xloc>
<yloc>544</yloc>
<attributes_hac/>
</action>
<action>
<name>init_job_name</name>
<description/>
<type>SET_VARIABLES</type>
<attributes/>
<fields>
<field>
<variable_name>JOB_NAME</variable_name>
<variable_type>CURRENT_WORKFLOW</variable_type>
<variable_value>employer_job</variable_value>
</field>
</fields>
<file_variable_type>CURRENT_WORKFLOW</file_variable_type>
<filename/>
<replacevars>N</replacevars>
<parallel>N</parallel>
<xloc>160</xloc>
<yloc>48</yloc>
<attributes_hac/>
</action>
</actions>
<hops>
<hop>
<from>check_if_job_execution_exists.hpl</from>
<from>check_if_employer_job_execution_exists.hpl</from>
<to>employer_job_execution_exists_check</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
@ -264,18 +240,11 @@
</hop>
<hop>
<from>Start</from>
<to>init_job_name</to>
<to>check_if_employer_job_execution_exists.hpl</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>init_job_name</from>
<to>check_if_job_execution_exists.hpl</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
</hops>
<notepads>
</notepads>

120
mappings/info_recruits/citizen_tables/work/employer/parallel/employer_flow.hpl
View file

@ -65,21 +65,6 @@
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>filter_non_actual_employers</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>filter_non_actual_employers</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -97,7 +82,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1168</xloc>
<xloc>944</xloc>
<yloc>448</yloc>
</GUI>
</transform>
@ -132,7 +117,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>944</xloc>
<xloc>720</xloc>
<yloc>448</yloc>
</GUI>
</transform>
@ -163,7 +148,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1168</xloc>
<xloc>944</xloc>
<yloc>112</yloc>
</GUI>
</transform>
@ -245,7 +230,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1168</xloc>
<xloc>944</xloc>
<yloc>288</yloc>
</GUI>
</transform>
@ -267,43 +252,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<yloc>288</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>employed</name>
<rename>actual_employer</rename>
</value>
</lookup>
<skip_lookup>Y</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>720</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>employer_input</name>
<type>TableInput</type>
@ -334,8 +282,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
false as actual_employer
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'predRabotodat' != 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
select fd.source_id,
fd.recruit_id,
to_date(rabotodat ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
@ -372,8 +319,7 @@ select fd.source_id,
then rabotodat -> 'extend' -> 'svedYUL' ->> 'kpp'
end as kpp,
coalesce(rabotodat -> 'svedYUL' -> 'extend' ->> 'priznakOpk', '0') = '1' as is_opk_org,
actual_employer,
coalesce(rabotodat -> 'extend' ->> 'statusRabotodat', '0') = '1' as status
actual_employer
from filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -448,10 +394,6 @@ from filteredData fd;</sql>
<column_name>is_opk_org</column_name>
<stream_name>is_opk_org</stream_name>
</field>
<field>
<column_name>status</column_name>
<stream_name>status</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
@ -468,59 +410,11 @@ from filteredData fd;</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>944</xloc>
<xloc>720</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform>
<name>filter_non_actual_employers</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>actual_employer</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Update</send_true_to>
<attributes/>
<GUI>
<xloc>528</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>employer_output</source_transform>
<target_transform>Change job status on error</target_transform>

74
mappings/info_recruits/citizen_tables/work/employer/parallel/employer_flow_delta.hpl
View file

@ -65,16 +65,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -265,46 +255,6 @@ and recruitment_id = '${IDM_ID}';</sql>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<key>
<condition>&lt;></condition>
<field>employed</field>
<name>actual_employer</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>employed</name>
<rename>actual_employer</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>672</xloc>
<yloc>544</yloc>
</GUI>
</transform>
<transform>
<name>employer_input</name>
<type>TableInput</type>
@ -376,8 +326,7 @@ select fd.source_id,
then rabotodat -> 'extend' -> 'svedYUL' ->> 'kpp'
end as kpp,
coalesce(rabotodat -> 'svedYUL' -> 'extend' ->> 'priznakOpk', '0') = '1' as is_opk_org,
actual_employer,
coalesce(rabotodat -> 'extend' ->> 'statusRabotodat', '0') = '1' as status
actual_employer
from filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -404,13 +353,11 @@ from filteredData fd;</sql>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<condition>=</condition>
<field>actual_employer</field>
<name>actual_employer</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>employer</table>
@ -479,11 +426,6 @@ from filteredData fd;</sql>
<rename>is_opk_org</rename>
<update>Y</update>
</value>
<value>
<name>status</name>
<rename>status</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -493,18 +435,6 @@ from filteredData fd;</sql>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>employer_upsert</source_transform>
<target_transform>Change job status on error</target_transform>

126
mappings/info_recruits/citizen_tables/work/employer/parallel/employer_flow_repeat.hpl
View file

@ -65,21 +65,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>filter_non_actual_employers</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>filter_non_actual_employers</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -97,7 +82,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1312</xloc>
<xloc>1072</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -132,7 +117,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1136</xloc>
<xloc>896</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -163,7 +148,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1312</xloc>
<xloc>1072</xloc>
<yloc>256</yloc>
</GUI>
</transform>
@ -181,8 +166,11 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
@ -193,6 +181,7 @@ DO UPDATE
SET status = 'PROCESSING',
error_description = null,
execution_datetime = current_timestamp;</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>352</xloc>
@ -249,7 +238,7 @@ SET status = 'PROCESSING',
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1312</xloc>
<xloc>1072</xloc>
<yloc>400</yloc>
</GUI>
</transform>
@ -271,41 +260,6 @@ SET status = 'PROCESSING',
<yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>employed</name>
<rename>actual_employer</rename>
</value>
</lookup>
<skip_lookup>Y</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>896</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>employer_input</name>
<type>TableInput</type>
@ -373,8 +327,7 @@ select fd.source_id,
then rabotodat -> 'extend' -> 'svedYUL' ->> 'kpp'
end as kpp,
coalesce(rabotodat -> 'svedYUL' -> 'extend' ->> 'priznakOpk', '0') = '1' as is_opk_org,
actual_employer,
coalesce(rabotodat -> 'extend' ->> 'statusRabotodat', '0') = '1' as status
actual_employer
from filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -401,7 +354,11 @@ from filteredData fd;</sql>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>actual_employer</field>
<name>actual_employer</name>
</key>
<schema>ervu_dashboard</schema>
<table>employer</table>
@ -470,68 +427,15 @@ from filteredData fd;</sql>
<rename>is_opk_org</rename>
<update>Y</update>
</value>
<value>
<name>status</name>
<rename>status</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1136</xloc>
<xloc>896</xloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>filter_non_actual_employers</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>actual_employer</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Update</send_true_to>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>employer_upsert</source_transform>
<target_transform>Change job status on error</target_transform>

13
mappings/info_recruits/citizen_tables/work/employer/recruitment_five_flow.hpl
View file

@ -21,33 +21,33 @@
</notepads>
<order>
<hop>
<from>Get all recruitments</from>
<from>Get all recruitments ordered by created_date</from>
<to>employer_flow.hpl</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Get all recruitments</from>
<from>Get all recruitments ordered by created_date</from>
<to>employer_flow.hpl 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Get all recruitments</from>
<from>Get all recruitments ordered by created_date</from>
<to>employer_flow.hpl 3</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Get all recruitments</from>
<from>Get all recruitments ordered by created_date</from>
<to>employer_flow.hpl 4</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Get all recruitments</from>
<from>Get all recruitments ordered by created_date</from>
<to>employer_flow.hpl 5</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Get all recruitments</name>
<name>Get all recruitments ordered by created_date</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
@ -59,7 +59,6 @@
</partitioning>
<connection>ervu-dashboard-test</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>SELECT
idm_id
FROM ervu_dashboard.recruitment;</sql>

71
mappings/info_recruits/citizen_tables/work/employer/recruitment_five_flow_delta.hpl
View file

@ -66,23 +66,22 @@
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_source_update_date</lookup>
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
<sql>SELECT
r.idm_id,
? max_source_update_date
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = 'employer_job'
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
AND ri.updated_at > (
SELECT MAX(execution_datetime)
FROM etl.job_execution
WHERE job_name = 'employer_job'
AND recruitment_id = r.idm_id
)
where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
@ -90,6 +89,29 @@ FROM ervu_dashboard.recruitment r
<yloc>336</yloc>
</GUI>
</transform>
<transform>
<name>get_max_source_update_date</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(source_update_date)
from employer;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>336</yloc>
</GUI>
</transform>
<transform>
<name>employer_flow_delta.hpl</name>
<type>PipelineExecutor</type>
@ -395,29 +417,6 @@ FROM ervu_dashboard.recruitment r
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>get_max_source_update_date</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(source_update_date)
from employer;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>336</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>

18
mappings/info_recruits/citizen_tables/support/check_if_job_execution_exists.hpl → mappings/info_recruits/citizen_tables/work/employer/support/check_if_employer_job_execution_exists.hpl
View file

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<pipeline>
<info>
<name>check_if_job_execution_exists</name>
<name>check_if_employer_job_execution_exists</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@ -13,9 +13,9 @@
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
<created_user>-</created_user>
<created_date>2025/09/08 16:38:31.637</created_date>
<created_date>2025/08/18 20:08:41.981</created_date>
<modified_user>-</modified_user>
<modified_date>2025/09/08 16:38:31.637</modified_date>
<modified_date>2025/08/18 20:08:41.981</modified_date>
</info>
<notepads>
</notepads>
@ -47,8 +47,8 @@
<use_formatting>Y</use_formatting>
<attributes/>
<GUI>
<xloc>592</xloc>
<yloc>176</yloc>
<xloc>816</xloc>
<yloc>336</yloc>
</GUI>
</transform>
<transform>
@ -68,13 +68,13 @@
<sql>SELECT EXISTS (
SELECT 1
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
WHERE job_name = 'employer_job'
) AS job_executed_flag;</sql>
<variables_active>Y</variables_active>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>304</xloc>
<yloc>176</yloc>
<xloc>528</xloc>
<yloc>336</yloc>
</GUI>
</transform>
<transform_error_handling>

16
mappings/info_recruits/citizen_tables/support/check_if_need_to_repeat.hpl → mappings/info_recruits/citizen_tables/work/employer/support/check_if_need_to_repeat.hpl
View file

@ -13,9 +13,9 @@
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
<created_user>-</created_user>
<created_date>2025/09/08 20:30:33.935</created_date>
<created_date>2025/08/18 20:14:41.794</created_date>
<modified_user>-</modified_user>
<modified_date>2025/09/08 20:30:33.935</modified_date>
<modified_date>2025/08/18 20:14:41.794</modified_date>
</info>
<notepads>
</notepads>
@ -47,8 +47,8 @@
<use_formatting>Y</use_formatting>
<attributes/>
<GUI>
<xloc>720</xloc>
<yloc>384</yloc>
<xloc>816</xloc>
<yloc>336</yloc>
</GUI>
</transform>
<transform>
@ -69,14 +69,14 @@
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = '${JOB_NAME}'
and job_name = 'employer_job'
where je.status is null or je.status in ('ERROR', 'PROCESSING')
) as need_to_repeat_job;</sql>
<variables_active>Y</variables_active>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>432</xloc>
<yloc>384</yloc>
<xloc>528</xloc>
<yloc>336</yloc>
</GUI>
</transform>
<transform_error_handling>

69
mappings/info_recruits/citizen_tables/work/individual_entrepreneur/individual_entrepreneur_job.hwf
View file

@ -31,7 +31,7 @@
<attributes_hac/>
</action>
<action>
<name>check_if_job_execution_exists.hpl</name>
<name>check_if_individual_entrepreneur_execution_exists.hpl</name>
<description/>
<type>PIPELINE</type>
<attributes/>
@ -41,7 +41,7 @@
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_job_execution_exists.hpl</filename>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/individual_entrepreneur/support/check_if_individual_entrepreneur_execution_exists.hpl</filename>
<logext/>
<logfile/>
<loglevel>Basic</loglevel>
@ -54,7 +54,7 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>384</xloc>
<xloc>256</xloc>
<yloc>224</yloc>
<attributes_hac/>
</action>
@ -71,7 +71,7 @@
<valuetype>variable</valuetype>
<variablename>JOB_EXECUTED_FLAG</variablename>
<parallel>N</parallel>
<xloc>720</xloc>
<xloc>592</xloc>
<yloc>224</yloc>
<attributes_hac/>
</action>
@ -87,6 +87,8 @@
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/individual_entrepreneur/recruitment_five_flow.hpl</filename>
<logext/>
<logfile/>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -97,7 +99,7 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>Y</parallel>
<xloc>1040</xloc>
<xloc>912</xloc>
<yloc>224</yloc>
<attributes_hac/>
</action>
@ -113,6 +115,8 @@
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/individual_entrepreneur/recruitment_five_flow_repeat.hpl</filename>
<logext/>
<logfile/>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -123,7 +127,7 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>960</xloc>
<xloc>832</xloc>
<yloc>528</yloc>
<attributes_hac/>
</action>
@ -138,7 +142,7 @@
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_need_to_repeat.hpl</filename>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/individual_entrepreneur/support/check_if_need_to_repeat.hpl</filename>
<logext/>
<logfile/>
<loglevel>Basic</loglevel>
@ -151,7 +155,7 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>720</xloc>
<xloc>592</xloc>
<yloc>384</yloc>
<attributes_hac/>
</action>
@ -168,7 +172,7 @@
<valuetype>variable</valuetype>
<variablename>NEED_TO_REPEAT_JOB</variablename>
<parallel>N</parallel>
<xloc>720</xloc>
<xloc>592</xloc>
<yloc>528</yloc>
<attributes_hac/>
</action>
@ -184,6 +188,8 @@
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/individual_entrepreneur/recruitment_five_flow_delta.hpl</filename>
<logext/>
<logfile/>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -194,34 +200,21 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>720</xloc>
<xloc>592</xloc>
<yloc>720</yloc>
<attributes_hac/>
</action>
<action>
<name>init_job_name</name>
<description/>
<type>SET_VARIABLES</type>
<attributes/>
<fields>
<field>
<variable_name>JOB_NAME</variable_name>
<variable_type>CURRENT_WORKFLOW</variable_type>
<variable_value>individual_entrepreneur_job</variable_value>
</field>
</fields>
<file_variable_type>CURRENT_WORKFLOW</file_variable_type>
<filename/>
<replacevars>N</replacevars>
<parallel>N</parallel>
<xloc>144</xloc>
<yloc>224</yloc>
<attributes_hac/>
</action>
</actions>
<hops>
<hop>
<from>check_if_job_execution_exists.hpl</from>
<from>Start</from>
<to>check_if_individual_entrepreneur_execution_exists.hpl</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>check_if_individual_entrepreneur_execution_exists.hpl</from>
<to>employer_job_execution_exists_check</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
@ -262,20 +255,6 @@
<evaluation>N</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>Start</from>
<to>init_job_name</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>init_job_name</from>
<to>check_if_job_execution_exists.hpl</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
</hops>
<notepads>
</notepads>

111
mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow.hpl
View file

@ -85,21 +85,6 @@
<to>filter_null_npd_dates</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>filter_non_actual_ip</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>filter_non_actual_ip</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -118,7 +103,7 @@
<attributes/>
<GUI>
<xloc>1648</xloc>
<yloc>704</yloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -153,7 +138,7 @@ and recruitment_id = '${IDM_ID}';
<attributes/>
<GUI>
<xloc>1440</xloc>
<yloc>704</yloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -287,77 +272,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>individual_entrepreneur</name>
<rename>actual</rename>
</value>
</lookup>
<skip_lookup>Y</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1184</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
<name>filter_non_actual_ip</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>actual</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Update</send_true_to>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
<name>filter_null_npd_dates</name>
<type>FilterRows</type>
@ -383,7 +297,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<attributes/>
<GUI>
<xloc>800</xloc>
<yloc>704</yloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -423,8 +337,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
false as actual
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'predRegIP' != 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
select fd.recruit_id,
fd.source_id,
ip_elem ->> 'ogrnip' as ogrnip,
@ -489,7 +402,7 @@ from filtered_data fd;</sql>
<attributes/>
<GUI>
<xloc>1184</xloc>
<yloc>704</yloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -576,22 +489,10 @@ from filtered_data fd;</sql>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>704</yloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>individual_entrepreneur_npd_output</source_transform>
<target_transform>Change job status on error</target_transform>

114
mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow_delta.hpl
View file

@ -85,21 +85,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>filter_non_actual_ip</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>filter_non_actual_ip</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -118,7 +103,7 @@
<attributes/>
<GUI>
<xloc>1616</xloc>
<yloc>752</yloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
@ -153,7 +138,7 @@ and recruitment_id = '${IDM_ID}';
<attributes/>
<GUI>
<xloc>1328</xloc>
<yloc>752</yloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
@ -315,7 +300,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<attributes/>
<GUI>
<xloc>736</xloc>
<yloc>752</yloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
@ -423,7 +408,7 @@ from filtered_data fd;</sql>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>752</yloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
@ -510,85 +495,6 @@ from filtered_data fd;</sql>
</field>
</fields>
<attributes/>
<GUI>
<xloc>912</xloc>
<yloc>752</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>individual_entrepreneur</field>
<name>actual</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>individual_entrepreneur</name>
<rename>actual</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>filter_non_actual_ip</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>actual</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Update</send_true_to>
<attributes/>
<GUI>
<xloc>912</xloc>
<yloc>592</yloc>
@ -619,18 +525,6 @@ from filtered_data fd;</sql>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
</transform_error_handling>
<attributes/>
</pipeline>

114
mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow_repeat.hpl
View file

@ -85,21 +85,6 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>filter_non_actual_ip</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>filter_non_actual_ip</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -118,7 +103,7 @@
<attributes/>
<GUI>
<xloc>1856</xloc>
<yloc>736</yloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
@ -153,7 +138,7 @@ and recruitment_id = '${IDM_ID}';
<attributes/>
<GUI>
<xloc>1568</xloc>
<yloc>736</yloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
@ -201,8 +186,11 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
@ -213,6 +201,7 @@ DO UPDATE
SET status = 'PROCESSING',
error_description = null,
execution_datetime = current_timestamp;</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>512</xloc>
@ -315,7 +304,7 @@ SET status = 'PROCESSING',
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>736</yloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
@ -419,7 +408,7 @@ from filtered_data fd;</sql>
<attributes/>
<GUI>
<xloc>1328</xloc>
<yloc>736</yloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
@ -508,80 +497,7 @@ from filtered_data fd;</sql>
<attributes/>
<GUI>
<xloc>1136</xloc>
<yloc>736</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>individual_entrepreneur</name>
<rename>actual</rename>
</value>
</lookup>
<skip_lookup>Y</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1328</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>filter_non_actual_ip</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>actual</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Update</send_true_to>
<attributes/>
<GUI>
<xloc>1136</xloc>
<yloc>592</yloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform_error_handling>
@ -609,18 +525,6 @@ from filtered_data fd;</sql>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
</transform_error_handling>
<attributes/>
</pipeline>

29
mappings/info_recruits/citizen_tables/work/individual_entrepreneur/recruitment_five_flow_delta.hpl
View file

@ -89,23 +89,22 @@
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>Get_max_source_update_date</lookup>
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id,
? max_source_update_date
<sql>SELECT
r.idm_id as recruitment_id,
? as max_source_update_date
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = 'individual_entrepreneur_job'
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
AND ri.updated_at > (
SELECT MAX(execution_datetime)
FROM etl.job_execution
WHERE job_name = 'individual_entrepreneur_job'
AND recruitment_id = r.idm_id
)
where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>

85
mappings/info_recruits/citizen_tables/work/individual_entrepreneur/support/check_if_need_to_repeat.hpl Normal file
View file

@ -0,0 +1,85 @@
<?xml version="1.0" encoding="UTF-8"?>
<pipeline>
<info>
<name>check_if_need_to_repeat</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<parameters>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
<created_user>-</created_user>
<created_date>2025/08/19 08:57:47.121</created_date>
<modified_user>-</modified_user>
<modified_date>2025/08/19 08:57:47.121</modified_date>
</info>
<notepads>
</notepads>
<order>
<hop>
<from>Table input</from>
<to>Set variables</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Set variables</name>
<type>SetVariable</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<fields>
<field>
<field_name>need_to_repeat_job</field_name>
<variable_name>NEED_TO_REPEAT_JOB</variable_name>
<variable_type>ROOT_WORKFLOW</variable_type>
</field>
</fields>
<use_formatting>Y</use_formatting>
<attributes/>
<GUI>
<xloc>736</xloc>
<yloc>240</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT EXISTS (SELECT 1
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = 'individual_entrepreneur_job'
where je.status is null or je.status in ('ERROR', 'PROCESSING')
) as need_to_repeat_job;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>448</xloc>
<yloc>240</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>
</pipeline>

130
mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow.hpl
View file

@ -65,21 +65,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>filter_non_actual_self_employed</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>filter_non_actual_self_employed</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -97,8 +82,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>496</yloc>
<xloc>1040</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
@ -118,8 +103,11 @@
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
@ -130,10 +118,11 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>896</xloc>
<yloc>496</yloc>
<xloc>832</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
@ -150,8 +139,11 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
@ -161,9 +153,10 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1104</xloc>
<xloc>1040</xloc>
<yloc>48</yloc>
</GUI>
</transform>
@ -181,8 +174,11 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
@ -190,6 +186,7 @@ and recruitment_id = '${IDM_ID}';
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>224</xloc>
@ -245,7 +242,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1104</xloc>
<xloc>1040</xloc>
<yloc>224</yloc>
</GUI>
</transform>
@ -267,77 +264,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>self_employed</name>
<rename>actual</rename>
</value>
</lookup>
<skip_lookup>Y</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>filter_non_actual_self_employed</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>actual</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Update</send_true_to>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>368</yloc>
</GUI>
</transform>
<transform>
<name>self_employed_input</name>
<type>TableInput</type>
@ -366,8 +292,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
false as actual
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedNPD' ->> 'predNPD' != 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
select fd.recruit_id,
npd_elem ->> 'id' as source_id,
to_date(npd_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
@ -425,34 +350,25 @@ from filtered_data fd;</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>self_employed</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>896</xloc>
<xloc>832</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>self_employed_output</source_transform>
<target_transform>Change job status on error</target_transform>

110
mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_delta.hpl
View file

@ -65,21 +65,6 @@
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>filter_non_actual_self_employed</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>filter_non_actual_self_employed</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -98,7 +83,7 @@
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>464</yloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
@ -133,7 +118,7 @@ and recruitment_id = '${IDM_ID}';
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>464</yloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
@ -270,85 +255,6 @@ and recruitment_id = '${IDM_ID}';</sql>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>self_employed</field>
<name>actual</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>self_employed</name>
<rename>actual</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>filter_non_actual_self_employed</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>actual</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Update</send_true_to>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>336</yloc>
</GUI>
</transform>
<transform>
<name>self_employed_input</name>
<type>TableInput</type>
@ -450,18 +356,6 @@ from filtered_data fd;</sql>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>self_employed_upsert</source_transform>
<target_transform>Change job status on error</target_transform>

104
mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_repeat.hpl
View file

@ -65,21 +65,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>filter_non_actual_self_employed</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>filter_non_actual_self_employed</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -98,7 +83,7 @@
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>656</yloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -133,7 +118,7 @@ and recruitment_id = '${IDM_ID}';
<attributes/>
<GUI>
<xloc>864</xloc>
<yloc>656</yloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -270,79 +255,6 @@ SET status = 'PROCESSING',
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>self_employed</name>
<rename>actual</rename>
</value>
</lookup>
<skip_lookup>Y</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>656</xloc>
<yloc>656</yloc>
</GUI>
</transform>
<transform>
<name>filter_non_actual_self_employed</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>actual</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Update</send_true_to>
<attributes/>
<GUI>
<xloc>656</xloc>
<yloc>544</yloc>
</GUI>
</transform>
<transform>
<name>self_employed_input</name>
<type>TableInput</type>
@ -441,18 +353,6 @@ from filtered_data fd;</sql>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>self_employed_output</source_transform>
<target_transform>Change job status on error</target_transform>

27
mappings/info_recruits/citizen_tables/work/self_employed/recruitment_five_flow_delta.hpl
View file

@ -89,23 +89,22 @@
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>Get_max_source_update_date</lookup>
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id,
? max_source_update_date
<sql>SELECT
r.idm_id AS recruitment_id,
? as max_source_update_date
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
AND je.job_name = 'self_employed_job'
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
AND ri.updated_at > (
SELECT MAX(execution_datetime)
FROM etl.job_execution
WHERE job_name = 'self_employed_job'
AND recruitment_id = r.idm_id
)
WHERE je.status IN ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>

59
mappings/info_recruits/citizen_tables/work/self_employed/self_employed_job.hwf
View file

@ -31,7 +31,7 @@
<attributes_hac/>
</action>
<action>
<name>check_if_job_execution_exists.hpl</name>
<name>check_if_self_employed_job_execution_exists.hpl</name>
<description/>
<type>PIPELINE</type>
<attributes/>
@ -41,9 +41,7 @@
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_job_execution_exists.hpl</filename>
<logext/>
<logfile/>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/self_employed/support/check_if_self_employed_job_execution_exists.hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -54,7 +52,7 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>912</xloc>
<xloc>704</xloc>
<yloc>224</yloc>
<attributes_hac/>
</action>
@ -71,7 +69,7 @@
<valuetype>variable</valuetype>
<variablename>JOB_EXECUTED_FLAG</variablename>
<parallel>N</parallel>
<xloc>1216</xloc>
<xloc>1008</xloc>
<yloc>224</yloc>
<attributes_hac/>
</action>
@ -97,7 +95,7 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>Y</parallel>
<xloc>1536</xloc>
<xloc>1328</xloc>
<yloc>224</yloc>
<attributes_hac/>
</action>
@ -113,6 +111,8 @@
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/self_employed/recruitments_five_flow_repeat.hpl</filename>
<logext/>
<logfile/>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -123,7 +123,7 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>1456</xloc>
<xloc>1248</xloc>
<yloc>528</yloc>
<attributes_hac/>
</action>
@ -138,9 +138,7 @@
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_need_to_repeat.hpl</filename>
<logext/>
<logfile/>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/self_employed/support/check_if_need_to_repeat.hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -151,7 +149,7 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>1216</xloc>
<xloc>1008</xloc>
<yloc>384</yloc>
<attributes_hac/>
</action>
@ -168,7 +166,7 @@
<valuetype>variable</valuetype>
<variablename>NEED_TO_REPEAT_JOB</variablename>
<parallel>N</parallel>
<xloc>1216</xloc>
<xloc>1008</xloc>
<yloc>528</yloc>
<attributes_hac/>
</action>
@ -184,6 +182,8 @@
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/self_employed/recruitment_five_flow_delta.hpl</filename>
<logext/>
<logfile/>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -194,34 +194,14 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>1216</xloc>
<xloc>1008</xloc>
<yloc>720</yloc>
<attributes_hac/>
</action>
<action>
<name>init_job_name</name>
<description/>
<type>SET_VARIABLES</type>
<attributes/>
<fields>
<field>
<variable_name>JOB_NAME</variable_name>
<variable_type>CURRENT_WORKFLOW</variable_type>
<variable_value>self_employed_job</variable_value>
</field>
</fields>
<file_variable_type>CURRENT_WORKFLOW</file_variable_type>
<filename/>
<replacevars>N</replacevars>
<parallel>N</parallel>
<xloc>640</xloc>
<yloc>224</yloc>
<attributes_hac/>
</action>
</actions>
<hops>
<hop>
<from>check_if_job_execution_exists.hpl</from>
<from>check_if_self_employed_job_execution_exists.hpl</from>
<to>employer_job_execution_exists_check</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
@ -264,18 +244,11 @@
</hop>
<hop>
<from>Start</from>
<to>init_job_name</to>
<to>check_if_self_employed_job_execution_exists.hpl</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>init_job_name</from>
<to>check_if_job_execution_exists.hpl</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
</hops>
<notepads>
</notepads>

86
mappings/info_recruits/citizen_tables/work/self_employed/support/check_if_need_to_repeat.hpl Normal file
View file

@ -0,0 +1,86 @@
<?xml version="1.0" encoding="UTF-8"?>
<pipeline>
<info>
<name>check_if_need_to_repeat</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<parameters>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
<created_user>-</created_user>
<created_date>2025/08/18 21:04:50.604</created_date>
<modified_user>-</modified_user>
<modified_date>2025/08/18 21:04:50.604</modified_date>
</info>
<notepads>
</notepads>
<order>
<hop>
<from>Table input</from>
<to>Set variables</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Set variables</name>
<type>SetVariable</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<fields>
<field>
<field_name>need_to_repeat_job</field_name>
<variable_name>NEED_TO_REPEAT_JOB</variable_name>
<variable_type>ROOT_WORKFLOW</variable_type>
</field>
</fields>
<use_formatting>Y</use_formatting>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT EXISTS (SELECT 1
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = 'self_employed_job'
where je.status is null or je.status in ('ERROR', 'PROCESSING')
limit 1
) as need_to_repeat_job;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>416</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>
</pipeline>

595
mappings/info_recruits/citizen_tables/work/test.hpl Normal file
View file

@ -0,0 +1,595 @@
<?xml version="1.0" encoding="UTF-8"?>
<pipeline>
<info>
<name>child_flow</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<parameters>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
<created_user>-</created_user>
<created_date>2025/08/05 14:31:22.799</created_date>
<modified_user>-</modified_user>
<modified_date>2025/08/05 14:31:22.799</modified_date>
</info>
<notepads>
</notepads>
<order>
<hop>
<from>Create job execution record </from>
<to>Table input </to>
<enabled>Y</enabled>
</hop>
<hop>
<from>User defined Java expression 2</from>
<to>Update 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update 2</from>
<to>Change job status on error </to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error </from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update 2</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success </to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update 2</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success </to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input </from>
<to>Table output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Group by 2</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>User defined Java expression 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<to>Change job status on error </to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<to>Group by 2</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT_WITH_ERROR</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>1024</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error </name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>800</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success </name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
error_description = null
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1744</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record </name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<attributes/>
<GUI>
<xloc>112</xloc>
<yloc>800</yloc>
</GUI>
</transform>
<transform>
<name>Detect empty stream</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>1568</xloc>
<yloc>480</yloc>
</GUI>
</transform>
<transform>
<name>Filter rows</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Change job status on success </send_true_to>
<attributes/>
<GUI>
<xloc>1392</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>Group by 2</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<aggregate>minors_count_raw</aggregate>
<subject>is_minor</subject>
<type>SUM</type>
</field>
<field>
<aggregate>has_minor_child_raw</aggregate>
<subject>is_minor</subject>
<type>MAX</type>
</field>
<field>
<aggregate>has_dead_child_raw</aggregate>
<subject>is_dead</subject>
<type>MAX</type>
</field>
</fields>
<give_back_row>N</give_back_row>
<group>
<field>
<name>recruit_id</name>
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>560</xloc>
<yloc>480</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>784</xloc>
<yloc>480</yloc>
</GUI>
</transform>
<transform>
<name>Table input </name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>SELECT
ri.recruit_id::uuid AS recruit_id,
ch->>'id' AS child_external_id,
NULLIF(ch->>'idERN', '') AS child_id_ern,
CASE NULLIF(ch->>'rodstvSvyazReb', '')::int
WHEN 1 THEN 3
WHEN 2 THEN 4
END AS kinship_type,
ch->'svedFLBS'->'fio'->>'familiya' AS last_name,
ch->'svedFLBS'->'fio'->>'imya' AS first_name,
ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name,
CONCAT_WS(' ',
ch->'svedFLBS'->'fio'->>'familiya',
ch->'svedFLBS'->'fio'->>'imya',
ch->'svedFLBS'->'fio'->>'otchestvo'
) AS full_name,
dates.birth_date,
dates.death_date,
ch->'svedSmert'->>'nomerZapis' AS death_az_number,
ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number,
CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead,
CASE
WHEN dates.death_date IS NOT NULL THEN 0 -- умершие не могут быть несовершеннолетними
WHEN dates.birth_date IS NULL THEN 0 -- если дата рождения неизвестна, считаем взрослым
WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1
ELSE 0
END AS is_minor
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch
CROSS JOIN LATERAL (
SELECT
MAKE_DATE(
NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int,
NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int,
NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int
) AS birth_date,
MAKE_DATE(
NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int,
NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int,
NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int
) AS death_date
) AS dates
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND'${M_R_CR_DATE}'::timestamp >= c.recruit_create_date
AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>304</xloc>
<yloc>800</yloc>
</GUI>
</transform>
<transform>
<name>Table output</name>
<type>TableOutput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>1000</commit>
<connection>ervu-dashboard</connection>
<fields>
<field>
<column_name>recruit_id</column_name>
<stream_name>recruit_id</stream_name>
</field>
<field>
<column_name>child_external_id</column_name>
<stream_name>child_external_id</stream_name>
</field>
<field>
<column_name>child_id_ern</column_name>
<stream_name>child_id_ern</stream_name>
</field>
<field>
<column_name>kinship_type</column_name>
<stream_name>kinship_type</stream_name>
</field>
<field>
<column_name>last_name</column_name>
<stream_name>last_name</stream_name>
</field>
<field>
<column_name>first_name</column_name>
<stream_name>first_name</stream_name>
</field>
<field>
<column_name>middle_name</column_name>
<stream_name>middle_name</stream_name>
</field>
<field>
<column_name>full_name</column_name>
<stream_name>full_name</stream_name>
</field>
<field>
<column_name>birth_date</column_name>
<stream_name>birth_date</stream_name>
</field>
<field>
<column_name>death_date</column_name>
<stream_name>death_date</stream_name>
</field>
<field>
<column_name>death_az_number</column_name>
<stream_name>death_az_number</stream_name>
</field>
<field>
<column_name>birth_az_number</column_name>
<stream_name>birth_az_number</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_monthly>Y</partitioning_monthly>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>citizen_child</table>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>560</xloc>
<yloc>800</yloc>
</GUI>
</transform>
<transform>
<name>Update 2</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>minors_count</name>
<rename>minors_count</rename>
</value>
<value>
<name>has_minor_child</name>
<rename>has_minor_child</rename>
</value>
<value>
<name>has_dead_child</name>
<rename>has_dead_child</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>480</yloc>
</GUI>
</transform>
<transform>
<name>User defined Java expression 2</name>
<type>Janino</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<formula>
<field_name>minors_count</field_name>
<formula_string>minors_count_raw == 1 ? "1" : minors_count_raw == 2 ? "2" : minors_count_raw >= 3 ? "3 и более" : "0"</formula_string>
<value_type>String</value_type>
<value_length>10</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<formula>
<field_name>has_minor_child</field_name>
<formula_string>has_minor_child_raw == 1 </formula_string>
<value_type>Boolean</value_type>
<value_length>-1</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<formula>
<field_name>has_dead_child</field_name>
<formula_string>has_dead_child_raw == 1 </formula_string>
<value_type>Boolean</value_type>
<value_length>-1</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<attributes/>
<GUI>
<xloc>1024</xloc>
<yloc>480</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Create job execution record </source_transform>
<target_transform>Change job status on error </target_transform>
<is_enabled>N</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>Table output</source_transform>
<target_transform>Change job status on error </target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename>error_code</codes_valuename>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>Update 2</source_transform>
<target_transform>Change job status on error </target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename>error_code</codes_valuename>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
</transform_error_handling>
<attributes/>
</pipeline>

287
mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow.hpl
View file

@ -26,7 +26,7 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>work_activity_output</from>
<from>citizen_work_activity_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
@ -40,6 +40,16 @@
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>citizen_work_activity_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
@ -50,24 +60,9 @@
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>employee_parental_leave_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Group by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Group by</from>
<to>work_activity_output</to>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
@ -77,17 +72,12 @@
</hop>
<hop>
<from>Filter_null_parental_leave_dates</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>employee_parental_leave_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>work_activity_output</from>
<to>Filter rows</to>
<from>employee_parental_leave_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
</order>
@ -107,30 +97,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1728</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>citizen_work_activity_output</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>928</xloc>
<yloc>624</yloc>
<xloc>1424</xloc>
<yloc>528</yloc>
</GUI>
</transform>
<transform>
@ -164,8 +132,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1504</xloc>
<yloc>624</yloc>
<xloc>1200</xloc>
<yloc>528</yloc>
</GUI>
</transform>
<transform>
@ -195,7 +163,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1504</xloc>
<xloc>1424</xloc>
<yloc>144</yloc>
</GUI>
</transform>
@ -277,8 +245,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1504</xloc>
<yloc>320</yloc>
<xloc>1200</xloc>
<yloc>144</yloc>
</GUI>
</transform>
<transform>
@ -302,92 +270,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<operator>-</operator>
</condition>
</compare>
<send_false_to>Change job status on success</send_false_to>
<send_true_to>Block until transforms finish</send_true_to>
<send_true_to>employee_parental_leave_output</send_true_to>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
<name>Group by</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<aggregate>recruit_id</aggregate>
<subject>recruit_id</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>employer_name</aggregate>
<subject>employer_name</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>position</aggregate>
<subject>position</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>personnel_event_date</aggregate>
<subject>personnel_event_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>personnel_event_type</aggregate>
<subject>personnel_event_type</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>source_update_date</aggregate>
<subject>source_update_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>actual_employer</aggregate>
<subject>actual_employer</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>parental_leave_start_date</aggregate>
<subject>parental_leave_start_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>parental_leave_end_date</aggregate>
<subject>parental_leave_end_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>last_row</aggregate>
<subject>last_row</subject>
<type>LAST_INCL_NULL</type>
</field>
</fields>
<give_back_row>N</give_back_row>
<group>
<field>
<name>source_id</name>
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>944</xloc>
<yloc>320</yloc>
<yloc>528</yloc>
</GUI>
</transform>
<transform>
@ -421,13 +308,22 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' as trud_arr
trud_elem,
uhod_elem
FROM recruits_info ri
CROSS JOIN LATERAL jsonb_array_elements(
ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat'
) AS trud_elem
LEFT JOIN LATERAL jsonb_array_elements(
CASE
WHEN jsonb_typeof(trud_elem -> 'svedUhodReb') = 'array'
THEN trud_elem -> 'svedUhodReb'
ELSE '[]'::jsonb
END
) AS uhod_elem ON true
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat') = 'array'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
SELECT fd.recruit_id,
trud_elem ->> 'id' AS source_id,
trud_elem -> 'svedOrg' ->> 'naimOrg' AS employer_name,
@ -438,66 +334,17 @@ SELECT fd.recruit_id,
(trud_elem ->> 'prAktMestRab') = '1' AS actual_employer,
to_date(uhod_elem ->> 'dataNachUhodReb', 'YYYY-MM-DD') AS parental_leave_start_date,
to_date(uhod_elem ->> 'dataKonUhodReb', 'YYYY-MM-DD') AS parental_leave_end_date
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(trud_arr) AS trud_elem
LEFT JOIN LATERAL jsonb_array_elements(COALESCE(trud_elem -> 'svedUhodReb', '[]'::jsonb)) AS uhod_elem
ON true;
FROM filteredData fd;
</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>496</xloc>
<xloc>512</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>employee_parental_leave_output</name>
<type>TableOutput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<fields>
<field>
<column_name>citizen_work_activity_source_id</column_name>
<stream_name>source_id</stream_name>
</field>
<field>
<column_name>start_date</column_name>
<stream_name>parental_leave_start_date</stream_name>
</field>
<field>
<column_name>end_date</column_name>
<stream_name>parental_leave_end_date</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_monthly>Y</partitioning_monthly>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>employee_parental_leave</table>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
<name>work_activity_output</name>
<name>citizen_work_activity_output</name>
<type>TableOutput</type>
<description/>
<distribute>Y</distribute>
@ -542,10 +389,6 @@ FROM filteredData fd
<column_name>actual_employer</column_name>
<stream_name>actual_employer</stream_name>
</field>
<field>
<column_name>source_update_date</column_name>
<stream_name>source_update_date</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
@ -555,7 +398,7 @@ FROM filteredData fd
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>work_activity</table>
<table>citizen_work_activity</table>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -566,9 +409,55 @@ FROM filteredData fd
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>employee_parental_leave_output</name>
<type>TableOutput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<fields>
<field>
<column_name>citizen_work_activity_source_id</column_name>
<stream_name>source_id</stream_name>
</field>
<field>
<column_name>parental_leave_start_date</column_name>
<stream_name>parental_leave_start_date</stream_name>
</field>
<field>
<column_name>parental_leave_end_date</column_name>
<stream_name>parental_leave_end_date</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_monthly>Y</partitioning_monthly>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>employee_parental_leave</table>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>960</xloc>
<yloc>528</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>employee_parental_leave_output</source_transform>
<source_transform>citizen_work_activity_output</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
@ -580,7 +469,7 @@ FROM filteredData fd
<min_pct_rows/>
</error>
<error>
<source_transform>work_activity_output</source_transform>
<source_transform>employee_parental_leave_output</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>

213
mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow_delta.hpl
View file

@ -20,31 +20,11 @@
<notepads>
</notepads>
<order>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>parental_leave_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Create job execution record</from>
<to>citizen_work_activity_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter_null_parental_leave_dates</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
@ -55,59 +35,47 @@
<to>Filter_null_parental_leave_dates</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Group by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Group by</from>
<from>Detect empty stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>citizen_work_activity_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<from>Filter_null_parental_leave_dates</from>
<to>parental_leave_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_upsert</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_upsert</from>
<to>Filter rows</to>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>parental_leave_upsert</to>
<from>parental_leave_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>citizen_work_activity_upsert</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>880</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -143,7 +111,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1552</xloc>
<xloc>1296</xloc>
<yloc>432</yloc>
</GUI>
</transform>
@ -174,7 +142,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1552</xloc>
<xloc>1296</xloc>
<yloc>48</yloc>
</GUI>
</transform>
@ -197,13 +165,16 @@ and recruitment_id = '${IDM_ID}';
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<sql>UPDATE etl.job_execution
SET
status = 'DELTA_PROCESSING',
execution_datetime = current_timestamp,
error_description = NULL
where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql>
<attributes/>
<GUI>
<xloc>224</xloc>
<xloc>336</xloc>
<yloc>224</yloc>
</GUI>
</transform>
@ -220,7 +191,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>640</xloc>
<xloc>752</xloc>
<yloc>48</yloc>
</GUI>
</transform>
@ -256,8 +227,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1552</xloc>
<yloc>224</yloc>
<xloc>1024</xloc>
<yloc>48</yloc>
</GUI>
</transform>
<transform>
@ -281,94 +252,13 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<operator>-</operator>
</condition>
</compare>
<send_false_to>Change job status on success</send_false_to>
<send_true_to>Block until transforms finish</send_true_to>
<send_true_to>parental_leave_upsert</send_true_to>
<attributes/>
<GUI>
<xloc>640</xloc>
<xloc>752</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Group by</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<aggregate>recruit_id</aggregate>
<subject>recruit_id</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>employer_name</aggregate>
<subject>employer_name</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>position</aggregate>
<subject>position</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>personnel_event_date</aggregate>
<subject>personnel_event_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>personnel_event_type</aggregate>
<subject>personnel_event_type</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>source_update_date</aggregate>
<subject>source_update_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>actual_employer</aggregate>
<subject>actual_employer</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>parental_leave_start_date</aggregate>
<subject>parental_leave_start_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>parental_leave_end_date</aggregate>
<subject>parental_leave_end_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>last_row</aggregate>
<subject>last_row</subject>
<type>LAST_INCL_NULL</type>
</field>
</fields>
<give_back_row>N</give_back_row>
<group>
<field>
<name>source_id</name>
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>880</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
@ -383,7 +273,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>640</xloc>
<xloc>752</xloc>
<yloc>224</yloc>
</GUI>
</transform>
@ -400,12 +290,23 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' as trud_arr
trud_elem,
uhod_elem
FROM recruits_info ri
CROSS JOIN LATERAL jsonb_array_elements(
ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat'
) AS trud_elem
LEFT JOIN LATERAL jsonb_array_elements(
CASE
WHEN jsonb_typeof(trud_elem -> 'svedUhodReb') = 'array'
THEN trud_elem -> 'svedUhodReb'
ELSE '[]'::jsonb
END
) AS uhod_elem ON true
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat') = 'array'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
--AND to_date(trud_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'
--AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
SELECT fd.recruit_id,
trud_elem ->> 'id' AS source_id,
@ -417,16 +318,12 @@ SELECT fd.recruit_id,
(trud_elem ->> 'prAktMestRab') = '1' AS actual_employer,
to_date(uhod_elem ->> 'dataNachUhodReb', 'YYYY-MM-DD') AS parental_leave_start_date,
to_date(uhod_elem ->> 'dataKonUhodReb', 'YYYY-MM-DD') AS parental_leave_end_date
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(trud_arr) AS trud_elem
LEFT JOIN LATERAL jsonb_array_elements(COALESCE(trud_elem -> 'svedUhodReb', '[]'::jsonb)) AS uhod_elem
ON true
where to_date(trud_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}';
FROM filteredData fd;
</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>432</xloc>
<xloc>544</xloc>
<yloc>224</yloc>
</GUI>
</transform>
@ -450,7 +347,7 @@ where to_date(trud_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DAT
<name>source_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>work_activity</table>
<table>citizen_work_activity</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
@ -495,7 +392,7 @@ where to_date(trud_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DAT
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1136</xloc>
<xloc>1040</xloc>
<yloc>224</yloc>
</GUI>
</transform>
@ -539,7 +436,7 @@ where to_date(trud_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DAT
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1136</xloc>
<xloc>1040</xloc>
<yloc>432</yloc>
</GUI>
</transform>

240
mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow_repeat.hpl
View file

@ -20,11 +20,46 @@
<notepads>
</notepads>
<order>
<hop>
<from>Detect empty stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter_null_parental_leave_dates</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Create job execution record</from>
<to>citizen_work_activity_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>citizen_work_activity_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter_null_parental_leave_dates</from>
<to>parental_leave_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>parental_leave_upsert</from>
<to>Change job status on error</to>
@ -37,77 +72,10 @@
</hop>
<hop>
<from>citizen_work_activity_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Create job execution record</from>
<to>citizen_work_activity_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter_null_parental_leave_dates</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter_null_parental_leave_dates</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Group by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Group by</from>
<to>citizen_work_activity_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>parental_leave_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>citizen_work_activity_upsert</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1088</xloc>
<yloc>704</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -143,7 +111,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1632</xloc>
<xloc>1312</xloc>
<yloc>704</yloc>
</GUI>
</transform>
@ -174,7 +142,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1632</xloc>
<xloc>1312</xloc>
<yloc>320</yloc>
</GUI>
</transform>
@ -198,12 +166,15 @@ and recruitment_id = '${IDM_ID}';
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}')
ON CONFLICT (job_name, recruitment_id)
DO UPDATE
SET status = 'PROCESSING',
error_description = null,
execution_datetime = current_timestamp;</sql>
<attributes/>
<GUI>
<xloc>432</xloc>
<xloc>304</xloc>
<yloc>496</yloc>
</GUI>
</transform>
@ -220,7 +191,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>848</xloc>
<xloc>768</xloc>
<yloc>320</yloc>
</GUI>
</transform>
@ -256,8 +227,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1632</xloc>
<yloc>496</yloc>
<xloc>1040</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
@ -281,94 +252,13 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<operator>-</operator>
</condition>
</compare>
<send_false_to>Change job status on success</send_false_to>
<send_true_to>Block until transforms finish</send_true_to>
<send_true_to>parental_leave_upsert</send_true_to>
<attributes/>
<GUI>
<xloc>848</xloc>
<xloc>768</xloc>
<yloc>704</yloc>
</GUI>
</transform>
<transform>
<name>Group by</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<aggregate>recruit_id</aggregate>
<subject>recruit_id</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>employer_name</aggregate>
<subject>employer_name</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>position</aggregate>
<subject>position</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>personnel_event_date</aggregate>
<subject>personnel_event_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>personnel_event_type</aggregate>
<subject>personnel_event_type</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>source_update_date</aggregate>
<subject>source_update_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>actual_employer</aggregate>
<subject>actual_employer</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>parental_leave_start_date</aggregate>
<subject>parental_leave_start_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>parental_leave_end_date</aggregate>
<subject>parental_leave_end_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>last_row</aggregate>
<subject>last_row</subject>
<type>LAST_INCL_NULL</type>
</field>
</fields>
<give_back_row>N</give_back_row>
<group>
<field>
<name>source_id</name>
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>1088</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
@ -383,7 +273,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>848</xloc>
<xloc>768</xloc>
<yloc>496</yloc>
</GUI>
</transform>
@ -400,13 +290,22 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' as trud_arr
trud_elem,
uhod_elem
FROM recruits_info ri
CROSS JOIN LATERAL jsonb_array_elements(
ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat'
) AS trud_elem
LEFT JOIN LATERAL jsonb_array_elements(
CASE
WHEN jsonb_typeof(trud_elem -> 'svedUhodReb') = 'array'
THEN trud_elem -> 'svedUhodReb'
ELSE '[]'::jsonb
END
) AS uhod_elem ON true
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat') = 'array'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
SELECT fd.recruit_id,
trud_elem ->> 'id' AS source_id,
trud_elem -> 'svedOrg' ->> 'naimOrg' AS employer_name,
@ -417,15 +316,12 @@ SELECT fd.recruit_id,
(trud_elem ->> 'prAktMestRab') = '1' AS actual_employer,
to_date(uhod_elem ->> 'dataNachUhodReb', 'YYYY-MM-DD') AS parental_leave_start_date,
to_date(uhod_elem ->> 'dataKonUhodReb', 'YYYY-MM-DD') AS parental_leave_end_date
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(trud_arr) AS trud_elem
LEFT JOIN LATERAL jsonb_array_elements(COALESCE(trud_elem -> 'svedUhodReb', '[]'::jsonb)) AS uhod_elem
ON true;
FROM filteredData fd;
</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>640</xloc>
<xloc>544</xloc>
<yloc>496</yloc>
</GUI>
</transform>
@ -449,7 +345,7 @@ FROM filteredData fd
<name>source_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>work_activity</table>
<table>citizen_work_activity</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
@ -494,7 +390,7 @@ FROM filteredData fd
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1360</xloc>
<xloc>1040</xloc>
<yloc>496</yloc>
</GUI>
</transform>
@ -538,7 +434,7 @@ FROM filteredData fd
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1360</xloc>
<xloc>1040</xloc>
<yloc>704</yloc>
</GUI>
</transform>

29
mappings/info_recruits/citizen_tables/work/work_activity/recruitment_five_flow_delta.hpl
View file

@ -66,23 +66,22 @@
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_source_update_date</lookup>
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id,
? max_source_update_date
<sql>SELECT
r.idm_id as recruitment_id,
? as max_source_update_date
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = 'work_activity_job'
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
AND ri.updated_at > (
SELECT MAX(execution_datetime)
FROM etl.job_execution
WHERE job_name = 'work_activity_job'
AND recruitment_id = r.idm_id
)
where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>

67
mappings/info_recruits/citizen_tables/work/work_activity/work_activity_job.hwf
View file

@ -52,7 +52,7 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>592</xloc>
<xloc>416</xloc>
<yloc>304</yloc>
<attributes_hac/>
</action>
@ -69,7 +69,7 @@
<valuetype>variable</valuetype>
<variablename>JOB_EXECUTED_FLAG</variablename>
<parallel>N</parallel>
<xloc>864</xloc>
<xloc>688</xloc>
<yloc>304</yloc>
<attributes_hac/>
</action>
@ -95,7 +95,7 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>Y</parallel>
<xloc>1216</xloc>
<xloc>1040</xloc>
<yloc>304</yloc>
<attributes_hac/>
</action>
@ -111,6 +111,8 @@
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/work_activity/recruitment_five_flow_repeat.hpl</filename>
<logext/>
<logfile/>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -121,7 +123,7 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>1104</xloc>
<xloc>928</xloc>
<yloc>608</yloc>
<attributes_hac/>
</action>
@ -147,7 +149,7 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>864</xloc>
<xloc>688</xloc>
<yloc>464</yloc>
<attributes_hac/>
</action>
@ -164,7 +166,7 @@
<valuetype>variable</valuetype>
<variablename>NEED_TO_REPEAT_JOB</variablename>
<parallel>N</parallel>
<xloc>864</xloc>
<xloc>688</xloc>
<yloc>608</yloc>
<attributes_hac/>
</action>
@ -190,32 +192,19 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>864</xloc>
<xloc>688</xloc>
<yloc>800</yloc>
<attributes_hac/>
</action>
<action>
<name>init_job_name</name>
<description/>
<type>SET_VARIABLES</type>
<attributes/>
<fields>
<field>
<variable_name>JOB_NAME</variable_name>
<variable_type>CURRENT_WORKFLOW</variable_type>
<variable_value>work_activity_job</variable_value>
</field>
</fields>
<file_variable_type>CURRENT_WORKFLOW</file_variable_type>
<filename/>
<replacevars>N</replacevars>
<parallel>N</parallel>
<xloc>384</xloc>
<yloc>304</yloc>
<attributes_hac/>
</action>
</actions>
<hops>
<hop>
<from>Start</from>
<to>check_if_work_activity_job_exists.hpl</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>check_if_work_activity_job_exists.hpl</from>
<to> work_activity_job_exists_check</to>
@ -258,20 +247,6 @@
<evaluation>N</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>Start</from>
<to>init_job_name</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>init_job_name</from>
<to>check_if_work_activity_job_exists.hpl</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
</hops>
<notepads>
<notepad>
@ -288,8 +263,8 @@
<fontitalic>N</fontitalic>
<fontsize>-1</fontsize>
<height>26</height>
<xloc>1200</xloc>
<yloc>256</yloc>
<xloc>880</xloc>
<yloc>224</yloc>
<note>Первичка</note>
<width>65</width>
</notepad>
@ -307,7 +282,7 @@
<fontitalic>N</fontitalic>
<fontsize>-1</fontsize>
<height>26</height>
<xloc>1088</xloc>
<xloc>800</xloc>
<yloc>560</yloc>
<note>Повторка</note>
<width>64</width>
@ -326,8 +301,8 @@
<fontitalic>N</fontitalic>
<fontsize>-1</fontsize>
<height>26</height>
<xloc>848</xloc>
<yloc>864</yloc>
<xloc>608</xloc>
<yloc>720</yloc>
<note>Дельта</note>
<width>49</width>
</notepad>