micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ERVU-480 : work flow

Browse source
This commit is contained in:
Fusionshh 2025-08-26 14:59:00 +03:00
parent e3e79bd30a
commit 2a953f4a6b
31 changed files with 2819 additions and 3321 deletions
Download patch file Download diff file Expand all files Collapse all files

23
mappings/info_recruits/citizen_tables/work/employer/employer_job.hwf
View file

@ -42,8 +42,6 @@
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/employer/support/check_if_employer_job_execution_exists.hpl</filename>
<logext/>
<logfile/>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -63,12 +61,7 @@
<description/>
<type>SIMPLE_EVAL</type>
<attributes/>
<comparevalue/>
<fieldname/>
<fieldtype>boolean</fieldtype>
<mask/>
<maxvalue/>
<minvalue/>
<successbooleancondition>false</successbooleancondition>
<successcondition>equal</successcondition>
<successnumbercondition>equal</successnumbercondition>
@ -92,8 +85,6 @@
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/employer/recruitment_five_flow.hpl</filename>
<logext/>
<logfile/>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -109,7 +100,7 @@
<attributes_hac/>
</action>
<action>
<name>recruitment_five_flow_on_error.hpl</name>
<name>recruitment_five_flow_repeat.hpl</name>
<description/>
<type>PIPELINE</type>
<attributes/>
@ -119,9 +110,7 @@
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/employer/recruitment_five_flow_on_error.hpl</filename>
<logext/>
<logfile/>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/employer/recruitment_five_flow_repeat.hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -148,8 +137,6 @@
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/employer/support/check_if_need_to_repeat.hpl</filename>
<logext/>
<logfile/>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -161,7 +148,7 @@
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>576</xloc>
<yloc>208</yloc>
<yloc>192</yloc>
<attributes_hac/>
</action>
<action>
@ -193,8 +180,6 @@
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/employer/recruitment_five_flow_delta.hpl</filename>
<logext/>
<logfile/>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -241,7 +226,7 @@
</hop>
<hop>
<from>Simple evaluation</from>
<to>recruitment_five_flow_on_error.hpl</to>
<to>recruitment_five_flow_repeat.hpl</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>

357
mappings/info_recruits/citizen_tables/work/employer/parallel/employer_flow.hpl
View file

@ -25,16 +25,6 @@
<to>employer_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>actual_employer_output</from>
<to>pred_employer_output</to>
<enabled>N</enabled>
</hop>
<hop>
<from>pred_employer_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<to>Abort</to>
@ -47,33 +37,33 @@
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>actual_employer_output</to>
<to>employer_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>actual_employer_output</from>
<from>employer_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>pred_employer_output</from>
<to>Detect empty stream</to>
<enabled>N</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>N</enabled>
</hop>
<hop>
<from>pred_employer_output</from>
<to>Filter rows</to>
<enabled>N</enabled>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>N</enabled>
<enabled>Y</enabled>
</hop>
<hop>
<from>employer_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
@ -92,7 +82,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1120</xloc>
<xloc>944</xloc>
<yloc>448</yloc>
</GUI>
</transform>
@ -112,7 +102,7 @@
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
@ -127,7 +117,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>896</xloc>
<xloc>720</xloc>
<yloc>448</yloc>
</GUI>
</transform>
@ -144,7 +134,7 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
@ -158,7 +148,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1120</xloc>
<xloc>944</xloc>
<yloc>112</yloc>
</GUI>
</transform>
@ -175,7 +165,7 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
@ -204,8 +194,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>1120</xloc>
<yloc>288</yloc>
<xloc>528</xloc>
<yloc>112</yloc>
</GUI>
</transform>
<transform>
@ -223,24 +213,32 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<condition>
<conditions>
</conditions>
<function>TRUE</function>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>String</type>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>896</xloc>
<yloc>112</yloc>
<xloc>944</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -254,95 +252,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<yloc>288</yloc>
</GUI>
</transform>
<transform>
<name>actual_employer_output</name>
<type>TableOutput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard-test</connection>
<fields>
<field>
<column_name>recruit_id</column_name>
<stream_name>recruit_id</stream_name>
</field>
<field>
<column_name>source_id</column_name>
<stream_name>source_id</stream_name>
</field>
<field>
<column_name>source_update_date</column_name>
<stream_name>akt_source_update_date</stream_name>
</field>
<field>
<column_name>address</column_name>
<stream_name>akt_address</stream_name>
</field>
<field>
<column_name>separate_unit_type_code</column_name>
<stream_name>akt_separate_unit_type</stream_name>
</field>
<field>
<column_name>separate_unit_address</column_name>
<stream_name>akt_separate_unit_address</stream_name>
</field>
<field>
<column_name>employer_category_name</column_name>
<stream_name>akt_employer_category_name</stream_name>
</field>
<field>
<column_name>name</column_name>
<stream_name>akt_name</stream_name>
</field>
<field>
<column_name>inn</column_name>
<stream_name>akt_inn</stream_name>
</field>
<field>
<column_name>ogrn</column_name>
<stream_name>akt_ogrn</stream_name>
</field>
<field>
<column_name>kpp</column_name>
<stream_name>akt_kpp</stream_name>
</field>
<field>
<column_name>actual_employer</column_name>
<stream_name>akt_actual_employer</stream_name>
</field>
<field>
<column_name>is_opk_org</column_name>
<stream_name>akt_is_opk_org</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>employer</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>720</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform>
<name>employer_input</name>
<type>TableInput</type>
@ -354,108 +263,76 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with filteredData as (
select ri.recruit_id,
(ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id')::uuid as source_id,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat' as aktRabotodat,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'predRabotodat' as predRabotodat
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRabotodat' ->> 'rabotodat' != 'null'
--and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
select
fd.recruit_id,
fd.source_id::uuid,
-- aktRabotodat
aktRabotodat -> 'dataSved' as akt_source_update_date,
aktRabotodat -> 'extend' ->> 'adresRabotodat' as akt_address,
aktRabotodat -> 'extend' ->> 'tipPodrazdRabotodat' as akt_separate_unit_type,
aktRabotodat -> 'extend' ->> 'adresPodrazdRabotodat' as akt_separate_unit_address,
case
when aktRabotodat ->> 'svedIP' != 'null' then 'Индивидуальный предприниматель'
when aktRabotodat ->&gt; 'svedYUL' != 'null' then 'Юридическое лицо'
when aktRabotodat ->> 'svedGlKFH' != 'null' then 'Глава КФХ'
when aktRabotodat ->> 'svedPlatFL' != 'null' then 'Физическое лицо'
END
as akt_employer_category_name,
case
when aktRabotodat ->> 'svedYUL' != 'null' then aktRabotodat -> 'svedYUL' -> 'naimOrg'
END
as akt_name,
case
when aktRabotodat ->> 'svedIP' != 'null' then aktRabotodat -> 'svedIP' ->> 'innfl'
when aktRabotodat ->> 'svedYUL' != 'null' then aktRabotodat -> 'svedYUL' ->> 'innyul'
when aktRabotodat ->> 'svedGlKFH' != 'null' then aktRabotodat -> 'svedGlKFH' ->> 'innfl'
when aktRabotodat ->> 'svedPlatFL' != 'null' then aktRabotodat -> 'svedPlatFL' ->> 'innfl'
END
as akt_inn,
case
when aktRabotodat ->> 'svedIP' != 'null' then aktRabotodat -> 'extend' -> 'svedYUL' ->> 'ogrn'
when aktRabotodat ->> 'svedYUL' != 'null' then aktRabotodat -> 'svedIP' ->> 'ogrnip'
END
as akt_ogrn,
case
when aktRabotodat ->> 'svedIP' != 'null' then aktRabotodat -> 'extend' -> 'svedYUL' ->> 'kpp'
END
as akt_kpp,
true as akt_actual_employer,
case
when aktRabotodat ->> 'svedYUL' != 'null' then aktRabotodat -> 'svedYUL' -> 'extend' -> 'priznakOpk'
END
as akt_is_opk_org,
-- predRabotodat
predRabotodat -> 'dataSved' as pred_source_update_date,
predRabotodat -> 'extend' ->> 'adresRabotodat' as pred_address,
predRabotodat -> 'extend' ->> 'tipPodrazdRabotodat' as pred_separate_unit_type_code,
predRabotodat -> 'extend' ->> 'adresPodrazdRabotodat' as pred_separate_unit_address,
case
when predRabotodat ->> 'svedIP' != 'null' then 'Индивидуальный предприниматель'
when predRabotodat ->> 'svedYUL' != 'null' then 'Юридическое лицо'
when predRabotodat ->> 'svedGlKFH' != 'null' then 'Глава КФХ'
when predRabotodat ->> 'svedPlatFL' != 'null' then 'Физическое лицо'
END
as pred_employer_category_name,
case
when predRabotodat ->> 'svedYUL' != 'null' then predRabotodat -> 'svedYUL' -> 'naimOrg'
END
as pred_name,
case
when predRabotodat ->> 'svedIP' != 'null' then predRabotodat -> 'svedIP' ->> 'innfl'
when predRabotodat ->> 'svedYUL' != 'null' then predRabotodat -> 'svedYUL' ->> 'innyul'
when predRabotodat ->> 'svedGlKFH' != 'null' then predRabotodat -> 'svedGlKFH' ->> 'innfl'
when predRabotodat ->> 'svedPlatFL' != 'null' then predRabotodat -> 'svedPlatFL' ->> 'innfl'
END
as pred_inn,
case
when predRabotodat ->> 'svedIP' != 'null' then predRabotodat -> 'extend' -> 'svedYUL' ->> 'ogrn'
when predRabotodat ->> 'svedYUL' != 'null' then predRabotodat -> 'svedIP' ->> 'ogrnip'
END
as pred_ogrn,
case
when predRabotodat ->> 'svedIP' != 'null' then predRabotodat -> 'extend' -> 'svedYUL' ->> 'kpp'
END
as pred_kpp,
false as pred_actual_employer,
case
when predRabotodat ->> 'svedYUL' != 'null' then predRabotodat -> 'svedYUL' -> 'extend' -> 'priznakOpk'
END
as pred_is_opk_org
<sql>with filteredData as (select ri.recruit_id,
(ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id') as source_id,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat' as rabotodat,
true as actual_employer
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'aktRabotodat' != 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
union all
select ri.recruit_id,
(ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id') as source_id,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'predRabotodat' as rabotodat,
false as actual_employer
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'predRabotodat' != 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
select fd.source_id,
fd.recruit_id,
to_date(rabotodat ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
COALESCE(rabotodat -> 'extend' ->> 'status', '0') = '1' AS status,
rabotodat -> 'extend' ->> 'adresRabotodat' as address,
rabotodat -> 'extend' ->> 'tipPodrazdRabotodat' as separate_unit_type,
rabotodat -> 'extend' ->> 'adresPodrazdRabotodat' as separate_unit_address,
case
when rabotodat ->> 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null'
then 'Индивидуальный предприниматель'
when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null' then 'Юридическое лицо'
when rabotodat ->> 'svedGlKFH' is not null and rabotodat ->> 'svedGlKFH' != 'null' then 'Глава КФХ'
when rabotodat ->> 'svedPlatFL' is not null and rabotodat ->> 'svedPlatFL' != 'null' then 'Физическое лицо'
end as employer_category_name,
rabotodat -> 'svedYUL' -> 'naimOrg' as name,
case
when rabotodat ->> 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null'
then rabotodat -> 'svedIP' ->> 'innfl'
when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null'
then rabotodat -> 'svedYUL' ->> 'innyul'
when rabotodat ->> 'svedGlKFH' is not null and rabotodat ->> 'svedGlKFH' != 'null'
then rabotodat -> 'svedGlKFH' ->> 'innfl'
when rabotodat ->> 'svedPlatFL' is not null and rabotodat ->> 'svedPlatFL' != 'null'
then rabotodat -> 'svedPlatFL' ->> 'innfl'
end as inn,
case
when rabotodat ->> 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null'
then rabotodat -> 'extend' -> 'svedYUL' ->> 'ogrn'
when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null'
then rabotodat -> 'svedIP' ->> 'ogrnip'
end as ogrn,
case
when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null'
then rabotodat -> 'extend' -> 'svedYUL' ->> 'kpp'
end as kpp,
coalesce(rabotodat -> 'svedYUL' -> 'extend' ->> 'priznakOpk', '0') = '1' as is_opk_org,
actual_employer
from filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>368</xloc>
<xloc>352</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform>
<name>pred_employer_output</name>
<name>employer_output</name>
<type>TableOutput</type>
<description/>
<distribute>N</distribute>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -463,55 +340,59 @@ from filteredData fd;</sql>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<fields>
<field>
<column_name>source_id</column_name>
<stream_name>source_id</stream_name>
</field>
<field>
<column_name>recruit_id</column_name>
<stream_name>recruit_id</stream_name>
</field>
<field>
<column_name>source_id</column_name>
<stream_name>source_id</stream_name>
</field>
<field>
<column_name>source_update_date</column_name>
<stream_name>pred_source_update_date</stream_name>
<stream_name>source_update_date</stream_name>
</field>
<field>
<column_name>address</column_name>
<stream_name>pred_address</stream_name>
<stream_name>address</stream_name>
</field>
<field>
<column_name>separate_unit_type_code</column_name>
<stream_name>separate_unit_type</stream_name>
</field>
<field>
<column_name>separate_unit_address</column_name>
<stream_name>pred_separate_unit_address</stream_name>
<stream_name>separate_unit_address</stream_name>
</field>
<field>
<column_name>employer_category_name</column_name>
<stream_name>pred_employer_category_name</stream_name>
<stream_name>employer_category_name</stream_name>
</field>
<field>
<column_name>name</column_name>
<stream_name>pred_name</stream_name>
<stream_name>name</stream_name>
</field>
<field>
<column_name>inn</column_name>
<stream_name>pred_inn</stream_name>
<stream_name>inn</stream_name>
</field>
<field>
<column_name>ogrn</column_name>
<stream_name>pred_ogrn</stream_name>
<stream_name>ogrn</stream_name>
</field>
<field>
<column_name>kpp</column_name>
<stream_name>pred_kpp</stream_name>
<stream_name>kpp</stream_name>
</field>
<field>
<column_name>actual_employer</column_name>
<stream_name>pred_actual_employer</stream_name>
<stream_name>actual_employer</stream_name>
</field>
<field>
<column_name>is_opk_org</column_name>
<stream_name>pred_is_opk_org</stream_name>
<stream_name>is_opk_org</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
@ -529,25 +410,13 @@ from filteredData fd;</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>896</xloc>
<xloc>720</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>actual_employer_output</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>pred_employer_output</source_transform>
<source_transform>employer_output</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>

746
mappings/info_recruits/citizen_tables/work/employer/parallel/employer_flow_delta.hpl
View file

@ -25,38 +25,23 @@
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update 2</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Create job execution record</from>
<to>employer_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Insert / update 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<from>employer_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Insert / update</to>
<to>employer_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<from>employer_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
@ -65,10 +50,25 @@
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Insert / update 2</name>
<type>InsertUpdate</type>
<name>Abort</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -77,396 +77,13 @@
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard-test</connection>
<lookup>
<key>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>employer</table>
<value>
<name>source_id</name>
<rename>source_id</rename>
<update>Y</update>
</value>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>pred_source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>address</name>
<rename>pred_address</rename>
<update>Y</update>
</value>
<value>
<name>separate_unit_type_code</name>
<rename>pred_separate_unit_type_code</rename>
<update>Y</update>
</value>
<value>
<name>separate_unit_address</name>
<rename>pred_separate_unit_address</rename>
<update>Y</update>
</value>
<value>
<name>employer_category_name</name>
<rename>pred_employer_category_name</rename>
<update>Y</update>
</value>
<value>
<name>name</name>
<rename>pred_name</rename>
<update>Y</update>
</value>
<value>
<name>inn</name>
<rename>pred_inn</rename>
<update>Y</update>
</value>
<value>
<name>ogrn</name>
<rename>pred_ogrn</rename>
<update>Y</update>
</value>
<value>
<name>kpp</name>
<rename>pred_kpp</rename>
<update>Y</update>
</value>
<value>
<name>actual_employer</name>
<rename>pred_actual_employer</rename>
<update>Y</update>
</value>
<value>
<name>is_opk_org</name>
<rename>pred_is_opk_org</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard-test</connection>
<lookup>
<key>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>employer</table>
<value>
<name>source_id</name>
<rename>source_id</rename>
<update>Y</update>
</value>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>akt_source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>address</name>
<rename>akt_address</rename>
<update>Y</update>
</value>
<value>
<name>separate_unit_type_code</name>
<rename>akt_separate_unit_type</rename>
<update>Y</update>
</value>
<value>
<name>separate_unit_address</name>
<rename>akt_separate_unit_address</rename>
<update>Y</update>
</value>
<value>
<name>employer_category_name</name>
<rename>akt_employer_category_name</rename>
<update>Y</update>
</value>
<value>
<name>name</name>
<rename>akt_name</rename>
<update>Y</update>
</value>
<value>
<name>inn</name>
<rename>akt_inn</rename>
<update>Y</update>
</value>
<value>
<name>ogrn</name>
<rename>akt_ogrn</rename>
<update>Y</update>
</value>
<value>
<name>kpp</name>
<rename>akt_kpp</rename>
<update>Y</update>
</value>
<value>
<name>actual_employer</name>
<rename>akt_actual_employer</rename>
<update>Y</update>
</value>
<value>
<name>is_opk_org</name>
<rename>akt_is_opk_org</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>864</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>employer_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with filtered_data as (select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id' as source_id,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat' as aktRabotodat,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'predRabotodat' as predRabotodat
from recruits_info ri
-- join на военкомат
where ri.info -> 'svedFL' -> 'svedRabotodat' ->> 'rabotodat' != 'null'
-- and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
and (ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat' ->> 'dataSved' >= '${MAX_SOURCE_DATE}'
or ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'predRabotodat' ->> 'dataSved' >= '${MAX_SOURCE_DATE}'))
select fd.source_id,
fd.recruit_id,
-- aktRabotodat
aktRabotodat -> 'dataSved' as akt_source_update_date,
aktRabotodat -> 'extend' ->> 'adresRabotodat' as akt_address,
aktRabotodat -> 'extend' ->> 'tipPodrazdRabotodat' as akt_separate_unit_type,
aktRabotodat -> 'extend' ->> 'adresPodrazdRabotodat' as akt_separate_unit_address,
case
when aktRabotodat -> 'svedIP' then 'Индивидуальный предприниматель'
when aktRabotodat -> 'svedYUL' then 'Юридическое лицо'
when aktRabotodat -> 'svedGlKFH' then 'Глава КФХ'
when aktRabotodat -> 'svedPlatFL' then 'Физическое лицо'
END
as akt_employer_category_name,
case
when aktRabotodat -> 'svedYUL' then aktRabotodat -> 'svedYUL' -> 'naimOrg'
END
as akt_name,
case
when aktRabotodat -> 'svedIP' then aktRabotodat -> 'svedIP' ->> 'innfl'
when aktRabotodat -&gt; 'svedYUL' then aktRabotodat -> 'svedYUL' ->> 'innyul'
when aktRabotodat -> 'svedGlKFH' then aktRabotodat -> 'svedGlKFH' ->> 'innfl'
when aktRabotodat -> 'svedPlatFL' then aktRabotodat -> 'svedPlatFL' ->> 'innfl'
END
as akt_inn,
case
when aktRabotodat -> 'svedIP' then aktRabotodat -> 'extend' -> 'svedYUL' ->> 'ogrn'
when aktRabotodat -> 'svedYUL' then aktRabotodat -> 'svedIP' ->> 'ogrnip'
END
as akt_ogrn,
case
when aktRabotodat -> 'svedIP' then aktRabotodat -> 'extend' -> 'svedYUL' ->> 'kpp'
END
as akt_kpp,
true as akt_actual_employer,
case
when aktRabotodat -> 'svedYUL' then aktRabotodat -> 'svedYUL' -> 'extend' -> 'priznakOpk'
END
as akt_is_opk_org,
-- predRabotodat
predRabotodat -> 'dataSved' as akt_source_update_date,
predRabotodat -> 'extend' ->> 'adresRabotodat' as akt_address,
predRabotodat -> 'extend' ->> 'tipPodrazdRabotodat' as akt_separate_unit_type,
predRabotodat -> 'extend' ->> 'adresPodrazdRabotodat' as akt_separate_unit_address,
case
when predRabotodat -> 'svedIP' then 'Индивидуальный предприниматель'
when predRabotodat -> 'svedYUL' then 'Юридическое лицо'
when predRabotodat -> 'svedGlKFH' then 'Глава КФХ'
when predRabotodat -> 'svedPlatFL' then 'Физическое лицо'
END
as akt_employer_category_name,
case
when predRabotodat -> 'svedYUL' then predRabotodat -> 'svedYUL' -> 'naimOrg'
END
as akt_name,
case
when predRabotodat -> 'svedIP' then predRabotodat -> 'svedIP' ->> 'innfl'
when predRabotodat -> 'svedYUL' then predRabotodat -> 'svedYUL' ->> 'innyul'
when predRabotodat -> 'svedGlKFH' then predRabotodat -> 'svedGlKFH' ->> 'innfl'
when predRabotodat -> 'svedPlatFL' then predRabotodat -> 'svedPlatFL' ->> 'innfl'
END
as akt_inn,
case
when predRabotodat -> 'svedIP' then predRabotodat -> 'extend' -> 'svedYUL' ->> 'ogrn'
when predRabotodat -> 'svedYUL' then predRabotodat -> 'svedIP' ->> 'ogrnip'
END
as akt_ogrn,
case
when predRabotodat -> 'svedIP' then predRabotodat -> 'extend' -> 'svedYUL' ->> 'kpp'
END
as akt_kpp,
true as akt_actual_employer,
case
when predRabotodat -> 'svedYUL' then predRabotodat -> 'svedYUL' -> 'extend' -> 'priznakOpk'
END
as akt_is_opk_org
from filtered_data fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>512</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>result</resultfieldname>
<attributes/>
<GUI>
<xloc>672</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Filter rows</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>TRUE</function>
<leftvalue>result</leftvalue>
<negated>N</negated>
<operator>-</operator>
</condition>
</compare>
<send_false_to>Change job status on success</send_false_to>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>864</xloc>
<yloc>240</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<attributes/>
<GUI>
<xloc>320</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS'
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>240</yloc>
<yloc>544</yloc>
</GUI>
</transform>
<transform>
@ -485,7 +102,7 @@ and recruitment_id = '${IDM_ID}';
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
@ -500,13 +117,13 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1040</xloc>
<xloc>864</xloc>
<yloc>544</yloc>
</GUI>
</transform>
<transform>
<name>Abort</name>
<type>Abort</type>
<name>Change job status on success</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -515,22 +132,315 @@ and recruitment_id = '${IDM_ID}';
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS'
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>544</yloc>
<xloc>1040</xloc>
<yloc>240</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET
status = 'DELTA_PROCESSING',
execution_datetime = current_timestamp,
error_description = NULL
where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql>
<attributes/>
<GUI>
<xloc>320</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Detect empty stream</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>672</xloc>
<yloc>240</yloc>
</GUI>
</transform>
<transform>
<name>Filter rows</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>672</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>employer_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with filteredData as (select ri.recruit_id,
(ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id') as source_id,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat' as rabotodat,
true as actual_employer
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'aktRabotodat' != 'null'
and to_date(ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat' ->>
'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
union all
select ri.recruit_id,
(ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id') as source_id,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'predRabotodat' as rabotodat,
false as actual_employer
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'predRabotodat' != 'null'
and to_date(ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'predRabotodat' ->>
'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
select fd.source_id,
fd.recruit_id,
to_date(rabotodat ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
COALESCE(rabotodat -> 'extend' ->> 'status', '0') = '1' AS status,
rabotodat -> 'extend' ->> 'adresRabotodat' as address,
rabotodat -> 'extend' ->> 'tipPodrazdRabotodat' as separate_unit_type,
rabotodat -> 'extend' ->> 'adresPodrazdRabotodat' as separate_unit_address,
case
when rabotodat ->> 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null'
then 'Индивидуальный предприниматель'
when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null' then 'Юридическое лицо'
when rabotodat ->> 'svedGlKFH' is not null and rabotodat ->> 'svedGlKFH' != 'null' then 'Глава КФХ'
when rabotodat ->> 'svedPlatFL' is not null and rabotodat ->> 'svedPlatFL' != 'null' then 'Физическое лицо'
end as employer_category_name,
rabotodat -> 'svedYUL' -> 'naimOrg' as name,
case
when rabotodat ->> 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null'
then rabotodat -> 'svedIP' ->> 'innfl'
when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null'
then rabotodat -> 'svedYUL' ->> 'innyul'
when rabotodat ->> 'svedGlKFH' is not null and rabotodat ->> 'svedGlKFH' != 'null'
then rabotodat -> 'svedGlKFH' ->> 'innfl'
when rabotodat ->> 'svedPlatFL' is not null and rabotodat ->> 'svedPlatFL' != 'null'
then rabotodat -> 'svedPlatFL' ->> 'innfl'
end as inn,
case
when rabotodat ->> 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null'
then rabotodat -> 'extend' -> 'svedYUL' ->> 'ogrn'
when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null'
then rabotodat -> 'svedIP' ->> 'ogrnip'
end as ogrn,
case
when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null'
then rabotodat -> 'extend' -> 'svedYUL' ->> 'kpp'
end as kpp,
coalesce(rabotodat -> 'svedYUL' -> 'extend' ->> 'priznakOpk', '0') = '1' as is_opk_org,
actual_employer
from filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>employer_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
</key>
<key>
<condition>=</condition>
<field>actual_employer</field>
<name>actual_employer</name>
</key>
<schema>ervu_dashboard</schema>
<table>employer</table>
<value>
<name>source_id</name>
<rename>source_id</rename>
<update>N</update>
</value>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>address</name>
<rename>address</rename>
<update>Y</update>
</value>
<value>
<name>separate_unit_type_code</name>
<rename>separate_unit_type</rename>
<update>Y</update>
</value>
<value>
<name>separate_unit_address</name>
<rename>separate_unit_address</rename>
<update>Y</update>
</value>
<value>
<name>employer_category_name</name>
<rename>employer_category_name</rename>
<update>Y</update>
</value>
<value>
<name>name</name>
<rename>name</rename>
<update>Y</update>
</value>
<value>
<name>inn</name>
<rename>inn</rename>
<update>Y</update>
</value>
<value>
<name>ogrn</name>
<rename>ogrn</rename>
<update>Y</update>
</value>
<value>
<name>kpp</name>
<rename>kpp</rename>
<update>Y</update>
</value>
<value>
<name>actual_employer</name>
<rename>actual_employer</rename>
<update>Y</update>
</value>
<value>
<name>is_opk_org</name>
<rename>is_opk_org</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>864</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Insert / update</source_transform>
<source_transform>employer_upsert</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>

365
mappings/info_recruits/citizen_tables/work/employer/parallel/employer_flow_repeat.hpl
View file

@ -30,11 +30,6 @@
<to>employer_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>employer_input</from>
<to>Identify last row in a stream</to>
@ -42,26 +37,31 @@
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>akt_employer_upsert</to>
<to>employer_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>akt_employer_upsert</from>
<to>pred_employer_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>akt_employer_upsert</from>
<from>employer_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>akt_employer_upsert</from>
<from>employer_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>pred_employer_upsert</from>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
@ -82,7 +82,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1248</xloc>
<xloc>1072</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -102,7 +102,7 @@
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
@ -117,7 +117,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1072</xloc>
<xloc>896</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -134,7 +134,7 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
@ -165,22 +165,46 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}')
ON CONFLICT (job_name, recruitment_id)
DO UPDATE
SET status = 'PROCESSING',
error_description = null,
execution_datetime = current_timestamp;</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>352</xloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>Detect empty stream</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>256</yloc>
</GUI>
</transform>
<transform>
<name>Filter rows</name>
<type>FilterRows</type>
@ -196,32 +220,40 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<condition>
<conditions>
</conditions>
<function>TRUE</function>
<leftvalue>result</leftvalue>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_false_to>Change job status on success</send_false_to>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>896</xloc>
<yloc>256</yloc>
<xloc>1072</xloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>result</resultfieldname>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>704</xloc>
@ -239,95 +271,63 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with filteredData as (
select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id' as source_id,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat' as aktRabotodat,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'predRabotodat' as predRabotodat
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRabotodat' ->> 'rabotodat' != 'null'
-- and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
select
fd.source_id,
fd.recruit_id,
-- aktRabotodat
aktRabotodat -> 'dataSved' as akt_source_update_date,
aktRabotodat -> 'extend' ->> 'adresRabotodat' as akt_address,
aktRabotodat -> 'extend' ->> 'tipPodrazdRabotodat' as akt_separate_unit_type,
aktRabotodat -> 'extend' ->> 'adresPodrazdRabotodat' as akt_separate_unit_address,
case
when aktRabotodat ->> 'svedIP' != 'null' then 'Индивидуальный предприниматель'
when aktRabotodat ->> 'svedYUL' != 'null' then 'Юридическое лицо'
when aktRabotodat ->> 'svedGlKFH' != 'null' then 'Глава КФХ'
when aktRabotodat ->> 'svedPlatFL' != 'null' then 'Физическое лицо'
END
as akt_employer_category_name,
case
when aktRabotodat ->> 'svedYUL' != 'null' then aktRabotodat -> 'svedYUL' -> 'naimOrg'
END
as akt_name,
case
when aktRabotodat ->> 'svedIP' != 'null' then aktRabotodat -> 'svedIP' ->> 'innfl'
when aktRabotodat ->> 'svedYUL' != 'null' then aktRabotodat -> 'svedYUL' ->> 'innyul'
when aktRabotodat ->> 'svedGlKFH' != 'null' then aktRabotodat -> 'svedGlKFH' ->> 'innfl'
when aktRabotodat ->> 'svedPlatFL' != 'null' then aktRabotodat -> 'svedPlatFL' ->> 'innfl'
END
as akt_inn,
case
when aktRabotodat ->> 'svedIP' != 'null' then aktRabotodat -> 'extend' -> 'svedYUL' ->> 'ogrn'
when aktRabotodat ->> 'svedYUL' != 'null' then aktRabotodat -> 'svedIP' ->> 'ogrnip'
END
as akt_ogrn,
case
when aktRabotodat ->> 'svedIP' != 'null' then aktRabotodat -> 'extend' -> 'svedYUL' ->> 'kpp'
END
as akt_kpp,
true as akt_actual_employer,
case
when aktRabotodat ->> 'svedYUL' != 'null' then aktRabotodat -> 'svedYUL' -> 'extend' -> 'priznakOpk'
END
as akt_is_opk_org,
-- predRabotodat
predRabotodat -> 'dataSved' as pred_source_update_date,
predRabotodat -> 'extend' ->> 'adresRabotodat' as pred_address,
predRabotodat -> 'extend' -&gt;&gt; 'tipPodrazdRabotodat' as pred_separate_unit_type_code,
predRabotodat -> 'extend' ->> 'adresPodrazdRabotodat' as pred_separate_unit_address,
case
when predRabotodat ->> 'svedIP' != 'null' then 'Индивидуальный предприниматель'
when predRabotodat ->> 'svedYUL' != 'null' then 'Юридическое лицо'
when predRabotodat ->> 'svedGlKFH' != 'null' then 'Глава КФХ'
when predRabotodat ->> 'svedPlatFL' != 'null' then 'Физическое лицо'
END
as pred_employer_category_name,
case
when predRabotodat ->> 'svedYUL' != 'null' then predRabotodat -> 'svedYUL' -> 'naimOrg'
END
as pred_name,
case
when predRabotodat ->> 'svedIP' != 'null' then predRabotodat -> 'svedIP' ->> 'innfl'
when predRabotodat ->> 'svedYUL' != 'null' then predRabotodat -> 'svedYUL' ->> 'innyul'
when predRabotodat ->> 'svedGlKFH' != 'null' then predRabotodat -> 'svedGlKFH' ->> 'innfl'
when predRabotodat ->> 'svedPlatFL' != 'null' then predRabotodat -> 'svedPlatFL' ->> 'innfl'
END
as pred_inn,
case
when predRabotodat ->> 'svedIP' != 'null' then predRabotodat -> 'extend' -> 'svedYUL' ->> 'ogrn'
when predRabotodat ->> 'svedYUL' != 'null' then predRabotodat -> 'svedIP' ->> 'ogrnip'
END
as pred_ogrn,
case
when predRabotodat ->> 'svedIP' != 'null' then predRabotodat -> 'extend' -> 'svedYUL' ->> 'kpp'
END
as pred_kpp,
false as pred_actual_employer,
case
when predRabotodat ->> 'svedYUL' != 'null' then predRabotodat -> 'svedYUL' -> 'extend' -> 'priznakOpk'
END
as pred_is_opk_org
<sql>with filteredData as (select ri.recruit_id,
(ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id') as source_id,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat' as rabotodat,
true as actual_employer
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'aktRabotodat' != 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
union all
select ri.recruit_id,
(ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id') as source_id,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'predRabotodat' as rabotodat,
false as actual_employer
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'predRabotodat' != 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
select fd.source_id,
fd.recruit_id,
to_date(rabotodat ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
COALESCE(rabotodat -> 'extend' ->> 'status', '0') = '1' AS status,
rabotodat -> 'extend' ->> 'adresRabotodat' as address,
rabotodat -> 'extend' ->> 'tipPodrazdRabotodat' as separate_unit_type,
rabotodat -> 'extend' ->> 'adresPodrazdRabotodat' as separate_unit_address,
case
when rabotodat ->> 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null'
then 'Индивидуальный предприниматель'
when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null' then 'Юридическое лицо'
when rabotodat ->> 'svedGlKFH' is not null and rabotodat ->> 'svedGlKFH' != 'null' then 'Глава КФХ'
when rabotodat ->> 'svedPlatFL' is not null and rabotodat ->> 'svedPlatFL' != 'null' then 'Физическое лицо'
end as employer_category_name,
rabotodat -> 'svedYUL' -> 'naimOrg' as name,
case
when rabotodat ->> 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null'
then rabotodat -> 'svedIP' ->> 'innfl'
when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null'
then rabotodat -> 'svedYUL' ->> 'innyul'
when rabotodat ->> 'svedGlKFH' is not null and rabotodat ->> 'svedGlKFH' != 'null'
then rabotodat -> 'svedGlKFH' ->> 'innfl'
when rabotodat ->> 'svedPlatFL' is not null and rabotodat ->> 'svedPlatFL' != 'null'
then rabotodat -> 'svedPlatFL' ->> 'innfl'
end as inn,
case
when rabotodat ->> 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null'
then rabotodat -> 'extend' -> 'svedYUL' ->> 'ogrn'
when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null'
then rabotodat -> 'svedIP' ->> 'ogrnip'
end as ogrn,
case
when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null'
then rabotodat -> 'extend' -> 'svedYUL' ->> 'kpp'
end as kpp,
coalesce(rabotodat -> 'svedYUL' -> 'extend' ->> 'priznakOpk', '0') = '1' as is_opk_org,
actual_employer
from filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -337,102 +337,7 @@ from filteredData fd;</sql>
</GUI>
</transform>
<transform>
<name>akt_employer_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard-test</connection>
<lookup>
<key>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>employer</table>
<value>
<name>source_id</name>
<rename>source_id</rename>
<update>Y</update>
</value>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>akt_source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>address</name>
<rename>akt_address</rename>
<update>Y</update>
</value>
<value>
<name>separate_unit_type_code</name>
<rename>akt_separate_unit_type</rename>
<update>Y</update>
</value>
<value>
<name>separate_unit_address</name>
<rename>akt_separate_unit_address</rename>
<update>Y</update>
</value>
<value>
<name>employer_category_name</name>
<rename>akt_employer_category_name</rename>
<update>Y</update>
</value>
<value>
<name>name</name>
<rename>akt_name</rename>
<update>Y</update>
</value>
<value>
<name>inn</name>
<rename>akt_inn</rename>
<update>Y</update>
</value>
<value>
<name>ogrn</name>
<rename>akt_ogrn</rename>
<update>Y</update>
</value>
<value>
<name>kpp</name>
<rename>akt_kpp</rename>
<update>Y</update>
</value>
<value>
<name>actual_employer</name>
<rename>akt_actual_employer</rename>
<update>Y</update>
</value>
<value>
<name>is_opk_org</name>
<rename>akt_is_opk_org</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>896</xloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>pred_employer_upsert</name>
<name>employer_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
@ -449,90 +354,94 @@ from filteredData fd;</sql>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>actual_employer</field>
<name>actual_employer</name>
</key>
<schema>ervu_dashboard</schema>
<table>employer</table>
<value>
<name>source_id</name>
<rename>source_id</rename>
<update>Y</update>
<update>N</update>
</value>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>Y</update>
<update>N</update>
</value>
<value>
<name>source_update_date</name>
<rename>pred_source_update_date</rename>
<rename>source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>address</name>
<rename>pred_address</rename>
<rename>address</rename>
<update>Y</update>
</value>
<value>
<name>separate_unit_type_code</name>
<rename>pred_separate_unit_type_code</rename>
<rename>separate_unit_type</rename>
<update>Y</update>
</value>
<value>
<name>separate_unit_address</name>
<rename>pred_separate_unit_address</rename>
<rename>separate_unit_address</rename>
<update>Y</update>
</value>
<value>
<name>employer_category_name</name>
<rename>pred_employer_category_name</rename>
<rename>employer_category_name</rename>
<update>Y</update>
</value>
<value>
<name>name</name>
<rename>pred_name</rename>
<rename>name</rename>
<update>Y</update>
</value>
<value>
<name>inn</name>
<rename>pred_inn</rename>
<rename>inn</rename>
<update>Y</update>
</value>
<value>
<name>ogrn</name>
<rename>pred_ogrn</rename>
<rename>ogrn</rename>
<update>Y</update>
</value>
<value>
<name>kpp</name>
<rename>pred_kpp</rename>
<rename>kpp</rename>
<update>Y</update>
</value>
<value>
<name>actual_employer</name>
<rename>pred_actual_employer</rename>
<rename>actual_employer</rename>
<update>Y</update>
</value>
<value>
<name>is_opk_org</name>
<rename>pred_is_opk_org</rename>
<rename>is_opk_org</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1072</xloc>
<xloc>896</xloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>akt_employer_upsert</source_transform>
<source_transform>employer_upsert</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>

15
mappings/info_recruits/citizen_tables/work/employer/recruitment_five_flow.hpl
View file

@ -59,15 +59,14 @@
</partitioning>
<connection>ervu-dashboard-test</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>SELECT
idm_id AS recruitment
idm_id
FROM ervu_dashboard.recruitment;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>416</xloc>
<yloc>432</yloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
@ -91,7 +90,7 @@ FROM ervu_dashboard.recruitment;</sql>
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
@ -147,7 +146,7 @@ FROM ervu_dashboard.recruitment;</sql>
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
@ -203,7 +202,7 @@ FROM ervu_dashboard.recruitment;</sql>
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
@ -259,7 +258,7 @@ FROM ervu_dashboard.recruitment;</sql>
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
@ -315,7 +314,7 @@ FROM ervu_dashboard.recruitment;</sql>
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>

91
mappings/info_recruits/citizen_tables/work/employer/recruitment_five_flow_delta.hpl
View file

@ -26,8 +26,8 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>employer_flow_delta.hpl 3</to>
<from>get_max_source_update_date</from>
<to>Table input</to>
<enabled>Y</enabled>
</hop>
<hop>
@ -35,6 +35,11 @@
<to>employer_flow_delta.hpl 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>employer_flow_delta.hpl 3</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>employer_flow_delta.hpl 4</to>
@ -60,12 +65,22 @@
<connection>ervu-dashboard-test</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_source_update_date</lookup>
<sql>SELECT
r.idm_id as recruitment_id
r.idm_id,
? max_source_update_date
FROM ervu_dashboard.recruitment r
LEFT JOIN public.job_execution je
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = 'employer_job'
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > (
SELECT MAX(execution_datetime)
FROM etl.job_execution
WHERE job_name = 'employer_job'
AND recruitment_id = r.idm_id
)
where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
<variables_active>N</variables_active>
<attributes/>
@ -74,6 +89,29 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<yloc>336</yloc>
</GUI>
</transform>
<transform>
<name>get_max_source_update_date</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(source_update_date)
from employer;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>336</yloc>
</GUI>
</transform>
<transform>
<name>employer_flow_delta.hpl</name>
<type>PipelineExecutor</type>
@ -95,13 +133,18 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>work_activity_job</input>
<input>employer_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
@ -151,13 +194,18 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>work_activity_job</input>
<input>employer_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
@ -207,13 +255,18 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>work_activity_job</input>
<input>employer_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
@ -263,13 +316,18 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>work_activity_job</input>
<input>employer_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
@ -319,13 +377,18 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>work_activity_job</input>
<input>employer_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>

22
mappings/info_recruits/citizen_tables/work/employer/recruitment_five_flow_on_error.hpl → mappings/info_recruits/citizen_tables/work/employer/recruitment_five_flow_repeat.hpl
View file

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<pipeline>
<info>
<name>recruitment_five_flow_on_error</name>
<name>recruitment_five_flow_repeat</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@ -60,13 +60,13 @@
<connection>ervu-dashboard-test</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT
r.idm_id as recruitment_id
<sql>SELECT r.idm_id AS recruitment_id
FROM ervu_dashboard.recruitment r
LEFT JOIN public.job_execution je
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = 'employer_job'
where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
AND je.job_name = 'employer_job'
WHERE je.id IS NULL
OR je.status IN ('ERROR', 'PROCESSING');</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
@ -101,7 +101,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>work_activity_job</input>
<input>employer_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
@ -157,7 +157,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>work_activity_job</input>
<input>employer_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
@ -213,7 +213,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>work_activity_job</input>
<input>employer_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
@ -269,7 +269,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>work_activity_job</input>
<input>employer_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
@ -325,7 +325,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>work_activity_job</input>
<input>employer_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>

11
mappings/info_recruits/citizen_tables/work/employer/support/check_if_employer_job_execution_exists.hpl
View file

@ -62,15 +62,14 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT EXISTS (
SELECT 1
FROM etl.job_execution
WHERE job_name = 'employer_job'
limit 1
) AS job_executed_flag;</sql>
SELECT 1
FROM etl.job_execution
WHERE job_name = 'employer_job'
) AS job_executed_flag;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>

3
mappings/info_recruits/citizen_tables/work/employer/support/check_if_need_to_repeat.hpl
View file

@ -62,7 +62,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT EXISTS (SELECT 1
@ -71,7 +71,6 @@ FROM ervu_dashboard.recruitment r
ON r.idm_id = je.recruitment_id
and job_name = 'employer_job'
where je.status is null or je.status in ('ERROR', 'PROCESSING')
limit 1
) as need_to_repeat_job;</sql>
<variables_active>N</variables_active>
<attributes/>

463
mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow.hpl
View file

@ -37,39 +37,54 @@
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>akt_individual_entrepreneur_output</to>
<to>individual_entrepreneur_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>akt_individual_entrepreneur_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>akt_individual_entrepreneur_output</from>
<from>individual_entrepreneur_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>akt_individual_entrepreneur_output</from>
<to>pred_individual_entrepreneur_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>pred_individual_entrepreneur_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>pred_individual_entrepreneur_output</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>filter_null_npd_dates</from>
<to>unique_by_rec_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>unique_by_rec_id</from>
<to>individual_entrepreneur_npd_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_npd_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>filter_null_npd_dates</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -87,8 +102,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1536</xloc>
<yloc>624</yloc>
<xloc>1648</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -107,7 +122,7 @@
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
@ -122,8 +137,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1360</xloc>
<yloc>624</yloc>
<xloc>1440</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -139,7 +154,7 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
@ -153,8 +168,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1360</xloc>
<yloc>272</yloc>
<xloc>1440</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform>
@ -170,7 +185,7 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
@ -186,6 +201,23 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>Detect empty stream</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>800</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform>
<name>Filter rows</name>
<type>FilterRows</type>
@ -201,41 +233,31 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<condition>
<conditions>
</conditions>
<function>TRUE</function>
<leftvalue>result</leftvalue>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>272</yloc>
<xloc>1440</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>result</resultfieldname>
<attributes/>
<GUI>
<xloc>800</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>akt_individual_entrepreneur_output</name>
<type>TableOutput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
@ -243,67 +265,16 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard-test</connection>
<fields>
<field>
<column_name>recruit_id</column_name>
<stream_name>recruit_id</stream_name>
</field>
<field>
<column_name>source_id</column_name>
<stream_name>source_id</stream_name>
</field>
<field>
<column_name>ogrnip</column_name>
<stream_name>akt_ogrnip</stream_name>
</field>
<field>
<column_name>source_update_date</column_name>
<stream_name>akt_source_update_date</stream_name>
</field>
<field>
<column_name>registration_date</column_name>
<stream_name>akt_registration_date</stream_name>
</field>
<field>
<column_name>reg_npd_date</column_name>
<stream_name>akt_reg_npd_date</stream_name>
</field>
<field>
<column_name>dereg_npd_date</column_name>
<stream_name>akt_dereg_npd_date</stream_name>
</field>
<field>
<column_name>actual</column_name>
<stream_name>akt_actual</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>individual_entrepreneur</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1040</xloc>
<xloc>800</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>pred_individual_entrepreneur_output</name>
<type>TableOutput</type>
<name>filter_null_npd_dates</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -312,66 +283,21 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard-test</connection>
<fields>
<field>
<column_name>recruit_id</column_name>
<stream_name>recruit_id</stream_name>
</field>
<field>
<column_name>source_id</column_name>
<stream_name>source_id</stream_name>
</field>
<field>
<column_name>ogrnip</column_name>
<stream_name>pred_ogrnip</stream_name>
</field>
<field>
<column_name>source_update_date</column_name>
<stream_name>pred_source_update_date</stream_name>
</field>
<field>
<column_name>registration_date</column_name>
<stream_name>pred_registration_date</stream_name>
</field>
<field>
<column_name>deregistration_date</column_name>
<stream_name>pred_deregistration_date</stream_name>
</field>
<field>
<column_name>reg_npd_date</column_name>
<stream_name>pred_reg_npd_date</stream_name>
</field>
<field>
<column_name>dereg_npd_date</column_name>
<stream_name>pred_dereg_npd_date</stream_name>
</field>
<field>
<column_name>actual</column_name>
<stream_name>pred_actual</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>individual_entrepreneur</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<compare>
<condition>
<conditions>
</conditions>
<function>IS NOT NULL</function>
<leftvalue>reg_npd_date</leftvalue>
<negated>N</negated>
<operator>-</operator>
</condition>
</compare>
<send_true_to>unique_by_rec_id</send_true_to>
<attributes/>
<GUI>
<xloc>1360</xloc>
<yloc>448</yloc>
<xloc>800</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -385,34 +311,42 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with filtered_data as (
select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'id' as source_id,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'aktRegIP' -> 0 as aktIp,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'predRegIP' as predIp
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRegIP' ->> 'regIP' != 'null'
-- and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
select
fd.recruit_id,
fd.source_id,
aktIp ->> 'ogrnip' as akt_ogrnip,
aktIp ->> 'dataSved' as akt_source_update_date,
aktIp ->> 'dataRegIP' as akt_registration_date,
aktIp -> 'extend' ->> 'dataUchNPD' as akt_reg_npd_date,
aktIp -> 'extend' ->> 'dataSnUchNPD' as akt_dereg_npd_date,
true as akt_actual,
predIp ->> 'ogrnip' as pred_ogrnip,
predIp ->> 'dataSved' as pred_source_update_date,
predIp ->> 'dataRegIP' as pred_registration_date,
predIp ->> 'dataINNNed' as pred_deregistration_date,
predIp -> 'extend' ->> 'dataUchNPD' as pred_reg_npd_date,
predIp -> 'extend' ->> 'dataSnUchNPD' as pred_dereg_npd_date,
false as pred_actual
<sql>with filtered_data as (select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'id' as source_id,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'aktRegIP' -> 0 as ip_elem,
to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataUchNPD',
'YYYY-MM-DD') as reg_npd_date,
to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataSnUchNPD',
'YYYY-MM-DD') as dereg_npd_date,
true as actual
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRegIP' ->> 'regIP' != 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
union all
select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'id' as source_id,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'predRegIP' as ip_elem,
to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataUchNPD',
'YYYY-MM-DD') as reg_npd_date,
to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataSnUchNPD',
'YYYY-MM-DD') as dereg_npd_date,
false as actual
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'predRegIP' != 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
select fd.recruit_id,
fd.source_id,
ip_elem ->> 'ogrnip' as ogrnip,
to_date(ip_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
to_date(ip_elem ->> 'dataRegIP', 'YYYY-MM-DD') as registration_date,
to_date(ip_elem ->> 'dataINNNed', 'YYYY-MM-DD') as deregistration_date,
reg_npd_date,
dereg_npd_date,
fd.actual as actual
from filtered_data fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -421,13 +355,150 @@ from filtered_data fd;</sql>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>individual_entrepreneur_npd_output</name>
<type>TableOutput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<fields>
<field>
<column_name>recruit_id</column_name>
<stream_name>recruit_id</stream_name>
</field>
<field>
<column_name>source_id</column_name>
<stream_name>source_id</stream_name>
</field>
<field>
<column_name>register_date</column_name>
<stream_name>reg_npd_date</stream_name>
</field>
<field>
<column_name>deregistration_date</column_name>
<stream_name>dereg_npd_date</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_monthly>Y</partitioning_monthly>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>individual_entrepreneur_npd</table>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1184</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
<name>individual_entrepreneur_output</name>
<type>TableOutput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard-test</connection>
<fields>
<field>
<column_name>recruit_id</column_name>
<stream_name>recruit_id</stream_name>
</field>
<field>
<column_name>source_id</column_name>
<stream_name>source_id</stream_name>
</field>
<field>
<column_name>ogrnip</column_name>
<stream_name>ogrnip</stream_name>
</field>
<field>
<column_name>source_update_date</column_name>
<stream_name>source_update_date</stream_name>
</field>
<field>
<column_name>registration_date</column_name>
<stream_name>registration_date</stream_name>
</field>
<field>
<column_name>deregistration_date</column_name>
<stream_name>deregistration_date</stream_name>
</field>
<field>
<column_name>actual</column_name>
<stream_name>actual</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_monthly>Y</partitioning_monthly>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>individual_entrepreneur</table>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1232</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>unique_by_rec_id</name>
<type>UniqueRowsByHashSet</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<store_values>N</store_values>
<reject_duplicate_row>N</reject_duplicate_row>
<error_description/>
<fields>
<field>
<name>recruit_id</name>
</field>
</fields>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>akt_individual_entrepreneur_output</source_transform>
<source_transform>individual_entrepreneur_npd_output</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
@ -435,11 +506,11 @@ from filtered_data fd;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>pred_individual_entrepreneur_output</source_transform>
<source_transform>individual_entrepreneur_output</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>

299
mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow_delta.hpl
View file

@ -37,7 +37,7 @@
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>akt_individual_entrepreneur_upsert</to>
<to>individual_entrepreneur_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
@ -46,27 +46,42 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>akt_individual_entrepreneur_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>akt_individual_entrepreneur_upsert</from>
<from>individual_entrepreneur_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>akt_individual_entrepreneur_upsert</from>
<to>pred_individual_entrepreneur_upsert</to>
<from>filter_null_npd_dates</from>
<to>unique_by_rec_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>pred_individual_entrepreneur_upsert</from>
<from>unique_by_rec_id</from>
<to>individual_entrepreneur_npd_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>filter_null_npd_dates</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>pred_individual_entrepreneur_upsert</from>
<from>individual_entrepreneur_npd_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
@ -87,8 +102,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1456</xloc>
<yloc>624</yloc>
<xloc>1616</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
@ -107,14 +122,14 @@
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
SET status = 'DELTA_ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
@ -122,8 +137,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1280</xloc>
<yloc>624</yloc>
<xloc>1328</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
@ -139,21 +154,21 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS'
SET status = 'DELTA_SUCCESS'
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1280</xloc>
<xloc>1616</xloc>
<yloc>272</yloc>
</GUI>
</transform>
@ -170,22 +185,42 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<sql>UPDATE etl.job_execution
SET
status = 'DELTA_PROCESSING',
execution_datetime = current_timestamp,
error_description = NULL
where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql>
<attributes/>
<GUI>
<xloc>272</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>Detect empty stream</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>736</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform>
<name>Filter rows</name>
<type>FilterRows</type>
@ -201,23 +236,49 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<condition>
<conditions>
</conditions>
<function>TRUE</function>
<leftvalue>result</leftvalue>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>992</xloc>
<yloc>272</yloc>
<xloc>1616</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>736</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>filter_null_npd_dates</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
@ -225,11 +286,21 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>result</resultfieldname>
<compare>
<condition>
<conditions>
</conditions>
<function>IS NOT NULL</function>
<leftvalue>reg_npd_date</leftvalue>
<negated>N</negated>
<operator>-</operator>
</condition>
</compare>
<send_true_to>unique_by_rec_id</send_true_to>
<attributes/>
<GUI>
<xloc>736</xloc>
<yloc>448</yloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
@ -243,36 +314,46 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with filtered_data as (
select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'id' as source_id,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'aktRegIP' -> 0 as aktIp,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'predRegIP' as predIp
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRegIP' ->> 'regIP' != 'null'
and (ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'aktRegIP' ->> 'dataSved' > '${MAX_SOURCE_UPDATE_DATE}'
or ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'predRegIP' ->> 'dataSved' > '${MAX_SOURCE_UPDATE_DATE}')
-- and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
select
fd.recruit_id,
fd.source_id,
aktIp ->> 'ogrnip' as akt_ogrnip,
aktIp ->> 'dataSved' as akt_source_update_date,
aktIp ->> 'dataRegIP' as akt_registration_date,
aktIp -> 'extend' ->> 'dataUchNPD' as akt_reg_npd_date,
aktIp -> 'extend' ->> 'dataSnUchNPD' as akt_dereg_npd_date,
true as akt_actual,
predIp ->> 'ogrnip' as pred_ogrnip,
predIp ->> 'dataSved' as pred_source_update_date,
predIp ->> 'dataRegIP' as pred_registration_date,
predIp ->> 'dataINNNed' as pred_deregistration_date,
predIp -> 'extend' ->> 'dataUchNPD' as pred_reg_npd_date,
predIp -> 'extend' ->> 'dataSnUchNPD' as pred_dereg_npd_date,
false as pred_actual
<sql>with filtered_data as (select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'id' as source_id,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'aktRegIP' -> 0 as ip_elem,
to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataUchNPD',
'YYYY-MM-DD') as reg_npd_date,
to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataSnUchNPD',
'YYYY-MM-DD') as dereg_npd_date,
true as actual
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'aktRegIP' != 'null'
and to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'aktRegIP' ->> 'dataSved',
'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
union all
select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'id' as source_id,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'predRegIP' as ip_elem,
to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataUchNPD',
'YYYY-MM-DD') as reg_npd_date,
to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataSnUchNPD',
'YYYY-MM-DD') as dereg_npd_date,
false as actual
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'predRegIP' != 'null'
and to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'predRegIP' ->> 'dataSved',
'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
select fd.recruit_id,
fd.source_id,
ip_elem ->> 'ogrnip' as ogrnip,
to_date(ip_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
to_date(ip_elem ->> 'dataRegIP', 'YYYY-MM-DD') as registration_date,
to_date(ip_elem ->> 'dataINNNed', 'YYYY-MM-DD') as deregistration_date,
reg_npd_date,
dereg_npd_date,
fd.actual as actual
from filtered_data fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -282,10 +363,10 @@ from filtered_data fd;</sql>
</GUI>
</transform>
<transform>
<name>akt_individual_entrepreneur_upsert</name>
<name>individual_entrepreneur_npd_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -293,16 +374,15 @@ from filtered_data fd;</sql>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
<name2/>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>individual_entrepreneur</table>
<table>individual_entrepreneur_npd</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
@ -314,45 +394,25 @@ from filtered_data fd;</sql>
<update>N</update>
</value>
<value>
<name>ogrnip</name>
<rename>akt_ogrnip</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>akt_source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>registration_date</name>
<rename>akt_registration_date</rename>
<update>Y</update>
</value>
<value>
<name>reg_npd_date</name>
<rename>akt_reg_npd_date</rename>
<name>register_date</name>
<rename>reg_npd_date</rename>
<update>Y</update>
</value>
<value>
<name>deregistration_date</name>
<rename>akt_dereg_npd_date</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>akt_actual</rename>
<rename>dereg_npd_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>992</xloc>
<yloc>448</yloc>
<xloc>1104</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>pred_individual_entrepreneur_upsert</name>
<name>individual_entrepreneur_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
@ -384,54 +444,69 @@ from filtered_data fd;</sql>
</value>
<value>
<name>ogrnip</name>
<rename>pred_ogrnip</rename>
<rename>ogrnip</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>pred_source_update_date</rename>
<rename>source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>registration_date</name>
<rename>pred_registration_date</rename>
<rename>registration_date</rename>
<update>Y</update>
</value>
<value>
<name>deregistration_date</name>
<rename>pred_deregistration_date</rename>
<update>Y</update>
</value>
<value>
<name>reg_npd_date</name>
<rename>pred_reg_npd_date</rename>
<update>Y</update>
</value>
<value>
<name>dereg_npd_date</name>
<rename>pred_dereg_npd_date</rename>
<rename>dereg_npd_date</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>pred_actual</rename>
<rename>actual</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1280</xloc>
<xloc>1328</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>unique_by_rec_id</name>
<type>UniqueRowsByHashSet</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<store_values>N</store_values>
<reject_duplicate_row>N</reject_duplicate_row>
<error_description/>
<fields>
<field>
<name>recruit_id</name>
</field>
</fields>
<attributes/>
<GUI>
<xloc>912</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>akt_individual_entrepreneur_upsert</source_transform>
<source_transform>individual_entrepreneur_npd_upsert</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
@ -439,11 +514,11 @@ from filtered_data fd;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>pred_individual_entrepreneur_upsert</source_transform>
<source_transform>individual_entrepreneur_upsert</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>

378
mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow_repeat.hpl
View file

@ -40,33 +40,48 @@
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>filter_null_npd_dates</from>
<to>unique_by_rec_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>akt_individual_entrepreneur_upsert</to>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>akt_individual_entrepreneur_upsert</from>
<to>pred_individual_entrepreneur_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>akt_individual_entrepreneur_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>pred_individual_entrepreneur_upsert</from>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>pred_individual_entrepreneur_upsert</from>
<from>Identify last row in a stream</from>
<to>individual_entrepreneur_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>akt_individual_entrepreneur_upsert</from>
<from>unique_by_rec_id</from>
<to>individual_entrepreneur_npd_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>filter_null_npd_dates</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_npd_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
@ -87,8 +102,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1696</xloc>
<yloc>640</yloc>
<xloc>1856</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
@ -107,7 +122,7 @@
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
@ -122,8 +137,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1520</xloc>
<yloc>640</yloc>
<xloc>1568</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
@ -139,7 +154,7 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
@ -153,8 +168,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1520</xloc>
<yloc>288</yloc>
<xloc>1856</xloc>
<yloc>304</yloc>
</GUI>
</transform>
<transform>
@ -170,22 +185,46 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}')
ON CONFLICT (job_name, recruitment_id)
DO UPDATE
SET status = 'PROCESSING',
error_description = null,
execution_datetime = current_timestamp;</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>512</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>Detect empty stream</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>304</yloc>
</GUI>
</transform>
<transform>
<name>Filter rows</name>
<type>FilterRows</type>
@ -201,23 +240,49 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<condition>
<conditions>
</conditions>
<function>TRUE</function>
<leftvalue>result</leftvalue>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1232</xloc>
<yloc>288</yloc>
<xloc>1856</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>filter_null_npd_dates</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
@ -225,11 +290,21 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>result</resultfieldname>
<compare>
<condition>
<conditions>
</conditions>
<function>IS NOT NULL</function>
<leftvalue>reg_npd_date</leftvalue>
<negated>N</negated>
<operator>-</operator>
</condition>
</compare>
<send_true_to>unique_by_rec_id</send_true_to>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>464</yloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
@ -243,34 +318,42 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with filtered_data as (
select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'id' as source_id,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'aktRegIP' -> 0 as aktIp,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'predRegIP' as predIp
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRegIP' ->> 'regIP' != 'null'
-- and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
select
fd.recruit_id,
fd.source_id,
aktIp ->> 'ogrnip' as akt_ogrnip,
aktIp ->> 'dataSved' as akt_source_update_date,
aktIp ->> 'dataRegIP' as akt_registration_date,
aktIp -> 'extend' ->> 'dataUchNPD' as akt_reg_npd_date,
aktIp -> 'extend' ->> 'dataSnUchNPD' as akt_dereg_npd_date,
true as akt_actual,
predIp ->> 'ogrnip' as pred_ogrnip,
predIp ->> 'dataSved' as pred_source_update_date,
predIp ->> 'dataRegIP' as pred_registration_date,
predIp ->> 'dataINNNed' as pred_deregistration_date,
predIp -> 'extend' ->> 'dataUchNPD' as pred_reg_npd_date,
predIp -> 'extend' ->> 'dataSnUchNPD' as pred_dereg_npd_date,
false as pred_actual
<sql>with filtered_data as (select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'id' as source_id,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'aktRegIP' -> 0 as ip_elem,
to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataUchNPD',
'YYYY-MM-DD') as reg_npd_date,
to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataSnUchNPD',
'YYYY-MM-DD') as dereg_npd_date,
true as actual
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'aktRegIP' != 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
union all
select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'id' as source_id,
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'predRegIP' as ip_elem,
to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataUchNPD',
'YYYY-MM-DD') as reg_npd_date,
to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataSnUchNPD',
'YYYY-MM-DD') as dereg_npd_date,
false as actual
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'predRegIP' != 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
select fd.recruit_id,
fd.source_id,
ip_elem ->> 'ogrnip' as ogrnip,
to_date(ip_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
to_date(ip_elem ->> 'dataRegIP', 'YYYY-MM-DD') as registration_date,
to_date(ip_elem ->> 'dataINNNed', 'YYYY-MM-DD') as deregistration_date,
reg_npd_date,
dereg_npd_date,
fd.actual as actual
from filtered_data fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -280,77 +363,7 @@ from filtered_data fd;</sql>
</GUI>
</transform>
<transform>
<name>akt_individual_entrepreneur_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard-test</connection>
<lookup>
<key>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>individual_entrepreneur</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>source_id</name>
<rename>source_id</rename>
<update>N</update>
</value>
<value>
<name>ogrnip</name>
<rename>akt_ogrnip</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>akt_source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>registration_date</name>
<rename>akt_registration_date</rename>
<update>Y</update>
</value>
<value>
<name>reg_npd_date</name>
<rename>akt_reg_npd_date</rename>
<update>Y</update>
</value>
<value>
<name>deregistration_date</name>
<rename>akt_dereg_npd_date</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>akt_actual</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1232</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>pred_individual_entrepreneur_upsert</name>
<name>individual_entrepreneur_npd_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
@ -361,13 +374,61 @@ from filtered_data fd;</sql>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>individual_entrepreneur_npd</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>source_id</name>
<rename>source_id</rename>
<update>N</update>
</value>
<value>
<name>register_date</name>
<rename>reg_npd_date</rename>
<update>Y</update>
</value>
<value>
<name>deregistration_date</name>
<rename>dereg_npd_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1328</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
<name>individual_entrepreneur_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>individual_entrepreneur</table>
@ -383,54 +444,69 @@ from filtered_data fd;</sql>
</value>
<value>
<name>ogrnip</name>
<rename>pred_ogrnip</rename>
<rename>ogrnip</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>pred_source_update_date</rename>
<rename>source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>registration_date</name>
<rename>pred_registration_date</rename>
<rename>registration_date</rename>
<update>Y</update>
</value>
<value>
<name>deregistration_date</name>
<rename>pred_deregistration_date</rename>
<update>Y</update>
</value>
<value>
<name>reg_npd_date</name>
<rename>pred_reg_npd_date</rename>
<update>Y</update>
</value>
<value>
<name>dereg_npd_date</name>
<rename>pred_dereg_npd_date</rename>
<rename>dereg_npd_date</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>pred_actual</rename>
<rename>actual</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1520</xloc>
<xloc>1568</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>unique_by_rec_id</name>
<type>UniqueRowsByHashSet</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<store_values>N</store_values>
<reject_duplicate_row>N</reject_duplicate_row>
<error_description/>
<fields>
<field>
<name>recruit_id</name>
</field>
</fields>
<attributes/>
<GUI>
<xloc>1136</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>akt_individual_entrepreneur_upsert</source_transform>
<source_transform>individual_entrepreneur_npd_upsert</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
@ -438,11 +514,11 @@ from filtered_data fd;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>pred_individual_entrepreneur_upsert</source_transform>
<source_transform>individual_entrepreneur_upsert</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>

67
mappings/info_recruits/citizen_tables/work/individual_entrepreneur/recruitment_five_flow_delta.hpl
View file

@ -25,6 +25,11 @@
<to>individual_entrepreneur_flow_delta.hpl</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Get_max_source_update_date</from>
<to>Table input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>individual_entrepreneur_flow_delta.hpl 2</to>
@ -46,6 +51,29 @@
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Get_max_source_update_date</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(source_update_date)
from individual_entrepreneur;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>368</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
@ -60,12 +88,22 @@
<connection>ervu-dashboard-test</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>Get_max_source_update_date</lookup>
<sql>SELECT
r.idm_id as recruitment_id
r.idm_id as recruitment_id,
? as max_source_update_date
FROM ervu_dashboard.recruitment r
LEFT JOIN public.job_execution je
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = 'individual_entrepreneur_job'
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > (
SELECT MAX(execution_datetime)
FROM etl.job_execution
WHERE job_name = 'individual_entrepreneur_job'
AND recruitment_id = r.idm_id
)
where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
<variables_active>N</variables_active>
<attributes/>
@ -103,6 +141,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<field/>
<input>individual_entrepreneur_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -159,6 +202,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<field/>
<input>individual_entrepreneur_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -215,6 +263,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<field/>
<input>individual_entrepreneur_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -271,6 +324,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<field/>
<input>individual_entrepreneur_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -327,6 +385,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<field/>
<input>individual_entrepreneur_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>

20
mappings/info_recruits/citizen_tables/work/individual_entrepreneur/recruitment_five_flow_repeat.hpl
View file

@ -60,9 +60,13 @@
<connection>ervu-dashboard-test</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT
idm_id AS recruitment
FROM ervu_dashboard.recruitment;</sql>
<sql>SELECT r.idm_id AS recruitment_id
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
AND je.job_name = 'individual_entrepreneur_job'
WHERE je.id IS NULL
OR je.status IN ('ERROR', 'PROCESSING');</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
@ -91,7 +95,7 @@ FROM ervu_dashboard.recruitment;</sql>
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment</field>
<field>recruitment_id</field>
<input/>
</variable_mapping>
<variable_mapping>
@ -147,7 +151,7 @@ FROM ervu_dashboard.recruitment;</sql>
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment</field>
<field>recruitment_id</field>
<input/>
</variable_mapping>
<variable_mapping>
@ -203,7 +207,7 @@ FROM ervu_dashboard.recruitment;</sql>
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment</field>
<field>recruitment_id</field>
<input/>
</variable_mapping>
<variable_mapping>
@ -259,7 +263,7 @@ FROM ervu_dashboard.recruitment;</sql>
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment</field>
<field>recruitment_id</field>
<input/>
</variable_mapping>
<variable_mapping>
@ -315,7 +319,7 @@ FROM ervu_dashboard.recruitment;</sql>
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment</field>
<field>recruitment_id</field>
<input/>
</variable_mapping>
<variable_mapping>

3
mappings/info_recruits/citizen_tables/work/individual_entrepreneur/support/check_if_individual_entrepreneur_execution_exists.hpl
View file

@ -62,14 +62,13 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT EXISTS (
SELECT 1
FROM etl.job_execution
WHERE job_name = 'individual_entrepreneur_job'
limit 1
) AS job_executed_flag;</sql>
<variables_active>N</variables_active>
<attributes/>

3
mappings/info_recruits/citizen_tables/work/individual_entrepreneur/support/check_if_need_to_repeat.hpl
View file

@ -62,7 +62,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT EXISTS (SELECT 1
@ -71,7 +71,6 @@ FROM ervu_dashboard.recruitment r
ON r.idm_id = je.recruitment_id
and job_name = 'individual_entrepreneur_job'
where je.status is null or je.status in ('ERROR', 'PROCESSING')
limit 1
) as need_to_repeat_job;</sql>
<variables_active>N</variables_active>
<attributes/>

523
mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow.hpl
View file

@ -30,11 +30,6 @@
<to>self_employed_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>akt_self_employed_output</from>
<to>pred_self_employed_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>self_employed_input</from>
<to>Identify last row in a stream</to>
@ -42,35 +37,129 @@
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>akt_self_employed_output</to>
<to>self_employed_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>akt_self_employed_output</from>
<from>self_employed_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>pred_self_employed_output</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>akt_self_employed_output</from>
<from>self_employed_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>pred_self_employed_output</from>
<to>Change job status on error</to>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>832</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS'
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>48</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
@ -84,7 +173,7 @@
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>N</execute_each_row>
<insert_field/>
@ -105,8 +194,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
<name>Detect empty stream</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -115,254 +204,10 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS'
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>48</yloc>
</GUI>
</transform>
<transform>
<name>Abort</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard-test</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>self_employed_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with filtered_data as (select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' as aktNpd,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' as predNpd
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedNPD' ->> 'npd' != 'null'
-- and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
limit 1)
select fd.recruit_id,
aktNpd ->> 'id' as akt_source_id,
aktNpd ->> 'dataSved' as akt_source_update_date,
aktNpd ->> 'dataUchNPD' as akt_register_date,
true as akt_actual,
predNpd ->> 'id' as pred_source_id,
predNpd ->> 'dataSved' as pred_source_update_date,
predNpd ->> 'dataUchNPD' as pred_register_date,
predNpd ->> 'dataSnUchNPD' as pred_deregistration_date,
false as pred_actual
from filtered_data fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>432</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>akt_self_employed_output</name>
<type>TableOutput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard-test</connection>
<fields>
<field>
<column_name>recruit_id</column_name>
<stream_name>recruit_id</stream_name>
</field>
<field>
<column_name>source_id</column_name>
<stream_name>akt_source_id</stream_name>
</field>
<field>
<column_name>source_update_date</column_name>
<stream_name>akt_source_update_date</stream_name>
</field>
<field>
<column_name>register_date</column_name>
<stream_name>akt_register_date</stream_name>
</field>
<field>
<column_name>actual</column_name>
<stream_name>akt_actual</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>self_employed</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>832</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>pred_self_employed_output</name>
<type>TableOutput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard-test</connection>
<fields>
<field>
<column_name>recruit_id</column_name>
<stream_name>recruit_id</stream_name>
</field>
<field>
<column_name>source_id</column_name>
<stream_name>pred_source_id</stream_name>
</field>
<field>
<column_name>source_update_date</column_name>
<stream_name>pred_source_update_date</stream_name>
</field>
<field>
<column_name>register_date</column_name>
<stream_name>pred_register_date</stream_name>
</field>
<field>
<column_name>deregistration_date</column_name>
<stream_name>pred_deregistration_date</stream_name>
</field>
<field>
<column_name>actual</column_name>
<stream_name>pred_actual</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>self_employed</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>result</resultfieldname>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>224</yloc>
<yloc>48</yloc>
</GUI>
</transform>
<transform>
@ -380,38 +225,156 @@ from filtered_data fd;</sql>
<condition>
<conditions>
</conditions>
<function>TRUE</function>
<leftvalue>result</leftvalue>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>self_employed_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with filtered_data as (select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' as npd_elem,
true as actual
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedNPD' ->> 'aktNPD' != 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
union all
select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' as npd_elem,
false as actual
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedNPD' ->> 'predNPD' != 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
select fd.recruit_id,
npd_elem ->> 'id' as source_id,
to_date(npd_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
to_date(npd_elem ->> 'dataUchNPD', 'YYYY-MM-DD') as register_date,
to_date(npd_elem ->> 'dataSnUchNPD', 'YYYY-MM-DD') as deregistration_date,
actual
from filtered_data fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>432</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>self_employed_output</name>
<type>TableOutput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<fields>
<field>
<column_name>recruit_id</column_name>
<stream_name>recruit_id</stream_name>
</field>
<field>
<column_name>source_id</column_name>
<stream_name>source_id</stream_name>
</field>
<field>
<column_name>source_update_date</column_name>
<stream_name>source_update_date</stream_name>
</field>
<field>
<column_name>register_date</column_name>
<stream_name>register_date</stream_name>
</field>
<field>
<column_name>deregistration_date</column_name>
<stream_name>deregistration_date</stream_name>
</field>
<field>
<column_name>actual</column_name>
<stream_name>actual</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>self_employed</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>832</xloc>
<yloc>48</yloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>akt_self_employed_output</source_transform>
<source_transform>self_employed_output</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>pred_self_employed_output</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>

394
mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_delta.hpl
View file

@ -22,21 +22,11 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>self_employed_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>self_employed_upsert</from>
<to>pred_self_employed_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>self_employed_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>pred_self_employed_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
@ -46,7 +36,7 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>self_employed_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
@ -66,11 +56,36 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>pred_self_employed_upsert</from>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT_WITH_ERROR</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -87,12 +102,9 @@
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard-test</connection>
<delete_field/>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
@ -103,11 +115,10 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>432</yloc>
<xloc>848</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
@ -123,7 +134,7 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
@ -154,7 +165,7 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
@ -163,20 +174,19 @@ and recruitment_id = '${IDM_ID}';
<sql>UPDATE etl.job_execution
SET
status = 'DELTA_PROCESSING',
execution_datetime = DEFAULT,
error_description = NULL,
error_code = NULL
execution_datetime = current_timestamp,
error_description = NULL
where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql>
<attributes/>
<GUI>
<xloc>256</xloc>
<xloc>272</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
<name>Detect empty stream</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -185,188 +195,10 @@ and recruitment_id = '${IDM_ID}';</sql>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with filtered_data as (select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' as aktNpd,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' as predNpd
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedNPD' ->> 'npd' != 'null'
-- and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
--and ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' ->> 'dataSved' > '${MAX_SOURCE_UPDATE_DATE}'
--or ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' ->> 'dataSved' > '${MAX_SOURCE_UPDATE_DATE}'
limit 1)
select fd.recruit_id,
aktNpd ->> 'id' as akt_source_id,
aktNpd ->> 'dataSved' as akt_source_update_date,
aktNpd ->> 'dataUchNPD' as akt_register_date,
true as akt_actual,
predNpd ->> 'id' as pred_source_id,
predNpd ->> 'dataSved' as pred_source_update_date,
predNpd ->> 'dataUchNPD' as pred_register_date,
predNpd ->> 'dataSnUchNPD' as pred_deregistration_date,
false as pred_actual
from filtered_data fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>464</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>self_employed_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard-test</connection>
<lookup>
<key>
<condition>=</condition>
<field>source_id</field>
<name>akt_source_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>self_employed</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>source_id</name>
<rename>akt_source_id</rename>
<update>N</update>
</value>
<value>
<name>source_update_date</name>
<rename>akt_source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>register_date</name>
<rename>akt_register_date</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>akt_actual</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>816</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>pred_self_employed_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard-test</connection>
<lookup>
<key>
<condition>=</condition>
<field>source_id</field>
<name>akt_source_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>self_employed</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>source_id</name>
<rename>pred_source_id</rename>
<update>N</update>
</value>
<value>
<name>source_update_date</name>
<rename>pred_source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>register_date</name>
<rename>pred_register_date</rename>
<update>Y</update>
</value>
<value>
<name>deregistration_date</name>
<rename>pred_deregistration_date</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>pred_actual</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Abort</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT_WITH_ERROR</abort_option>
<always_log_rows>Y</always_log_rows>
<message/>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1296</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>result</resultfieldname>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>224</yloc>
<yloc>16</yloc>
</GUI>
</transform>
<transform>
@ -384,17 +216,143 @@ from filtered_data fd;</sql>
<condition>
<conditions>
</conditions>
<function>TRUE</function>
<leftvalue>result</leftvalue>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>816</xloc>
<yloc>16</yloc>
<xloc>1040</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>self_employed_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with filtered_data as (select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' as npd_elem,
true as actual
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedNPD' ->> 'aktNPD' != 'null'
and
to_date(ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
union all
select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' as npd_elem,
false as actual
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedNPD' ->> 'predNPD' != 'null'
and to_date(ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
select fd.recruit_id,
npd_elem ->> 'id' as source_id,
to_date(npd_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
to_date(npd_elem ->> 'dataUchNPD', 'YYYY-MM-DD') as register_date,
to_date(npd_elem ->> 'dataSnUchNPD', 'YYYY-MM-DD') as deregistration_date,
actual
from filtered_data fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>448</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>self_employed_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>self_employed</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>source_id</name>
<rename>source_id</rename>
<update>N</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>register_date</name>
<rename>register_date</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>actual</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform_error_handling>
@ -403,19 +361,7 @@ from filtered_data fd;</sql>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>pred_self_employed_upsert</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>

218
mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_on_error.hpl → mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_repeat.hpl
View file

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<pipeline>
<info>
<name>self_employed_flow_on_error</name>
<name>self_employed_flow_repeat</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@ -42,34 +42,29 @@
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>akt_self_employed_output</to>
<to>self_employed_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>akt_self_employed_output</from>
<to>pred_self_employed_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>akt_self_employed_output</from>
<from>self_employed_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>pred_self_employed_output</from>
<from>self_employed_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>pred_self_employed_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>akt_self_employed_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -87,8 +82,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1248</xloc>
<yloc>592</yloc>
<xloc>1072</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -107,7 +102,7 @@
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
@ -122,8 +117,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>592</yloc>
<xloc>864</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -139,7 +134,7 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
@ -170,22 +165,42 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}')
ON CONFLICT (job_name, recruitment_id)
DO UPDATE
SET status = 'PROCESSING',
error_description = null,
execution_datetime = current_timestamp;</sql>
<attributes/>
<GUI>
<xloc>256</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Detect empty stream</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>656</xloc>
<yloc>240</yloc>
</GUI>
</transform>
<transform>
<name>Filter rows</name>
<type>FilterRows</type>
@ -202,30 +217,38 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<conditions>
</conditions>
<function>TRUE</function>
<leftvalue>result</leftvalue>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>864</xloc>
<yloc>240</yloc>
<xloc>1072</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>result</resultfieldname>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>656</xloc>
@ -243,26 +266,30 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with filtered_data as (select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' as aktNpd,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' as predNpd
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' as npd_elem,
true as actual
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedNPD' ->> 'npd' != 'null'
-- and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
limit 1)
where ri.info -> 'svedFL' -> 'svedNPD' ->> 'aktNPD' != 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
union all
select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' as npd_elem,
false as actual
from recruits_info ri
where ri.info -> 'svedFL' -> 'svedNPD' ->> 'predNPD' != 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
select fd.recruit_id,
aktNpd ->> 'id' as akt_source_id,
aktNpd ->> 'dataSved' as akt_source_update_date,
aktNpd ->> 'dataUchNPD' as akt_register_date,
true as akt_actual,
predNpd ->> 'id' as pred_source_id,
predNpd ->> 'dataSved' as pred_source_update_date,
predNpd ->> 'dataUchNPD' as pred_register_date,
predNpd ->> 'dataSnUchNPD' as pred_deregistration_date,
false as pred_actual
npd_elem ->> 'id' as source_id,
to_date(npd_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
to_date(npd_elem ->> 'dataUchNPD', 'YYYY-MM-DD') as register_date,
to_date(npd_elem ->> 'dataSnUchNPD', 'YYYY-MM-DD') as deregistration_date,
actual
from filtered_data fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -272,7 +299,7 @@ from filtered_data fd;</sql>
</GUI>
</transform>
<transform>
<name>akt_self_employed_output</name>
<name>self_employed_output</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
@ -283,13 +310,12 @@ from filtered_data fd;</sql>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>source_id</field>
<name>akt_source_id</name>
<name2/>
<name>source_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>self_employed</table>
@ -300,22 +326,22 @@ from filtered_data fd;</sql>
</value>
<value>
<name>source_id</name>
<rename>akt_source_id</rename>
<update>Y</update>
<rename>source_id</rename>
<update>N</update>
</value>
<value>
<name>source_update_date</name>
<rename>akt_source_update_date</rename>
<rename>source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>register_date</name>
<rename>akt_register_date</rename>
<rename>register_date</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>akt_actual</rename>
<rename>actual</rename>
<update>Y</update>
</value>
</lookup>
@ -326,85 +352,13 @@ from filtered_data fd;</sql>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>pred_self_employed_output</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard-test</connection>
<lookup>
<key>
<condition>=</condition>
<field>source_id</field>
<name>pred_source_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>self_employed</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>source_id</name>
<rename>pred_source_id</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>pred_source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>register_date</name>
<rename>pred_register_date</rename>
<update>Y</update>
</value>
<value>
<name>deregistration_date</name>
<rename>pred_deregistration_date</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>pred_actual</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>akt_self_employed_output</source_transform>
<source_transform>self_employed_output</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>pred_self_employed_output</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>

75
mappings/info_recruits/citizen_tables/work/self_employed/recruitment_five_flow_delta.hpl
View file

@ -20,6 +20,11 @@
<notepads>
</notepads>
<order>
<hop>
<from>Get_max_source_update_date</from>
<to>Table input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>self_employed_flow_delta.hpl</to>
@ -46,6 +51,29 @@
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Get_max_source_update_date</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(source_update_date)
from self_employed;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>208</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
@ -57,20 +85,30 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>Get_max_source_update_date</lookup>
<sql>SELECT
r.idm_id as recruitment_id
r.idm_id AS recruitment_id,
? as max_source_update_date
FROM ervu_dashboard.recruitment r
LEFT JOIN public.job_execution je
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = 'self_employed_job'
where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
AND je.job_name = 'self_employed_job'
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > (
SELECT MAX(execution_datetime)
FROM etl.job_execution
WHERE job_name = 'self_employed_job'
AND recruitment_id = r.idm_id
)
WHERE je.status IN ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>416</xloc>
<xloc>384</xloc>
<yloc>496</yloc>
</GUI>
</transform>
@ -103,6 +141,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<field/>
<input>self_employed_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -159,6 +202,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<field/>
<input>self_employed_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -215,6 +263,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<field/>
<input>self_employed_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -271,6 +324,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<field/>
<input>self_employed_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -327,6 +385,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<field/>
<input>self_employed_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>

40
mappings/info_recruits/citizen_tables/work/self_employed/recruitments_five_flow_repeat.hpl
View file

@ -22,27 +22,27 @@
<order>
<hop>
<from>Table input</from>
<to>self_employed_flow_on_error.hpl</to>
<to>self_employed_flow_repeat.hpl</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>self_employed_flow_on_error.hpl 2</to>
<to>self_employed_flow_repeat.hpl 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>self_employed_flow_on_error.hpl 3</to>
<to>self_employed_flow_repeat.hpl 3</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>self_employed_flow_on_error.hpl 4</to>
<to>self_employed_flow_repeat.hpl 4</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>self_employed_flow_on_error.hpl 5</to>
<to>self_employed_flow_repeat.hpl 5</to>
<enabled>Y</enabled>
</hop>
</order>
@ -75,7 +75,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
</GUI>
</transform>
<transform>
<name>self_employed_flow_on_error.hpl</name>
<name>self_employed_flow_repeat.hpl</name>
<type>PipelineExecutor</type>
<description/>
<distribute>Y</distribute>
@ -86,7 +86,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<schema_name/>
</partitioning>
<run_configuration>local</run_configuration>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_on_error.hpl</filename>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_repeat.hpl</filename>
<filenameInField>N</filenameInField>
<filenameField/>
<group_size>1</group_size>
@ -101,7 +101,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>work_activity_job</input>
<input>self_employed_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
@ -131,7 +131,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
</GUI>
</transform>
<transform>
<name>self_employed_flow_on_error.hpl 2</name>
<name>self_employed_flow_repeat.hpl 2</name>
<type>PipelineExecutor</type>
<description/>
<distribute>Y</distribute>
@ -142,7 +142,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<schema_name/>
</partitioning>
<run_configuration>local</run_configuration>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_on_error.hpl</filename>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_repeat.hpl</filename>
<filenameInField>N</filenameInField>
<filenameField/>
<group_size>1</group_size>
@ -157,7 +157,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>work_activity_job</input>
<input>self_employed_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
@ -187,7 +187,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
</GUI>
</transform>
<transform>
<name>self_employed_flow_on_error.hpl 3</name>
<name>self_employed_flow_repeat.hpl 3</name>
<type>PipelineExecutor</type>
<description/>
<distribute>Y</distribute>
@ -198,7 +198,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<schema_name/>
</partitioning>
<run_configuration>local</run_configuration>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_on_error.hpl</filename>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_repeat.hpl</filename>
<filenameInField>N</filenameInField>
<filenameField/>
<group_size>1</group_size>
@ -213,7 +213,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>work_activity_job</input>
<input>self_employed_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
@ -243,7 +243,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
</GUI>
</transform>
<transform>
<name>self_employed_flow_on_error.hpl 4</name>
<name>self_employed_flow_repeat.hpl 4</name>
<type>PipelineExecutor</type>
<description/>
<distribute>Y</distribute>
@ -254,7 +254,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<schema_name/>
</partitioning>
<run_configuration>local</run_configuration>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_on_error.hpl</filename>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_repeat.hpl</filename>
<filenameInField>N</filenameInField>
<filenameField/>
<group_size>1</group_size>
@ -269,7 +269,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>work_activity_job</input>
<input>self_employed_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
@ -299,7 +299,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
</GUI>
</transform>
<transform>
<name>self_employed_flow_on_error.hpl 5</name>
<name>self_employed_flow_repeat.hpl 5</name>
<type>PipelineExecutor</type>
<description/>
<distribute>Y</distribute>
@ -310,7 +310,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<schema_name/>
</partitioning>
<run_configuration>local</run_configuration>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_on_error.hpl</filename>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_repeat.hpl</filename>
<filenameInField>N</filenameInField>
<filenameField/>
<group_size>1</group_size>
@ -325,7 +325,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>work_activity_job</input>
<input>self_employed_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>

2
mappings/info_recruits/citizen_tables/work/self_employed/support/check_if_need_to_repeat.hpl
View file

@ -62,7 +62,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT EXISTS (SELECT 1

2
mappings/info_recruits/citizen_tables/work/self_employed/support/check_if_self_employed_job_execution_exists.hpl
View file

@ -62,7 +62,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT EXISTS (

839
mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow.hpl
View file

File diff suppressed because it is too large Load diff

514
mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow_delta.hpl
View file

@ -22,22 +22,57 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>citizen_work_activity_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Table output</to>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<from>Identify last row in a stream</from>
<to>Filter_null_parental_leave_dates</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>citizen_work_activity_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter_null_parental_leave_dates</from>
<to>parental_leave_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_upsert</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<to>Change job status on success</to>
<from>parental_leave_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
</order>
@ -60,12 +95,9 @@
<name>error_code</name>
</argument>
</arguments>
<connection>ervu-dashboard-test</connection>
<delete_field/>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
@ -77,11 +109,10 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>416</yloc>
<xloc>1296</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
@ -97,12 +128,9 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<delete_field/>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
@ -112,11 +140,10 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>224</yloc>
<xloc>1296</xloc>
<yloc>48</yloc>
</GUI>
</transform>
<transform>
@ -132,24 +159,19 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<delete_field/>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET
status = 'DELTA_PROCESSING',
execution_datetime = DEFAULT,
error_description = NULL,
error_code = NULL
execution_datetime = current_timestamp,
error_description = NULL
where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>336</xloc>
@ -157,7 +179,106 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>Table input</name>
<name>Detect empty stream</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>48</yloc>
</GUI>
</transform>
<transform>
<name>Filter rows</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1024</xloc>
<yloc>48</yloc>
</GUI>
</transform>
<transform>
<name>Filter_null_parental_leave_dates</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>IS NOT NULL</function>
<leftvalue>parental_leave_start_date</leftvalue>
<negated>N</negated>
<operator>-</operator>
</condition>
</compare>
<send_true_to>parental_leave_upsert</send_true_to>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>citizen_work_activity_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
@ -167,87 +288,37 @@ and recruitment_id = '${IDM_ID}';</sql>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>10</limit>
<sql>WITH trud_info AS (SELECT ri.info,
ri.recruit_id::uuid,
CASE
WHEN ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' IS NULL THEN NULL
WHEN jsonb_typeof(ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat') = 'array' THEN
ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' -> 0
END AS trud_deyat,
CASE
WHEN ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' IS NULL THEN NULL
WHEN
jsonb_typeof(ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' -> 0 -> 'svedUhodReb') =
'array' THEN
ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' -> 0 -> 'svedUhodReb' -> 0
END AS parental_leave_info,
COALESCE(
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat',
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'predRabotodat'
) AS employer,
CASE
WHEN ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' IS NULL THEN NULL
WHEN jsonb_typeof(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'aktRegIP') = 'array'
THEN
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'aktRegIP' -> 0
ELSE
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'predRegIP'
END AS reg_ip,
COALESCE(
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD',
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD'
) AS self_employment
FROM ervu_dashboard.recruits_info ri
<sql>WITH filteredData AS (SELECT ri.recruit_id,
trud_elem,
uhod_elem
FROM recruits_info ri
CROSS JOIN LATERAL jsonb_array_elements(
ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat'
) AS trud_elem
LEFT JOIN LATERAL jsonb_array_elements(
CASE
WHEN jsonb_typeof(trud_elem -> 'svedUhodReb') = 'array'
THEN trud_elem -> 'svedUhodReb'
ELSE '[]'::jsonb
END
) AS uhod_elem ON true
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat') = 'array'
--AND to_date(trud_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'
--AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
SELECT ti.recruit_id::uuid,
(ti.trud_deyat ->> 'dataMeropr')::date AS personnel_event_date,
ti.trud_deyat ->> 'naimTipKadrMeropr' AS personnel_event_type,
(ti.trud_deyat ->> 'prAktMestRab' = '1') AS current_workplace,
ti.trud_deyat ->> 'trudFunkcziya' AS position,
ti.trud_deyat -> 'svedOrg' ->> 'naimOrg' AS employer_name,
ti.employer -> 'svedYUL' ->> 'kpp' AS employer_kpp,
CASE
WHEN ti.employer->'svedYUL' IS NOT NULL THEN 'Юридическое лицо'
WHEN ti.employer->'svedIP' IS NOT NULL THEN 'Индивидуальный предприниматель'
END AS employer_category,
COALESCE(
ti.employer -> 'svedYUL' ->> 'innyul',
ti.employer -> 'svedIP' ->> 'innyfl'
) AS employer_inn,
ti.employer -> 'svedYUL' -> 'svedOPK' ->> 'region' AS employer_address,
COALESCE(
ti.employer -> 'svedIP' ->> 'ogrnip',
ti.employer -> 'svedYUL' -> 'extend' ->> 'ogrn'
) AS employer_ogrnip,
ti.employer -> 'extend' ->> 'tipPodrazdRabotodat' AS employer_unit_type,
ti.employer -> 'extend' ->> 'adresPodrazdRabotodat' AS employer_unit_address,
(ti.parental_leave_info ->> 'dataNachUhodReb')::date AS parental_leave_start_date,
(ti.parental_leave_info ->> 'dataKonUhodReb')::date AS parental_leave_end_date,
(
(ti.parental_leave_info ->> 'dataNachUhodReb')::date &lt;= current_date AND
(
(ti.parental_leave_info ->> 'dataKonUhodReb') IS NULL OR
(ti.parental_leave_info ->> 'dataKonUhodReb')::date > current_date
)
) AS parental_leave_active,
(ti.reg_ip ->> 'dataRegIP')::date AS reg_ip_date,
(ti.reg_ip ->> 'dataINNNed')::date AS dereg_ip_date,
ti.reg_ip ->> 'ogrnip' AS ogrnip_ip,
ti.reg_ip IS NOT NULL AS active_ip,
ti.self_employment ->> 'dataUchNPD' AS reg_self_employment_date,
ti.self_employment ->> 'dataSnUchNPD' AS dereg_self_employment_date,
ti.self_employment IS NOT NULL AS active_self_employment
FROM trud_info ti;
SELECT fd.recruit_id,
trud_elem ->> 'id' AS source_id,
trud_elem -> 'svedOrg' ->> 'naimOrg' AS employer_name,
trud_elem ->> 'trudFunkcziya' AS position,
to_date(trud_elem ->> 'dataMeropr', 'YYYY-MM-DD') AS personnel_event_date,
trud_elem ->> 'naimTipKadrMeropr' AS personnel_event_type,
to_date(trud_elem ->> 'dataSved', 'YYYY-MM-DD') AS source_update_date,
(trud_elem ->> 'prAktMestRab') = '1' AS actual_employer,
to_date(uhod_elem ->> 'dataNachUhodReb', 'YYYY-MM-DD') AS parental_leave_start_date,
to_date(uhod_elem ->> 'dataKonUhodReb', 'YYYY-MM-DD') AS parental_leave_end_date
FROM filteredData fd;
</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -257,8 +328,8 @@ FROM trud_info ti;
</GUI>
</transform>
<transform>
<name>Table output</name>
<type>TableOutput</type>
<name>citizen_work_activity_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -267,130 +338,129 @@ FROM trud_info ti;
<method>none</method>
<schema_name/>
</partitioning>
<commit>1000</commit>
<connection>ervu-dashboard-test</connection>
<fields>
<field>
<column_name>recruit_id</column_name>
<stream_name>recruit_id</stream_name>
</field>
<field>
<column_name>personnel_event_date</column_name>
<stream_name>personnel_event_date</stream_name>
</field>
<field>
<column_name>personnel_event_type</column_name>
<stream_name>personnel_event_type</stream_name>
</field>
<field>
<column_name>current_workplace</column_name>
<stream_name>current_workplace</stream_name>
</field>
<field>
<column_name>position</column_name>
<stream_name>position</stream_name>
</field>
<field>
<column_name>employer_name</column_name>
<stream_name>employer_name</stream_name>
</field>
<field>
<column_name>employer_kpp</column_name>
<stream_name>employer_kpp</stream_name>
</field>
<field>
<column_name>employer_category</column_name>
<stream_name>employer_category</stream_name>
</field>
<field>
<column_name>employer_inn</column_name>
<stream_name>employer_inn</stream_name>
</field>
<field>
<column_name>employer_address</column_name>
<stream_name>employer_address</stream_name>
</field>
<field>
<column_name>employer_ogrnip</column_name>
<stream_name>employer_ogrnip</stream_name>
</field>
<field>
<column_name>employer_unit_type</column_name>
<stream_name>employer_unit_type</stream_name>
</field>
<field>
<column_name>employer_unit_address</column_name>
<stream_name>employer_unit_address</stream_name>
</field>
<field>
<column_name>parental_leave_start_date</column_name>
<stream_name>parental_leave_start_date</stream_name>
</field>
<field>
<column_name>parental_leave_end_date</column_name>
<stream_name>parental_leave_end_date</stream_name>
</field>
<field>
<column_name>parental_leave_active</column_name>
<stream_name>parental_leave_active</stream_name>
</field>
<field>
<column_name>reg_ip_date</column_name>
<stream_name>reg_ip_date</stream_name>
</field>
<field>
<column_name>dereg_ip_date</column_name>
<stream_name>dereg_ip_date</stream_name>
</field>
<field>
<column_name>ogrnip_ip</column_name>
<stream_name>ogrnip_ip</stream_name>
</field>
<field>
<column_name>active_ip</column_name>
<stream_name>active_ip</stream_name>
</field>
<field>
<column_name>reg_self_employment_date</column_name>
<stream_name>reg_self_employment_date</stream_name>
</field>
<field>
<column_name>dereg_self_employment_date</column_name>
<stream_name>dereg_self_employment_date</stream_name>
</field>
<field>
<column_name>active_self_employment</column_name>
<stream_name>active_self_employment</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_monthly>Y</partitioning_monthly>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>citizen_work_activity</table>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen_work_activity</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>source_id</name>
<rename>source_id</rename>
<update>N</update>
</value>
<value>
<name>employer_name</name>
<rename>employer_name</rename>
<update>Y</update>
</value>
<value>
<name>position</name>
<rename>position</rename>
<update>Y</update>
</value>
<value>
<name>personnel_event_date</name>
<rename>personnel_event_date</rename>
<update>Y</update>
</value>
<value>
<name>personnel_event_type</name>
<rename>personnel_event_type</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>actual_employer</name>
<rename>actual_employer</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>768</xloc>
<xloc>1040</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>parental_leave_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>citizen_work_activity_source_id</field>
<name>source_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>employee_parental_leave</table>
<value>
<name>start_date</name>
<rename>parental_leave_start_date</rename>
<update>Y</update>
</value>
<value>
<name>end_date</name>
<rename>parental_leave_end_date</rename>
<update>Y</update>
</value>
<value>
<name>citizen_work_activity_source_id</name>
<rename>source_id</rename>
<update>N</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Table output</source_transform>
<source_transform>citizen_work_activity_upsert</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename>error_code</codes_valuename>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>parental_leave_upsert</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>

553
mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow_repeat.hpl
View file

@ -20,23 +20,58 @@
<notepads>
</notepads>
<order>
<hop>
<from>Detect empty stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter_null_parental_leave_dates</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>citizen_work_activity_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Table output</to>
<from>Identify last row in a stream</from>
<to>citizen_work_activity_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<from>Filter_null_parental_leave_dates</from>
<to>parental_leave_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>parental_leave_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<from>citizen_work_activity_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_upsert</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
@ -60,27 +95,24 @@
<name>error_code</name>
</argument>
</arguments>
<connection>ervu-dashboard-test</connection>
<delete_field/>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?
SET status = 'DELTA_ERROR',
error_description = ?,
error_code = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>400</yloc>
<xloc>1312</xloc>
<yloc>704</yloc>
</GUI>
</transform>
<transform>
@ -96,26 +128,22 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<delete_field/>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS'
SET status = 'DELTA_SUCCESS'
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>208</yloc>
<xloc>1312</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
@ -131,48 +159,126 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard-test</connection>
<delete_field/>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (
id,
job_name,
status,
execution_datetime,
error_description,
error_code,
recruitment_id
)
VALUES (
DEFAULT,
'${JOB_NAME}',
'PROCESSING',
DEFAULT,
NULL,
NULL,
'${IDM_ID}'
)
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}')
ON CONFLICT (job_name, recruitment_id)
DO UPDATE SET
status = 'PROCESSING',
execution_datetime = DEFAULT,
error_description = NULL,
error_code = NULL;</sql>
<update_field/>
DO UPDATE
SET status = 'PROCESSING',
error_description = null,
execution_datetime = current_timestamp;</sql>
<attributes/>
<GUI>
<xloc>272</xloc>
<yloc>208</yloc>
<xloc>304</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<name>Detect empty stream</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>768</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>Filter rows</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>Filter_null_parental_leave_dates</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>IS NOT NULL</function>
<leftvalue>parental_leave_start_date</leftvalue>
<negated>N</negated>
<operator>-</operator>
</condition>
</compare>
<send_true_to>parental_leave_upsert</send_true_to>
<attributes/>
<GUI>
<xloc>768</xloc>
<yloc>704</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>768</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>citizen_work_activity_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
@ -182,98 +288,46 @@ DO UPDATE SET
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>10</limit>
<sql>WITH trud_info AS (SELECT ri.info,
ri.recruit_id::uuid,
CASE
WHEN ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' IS NULL THEN NULL
WHEN jsonb_typeof(ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat') = 'array' THEN
ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' -> 0
END AS trud_deyat,
CASE
WHEN ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' IS NULL THEN NULL
WHEN
jsonb_typeof(ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' -> 0 -> 'svedUhodReb') =
'array' THEN
ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' -> 0 -> 'svedUhodReb' -> 0
END AS parental_leave_info,
COALESCE(
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat',
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'predRabotodat'
) AS employer,
CASE
WHEN ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' IS NULL THEN NULL
WHEN jsonb_typeof(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'aktRegIP') = 'array'
THEN
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'aktRegIP' -> 0
ELSE
ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'predRegIP'
END AS reg_ip,
COALESCE(
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD',
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD'
) AS self_employment
FROM ervu_dashboard.recruits_info ri
)
SELECT ti.recruit_id::uuid,
(ti.trud_deyat ->> 'dataMeropr')::date AS personnel_event_date,
ti.trud_deyat ->> 'naimTipKadrMeropr' AS personnel_event_type,
(ti.trud_deyat ->> 'prAktMestRab' = '1') AS current_workplace,
ti.trud_deyat ->> 'trudFunkcziya' AS position,
ti.trud_deyat -> 'svedOrg' ->> 'naimOrg' AS employer_name,
ti.employer -> 'svedYUL' ->> 'kpp' AS employer_kpp,
CASE
WHEN ti.employer->'svedYUL' IS NOT NULL THEN 'Юридическое лицо'
WHEN ti.employer->'svedIP' IS NOT NULL THEN 'Индивидуальный предприниматель'
END AS employer_category,
COALESCE(
ti.employer -> 'svedYUL' ->> 'innyul',
ti.employer -> 'svedIP' ->> 'innyfl'
) AS employer_inn,
ti.employer -> 'svedYUL' -> 'svedOPK' ->> 'region' AS employer_address,
COALESCE(
ti.employer -> 'svedIP' ->> 'ogrnip',
ti.employer -> 'svedYUL' -> 'extend' ->> 'ogrn'
) AS employer_ogrnip,
ti.employer -> 'extend' ->> 'tipPodrazdRabotodat' AS employer_unit_type,
ti.employer -> 'extend' ->> 'adresPodrazdRabotodat' AS employer_unit_address,
(ti.parental_leave_info ->> 'dataNachUhodReb')::date AS parental_leave_start_date,
(ti.parental_leave_info ->> 'dataKonUhodReb')::date AS parental_leave_end_date,
(
(ti.parental_leave_info ->> 'dataNachUhodReb')::date &lt;= current_date AND
(
(ti.parental_leave_info ->> 'dataKonUhodReb') IS NULL OR
(ti.parental_leave_info ->> 'dataKonUhodReb')::date > current_date
)
) AS parental_leave_active,
(ti.reg_ip ->> 'dataRegIP')::date AS reg_ip_date,
(ti.reg_ip ->> 'dataINNNed')::date AS dereg_ip_date,
ti.reg_ip ->> 'ogrnip' AS ogrnip_ip,
ti.reg_ip IS NOT NULL AS active_ip,
ti.self_employment ->> 'dataUchNPD' AS reg_self_employment_date,
ti.self_employment ->> 'dataSnUchNPD' AS dereg_self_employment_date,
ti.self_employment IS NOT NULL AS active_self_employment
FROM trud_info ti;
<sql>WITH filteredData AS (SELECT ri.recruit_id,
trud_elem,
uhod_elem
FROM recruits_info ri
CROSS JOIN LATERAL jsonb_array_elements(
ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat'
) AS trud_elem
LEFT JOIN LATERAL jsonb_array_elements(
CASE
WHEN jsonb_typeof(trud_elem -> 'svedUhodReb') = 'array'
THEN trud_elem -> 'svedUhodReb'
ELSE '[]'::jsonb
END
) AS uhod_elem ON true
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat') = 'array'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
SELECT fd.recruit_id,
trud_elem ->> 'id' AS source_id,
trud_elem -> 'svedOrg' ->> 'naimOrg' AS employer_name,
trud_elem ->> 'trudFunkcziya' AS position,
to_date(trud_elem ->> 'dataMeropr', 'YYYY-MM-DD') AS personnel_event_date,
trud_elem ->> 'naimTipKadrMeropr' AS personnel_event_type,
to_date(trud_elem ->> 'dataSved', 'YYYY-MM-DD') AS source_update_date,
(trud_elem ->> 'prAktMestRab') = '1' AS actual_employer,
to_date(uhod_elem ->> 'dataNachUhodReb', 'YYYY-MM-DD') AS parental_leave_start_date,
to_date(uhod_elem ->> 'dataKonUhodReb', 'YYYY-MM-DD') AS parental_leave_end_date
FROM filteredData fd;
</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>480</xloc>
<yloc>208</yloc>
<xloc>544</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>Table output</name>
<type>TableOutput</type>
<name>citizen_work_activity_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -282,130 +336,129 @@ FROM trud_info ti;
<method>none</method>
<schema_name/>
</partitioning>
<commit>1000</commit>
<connection>ervu-dashboard-test</connection>
<fields>
<field>
<column_name>recruit_id</column_name>
<stream_name>recruit_id</stream_name>
</field>
<field>
<column_name>personnel_event_date</column_name>
<stream_name>personnel_event_date</stream_name>
</field>
<field>
<column_name>personnel_event_type</column_name>
<stream_name>personnel_event_type</stream_name>
</field>
<field>
<column_name>current_workplace</column_name>
<stream_name>current_workplace</stream_name>
</field>
<field>
<column_name>position</column_name>
<stream_name>position</stream_name>
</field>
<field>
<column_name>employer_name</column_name>
<stream_name>employer_name</stream_name>
</field>
<field>
<column_name>employer_kpp</column_name>
<stream_name>employer_kpp</stream_name>
</field>
<field>
<column_name>employer_category</column_name>
<stream_name>employer_category</stream_name>
</field>
<field>
<column_name>employer_inn</column_name>
<stream_name>employer_inn</stream_name>
</field>
<field>
<column_name>employer_address</column_name>
<stream_name>employer_address</stream_name>
</field>
<field>
<column_name>employer_ogrnip</column_name>
<stream_name>employer_ogrnip</stream_name>
</field>
<field>
<column_name>employer_unit_type</column_name>
<stream_name>employer_unit_type</stream_name>
</field>
<field>
<column_name>employer_unit_address</column_name>
<stream_name>employer_unit_address</stream_name>
</field>
<field>
<column_name>parental_leave_start_date</column_name>
<stream_name>parental_leave_start_date</stream_name>
</field>
<field>
<column_name>parental_leave_end_date</column_name>
<stream_name>parental_leave_end_date</stream_name>
</field>
<field>
<column_name>parental_leave_active</column_name>
<stream_name>parental_leave_active</stream_name>
</field>
<field>
<column_name>reg_ip_date</column_name>
<stream_name>reg_ip_date</stream_name>
</field>
<field>
<column_name>dereg_ip_date</column_name>
<stream_name>dereg_ip_date</stream_name>
</field>
<field>
<column_name>ogrnip_ip</column_name>
<stream_name>ogrnip_ip</stream_name>
</field>
<field>
<column_name>active_ip</column_name>
<stream_name>active_ip</stream_name>
</field>
<field>
<column_name>reg_self_employment_date</column_name>
<stream_name>reg_self_employment_date</stream_name>
</field>
<field>
<column_name>dereg_self_employment_date</column_name>
<stream_name>dereg_self_employment_date</stream_name>
</field>
<field>
<column_name>active_self_employment</column_name>
<stream_name>active_self_employment</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_monthly>Y</partitioning_monthly>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>citizen_work_activity</table>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen_work_activity</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>source_id</name>
<rename>source_id</rename>
<update>N</update>
</value>
<value>
<name>employer_name</name>
<rename>employer_name</rename>
<update>Y</update>
</value>
<value>
<name>position</name>
<rename>position</rename>
<update>Y</update>
</value>
<value>
<name>personnel_event_date</name>
<rename>personnel_event_date</rename>
<update>Y</update>
</value>
<value>
<name>personnel_event_type</name>
<rename>personnel_event_type</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>actual_employer</name>
<rename>actual_employer</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>208</yloc>
<xloc>1040</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>parental_leave_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>citizen_work_activity_source_id</field>
<name>source_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>employee_parental_leave</table>
<value>
<name>start_date</name>
<rename>parental_leave_start_date</rename>
<update>Y</update>
</value>
<value>
<name>end_date</name>
<rename>parental_leave_end_date</rename>
<update>Y</update>
</value>
<value>
<name>citizen_work_activity_source_id</name>
<rename>source_id</rename>
<update>N</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>704</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Table output</source_transform>
<source_transform>citizen_work_activity_upsert</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename>error_code</codes_valuename>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>parental_leave_upsert</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>

67
mappings/info_recruits/citizen_tables/work/work_activity/recruitment_five_flow_delta.hpl
View file

@ -25,6 +25,11 @@
<to>work_activity_flow_delta.hpl</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>get_max_source_update_date</from>
<to>Table input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>work_activity_flow_delta.hpl 2</to>
@ -60,12 +65,22 @@
<connection>ervu-dashboard-test</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_source_update_date</lookup>
<sql>SELECT
r.idm_id as recruitment_id
r.idm_id as recruitment_id,
? as max_source_update_date
FROM ervu_dashboard.recruitment r
LEFT JOIN public.job_execution je
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = 'work_activity_job'
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > (
SELECT MAX(execution_datetime)
FROM etl.job_execution
WHERE job_name = 'work_activity_job'
AND recruitment_id = r.idm_id
)
where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
<variables_active>N</variables_active>
<attributes/>
@ -74,6 +89,29 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<yloc>288</yloc>
</GUI>
</transform>
<transform>
<name>get_max_source_update_date</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(source_update_date)
FROM citizen_work_activity;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>160</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform>
<name>work_activity_flow_delta.hpl</name>
<type>PipelineExecutor</type>
@ -103,6 +141,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<field/>
<input>work_activity_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -159,6 +202,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<field/>
<input>work_activity_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -215,6 +263,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<field/>
<input>work_activity_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -271,6 +324,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<field/>
<input>work_activity_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -327,6 +385,11 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<field/>
<input>work_activity_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>

2
mappings/info_recruits/citizen_tables/work/work_activity/recruitment_five_flow_on_error.hpl → mappings/info_recruits/citizen_tables/work/work_activity/recruitment_five_flow_repeat.hpl
View file

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<pipeline>
<info>
<name>recruitment_five_flow_on_error</name>
<name>recruitment_five_flow_repeat</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>

3
mappings/info_recruits/citizen_tables/work/work_activity/support/check_if_need_to_repeat_job.hpl
View file

@ -62,7 +62,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT EXISTS (SELECT 1
@ -71,7 +71,6 @@ FROM ervu_dashboard.recruitment r
ON r.idm_id = je.recruitment_id
and job_name = 'work_activity_job'
where je.status is null or je.status in ('ERROR', 'PROCESSING')
limit 1
) as need_to_repeat_job;</sql>
<variables_active>N</variables_active>
<attributes/>

32
mappings/info_recruits/citizen_tables/work/work_activity/support/check_if_work_activity_job_exists.hpl
View file

@ -20,11 +20,6 @@
<notepads>
</notepads>
<order>
<hop>
<from>Set variables</from>
<to>Write to log</to>
<enabled>N</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Set variables</to>
@ -67,14 +62,13 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT EXISTS (
SELECT 1
FROM etl.job_execution
WHERE job_name = 'work_activity_job'
limit 1
) AS job_executed_flag;</sql>
<variables_active>N</variables_active>
<attributes/>
@ -83,30 +77,6 @@
<yloc>336</yloc>
</GUI>
</transform>
<transform>
<name>Write to log</name>
<type>WriteToLog</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<loglevel>Basic</loglevel>
<displayHeader>Y</displayHeader>
<limitRows>N</limitRows>
<limitRowsNumber>0</limitRowsNumber>
<logmessage>${STATUS}</logmessage>
<fields>
</fields>
<attributes/>
<GUI>
<xloc>944</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>

10
mappings/info_recruits/citizen_tables/work/work_activity/work_activity_job.hwf
View file

@ -85,8 +85,6 @@
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/work_activity/recruitments_five_flow.hpl</filename>
<logext/>
<logfile/>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -102,7 +100,7 @@
<attributes_hac/>
</action>
<action>
<name>recruitment_five_flow_on_error.hpl</name>
<name>recruitment_five_flow_repeat.hpl</name>
<description/>
<type>PIPELINE</type>
<attributes/>
@ -112,7 +110,9 @@
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/work_activity/recruitment_five_flow_on_error.hpl</filename>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/work/work_activity/recruitment_five_flow_repeat.hpl</filename>
<logext/>
<logfile/>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -235,7 +235,7 @@
</hop>
<hop>
<from>Simple evaluation</from>
<to>recruitment_five_flow_on_error.hpl</to>
<to>recruitment_five_flow_repeat.hpl</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>