micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lots of fixes

Browse source
This commit is contained in:
Fusionshh 2025-10-09 11:26:18 +03:00
parent 8ee1c35c31
commit 62bcbbfd97
60 changed files with 3367 additions and 2149 deletions
Download patch file Download diff file Expand all files Collapse all files

3
mappings/info_recruits/citizen_tables/deputy/deputy_job.hwf
View file

@ -42,8 +42,6 @@
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_job_execution_exists.hpl</filename>
<logext/>
<logfile/>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -209,6 +207,7 @@
</field>
</fields>
<file_variable_type>CURRENT_WORKFLOW</file_variable_type>
<filename/>
<replacevars>N</replacevars>
<parallel>N</parallel>
<xloc>864</xloc>

74
mappings/info_recruits/citizen_tables/deputy/parallel/deputy_flow_delta.hpl
View file

@ -20,11 +20,6 @@
<notepads>
</notepads>
<order>
<hop>
<from>Block until transforms finish</from>
<to>deputy_flag_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<to>Abort</to>
@ -45,11 +40,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
@ -76,7 +66,17 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_flag_update</from>
<from>Identify last row in a stream</from>
<to>deputy_flag_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_upsert</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
@ -97,7 +97,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1552</xloc>
<xloc>1360</xloc>
<yloc>688</yloc>
</GUI>
</transform>
@ -114,13 +114,12 @@
</partitioning>
<transforms>
<transform>
<CopyNr/>
<name>deputy_upsert</name>
<name>deputy_flag_update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>896</xloc>
<xloc>1152</xloc>
<yloc>416</yloc>
</GUI>
</transform>
@ -147,7 +146,7 @@
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
SET status = 'DELTA_ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
@ -155,7 +154,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1136</xloc>
<xloc>944</xloc>
<yloc>688</yloc>
</GUI>
</transform>
@ -179,14 +178,14 @@ and recruitment_id = '${IDM_ID}';
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS'
SET status = 'DELTA_SUCCESS'
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1552</xloc>
<xloc>1360</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -209,10 +208,13 @@ and recruitment_id = '${IDM_ID}';
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<sql>UPDATE etl.job_execution
SET
status = 'DELTA_PROCESSING',
execution_datetime = current_timestamp,
error_description = NULL
where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql>
<attributes/>
<GUI>
<xloc>208</xloc>
@ -268,7 +270,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1552</xloc>
<xloc>1360</xloc>
<yloc>416</yloc>
</GUI>
</transform>
@ -310,11 +312,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<key>
<condition>&lt;></condition>
<field>has_deputy_info</field>
<name>has_deputy_info</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
@ -326,8 +323,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1136</xloc>
<yloc>416</yloc>
<xloc>624</xloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
@ -343,12 +340,16 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedDeputat' -> 'deputat' as deputy_elem
FROM recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedDeputat' ->> 'deputat' &lt;&gt; 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
and to_date(ri.info -> 'svedFL' -> 'svedDeputat' -> 'deputat' ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'
and case when '${MAX_SOURCE_UPDATE_DATE}' &lt;&gt; ''
then to_date(ri.info -> 'svedFL' -> 'svedDeputat' -> 'deputat' ->> 'dataSved', 'YYYY-MM-DD') >
to_date('${MAX_SOURCE_UPDATE_DATE}', 'YYYY-MM-DD')
else true end
)
SELECT fd.recruit_id,
deputy_elem ->> 'id' AS source_id,
@ -359,8 +360,9 @@ SELECT fd.recruit_id,
to_date(deputy_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS end_date,
deputy_elem ->> 'prichinaDosrochKod' AS termination_reason_code,
to_date(deputy_elem ->> 'dataDosroch', 'YYYY-MM-DD') AS termination_date,
true as has_deputy_info
FROM filteredData fd;</sql>
true as has_deputy_info
FROM filteredData fd;
</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -438,8 +440,8 @@ FROM filteredData fd;</sql>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>896</xloc>
<yloc>688</yloc>
<xloc>944</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform_error_handling>

84
mappings/info_recruits/citizen_tables/deputy/parallel/deputy_flow_repeat.hpl
View file

@ -7,7 +7,18 @@
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<pipeline_status>0</pipeline_status>
<parameters>
<parameter>
<name>IDM_ID</name>
<default_value>b5265855-1972-4ff5-8b5e-0f746127fa70</default_value>
<description/>
</parameter>
<parameter>
<name>JOB_NAME</name>
<default_value>deputy_job</default_value>
<description/>
</parameter>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
@ -20,11 +31,6 @@
<notepads>
</notepads>
<order>
<hop>
<from>Block until transforms finish</from>
<to>deputy_flag_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<to>Abort</to>
@ -45,11 +51,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
@ -60,11 +61,6 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_flag_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_input</from>
<to>Identify last row in a stream</to>
@ -80,6 +76,21 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_upsert</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>deputy_flag_update</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -97,7 +108,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1584</xloc>
<xloc>1440</xloc>
<yloc>784</yloc>
</GUI>
</transform>
@ -114,13 +125,12 @@
</partitioning>
<transforms>
<transform>
<CopyNr/>
<name>deputy_upsert</name>
<name>deputy_flag_update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1056</xloc>
<xloc>1248</xloc>
<yloc>512</yloc>
</GUI>
</transform>
@ -155,7 +165,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1296</xloc>
<xloc>1072</xloc>
<yloc>784</yloc>
</GUI>
</transform>
@ -186,7 +196,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1584</xloc>
<xloc>1440</xloc>
<yloc>336</yloc>
</GUI>
</transform>
@ -209,13 +219,19 @@ and recruitment_id = '${IDM_ID}';
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET
status = 'PROCESSING',
execution_datetime = current_timestamp,
error_description = NULL
where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql>
<sql>INSERT INTO etl.job_execution (job_name, recruitment_id, status, execution_datetime, error_description)
VALUES (
'${JOB_NAME}',
'${IDM_ID}',
'PROCESSING',
current_timestamp,
NULL
)
ON CONFLICT (job_name, recruitment_id)
DO UPDATE SET
status = EXCLUDED.status,
execution_datetime = EXCLUDED.execution_datetime,
error_description = EXCLUDED.error_description;</sql>
<attributes/>
<GUI>
<xloc>368</xloc>
@ -271,7 +287,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1584</xloc>
<xloc>1440</xloc>
<yloc>512</yloc>
</GUI>
</transform>
@ -324,8 +340,8 @@ and recruitment_id = '${IDM_ID}';</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1296</xloc>
<yloc>512</yloc>
<xloc>784</xloc>
<yloc>784</yloc>
</GUI>
</transform>
<transform>
@ -361,7 +377,7 @@ FROM filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>592</xloc>
<xloc>576</xloc>
<yloc>512</yloc>
</GUI>
</transform>
@ -435,8 +451,8 @@ FROM filteredData fd;</sql>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1056</xloc>
<yloc>784</yloc>
<xloc>1072</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform_error_handling>

29
mappings/info_recruits/citizen_tables/deputy/recruitment_five_flow_delta.hpl
View file

@ -65,22 +65,27 @@
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id
<sql>WITH mud AS (SELECT recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS', 'DELTA_SUCCESS')
GROUP BY recruitment_id)
SELECT r.idm_id
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
AND ri.updated_at > mud.max_upd_date
union
select r2.idm_id
from recruitment r2
join etl.job_execution je
on r2.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>

4
mappings/info_recruits/citizen_tables/deputy/recruitment_five_flow_repeat.hpl
View file

@ -64,10 +64,10 @@
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
AND je.job_name = 'disability_job'
AND je.job_name = '${JOB_NAME}'
WHERE je.id IS NULL
OR je.status IN ('ERROR', 'PROCESSING');</sql>
<variables_active>N</variables_active>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>656</xloc>

149
mappings/info_recruits/citizen_tables/education/educational_institution_learning/educational_institution_learning_job.hwf
View file

@ -27,8 +27,8 @@
<schedulerType>0</schedulerType>
<weekDay>1</weekDay>
<parallel>N</parallel>
<xloc>384</xloc>
<yloc>288</yloc>
<xloc>992</xloc>
<yloc>304</yloc>
<attributes_hac/>
</action>
<action>
@ -195,101 +195,6 @@
<yloc>800</yloc>
<attributes_hac/>
</action>
<action>
<name>check_if_constants_exists.hpl</name>
<description/>
<type>PIPELINE</type>
<attributes/>
<add_date>N</add_date>
<add_time>N</add_time>
<clear_files>N</clear_files>
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/education/foreign_diploma/constants/support/check_if_constants_exists.hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<params_from_previous>N</params_from_previous>
<run_configuration>local</run_configuration>
<set_append_logfile>N</set_append_logfile>
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>576</xloc>
<yloc>288</yloc>
<attributes_hac/>
</action>
<action>
<name>constants_exists</name>
<description/>
<type>SIMPLE_EVAL</type>
<attributes/>
<fieldtype>boolean</fieldtype>
<successbooleancondition>true</successbooleancondition>
<successcondition>equal</successcondition>
<successnumbercondition>equal</successnumbercondition>
<successwhenvarset>N</successwhenvarset>
<valuetype>variable</valuetype>
<variablename>CONSTANTS_EXISTS</variablename>
<parallel>N</parallel>
<xloc>752</xloc>
<yloc>288</yloc>
<attributes_hac/>
</action>
<action>
<name>education_constants_flow.hpl</name>
<description/>
<type>PIPELINE</type>
<attributes/>
<add_date>N</add_date>
<add_time>N</add_time>
<clear_files>N</clear_files>
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow.hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<params_from_previous>N</params_from_previous>
<run_configuration>local</run_configuration>
<set_append_logfile>N</set_append_logfile>
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>944</xloc>
<yloc>208</yloc>
<attributes_hac/>
</action>
<action>
<name>education_constants_flow_repeat.hpl</name>
<description/>
<type>PIPELINE</type>
<attributes/>
<add_date>N</add_date>
<add_time>N</add_time>
<clear_files>N</clear_files>
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/education/educational_institution_learning/constants/education_constants_flow_repeat.hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<params_from_previous>N</params_from_previous>
<run_configuration>local</run_configuration>
<set_append_logfile>N</set_append_logfile>
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>944</xloc>
<yloc>384</yloc>
<attributes_hac/>
</action>
<action>
<name>init_job_name</name>
<description/>
@ -303,6 +208,7 @@
</field>
</fields>
<file_variable_type>CURRENT_WORKFLOW</file_variable_type>
<filename/>
<replacevars>N</replacevars>
<parallel>N</parallel>
<xloc>1184</xloc>
@ -332,48 +238,6 @@
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>check_if_constants_exists.hpl</from>
<to>constants_exists</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>constants_exists</from>
<to>education_constants_flow.hpl</to>
<enabled>Y</enabled>
<evaluation>N</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>constants_exists</from>
<to>education_constants_flow_repeat.hpl</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>Start</from>
<to>check_if_constants_exists.hpl</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>education_constants_flow.hpl</from>
<to>init_job_name</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>education_constants_flow_repeat.hpl</from>
<to>init_job_name</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>init_job_name</from>
<to>check_if_job_execution_exists.hpl</to>
@ -402,6 +266,13 @@
<evaluation>N</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>Start</from>
<to>init_job_name</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
</hops>
<notepads>
</notepads>

2
mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl
View file

@ -1060,7 +1060,7 @@ SELECT education_elem ->> 'id'
to_date(coalesce(education_elem -> 'extend' ->> 'dataZavershObuch', education_elem ->> 'dataOkonch'),
'YYYY-MM-DD') as completion_date,
education_elem -> 'svedObrProg' ->> 'naimFormaObuch' as education_form,
education_elem -> 'svedObrProg' ->> 'dataNachObuch' as education_program_start_date,
to_date(education_elem -> 'svedObrProg' ->> 'dataNachObuch', 'YYYY-MM-DD') as education_program_start_date,
to_date(education_elem -> 'extend' ->> 'dataNachObuch', 'YYYY-MM-DD') as education_start_date,
to_date(coalesce(education_elem -> 'svedObrProg' ->> 'planDataOkonch', education_elem ->> 'dataOzhid'),
'YYYY-MM-DD') as planned_completion_date,

772
mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl
View file

@ -16,12 +16,12 @@
</parameter>
<parameter>
<name>JOB_NAME</name>
<default_value>education_test_job</default_value>
<default_value>educational_institution_learning_job</default_value>
<description/>
</parameter>
<parameter>
<name>MAX_SOURCE_UPDATE_DATE</name>
<default_value>2025-10-02</default_value>
<default_value>2025/04/08 00:00:00.000</default_value>
<description/>
</parameter>
</parameters>
@ -61,41 +61,16 @@
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter_null_parental_leave_dates</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Group by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>sort_by_recruit_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>block_until_all_complete</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Group by</from>
<to>educ_info_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>educ_info_upsert</from>
<to>block_until_all_complete</to>
@ -131,6 +106,31 @@
<to>Change job status on error 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_input</from>
<to>Filter_null_parental_leave_dates</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_input</from>
<to>sort_by_recruit_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_input</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>educ_info_upsert</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -185,7 +185,7 @@
</partitioning>
<transforms>
<transform>
<name>citizen_education_output</name>
<name>educ_info_upsert</name>
</transform>
</transforms>
<attributes/>
@ -217,7 +217,7 @@
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
SET status = 'DELTA_ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
@ -286,8 +286,7 @@ and recruitment_id = '${IDM_ID}';
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS'
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
@ -340,7 +339,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>832</xloc>
<xloc>1152</xloc>
<yloc>288</yloc>
</GUI>
</transform>
@ -408,271 +407,6 @@ and recruitment_id = '${IDM_ID}';</sql>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
<name>Group by</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<aggregate>source_id</aggregate>
<subject>source_id</subject>
<type>-</type>
</field>
<field>
<aggregate>recruit_id</aggregate>
<subject>recruit_id</subject>
<type>-</type>
</field>
<field>
<aggregate>source_update_date</aggregate>
<subject>source_update_date</subject>
<type>-</type>
</field>
<field>
<aggregate>educational_institution_type</aggregate>
<subject>educational_institution_type</subject>
<type>-</type>
</field>
<field>
<aggregate>org_type</aggregate>
<subject>org_type</subject>
<type>-</type>
</field>
<field>
<aggregate>inn</aggregate>
<subject>inn</subject>
<type>-</type>
</field>
<field>
<aggregate>kpp</aggregate>
<subject>kpp</subject>
<type>-</type>
</field>
<field>
<aggregate>ogrn</aggregate>
<subject>ogrn</subject>
<type>-</type>
</field>
<field>
<aggregate>educational_institution_address</aggregate>
<subject>educational_institution_address</subject>
<type>-</type>
</field>
<field>
<aggregate>educational_institution_name</aggregate>
<subject>educational_institution_name</subject>
<type>-</type>
</field>
<field>
<aggregate>educational_institution_status_code</aggregate>
<subject>educational_institution_status_code</subject>
<type>-</type>
</field>
<field>
<aggregate>educational_organization_separate_unit_type</aggregate>
<subject>educational_organization_separate_unit_type</subject>
<type>-</type>
</field>
<field>
<aggregate>educational_organization_separate_unit_address</aggregate>
<subject>educational_organization_separate_unit_address</subject>
<type>-</type>
</field>
<field>
<aggregate>student_status</aggregate>
<subject>student_status</subject>
<type>-</type>
</field>
<field>
<aggregate>grade_number</aggregate>
<subject>grade_number</subject>
<type>-</type>
</field>
<field>
<aggregate>education_level_code</aggregate>
<subject>education_level_code</subject>
<type>-</type>
</field>
<field>
<aggregate>admission_date</aggregate>
<subject>admission_date</subject>
<type>-</type>
</field>
<field>
<aggregate>enrollment_date</aggregate>
<subject>enrollment_date</subject>
<type>-</type>
</field>
<field>
<aggregate>enrollment_order_number</aggregate>
<subject>enrollment_order_number</subject>
<type>-</type>
</field>
<field>
<aggregate>enrollment_order_date</aggregate>
<subject>enrollment_order_date</subject>
<type>-</type>
</field>
<field>
<aggregate>completion_date</aggregate>
<subject>completion_date</subject>
<type>-</type>
</field>
<field>
<aggregate>education_form</aggregate>
<subject>education_form</subject>
<type>-</type>
</field>
<field>
<aggregate>education_program_start_date</aggregate>
<subject>education_program_start_date</subject>
<type>-</type>
</field>
<field>
<aggregate>education_start_date</aggregate>
<subject>education_start_date</subject>
<type>-</type>
</field>
<field>
<aggregate>planned_completion_date</aggregate>
<subject>planned_completion_date</subject>
<type>-</type>
</field>
<field>
<aggregate>minobr_speciality_code</aggregate>
<subject>minobr_speciality_code</subject>
<type>-</type>
</field>
<field>
<aggregate>minprosv_speciality_code</aggregate>
<subject>minprosv_speciality_code</subject>
<type>-</type>
</field>
<field>
<aggregate>recovery_info</aggregate>
<subject>recovery_info</subject>
<type>-</type>
</field>
<field>
<aggregate>recovery_date</aggregate>
<subject>recovery_date</subject>
<type>-</type>
</field>
<field>
<aggregate>recovery_order_date</aggregate>
<subject>recovery_order_date</subject>
<type>-</type>
</field>
<field>
<aggregate>recovery_order_number</aggregate>
<subject>recovery_order_number</subject>
<type>-</type>
</field>
<field>
<aggregate>expulsion_info</aggregate>
<subject>expulsion_info</subject>
<type>-</type>
</field>
<field>
<aggregate>expulsion_date</aggregate>
<subject>expulsion_date</subject>
<type>-</type>
</field>
<field>
<aggregate>expulsion_order_date</aggregate>
<subject>expulsion_order_date</subject>
<type>-</type>
</field>
<field>
<aggregate>expulsion_order_number</aggregate>
<subject>expulsion_order_number</subject>
<type>-</type>
</field>
<field>
<aggregate>academical_leave_info</aggregate>
<subject>academical_leave_info</subject>
<type>-</type>
</field>
<field>
<aggregate>first_education</aggregate>
<subject>first_education</subject>
<type>-</type>
</field>
<field>
<aggregate>vus_spec</aggregate>
<subject>vus_spec</subject>
<type>-</type>
</field>
<field>
<aggregate>military_center_education_info</aggregate>
<subject>military_center_education_info</subject>
<type>-</type>
</field>
<field>
<aggregate>military_center_education_start_date</aggregate>
<subject>military_center_education_start_date</subject>
<type>-</type>
</field>
<field>
<aggregate>military_center_education_end_date</aggregate>
<subject>military_center_education_end_date</subject>
<type>-</type>
</field>
<field>
<aggregate>military_center_education_period</aggregate>
<subject>military_center_education_period</subject>
<type>-</type>
</field>
<field>
<aggregate>military_center_completion</aggregate>
<subject>military_center_completion</subject>
<type>-</type>
</field>
<field>
<aggregate>military_department_education_info</aggregate>
<subject>military_department_education_info</subject>
<type>-</type>
</field>
<field>
<aggregate>military_department_start_date</aggregate>
<subject>military_department_start_date</subject>
<type>-</type>
</field>
<field>
<aggregate>military_department_end_date</aggregate>
<subject>military_department_end_date</subject>
<type>-</type>
</field>
<field>
<aggregate>last_row</aggregate>
<subject>last_row</subject>
<type>-</type>
</field>
</fields>
<give_back_row>N</give_back_row>
<group>
<field>
<name>source_id</name>
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>1152</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
@ -687,7 +421,41 @@ and recruitment_id = '${IDM_ID}';</sql>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>832</xloc>
<xloc>1152</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>source_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>944</xloc>
<yloc>464</yloc>
</GUI>
</transform>
@ -739,10 +507,10 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<transforms>
<transform>
<name>education_leave_output</name>
<name>Update</name>
</transform>
<transform>
<name>Update</name>
<name>edu_leave_upsert</name>
</transform>
</transforms>
<attributes/>
@ -802,73 +570,8 @@ and recruitment_id = '${IDM_ID}';</sql>
<schema>ervu_dashboard</schema>
<table>educational_institution_learning</table>
<value>
<name>source_id</name>
<rename>source_id</rename>
<update>N</update>
</value>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>educational_institution_type</name>
<rename>educational_institution_type</rename>
<update>Y</update>
</value>
<value>
<name>org_type</name>
<rename>org_type</rename>
<update>Y</update>
</value>
<value>
<name>inn</name>
<rename>inn</rename>
<update>Y</update>
</value>
<value>
<name>kpp</name>
<rename>kpp</rename>
<update>Y</update>
</value>
<value>
<name>ogrn</name>
<rename>ogrn</rename>
<update>Y</update>
</value>
<value>
<name>educational_institution_address</name>
<rename>educational_institution_address</rename>
<update>Y</update>
</value>
<value>
<name>educational_organization_separate_unit_type</name>
<rename>educational_organization_separate_unit_type</rename>
<update>Y</update>
</value>
<value>
<name>educational_organization_separate_unit_address</name>
<rename>educational_organization_separate_unit_address</rename>
<update>Y</update>
</value>
<value>
<name>student_status</name>
<rename>student_status</rename>
<update>Y</update>
</value>
<value>
<name>grade_number</name>
<rename>grade_number</rename>
<update>Y</update>
</value>
<value>
<name>education_level_code</name>
<rename>education_level_code</rename>
<name>academical_leave_info</name>
<rename>academical_leave_info</rename>
<update>Y</update>
</value>
<value>
@ -876,21 +579,6 @@ and recruitment_id = '${IDM_ID}';</sql>
<rename>admission_date</rename>
<update>Y</update>
</value>
<value>
<name>enrollment_date</name>
<rename>enrollment_date</rename>
<update>Y</update>
</value>
<value>
<name>enrollment_order_number</name>
<rename>enrollment_order_number</rename>
<update>Y</update>
</value>
<value>
<name>enrollment_order_date</name>
<rename>enrollment_order_date</rename>
<update>Y</update>
</value>
<value>
<name>completion_date</name>
<rename>completion_date</rename>
@ -901,6 +589,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<rename>education_form</rename>
<update>Y</update>
</value>
<value>
<name>education_level_code</name>
<rename>education_level_code</rename>
<update>Y</update>
</value>
<value>
<name>education_program_start_date</name>
<rename>education_program_start_date</rename>
@ -912,43 +605,48 @@ and recruitment_id = '${IDM_ID}';</sql>
<update>Y</update>
</value>
<value>
<name>planned_completion_date</name>
<rename>planned_completion_date</rename>
<name>educational_institution_address</name>
<rename>educational_institution_address</rename>
<update>Y</update>
</value>
<value>
<name>minobr_speciality_code</name>
<rename>minobr_speciality_code</rename>
<name>educational_institution_name</name>
<rename>educational_institution_name</rename>
<update>Y</update>
</value>
<value>
<name>minprosv_speciality_code</name>
<rename>minprosv_speciality_code</rename>
<name>educational_institution_status_code</name>
<rename>educational_institution_status_code</rename>
<update>Y</update>
</value>
<value>
<name>recovery_info</name>
<rename>recovery_info</rename>
<name>educational_institution_type</name>
<rename>educational_institution_type</rename>
<update>Y</update>
</value>
<value>
<name>recovery_date</name>
<rename>recovery_date</rename>
<name>educational_organization_separate_unit_address</name>
<rename>educational_organization_separate_unit_address</rename>
<update>Y</update>
</value>
<value>
<name>recovery_order_date</name>
<rename>recovery_order_date</rename>
<name>educational_organization_separate_unit_type</name>
<rename>educational_organization_separate_unit_type</rename>
<update>Y</update>
</value>
<value>
<name>recovery_order_number</name>
<rename>recovery_order_number</rename>
<name>enrollment_date</name>
<rename>enrollment_date</rename>
<update>Y</update>
</value>
<value>
<name>expulsion_info</name>
<rename>expulsion_info</rename>
<name>enrollment_order_date</name>
<rename>enrollment_order_date</rename>
<update>Y</update>
</value>
<value>
<name>enrollment_order_number</name>
<rename>enrollment_order_number</rename>
<update>Y</update>
</value>
<value>
@ -956,6 +654,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<rename>expulsion_date</rename>
<update>Y</update>
</value>
<value>
<name>expulsion_info</name>
<rename>expulsion_info</rename>
<update>Y</update>
</value>
<value>
<name>expulsion_order_date</name>
<rename>expulsion_order_date</rename>
@ -966,39 +669,24 @@ and recruitment_id = '${IDM_ID}';</sql>
<rename>expulsion_order_number</rename>
<update>Y</update>
</value>
<value>
<name>academical_leave_info</name>
<rename>academical_leave_info</rename>
<update>Y</update>
</value>
<value>
<name>first_education</name>
<rename>first_education</rename>
<update>Y</update>
</value>
<value>
<name>vus_spec</name>
<rename>vus_spec</rename>
<name>grade_number</name>
<rename>grade_number</rename>
<update>Y</update>
</value>
<value>
<name>military_center_education_info</name>
<rename>military_center_education_info</rename>
<name>inn</name>
<rename>inn</rename>
<update>Y</update>
</value>
<value>
<name>military_center_education_start_date</name>
<rename>military_center_education_start_date</rename>
<update>Y</update>
</value>
<value>
<name>military_center_education_end_date</name>
<rename>military_center_education_end_date</rename>
<update>Y</update>
</value>
<value>
<name>military_center_education_period</name>
<rename>military_center_education_period</rename>
<name>kpp</name>
<rename>kpp</rename>
<update>Y</update>
</value>
<value>
@ -1007,13 +695,28 @@ and recruitment_id = '${IDM_ID}';</sql>
<update>Y</update>
</value>
<value>
<name>military_department_education_info</name>
<rename>military_department_education_info</rename>
<name>military_center_education_end_date</name>
<rename>military_center_education_end_date</rename>
<update>Y</update>
</value>
<value>
<name>military_department_start_date</name>
<rename>military_department_start_date</rename>
<name>military_center_education_info</name>
<rename>military_center_education_info</rename>
<update>Y</update>
</value>
<value>
<name>military_center_education_period</name>
<rename>military_center_education_period</rename>
<update>Y</update>
</value>
<value>
<name>military_center_education_start_date</name>
<rename>military_center_education_start_date</rename>
<update>Y</update>
</value>
<value>
<name>military_department_education_info</name>
<rename>military_department_education_info</rename>
<update>Y</update>
</value>
<value>
@ -1021,34 +724,79 @@ and recruitment_id = '${IDM_ID}';</sql>
<rename>military_department_end_date</rename>
<update>Y</update>
</value>
<value>
<name>military_department_start_date</name>
<rename>military_department_start_date</rename>
<update>Y</update>
</value>
<value>
<name>minobr_speciality_code</name>
<rename>minobr_speciality_code</rename>
<update>Y</update>
</value>
<value>
<name>minprosv_speciality_code</name>
<rename>minprosv_speciality_code</rename>
<update>Y</update>
</value>
<value>
<name>ogrn</name>
<rename>ogrn</rename>
<update>Y</update>
</value>
<value>
<name>org_type</name>
<rename>last_row</rename>
<rename>org_type</rename>
<update>Y</update>
</value>
<value>
<name>education_start_date</name>
<rename>academ_start_date</rename>
<name>planned_completion_date</name>
<rename>planned_completion_date</rename>
<update>Y</update>
</value>
<value>
<name>admission_date</name>
<rename>academ_end_date</rename>
<name>recovery_date</name>
<rename>recovery_date</rename>
<update>Y</update>
</value>
<value>
<name>grade_number</name>
<rename>academ_reason</rename>
<name>recovery_info</name>
<rename>recovery_info</rename>
<update>Y</update>
</value>
<value>
<name>educational_institution_name</name>
<rename>educational_institution_name</rename>
<name>recovery_order_date</name>
<rename>recovery_order_date</rename>
<update>Y</update>
</value>
<value>
<name>educational_institution_status_code</name>
<rename>educational_institution_status_code</rename>
<name>recovery_order_number</name>
<rename>recovery_order_number</rename>
<update>Y</update>
</value>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>source_id</name>
<rename>source_id</rename>
<update>N</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
<value>
<name>student_status</name>
<rename>student_status</rename>
<update>Y</update>
</value>
<value>
<name>vus_spec</name>
<rename>vus_spec</rename>
<update>Y</update>
</value>
</lookup>
@ -1063,7 +811,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<name>education_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -1072,127 +820,111 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
education_elem,
academ_elem,
ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz' as education_arr,
'Образовательная организация высшего образования' as educational_institution_type
FROM recruits_info ri
CROSS JOIN LATERAL jsonb_array_elements(
ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') AS education_elem
LEFT JOIN LATERAL jsonb_array_elements(
CASE
WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array'
THEN education_elem -> 'svedOtpusk'
ELSE '[]'::jsonb
END
) AS academ_elem ON true
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') = 'array'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
union all
SELECT ri.recruit_id,
education_elem,
academ_elem,
'Профессиональная образовательная организация' as educational_institution_type
ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh' as education_arr,
'Профессиональная образовательная организация' as educational_institution_type
FROM recruits_info ri
CROSS JOIN LATERAL jsonb_array_elements(
ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') AS education_elem
LEFT JOIN LATERAL jsonb_array_elements(
CASE
WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array'
THEN education_elem -> 'svedOtpusk'
ELSE '[]'::jsonb
END
) AS academ_elem ON true
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') = 'array'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
union all
SELECT ri.recruit_id,
education_elem,
academ_elem,
'Общеобразовательная организация' as educational_institution_type
ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola' as education_arr,
'Общеобразовательная организация' as educational_institution_type
FROM recruits_info ri
CROSS JOIN LATERAL jsonb_array_elements(
ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') AS education_elem
LEFT JOIN LATERAL jsonb_array_elements(
CASE
WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array'
THEN education_elem -> 'svedOtpusk'
ELSE '[]'::jsonb
END
) AS academ_elem ON true
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') = 'array'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
SELECT education_elem ->> 'id' AS source_id,
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
SELECT education_elem ->> 'id' AS source_id,
fd.recruit_id,
to_date(education_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
to_date(education_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
educational_institution_type,
education_elem -> 'svedOrg' ->> 'naimTipOrg' as org_type,
education_elem -> 'svedOrg' ->> 'inn' as inn,
education_elem -> 'svedOrg' ->> 'kpp' as kpp,
education_elem -> 'svedOrg' ->> 'naimTipOrg' as org_type,
education_elem -> 'svedOrg' ->> 'inn' as inn,
education_elem -> 'svedOrg' ->> 'kpp' as kpp,
coalesce(education_elem -> 'svedOrg' ->> 'ogrn', education_elem -> 'svedOrg' ->>
'ogrnip') as ogrn,
education_elem -> 'svedOrg' ->> 'adresOrg' as educational_institution_address,
education_elem -> 'svedOrg' ->> 'naimOrg' as educational_institution_name,
education_elem -> 'svedOrg' -> 'extend' ->> 'statusOrg' as educational_institution_status_code,
education_elem -> 'svedOrg' -> 'extend' ->> 'vidPodrazdOrg' as educational_organization_separate_unit_type,
education_elem -> 'svedOrg' -> 'extend' ->> 'adresPodrazdOrg' as educational_organization_separate_unit_address,
education_elem ->> 'naimStatus' as student_status,
'ogrnip') as ogrn,
education_elem -> 'svedOrg' ->> 'adresOrg' as educational_institution_address,
education_elem -> 'svedOrg' ->> 'naimOrg' as educational_institution_name,
education_elem -> 'svedOrg' -> 'extend' ->> 'statusOrg' as educational_institution_status_code,
education_elem -> 'svedOrg' -> 'extend' ->> 'vidPodrazdOrg' as educational_organization_separate_unit_type,
education_elem -> 'svedOrg' -> 'extend' ->> 'adresPodrazdOrg' as educational_organization_separate_unit_address,
education_elem ->> 'naimStatus' as student_status,
case
when educational_institution_type = 'Образовательная организация высшего образования'
then education_elem ->> 'nomerKursa'
when educational_institution_type = 'Профессиональная образовательная организация'
then education_elem -> 'extend' ->> 'klass'
when educational_institution_type = 'Общеобразовательная организация' then education_elem ->> 'klass'
end as grade_number,
education_elem ->> 'kodUrovObr' as education_level_code,
end as grade_number,
education_elem ->> 'kodUrovObr' as education_level_code,
to_date(coalesce(education_elem -> 'extend' ->> 'dataPostup', education_elem ->> 'dataPostup'),
'YYYY-MM-DD') as admission_date,
to_date(education_elem ->> 'dataZachisl', 'YYYY-MM-DD') as enrollment_date,
education_elem ->> 'nomPrikazZachisl' as enrollment_order_number,
to_date(education_elem ->> 'dataPrikazZachisl', 'YYYY-MM-DD') as enrollment_order_date,
'YYYY-MM-DD') as admission_date,
to_date(education_elem ->> 'dataZachisl', 'YYYY-MM-DD') as enrollment_date,
education_elem ->> 'nomPrikazZachisl' as enrollment_order_number,
to_date(education_elem ->> 'dataPrikazZachisl', 'YYYY-MM-DD') as enrollment_order_date,
to_date(coalesce(education_elem -> 'extend' ->> 'dataZavershObuch', education_elem ->> 'dataOkonch'),
'YYYY-MM-DD') as completion_date,
education_elem -> 'svedObrProg' ->> 'naimFormaObuch' as education_form,
education_elem -> 'svedObrProg' ->> 'dataNachObuch' as education_program_start_date,
to_date(education_elem -> 'extend' ->> 'dataNachObuch', 'YYYY-MM-DD') as education_start_date,
'YYYY-MM-DD') as completion_date,
education_elem -> 'svedObrProg' ->> 'naimFormaObuch' as education_form,
to_date(education_elem -> 'svedObrProg' ->> 'dataNachObuch', 'YYYY-MM-DD') as education_program_start_date,
to_date(education_elem -> 'extend' ->> 'dataNachObuch', 'YYYY-MM-DD') as education_start_date,
to_date(coalesce(education_elem -> 'svedObrProg' ->> 'planDataOkonch', education_elem ->> 'dataOzhid'),
'YYYY-MM-DD') as planned_completion_date,
education_elem -> 'svedObrProg' -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code,
education_elem -> 'svedObrProg' -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code,
coalesce(education_elem ->> 'svedVosst' &lt;&gt; 'null', false) as recovery_info,
to_date(education_elem -> 'svedVosst' ->> 'dataVosst', 'YYYY-MM-DD') as recovery_date,
'YYYY-MM-DD') as planned_completion_date,
education_elem -> 'svedObrProg' -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code,
education_elem -> 'svedObrProg' -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code,
coalesce(education_elem ->> 'svedVosst' &lt;&gt; 'null', false) as recovery_info,
to_date(education_elem -> 'svedVosst' ->> 'dataVosst', 'YYYY-MM-DD') as recovery_date,
to_date(education_elem -> 'svedVosst' ->> 'dataPrikazVosst',
'YYYY-MM-DD') as recovery_order_date,
education_elem -> 'svedVosst' ->> 'nomPrikazVosst' as recovery_order_number,
coalesce(education_elem ->> 'svedOtchisl' &lt;&gt; 'null', false) as expulsion_info,
to_date(education_elem -> 'svedOtchisl' ->> 'dataOtchisl', 'YYYY-MM-DD') as expulsion_date,
'YYYY-MM-DD') as recovery_order_date,
education_elem -> 'svedVosst' ->> 'nomPrikazVosst' as recovery_order_number,
coalesce(education_elem ->> 'svedOtchisl' &lt;&gt; 'null', false) as expulsion_info,
to_date(education_elem -> 'svedOtchisl' ->> 'dataOtchisl', 'YYYY-MM-DD') as expulsion_date,
to_date(education_elem -> 'svedOtchisl' ->> 'dataPrikazOtchisl',
'YYYY-MM-DD') as expulsion_order_date,
education_elem -> 'svedOtchisl' ->> 'nomPrikazOtchisl' as expulsion_order_number,
coalesce(education_elem ->> 'svedOtpusk' &lt;&gt; 'null', false) as academical_leave_info,
coalesce(education_elem ->> 'prPervObuch' = '1', false) as first_education,
coalesce(education_elem -> 'svedObrProg' ->> 'prVUSpecz' = '1', false) as vus_spec,
coalesce(education_elem ->> 'svedVoenObuch' &lt;&gt; 'null', false) as military_center_education_info,
to_date(education_elem -> 'svedVoenObuch' ->> 'dataNach', 'YYYY-MM-DD') as military_center_education_start_date,
'YYYY-MM-DD') as expulsion_order_date,
education_elem -> 'svedOtchisl' ->> 'nomPrikazOtchisl' as expulsion_order_number,
coalesce(education_elem ->> 'svedOtpusk' &lt;&gt; 'null', false) as academical_leave_info,
coalesce(education_elem ->> 'prPervObuch' = '1', false) as first_education,
coalesce(education_elem -> 'svedObrProg' ->> 'prVUSpecz' = '1', false) as vus_spec,
coalesce(education_elem ->> 'svedVoenObuch' &lt;&gt; 'null', false) as military_center_education_info,
to_date(education_elem -> 'svedVoenObuch' ->> 'dataNach', 'YYYY-MM-DD') as military_center_education_start_date,
to_date(education_elem -> 'svedVoenObuch' ->> 'dataOkonch',
'YYYY-MM-DD') as military_center_education_end_date,
education_elem -> 'svedVoenObuch' -> 'extend' ->> 'srokObuch' as military_center_education_period,
'YYYY-MM-DD') as military_center_education_end_date,
education_elem -> 'svedVoenObuch' -> 'extend' ->> 'srokObuch' as military_center_education_period,
coalesce(education_elem -> 'svedVoenObuch' -> 'extend' ->> 'svedUspeshZaverObuch' = '1',
false) as military_center_completion,
coalesce(education_elem -> 'extend' ->> 'svedVoenKaf' &lt;&gt; 'null', false) as military_department_education_info,
false) as military_center_completion,
coalesce(education_elem -> 'extend' ->> 'svedVoenKaf' &lt;&gt; 'null', false) as military_department_education_info,
to_date(education_elem -> 'extend' -> 'svedVoenKaf' ->> 'dataNach',
'YYYY-MM-DD') as military_department_start_date,
'YYYY-MM-DD') as military_department_start_date,
to_date(education_elem -> 'extend' -> 'svedVoenKaf' ->> 'dataOkonch',
'YYYY-MM-DD') as military_department_end_date,
to_date(academ_elem ->> 'dataNach', 'YYYY-MM-DD') as academ_start_date,
to_date(academ_elem ->> 'dataOkonch', 'YYYY-MM-DD') as academ_end_date,
academ_elem -> 'naimPrichOtpusk' as academ_reason,
true as has_educational_institution_info
FROM filteredData fd;</sql>
'YYYY-MM-DD') as military_department_end_date,
to_date(academ_elem ->> 'dataNach', 'YYYY-MM-DD') as academ_start_date,
to_date(academ_elem ->> 'dataOkonch', 'YYYY-MM-DD') as academ_end_date,
academ_elem -> 'naimPrichOtpusk' as academ_reason,
true as has_educational_institution_info
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(education_arr) AS education_elem
LEFT JOIN LATERAL jsonb_array_elements(
CASE
WHEN jsonb_typeof(education_elem -> 'svedOtpusk') = 'array'
THEN education_elem -> 'svedOtpusk'
ELSE '[]'::jsonb
END) AS academ_elem ON true
where case
when '${MAX_SOURCE_UPDATE_DATE}' &lt;&gt; ''
then to_date(education_elem ->> 'dataSved', 'YYYY-MM-DD') >
to_date('${MAX_SOURCE_UPDATE_DATE}', 'YYYY-MM-DD')
else true end;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>

26
mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl
View file

@ -11,7 +11,7 @@
<parameters>
<parameter>
<name>IDM_ID</name>
<default_value>0e0535ab-5303-43ea-b36a-8c0a2a8d0ed5</default_value>
<default_value>8b100965-1bdf-408e-bf83-86d017b79e61</default_value>
<description/>
</parameter>
</parameters>
@ -287,10 +287,19 @@ and recruitment_id = '${IDM_ID}';
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<sql>INSERT INTO etl.job_execution (job_name, recruitment_id, status, execution_datetime, error_description)
VALUES (
'${JOB_NAME}',
'${IDM_ID}',
'PROCESSING',
current_timestamp,
NULL
)
ON CONFLICT (job_name, recruitment_id)
DO UPDATE SET
status = EXCLUDED.status,
execution_datetime = EXCLUDED.execution_datetime,
error_description = EXCLUDED.error_description;</sql>
<attributes/>
<GUI>
<xloc>640</xloc>
@ -704,7 +713,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<transforms>
<transform>
<name>education_leave_output</name>
<name>upsert_educational_leave</name>
</transform>
<transform>
<name>Update</name>
@ -729,7 +738,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<transforms>
<transform>
<name>citizen_education_output</name>
<name>upsert_edu_inst_info</name>
</transform>
</transforms>
<attributes/>
@ -751,6 +760,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
education_elem,
academ_elem,
@ -836,7 +846,7 @@ SELECT fd.recruit_id,
to_date(coalesce(education_elem -> 'extend' ->> 'dataZavershObuch', education_elem ->> 'dataOkonch'),
'YYYY-MM-DD') as completion_date,
education_elem -> 'svedObrProg' ->> 'naimFormaObuch' as education_form,
education_elem -> 'svedObrProg' ->> 'dataNachObuch' as education_program_start_date,
to_date(education_elem -> 'svedObrProg' ->> 'dataNachObuch', 'YYYY-MM-DD') as education_program_start_date,
to_date(education_elem -> 'extend' ->> 'dataNachObuch', 'YYYY-MM-DD') as education_start_date,
to_date(coalesce(education_elem -> 'svedObrProg' ->> 'planDataOkonch', education_elem ->> 'dataOzhid'),
'YYYY-MM-DD') as planned_completion_date,

47
mappings/info_recruits/citizen_tables/education/educational_institution_learning/recruitment_five_flow_delta.hpl
View file

@ -7,7 +7,13 @@
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<pipeline_status>0</pipeline_status>
<parameters>
<parameter>
<name>JOB_NAME</name>
<default_value>educational_institution_learning_job</default_value>
<description/>
</parameter>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
@ -21,13 +27,13 @@
</notepads>
<order>
<hop>
<from>Table input</from>
<to>citizen_education_flow_delta.hpl</to>
<from>get_max_source_update_date</from>
<to>Table input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>get_max_source_update_date</from>
<to>Table input</to>
<from>Table input</from>
<to>citizen_education_flow_delta.hpl</to>
<enabled>Y</enabled>
</hop>
<hop>
@ -65,24 +71,27 @@
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_source_update_date</lookup>
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id,
? max_source_update_date
<sql>WITH mud AS (SELECT recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS', 'DELTA_SUCCESS')
GROUP BY recruitment_id)
SELECT r.idm_id
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
AND ri.updated_at > mud.max_upd_date
union
select r2.idm_id
from recruitment r2
join etl.job_execution je
on r2.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -385,7 +394,7 @@ FROM ervu_dashboard.recruitment r
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(source_update_date)
from ervu_dashboard.education_institution_learning;</sql>
from ervu_dashboard.educational_institution_learning;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>

45
mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl
View file

@ -50,11 +50,6 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_education_diploma_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>foerign_diploma_input</from>
<to>Identify last row in a stream</to>
@ -80,6 +75,16 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_education_diploma_output</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -101,6 +106,28 @@
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1280</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -163,7 +190,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1248</xloc>
<xloc>1488</xloc>
<yloc>256</yloc>
</GUI>
</transform>
@ -245,7 +272,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1248</xloc>
<xloc>1488</xloc>
<yloc>432</yloc>
</GUI>
</transform>
@ -415,14 +442,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>foreign_diploma</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -446,7 +470,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr
FROM recruits_info ri

172
mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl
View file

@ -27,7 +27,7 @@
</hop>
<hop>
<from>Create job execution record</from>
<to>foerign_diploma_input</to>
<to>foreign_diploma_input</to>
<enabled>Y</enabled>
</hop>
<hop>
@ -46,7 +46,7 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>foerign_diploma_input</from>
<from>foreign_diploma_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
@ -56,13 +56,33 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>foreign_diploma_upsert</from>
<to>Filter rows</to>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<from>Sort rows</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>foreign_diploma_upsert</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
@ -82,10 +102,32 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1280</xloc>
<xloc>1472</xloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1232</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -117,7 +159,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1056</xloc>
<xloc>1280</xloc>
<yloc>688</yloc>
</GUI>
</transform>
@ -148,7 +190,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1280</xloc>
<xloc>1472</xloc>
<yloc>320</yloc>
</GUI>
</transform>
@ -233,7 +275,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1280</xloc>
<xloc>1472</xloc>
<yloc>496</yloc>
</GUI>
</transform>
@ -256,7 +298,76 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>foerign_diploma_input</name>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>768</xloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_foreign_diploma</name>
<rename>has_foreign_diploma</rename>
</value>
</lookup>
<skip_lookup>Y</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>992</xloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
<name>foreign_diploma_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
@ -268,22 +379,19 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
diploma_elem
ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr
FROM recruits_info ri
CROSS JOIN LATERAL jsonb_array_elements(
ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr') AS diploma_elem
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr') = 'array'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
and to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'
)
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
SELECT fd.recruit_id,
diploma_elem ->> 'id' AS source_id,
to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
diploma_elem ->> 'kodUrObr' as education_level_code,
diploma_elem ->> 'naimUchZaved' as educational_institution_name,
diploma_elem ->> 'naimInoDok' as foreign_doc_name,
diploma_elem ->> 'serNomDok' as ser_num_doc,
diploma_elem ->> 'naimInoDok' as foreign_doc_name,
diploma_elem ->> 'serNomDok' as ser_num_doc,
diploma_elem ->> 'regNomDok' as reg_num,
diploma_elem ->> 'nomSved' as recognition_certificate_number,
to_date(diploma_elem ->> 'dataPriz', 'YYYY-MM-DD') as recognition_date,
@ -291,8 +399,15 @@ SELECT fd.recruit_id,
diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code,
diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code,
diploma_elem ->> 'kodOKSM' as oksm_code,
coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec
FROM filteredData fd;</sql>
coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec,
true as has_foreign_diploma
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem
where case
when '${MAX_SOURCE_UPDATE_DATE}' &lt;&gt; ''
then to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') >
to_date('MAX_SOURCE_UPDATE_DATE', 'YYYY-MM-DD')
else true end;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -318,7 +433,6 @@ FROM filteredData fd;</sql>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>foreign_diploma</table>
@ -401,11 +515,23 @@ FROM filteredData fd;</sql>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1056</xloc>
<xloc>992</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>foreign_diploma_upsert</source_transform>
<target_transform>Change job status on error</target_transform>

173
mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl
View file

@ -50,11 +50,6 @@
<to>foreign_diploma_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>foreign_diploma_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>foreign_diploma_upsert</from>
<to>Change job status on error</to>
@ -65,6 +60,31 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>foreign_diploma_upsert</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -81,9 +101,31 @@
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1488</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1248</xloc>
<yloc>624</yloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
@ -117,8 +159,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1024</xloc>
<yloc>624</yloc>
<xloc>1232</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
@ -148,7 +190,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1248</xloc>
<xloc>1488</xloc>
<yloc>272</yloc>
</GUI>
</transform>
@ -166,18 +208,28 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET
status = 'PROCESSING',
execution_datetime = current_timestamp,
error_description = NULL
where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql>
<sql>INSERT INTO etl.job_execution (job_name, recruitment_id, status, execution_datetime, error_description)
VALUES (
'${JOB_NAME}',
'${IDM_ID}',
'PROCESSING',
current_timestamp,
NULL
)
ON CONFLICT (job_name, recruitment_id)
DO UPDATE SET
status = EXCLUDED.status,
execution_datetime = EXCLUDED.execution_datetime,
error_description = EXCLUDED.error_description;</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>320</xloc>
@ -233,7 +285,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1248</xloc>
<xloc>1488</xloc>
<yloc>448</yloc>
</GUI>
</transform>
@ -255,6 +307,75 @@ and recruitment_id = '${IDM_ID}';</sql>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>736</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_foreign_diploma</name>
<rename>has_foreign_diploma</rename>
</value>
</lookup>
<skip_lookup>Y</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>960</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>foerign_diploma_input</name>
<type>TableInput</type>
@ -268,7 +389,6 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr
FROM recruits_info ri
@ -289,7 +409,8 @@ SELECT fd.recruit_id,
diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code,
diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code,
diploma_elem ->> 'kodOKSM' as oksm_code,
coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec
coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec,
true as has_foreign_diploma
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem;
</sql>
@ -400,11 +521,23 @@ FROM filteredData fd
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1024</xloc>
<xloc>960</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>foreign_diploma_upsert</source_transform>
<target_transform>Change job status on error</target_transform>

38
mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_delta.hpl
View file

@ -65,24 +65,28 @@
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_source_update_date</lookup>
<sql>SELECT
r.idm_id,
? max_source_update_date
<sql>WITH mud AS (SELECT recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS', 'DELTA_SUCCESS')
GROUP BY recruitment_id)
SELECT r.idm_id
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = '${JOB_NAME}'
JOIN recruits_info ri
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > (
SELECT MAX(execution_datetime)
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND recruitment_id = r.idm_id
)
where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
<variables_active>N</variables_active>
AND ri.updated_at > mud.max_upd_date
union
select r2.idm_id
from recruitment r2
join etl.job_execution je
on r2.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>608</xloc>
@ -383,7 +387,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(source_update_date)
<sql>select max(source_update_date) as max_source_update_date
from ervu_dashboard.foreign_diploma;</sql>
<variables_active>N</variables_active>
<attributes/>

4
mappings/info_recruits/citizen_tables/education/foreign_diploma/recruitment_five_flow_repeat.hpl
View file

@ -64,10 +64,10 @@
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
AND je.job_name = 'foreign_diploma_job'
AND je.job_name = '${JOB_NAME}'
WHERE je.id IS NULL
OR je.status IN ('ERROR', 'PROCESSING');</sql>
<variables_active>N</variables_active>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>624</xloc>

64
mappings/info_recruits/citizen_tables/education/russian_diploma/constants/russian_diploma_constants_flow.hpl
View file

@ -42,7 +42,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>nsi</connection>
<connection>ervu_nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT
@ -59,34 +59,6 @@ WHERE code = 'educationNaimVidDocMinobr';</sql>
<yloc>272</yloc>
</GUI>
</transform>
<transform>
<name>minprosv_doc_type_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT
key as code,
value ->> 'value' as value,
NOT hidden as actual,
updated_at as update_date
FROM classifier_records
WHERE code = 'educationNaimVidDocMinprosv';</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>384</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>minobr_doc_type_output</name>
<type>TableOutput</type>
@ -122,14 +94,11 @@ WHERE code = 'educationNaimVidDocMinprosv';</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>minobr_doc_type</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -140,6 +109,34 @@ WHERE code = 'educationNaimVidDocMinprosv';</sql>
<yloc>272</yloc>
</GUI>
</transform>
<transform>
<name>minprosv_doc_type_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu_nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT
key as code,
value ->> 'value' as value,
NOT hidden as actual,
updated_at as update_date
FROM classifier_records
WHERE code = 'educationNaimVidDocMinprosv';</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>384</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>minprosv_doc_type_output</name>
<type>TableOutput</type>
@ -175,14 +172,11 @@ WHERE code = 'educationNaimVidDocMinprosv';</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>minprosv_doc_type</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>

316
mappings/info_recruits/citizen_tables/education/russian_diploma/constants/russian_diploma_constants_flow_repeat.hpl
View file

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<pipeline>
<info>
<name>foreign_diploma_constants_flow</name>
<name>russian_diploma_constants_flow_repeat</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
@ -41,164 +41,6 @@
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>minobr_doc_type_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_minobr_doc_type_update_date</lookup>
<sql>SELECT
key as code,
value ->> 'value' as value,
NOT hidden as actual
FROM classifier_records
WHERE code = 'educationNaimVidDocMinobr'
AND updated_at > ?;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>720</xloc>
<yloc>256</yloc>
</GUI>
</transform>
<transform>
<name>minobr_doc_type_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>code</field>
<name>code</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>minobr_doc_type</table>
<value>
<name>code</name>
<rename>code</rename>
<update>N</update>
</value>
<value>
<name>value</name>
<rename>value</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>update_date</name>
<rename>update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>992</xloc>
<yloc>256</yloc>
</GUI>
</transform>
<transform>
<name>minprosv_doc_type_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_minprosv_doc_type_date</lookup>
<sql>SELECT
key as code,
value ->> 'value' as value,
NOT hidden as actual
FROM classifier_records
WHERE code = 'educationNaimVidDocMinprosv'
AND updated_at > ?;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>720</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>minprosv_doc_type_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>code</field>
<name>code</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>minprosv_doc_type</table>
<value>
<name>code</name>
<rename>code</rename>
<update>N</update>
</value>
<value>
<name>value</name>
<rename>value</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>update_date</name>
<rename>update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>992</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>get_max_minobr_doc_type_update_date</name>
<type>TableInput</type>
@ -249,6 +91,162 @@ FROM minprosv_doc_type;
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>minobr_doc_type_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu_nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_minobr_doc_type_update_date</lookup>
<sql>SELECT
key as code,
value ->> 'value' as value,
NOT hidden as actual
FROM classifier_records
WHERE code = 'educationNaimVidDocMinobr'
AND updated_at > ?;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>720</xloc>
<yloc>256</yloc>
</GUI>
</transform>
<transform>
<name>minobr_doc_type_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>code</field>
<name>code</name>
</key>
<schema>ervu_dashboard</schema>
<table>minobr_doc_type</table>
<value>
<name>code</name>
<rename>code</rename>
<update>N</update>
</value>
<value>
<name>value</name>
<rename>value</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>update_date</name>
<rename>update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>992</xloc>
<yloc>256</yloc>
</GUI>
</transform>
<transform>
<name>minprosv_doc_type_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu_nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_minprosv_doc_type_date</lookup>
<sql>SELECT
key as code,
value ->> 'value' as value,
NOT hidden as actual
FROM classifier_records
WHERE code = 'educationNaimVidDocMinprosv'
AND updated_at > ?;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>720</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>minprosv_doc_type_upsert</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>code</field>
<name>code</name>
</key>
<schema>ervu_dashboard</schema>
<table>minprosv_doc_type</table>
<value>
<name>code</name>
<rename>code</rename>
<update>N</update>
</value>
<value>
<name>value</name>
<rename>value</rename>
<update>Y</update>
</value>
<value>
<name>actual</name>
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>update_date</name>
<rename>update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>992</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>

146
mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl
View file

@ -56,13 +56,33 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_education_diploma_output</from>
<to>Filter rows</to>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<from>Sort rows</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_education_diploma_output</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
@ -82,7 +102,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1152</xloc>
<xloc>1328</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -117,7 +137,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>928</xloc>
<xloc>1088</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -148,7 +168,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1152</xloc>
<xloc>1328</xloc>
<yloc>224</yloc>
</GUI>
</transform>
@ -230,7 +250,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1152</xloc>
<xloc>1328</xloc>
<yloc>400</yloc>
</GUI>
</transform>
@ -252,6 +272,75 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_russian_diploma </name>
<rename>has_russian_diploma </rename>
</value>
</lookup>
<skip_lookup>Y</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>832</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>citizen_education_diploma_output</name>
<type>TableOutput</type>
@ -335,14 +424,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>russian_diploma</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -366,7 +452,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr
FROM recruits_info ri
@ -388,7 +473,8 @@ SELECT fd.recruit_id,
diploma_elem -> 'svedSpecz' ->> 'naimSpecz' as speciality_name,
diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code,
diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code,
coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec
coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec,
true as has_russian_diploma
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem;</sql>
<variables_active>Y</variables_active>
@ -398,7 +484,41 @@ FROM filteredData fd
<yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1152</xloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>citizen_education_diploma_output</source_transform>
<target_transform>Change job status on error</target_transform>

177
mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl
View file

@ -7,7 +7,23 @@
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<pipeline_status>0</pipeline_status>
<parameters>
<parameter>
<name>IDM_ID</name>
<default_value>e08dd467-2d0d-4217-880d-9938dbb59afe</default_value>
<description/>
</parameter>
<parameter>
<name>JOB_NAME</name>
<default_value>russian_diploma_job</default_value>
<description/>
</parameter>
<parameter>
<name>MAX_SOURCE_UPDATE_DATE</name>
<default_value>2025/04/08 00:00:00.000</default_value>
<description/>
</parameter>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
@ -56,13 +72,33 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Filter rows</to>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<from>Sort rows</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
@ -82,10 +118,32 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1392</xloc>
<xloc>1472</xloc>
<yloc>656</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1312</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -117,7 +175,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1168</xloc>
<xloc>1248</xloc>
<yloc>656</yloc>
</GUI>
</transform>
@ -148,7 +206,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1392</xloc>
<xloc>1472</xloc>
<yloc>288</yloc>
</GUI>
</transform>
@ -233,7 +291,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1392</xloc>
<xloc>1472</xloc>
<yloc>464</yloc>
</GUI>
</transform>
@ -360,10 +418,79 @@ and recruitment_id = '${IDM_ID}';</sql>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1168</xloc>
<xloc>1120</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>880</xloc>
<yloc>656</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_russian_diploma</name>
<rename>has_russian_diploma</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>656</yloc>
</GUI>
</transform>
<transform>
<name>education_input</name>
<type>TableInput</type>
@ -377,12 +504,14 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr
<sql>
WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' as diploma_arr
FROM recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedRObr' -> 'robr') = 'array'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '0e0535ab-5303-43ea-b36a-8c0a2a8d0ed5'
)
SELECT fd.recruit_id,
diploma_elem ->> 'id' AS source_id,
to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
@ -398,10 +527,16 @@ SELECT fd.recruit_id,
diploma_elem -> 'svedSpecz' ->> 'naimSpecz' as speciality_name,
diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code,
diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code,
coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec
coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec,
true as has_russian_diploma
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem
where to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}';</sql>
where case
when '${MAX_SOURCE_UPDATE_DATE}' &lt;&gt; ''
then to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') >
to_date('MAX_SOURCE_UPDATE_DATE', 'YYYY-MM-DD')
else true end;
</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -422,6 +557,18 @@ where to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
</transform_error_handling>
<attributes/>
</pipeline>

175
mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl
View file

@ -7,7 +7,13 @@
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<pipeline_status>0</pipeline_status>
<parameters>
<parameter>
<name>IDM_ID</name>
<default_value>591263f8-c337-418a-9cee-d23a0b9c0a54</default_value>
<description/>
</parameter>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
@ -56,13 +62,33 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Filter rows</to>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<from>Sort rows</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
@ -82,8 +108,30 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1136</xloc>
<yloc>544</yloc>
<xloc>1216</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
@ -117,8 +165,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>912</xloc>
<yloc>544</yloc>
<xloc>992</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
@ -148,7 +196,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1136</xloc>
<xloc>1216</xloc>
<yloc>208</yloc>
</GUI>
</transform>
@ -171,13 +219,19 @@ and recruitment_id = '${IDM_ID}';
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}')
<sql>INSERT INTO etl.job_execution (job_name, recruitment_id, status, execution_datetime, error_description)
VALUES (
'${JOB_NAME}',
'${IDM_ID}',
'PROCESSING',
current_timestamp,
NULL
)
ON CONFLICT (job_name, recruitment_id)
DO UPDATE
SET status = 'PROCESSING',
error_description = null,
execution_datetime = current_timestamp;</sql>
DO UPDATE SET
status = EXCLUDED.status,
execution_datetime = EXCLUDED.execution_datetime,
error_description = EXCLUDED.error_description;</sql>
<attributes/>
<GUI>
<xloc>208</xloc>
@ -233,7 +287,7 @@ SET status = 'PROCESSING',
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1136</xloc>
<xloc>1216</xloc>
<yloc>384</yloc>
</GUI>
</transform>
@ -360,10 +414,81 @@ SET status = 'PROCESSING',
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>912</xloc>
<xloc>864</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_russian_diploma</name>
<rename>has_russian_diploma</rename>
</value>
</lookup>
<skip_lookup>Y</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>816</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>education_input</name>
<type>TableInput</type>
@ -377,7 +502,6 @@ SET status = 'PROCESSING',
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr
FROM recruits_info ri
@ -399,7 +523,8 @@ SELECT fd.recruit_id,
diploma_elem -> 'svedSpecz' ->> 'naimSpecz' as speciality_name,
diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code,
diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code,
coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec
coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec,
true as has_russian_diploma
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem;</sql>
<variables_active>Y</variables_active>
@ -422,6 +547,18 @@ FROM filteredData fd
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
</transform_error_handling>
<attributes/>
</pipeline>

3
mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow.hpl
View file

@ -57,8 +57,9 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>SELECT
idm_id
FROM ervu_dashboard.recruitment;</sql>

65
mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow_delta.hpl
View file

@ -62,27 +62,31 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_source_update_date</lookup>
<sql>SELECT
r.idm_id,
? max_source_update_date
<sql>WITH mud AS (SELECT recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS', 'DELTA_SUCCESS')
GROUP BY recruitment_id)
SELECT r.idm_id
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = '${JOB_NAME}'
JOIN recruits_info ri
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > (
SELECT MAX(execution_datetime)
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND recruitment_id = r.idm_id
)
where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
<variables_active>N</variables_active>
AND ri.updated_at > mud.max_upd_date
union
select r2.idm_id
from recruitment r2
join etl.job_execution je
on r2.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>688</xloc>
@ -103,7 +107,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(source_update_date)
<sql>select max(source_update_date) as max_source_update_date
from ervu_dashboard.russian_diploma;</sql>
<variables_active>N</variables_active>
<attributes/>
@ -136,11 +140,6 @@ from ervu_dashboard.russian_diploma;</sql>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>russian_diploma_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
@ -197,11 +196,6 @@ from ervu_dashboard.russian_diploma;</sql>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>russian_diploma_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
@ -258,11 +252,6 @@ from ervu_dashboard.russian_diploma;</sql>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>russian_diploma_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
@ -319,11 +308,6 @@ from ervu_dashboard.russian_diploma;</sql>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>russian_diploma_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
@ -380,11 +364,6 @@ from ervu_dashboard.russian_diploma;</sql>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>russian_diploma_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>

16
mappings/info_recruits/citizen_tables/education/russian_diploma/recruitment_five_flow_repeat.hpl
View file

@ -57,17 +57,17 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT r.idm_id AS recruitment_id
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
AND je.job_name = 'russian_diploma_job'
AND je.job_name = '${JOB_NAME}'
WHERE je.id IS NULL
OR je.status IN ('ERROR', 'PROCESSING');</sql>
<variables_active>N</variables_active>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>368</xloc>
@ -94,7 +94,7 @@ WHERE je.id IS NULL
<group_time/>
<parameters>
<variable_mapping>
<variable/>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<input/>
</variable_mapping>
@ -145,7 +145,7 @@ WHERE je.id IS NULL
<group_time/>
<parameters>
<variable_mapping>
<variable/>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<input/>
</variable_mapping>
@ -196,7 +196,7 @@ WHERE je.id IS NULL
<group_time/>
<parameters>
<variable_mapping>
<variable/>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<input/>
</variable_mapping>
@ -247,7 +247,7 @@ WHERE je.id IS NULL
<group_time/>
<parameters>
<variable_mapping>
<variable/>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<input/>
</variable_mapping>
@ -298,7 +298,7 @@ WHERE je.id IS NULL
<group_time/>
<parameters>
<variable_mapping>
<variable/>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<input/>
</variable_mapping>

66
mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_delta.hpl
View file

@ -7,6 +7,7 @@
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<pipeline_status>0</pipeline_status>
<parameters>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
@ -20,11 +21,6 @@
<notepads>
</notepads>
<order>
<hop>
<from>Block until transforms finish</from>
<to>election_candidate_flag_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<to>Abort</to>
@ -45,11 +41,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
@ -76,7 +67,17 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>election_candidate_flag_update</from>
<from>Identify last row in a stream</from>
<to>election_candidate_flag_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>election_candidate_upsert</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
@ -97,7 +98,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1552</xloc>
<xloc>1296</xloc>
<yloc>688</yloc>
</GUI>
</transform>
@ -114,12 +115,12 @@
</partitioning>
<transforms>
<transform>
<name>deputy_upsert</name>
<name>election_candidate_upsert</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>896</xloc>
<xloc>1104</xloc>
<yloc>416</yloc>
</GUI>
</transform>
@ -146,7 +147,7 @@
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
SET status = 'DELTA_ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
@ -154,7 +155,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1136</xloc>
<xloc>896</xloc>
<yloc>688</yloc>
</GUI>
</transform>
@ -178,14 +179,14 @@ and recruitment_id = '${IDM_ID}';
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS'
SET status = 'DELTA_SUCCESS'
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1552</xloc>
<xloc>1296</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -208,10 +209,13 @@ and recruitment_id = '${IDM_ID}';
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<sql>UPDATE etl.job_execution
SET
status = 'DELTA_PROCESSING',
execution_datetime = current_timestamp,
error_description = NULL
where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql>
<attributes/>
<GUI>
<xloc>208</xloc>
@ -267,7 +271,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1552</xloc>
<xloc>1296</xloc>
<yloc>416</yloc>
</GUI>
</transform>
@ -325,8 +329,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1136</xloc>
<yloc>416</yloc>
<xloc>624</xloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
@ -346,13 +350,16 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem
FROM recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedKandidat' ->> 'kandidat' &lt;&gt; 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
and to_date(ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = 'b5265855-1972-4ff5-8b5e-0f746127fa70'
and case when '${MAX_SOURCE_UPDATE_DATE}' &lt;&gt; ''
then to_date(ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' ->> 'dataSved', 'YYYY-MM-DD') >
to_date('MAX_SOURCE_UPDATE_DATE', 'YYYY-MM-DD')
else true end
)
SELECT fd.recruit_id,
deputy_elem ->> 'id' AS source_id,
to_date(deputy_elem ->> 'dataSved', 'YYYY-MM-DD') AS source_update_date,
deputy_elem -> 'extend' ->> 'unikIdentGrazhd' AS citizen_unique_id,
deputy_elem -&gt; 'extend' ->> 'unikIdentGrazhd' AS citizen_unique_id,
deputy_elem ->> 'priznakAktualNaim' AS actual_status_name,
to_date(deputy_elem ->> 'dataReg', 'YYYY-MM-DD') AS registration_date,
deputy_elem ->> 'prichinaSnyatNaim' AS termination_reason,
@ -384,7 +391,6 @@ FROM filteredData fd;</sql>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>election_candidate</table>
@ -433,7 +439,7 @@ FROM filteredData fd;</sql>
<attributes/>
<GUI>
<xloc>896</xloc>
<yloc>688</yloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform_error_handling>

52
mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_flow.hpl
View file

@ -7,7 +7,13 @@
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<pipeline_status>0</pipeline_status>
<parameters>
<parameter>
<name>IDM_ID</name>
<default_value>009f24ac-0338-4982-aa66-0f60e6ada467</default_value>
<description/>
</parameter>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
@ -25,21 +31,11 @@
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>election_candidate_flag_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Create job execution record</from>
<to>election_candidate_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
@ -55,11 +51,6 @@
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>election_candidate_flag_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>election_candidate_output</to>
@ -80,6 +71,21 @@
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>election_candidate_output</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>election_candidate_flag_update</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -97,7 +103,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1504</xloc>
<xloc>1248</xloc>
<yloc>592</yloc>
</GUI>
</transform>
@ -114,12 +120,13 @@
</partitioning>
<transforms>
<transform>
<name>deputy_output</name>
<CopyNr/>
<name>election_candidate_flag_update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>976</xloc>
<xloc>1248</xloc>
<yloc>416</yloc>
</GUI>
</transform>
@ -154,7 +161,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1216</xloc>
<xloc>976</xloc>
<yloc>592</yloc>
</GUI>
</transform>
@ -320,8 +327,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>416</yloc>
<xloc>704</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
@ -337,7 +344,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem
FROM recruits_info ri
@ -424,7 +430,7 @@ FROM filteredData fd;</sql>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>592</yloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform_error_handling>

131
mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_repeat.hpl
View file

@ -7,7 +7,18 @@
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<pipeline_status>0</pipeline_status>
<parameters>
<parameter>
<name>IDM_ID</name>
<default_value>b9ed42f3-8a58-48f2-8e2f-e4d64a3a6869</default_value>
<description/>
</parameter>
<parameter>
<name>JOB_NAME</name>
<default_value>election_candidate_job</default_value>
<description/>
</parameter>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
@ -20,11 +31,6 @@
<notepads>
</notepads>
<order>
<hop>
<from>Block until transforms finish</from>
<to>election_candidate_flag_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<to>Abort</to>
@ -40,11 +46,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
@ -55,11 +56,6 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>election_candidate_flag_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>election_candidate_upsert</to>
@ -80,6 +76,21 @@
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>election_candidate_flag_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>election_candidate_upsert</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -98,7 +109,7 @@
<attributes/>
<GUI>
<xloc>1584</xloc>
<yloc>784</yloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
@ -114,12 +125,12 @@
</partitioning>
<transforms>
<transform>
<name>deputy_upsert</name>
<name>election_candidate_flag_update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1056</xloc>
<xloc>1408</xloc>
<yloc>512</yloc>
</GUI>
</transform>
@ -154,8 +165,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1296</xloc>
<yloc>784</yloc>
<xloc>1152</xloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
@ -306,13 +317,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
@ -325,7 +334,43 @@ and recruitment_id = '${IDM_ID}';</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1296</xloc>
<xloc>784</xloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
<name>election_candidate_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem
FROM recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedKandidat' ->> 'kandidat' &lt;&gt; 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
SELECT fd.recruit_id,
deputy_elem ->> 'id' AS source_id,
to_date(deputy_elem ->&gt; 'dataSved', 'YYYY-MM-DD') AS source_update_date,
deputy_elem -> 'extend' ->> 'unikIdentGrazhd' AS citizen_unique_id,
deputy_elem ->> 'priznakAktualNaim' AS actual_status_name,
to_date(deputy_elem ->> 'dataReg', 'YYYY-MM-DD') AS registration_date,
deputy_elem ->> 'prichinaSnyatNaim' AS termination_reason,
to_date(deputy_elem ->> 'dataSnyatKandidat', 'YYYY-MM-DD') AS termination_date,
true as has_election_candidate_info
FROM filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>576</xloc>
<yloc>512</yloc>
</GUI>
</transform>
@ -347,7 +392,6 @@ and recruitment_id = '${IDM_ID}';</sql>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>election_candidate</table>
@ -395,44 +439,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1056</xloc>
<yloc>784</yloc>
</GUI>
</transform>
<transform>
<name>election_candidate_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem
FROM recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedKandidat' ->> 'kandidat' &lt;&gt; 'null'
--and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
SELECT fd.recruit_id,
deputy_elem ->> 'id' AS source_id,
to_date(deputy_elem ->> 'dataSved', 'YYYY-MM-DD') AS source_update_date,
deputy_elem -> 'extend' ->> 'unikIdentGrazhd' AS citizen_unique_id,
deputy_elem ->> 'priznakAktualNaim' AS actual_status_name,
to_date(deputy_elem ->> 'dataReg', 'YYYY-MM-DD') AS registration_date,
deputy_elem ->> 'prichinaSnyatNaim' AS termination_reason,
to_date(deputy_elem ->> 'dataSnyatKandidat', 'YYYY-MM-DD') AS termination_date,
true as has_election_candidate_info
FROM filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>576</xloc>
<xloc>1152</xloc>
<yloc>512</yloc>
</GUI>
</transform>

3
mappings/info_recruits/citizen_tables/election_candidate/recruitment_five_flow.hpl
View file

@ -57,8 +57,9 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>SELECT
idm_id
FROM ervu_dashboard.recruitment;</sql>

39
mappings/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_delta.hpl
View file

@ -7,7 +7,13 @@
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<pipeline_status>0</pipeline_status>
<parameters>
<parameter>
<name>JOB_NAME</name>
<default_value>election_candidate_job</default_value>
<description/>
</parameter>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
@ -62,27 +68,30 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_source_update_date</lookup>
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id,
? max_source_update_date
<sql>WITH mud AS (SELECT recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS', 'DELTA_SUCCESS')
GROUP BY recruitment_id)
SELECT r.idm_id
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
AND ri.updated_at > mud.max_upd_date
union
select r2.idm_id
from recruitment r2
join etl.job_execution je
on r2.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>

36
mappings/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_repeat.hpl
View file

@ -22,27 +22,27 @@
<order>
<hop>
<from>Table input</from>
<to>recruitment_five_flow_repeat.hpl</to>
<to>election_candidate_repeat.hpl</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>recruitment_five_flow_repeat.hpl 2</to>
<to>election_candidate_repeat.hpl 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>recruitment_five_flow_repeat.hpl 3</to>
<to>election_candidate_repeat.hpl 3</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>recruitment_five_flow_repeat.hpl 4</to>
<to>election_candidate_repeat.hpl 4</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>recruitment_five_flow_repeat.hpl 5</to>
<to>election_candidate_repeat.hpl 5</to>
<enabled>Y</enabled>
</hop>
</order>
@ -57,17 +57,17 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT r.idm_id AS recruitment_id
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
AND je.job_name = 'disability_job'
AND je.job_name = '${JOB_NAME}'
WHERE je.id IS NULL
OR je.status IN ('ERROR', 'PROCESSING');</sql>
<variables_active>N</variables_active>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>656</xloc>
@ -75,7 +75,7 @@ WHERE je.id IS NULL
</GUI>
</transform>
<transform>
<name>recruitment_five_flow_repeat.hpl</name>
<name>election_candidate_repeat.hpl</name>
<type>PipelineExecutor</type>
<description/>
<distribute>Y</distribute>
@ -86,7 +86,7 @@ WHERE je.id IS NULL
<schema_name/>
</partitioning>
<run_configuration>local</run_configuration>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_repeat.hpl</filename>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_repeat.hpl</filename>
<filenameInField>N</filenameInField>
<filenameField/>
<group_size>1</group_size>
@ -126,7 +126,7 @@ WHERE je.id IS NULL
</GUI>
</transform>
<transform>
<name>recruitment_five_flow_repeat.hpl 2</name>
<name>election_candidate_repeat.hpl 2</name>
<type>PipelineExecutor</type>
<description/>
<distribute>Y</distribute>
@ -137,7 +137,7 @@ WHERE je.id IS NULL
<schema_name/>
</partitioning>
<run_configuration>local</run_configuration>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_repeat.hpl</filename>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_repeat.hpl</filename>
<filenameInField>N</filenameInField>
<filenameField/>
<group_size>1</group_size>
@ -177,7 +177,7 @@ WHERE je.id IS NULL
</GUI>
</transform>
<transform>
<name>recruitment_five_flow_repeat.hpl 3</name>
<name>election_candidate_repeat.hpl 3</name>
<type>PipelineExecutor</type>
<description/>
<distribute>Y</distribute>
@ -188,7 +188,7 @@ WHERE je.id IS NULL
<schema_name/>
</partitioning>
<run_configuration>local</run_configuration>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_repeat.hpl</filename>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_repeat.hpl</filename>
<filenameInField>N</filenameInField>
<filenameField/>
<group_size>1</group_size>
@ -228,7 +228,7 @@ WHERE je.id IS NULL
</GUI>
</transform>
<transform>
<name>recruitment_five_flow_repeat.hpl 4</name>
<name>election_candidate_repeat.hpl 4</name>
<type>PipelineExecutor</type>
<description/>
<distribute>Y</distribute>
@ -239,7 +239,7 @@ WHERE je.id IS NULL
<schema_name/>
</partitioning>
<run_configuration>local</run_configuration>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_repeat.hpl</filename>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_repeat.hpl</filename>
<filenameInField>N</filenameInField>
<filenameField/>
<group_size>1</group_size>
@ -279,7 +279,7 @@ WHERE je.id IS NULL
</GUI>
</transform>
<transform>
<name>recruitment_five_flow_repeat.hpl 5</name>
<name>election_candidate_repeat.hpl 5</name>
<type>PipelineExecutor</type>
<description/>
<distribute>Y</distribute>
@ -290,7 +290,7 @@ WHERE je.id IS NULL
<schema_name/>
</partitioning>
<run_configuration>local</run_configuration>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/recruitment_five_flow_repeat.hpl</filename>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_repeat.hpl</filename>
<filenameInField>N</filenameInField>
<filenameField/>
<group_size>1</group_size>

7
mappings/info_recruits/citizen_tables/medicine/disability/disability_job.hwf
View file

@ -41,7 +41,7 @@
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/support/check_if_job_execution_exists.hpl</filename>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_job_execution_exists.hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -136,7 +136,9 @@
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/medicine/disability/support/check_if_need_to_repeat.hpl</filename>
<filename>${PROJECT_HOME}/info_recruits/citizen_tables/support/check_if_need_to_repeat.hpl</filename>
<logext/>
<logfile/>
<loglevel>Basic</loglevel>
<parameters>
<pass_all_parameters>Y</pass_all_parameters>
@ -207,7 +209,6 @@
</field>
</fields>
<file_variable_type>CURRENT_WORKFLOW</file_variable_type>
<filename/>
<replacevars>N</replacevars>
<parallel>N</parallel>
<xloc>272</xloc>

14
mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow.hpl
View file

@ -7,7 +7,18 @@
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<pipeline_status>0</pipeline_status>
<parameters>
<parameter>
<name>IDM_ID</name>
<default_value>c0c163d2-306e-4108-9fe4-dbb6afd70c82</default_value>
<description/>
</parameter>
<parameter>
<name>JOB_NAME</name>
<default_value>disability_job</default_value>
<description/>
</parameter>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
@ -275,7 +286,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' AS invalid_elem
FROM recruits_info ri
@ -384,13 +394,11 @@ FROM filteredData fd;</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

56
mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_delta.hpl
View file

@ -7,7 +7,13 @@
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<pipeline_status>0</pipeline_status>
<parameters>
<parameter>
<name>IDM_ID</name>
<default_value>42c79094-4767-4898-9c45-e2453d0665c7</default_value>
<description/>
</parameter>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
@ -45,11 +51,6 @@
<to>disability_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>disability_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>disability_upsert</from>
<to>Change job status on error</to>
@ -75,6 +76,11 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>disability_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -92,7 +98,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1936</xloc>
<xloc>1968</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -127,7 +133,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1712</xloc>
<xloc>1744</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -158,7 +164,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1936</xloc>
<xloc>1968</xloc>
<yloc>208</yloc>
</GUI>
</transform>
@ -207,7 +213,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>1232</xloc>
<xloc>1264</xloc>
<yloc>208</yloc>
</GUI>
</transform>
@ -243,7 +249,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1936</xloc>
<xloc>1968</xloc>
<yloc>384</yloc>
</GUI>
</transform>
@ -261,7 +267,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1232</xloc>
<xloc>1264</xloc>
<yloc>384</yloc>
</GUI>
</transform>
@ -278,12 +284,18 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' AS invalid_elem
FROM recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' &lt;&gt; 'null'
WHERE ri.info -> 'svedFL' -> 'svedInvalid' ->> 'invalid' &lt;&gt; 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
and to_date(ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'
and case
when '${MAX_SOURCE_UPDATE_DATE}' &lt;&gt; ''
then to_date(ri.info -> ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' ->> 'dataSved',
'YYYY-MM-DD') >
to_date('MAX_SOURCE_UPDATE_DATE', 'YYYY-MM-DD')
else true end
)
SELECT fd.recruit_id,
invalid_elem ->> 'id' AS source_id,
@ -293,7 +305,8 @@ SELECT fd.recruit_id,
to_date(invalid_elem -> 'ustanInvalid' ->> 'dataUstan', 'YYYY-MM-DD') as register_date,
to_date(invalid_elem -> 'ustanInvalid' ->> 'dataPodtverzhdPlan', 'YYYY-MM-DD') as confirmation_date,
to_date(invalid_elem ->> 'dataSnyat', 'YYYY-MM-DD') as deregistration_date,
coalesce(invalid_elem -> 'ustanInvalid' ->> 'prBessrInvl' = '1', false) as permanent
coalesce(invalid_elem -> 'ustanInvalid' ->> 'prBessrInvl' = '1', false) as permanent,
true as has_disability
FROM filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -368,16 +381,11 @@ FROM filteredData fd;</sql>
<rename>permanent</rename>
<update>Y</update>
</value>
<value>
<name>last_row</name>
<rename>last_row</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1520</xloc>
<xloc>1552</xloc>
<yloc>384</yloc>
</GUI>
</transform>
@ -403,12 +411,6 @@ FROM filteredData fd;</sql>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_disability</field>
<name>has_disability</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
@ -420,7 +422,7 @@ FROM filteredData fd;</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1712</xloc>
<xloc>1744</xloc>
<yloc>384</yloc>
</GUI>
</transform>

14
mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_repeat.hpl
View file

@ -319,6 +319,7 @@ FROM filteredData fd;</sql>
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>disability</table>
@ -367,11 +368,6 @@ FROM filteredData fd;</sql>
<rename>permanent</rename>
<update>Y</update>
</value>
<value>
<name>last_row</name>
<rename>last_row</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -394,19 +390,11 @@ FROM filteredData fd;</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_disability</field>
<name>has_disability</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

2
mappings/info_recruits/citizen_tables/medicine/disability/recruitment_five_flow.hpl
View file

@ -57,7 +57,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>SELECT
idm_id

35
mappings/info_recruits/citizen_tables/medicine/disability/recruitment_five_flow_delta.hpl
View file

@ -62,27 +62,30 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_source_update_date</lookup>
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id,
? max_source_update_date
<sql>WITH mud AS (SELECT recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS', 'DELTA_SUCCESS')
GROUP BY recruitment_id)
SELECT r.idm_id
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
AND ri.updated_at > mud.max_upd_date
union
select r2.idm_id
from recruitment r2
join etl.job_execution je
on r2.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -384,7 +387,7 @@ FROM ervu_dashboard.recruitment r
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(source_update_date)
<sql>select max(source_update_date) as max_source_update_date
from ervu_dashboard.disability;</sql>
<variables_active>N</variables_active>
<attributes/>

6
mappings/info_recruits/citizen_tables/medicine/disability/recruitment_five_flow_repeat.hpl
View file

@ -57,17 +57,17 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT r.idm_id AS recruitment_id
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
AND je.job_name = 'disability_job'
AND je.job_name = '${JOB_NAME}'
WHERE je.id IS NULL
OR je.status IN ('ERROR', 'PROCESSING');</sql>
<variables_active>N</variables_active>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>656</xloc>

2
mappings/info_recruits/citizen_tables/medicine/health_state/constants/health_state_constants_flow.hpl
View file

@ -87,7 +87,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>nsi</connection>
<connection>ervu_nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT

3
mappings/info_recruits/citizen_tables/medicine/health_state/constants/health_state_constants_flow_repeat.hpl
View file

@ -42,7 +42,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>nsi</connection>
<connection>ervu_nsi</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_disease_update_date</lookup>
@ -104,7 +104,6 @@ FROM disease;
<condition>=</condition>
<field>code</field>
<name>code</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>disease</table>

318
mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow.hpl
View file

@ -45,11 +45,6 @@
<to>health_state_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_output</from>
<to>Change job status on error</to>
@ -60,21 +55,46 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows 2</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update 2</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_output</from>
<to>has_health_state_update</to>
<to>Update 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>has_health_state_update</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>has_health_state_update</from>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_input</from>
<to>Filter rows 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows 2</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error 2</from>
<to>Abort 2</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -92,7 +112,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1264</xloc>
<xloc>1472</xloc>
<yloc>672</yloc>
</GUI>
</transform>
@ -127,7 +147,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1040</xloc>
<xloc>1248</xloc>
<yloc>672</yloc>
</GUI>
</transform>
@ -158,7 +178,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1504</xloc>
<xloc>1744</xloc>
<yloc>288</yloc>
</GUI>
</transform>
@ -204,7 +224,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>752</xloc>
<xloc>960</xloc>
<yloc>288</yloc>
</GUI>
</transform>
@ -240,7 +260,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1504</xloc>
<xloc>1744</xloc>
<yloc>464</yloc>
</GUI>
</transform>
@ -258,42 +278,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>has_health_state_update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_health_state_info</name>
<rename>has_health_state_info</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1264</xloc>
<xloc>960</xloc>
<yloc>464</yloc>
</GUI>
</transform>
@ -313,17 +298,24 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved' as source_update_date,
ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov' as zdorov_arr
ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov' as zdorov_arr,
coalesce(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'priznakNalichSved' =
'1', false) as has_health_state_info
FROM recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov') = 'array'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
WHERE ri.info -> 'svedFL' -> 'svedSostZdorov' ->> 'svedZdorov' &lt;&gt; 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
SELECT fd.recruit_id,
has_health_state_info,
zdorov_elem ->> 'id' AS source_id,
source_update_date,
zdorov_elem ->> 'kodZabol' AS disease_code,
to_date(zdorov_elem ->> 'dataDiagn', 'YYYY-MM-DD') AS diagnosis_date
to_date(zdorov_elem ->> 'dataDiagn', 'YYYY-MM-DD') AS diagnosis_date,
coalesce(zdorov_elem ->> 'kodZabol' &lt;&gt; 'null', false) as has_disease
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(zdorov_arr) AS zdorov_elem;</sql>
CROSS JOIN LATERAL jsonb_array_elements(COALESCE(
NULLIF(zdorov_arr, 'null'::jsonb),
'[]'::jsonb)) AS zdorov_elem;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -381,23 +373,195 @@ FROM filteredData fd
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1040</xloc>
<xloc>1248</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error 2</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
<name>Filter rows 2</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>has_disease</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_false_to>Update</send_false_to>
<send_true_to>Identify last row in a stream</send_true_to>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_health_state_info</name>
<rename>has_health_state_info</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
<name>Abort 2</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>928</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1584</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>Update 2</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_disease</name>
<rename>has_disease</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1392</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>has_health_state_update</source_transform>
<target_transform>Abort</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>health_state_output</source_transform>
<target_transform>Change job status on error</target_transform>
@ -410,6 +574,18 @@ FROM filteredData fd
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error 2</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
</transform_error_handling>
<attributes/>
</pipeline>

310
mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_delta.hpl
View file

@ -7,7 +7,18 @@
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<pipeline_status>0</pipeline_status>
<parameters>
<parameter>
<name>IDM_ID</name>
<default_value>42c79094-4767-4898-9c45-e2453d0665c7</default_value>
<description/>
</parameter>
<parameter>
<name>MAX_SOURCE_UPDATE_DATE</name>
<default_value>2024-05-10</default_value>
<description/>
</parameter>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
@ -45,24 +56,59 @@
<to>health_state_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_upsert</from>
<to>Filter rows</to>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<from>Filter rows 2</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows 2</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_input</from>
<to>Filter rows 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_upsert</from>
<to>Update 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update 2</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error 2</from>
<to>Abort 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update 2</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
@ -82,10 +128,52 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1584</xloc>
<xloc>2256</xloc>
<yloc>704</yloc>
</GUI>
</transform>
<transform>
<name>Abort 2</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1328</xloc>
<yloc>848</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>2064</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -117,10 +205,45 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1360</xloc>
<xloc>1856</xloc>
<yloc>704</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error 2</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1120</xloc>
<yloc>848</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
@ -148,7 +271,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1584</xloc>
<xloc>2256</xloc>
<yloc>336</yloc>
</GUI>
</transform>
@ -197,7 +320,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>1072</xloc>
<xloc>1376</xloc>
<yloc>336</yloc>
</GUI>
</transform>
@ -233,7 +356,44 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1584</xloc>
<xloc>2256</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
<name>Filter rows 2</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>has_disease</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_false_to>Update</send_false_to>
<send_true_to>Identify last row in a stream</send_true_to>
<attributes/>
<GUI>
<xloc>1120</xloc>
<yloc>512</yloc>
</GUI>
</transform>
@ -251,7 +411,77 @@ and recruitment_id = '${IDM_ID}';</sql>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1072</xloc>
<xloc>1376</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_health_state_info</name>
<rename>has_health_state_info</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1120</xloc>
<yloc>704</yloc>
</GUI>
</transform>
<transform>
<name>Update 2</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_disease</name>
<rename>has_disease</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1856</xloc>
<yloc>512</yloc>
</GUI>
</transform>
@ -271,20 +501,30 @@ and recruitment_id = '${IDM_ID}';</sql>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved' as source_update_date,
ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov' as zdorov_arr
ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov' as zdorov_arr,
coalesce(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'priznakNalichSved' =
'1', false) as has_health_state_info
FROM recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov') =
'array'
WHERE ri.info -> 'svedFL' -> 'svedSostZdorov' ->> 'svedZdorov' &lt;&gt; 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
and to_date(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved',
'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}')
and case
when '${MAX_SOURCE_UPDATE_DATE}' &lt;&gt; ''
then to_date(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved',
'YYYY-MM-DD') >
to_date('${MAX_SOURCE_UPDATE_DATE}', 'YYYY-MM-DD')
else true end
)
SELECT fd.recruit_id,
has_health_state_info,
zdorov_elem ->> 'id' AS source_id,
source_update_date,
zdorov_elem ->> 'kodZabol' AS disease_code,
to_date(zdorov_elem ->> 'dataDiagn', 'YYYY-MM-DD') AS diagnosis_date
to_date(zdorov_elem ->> 'dataDiagn', 'YYYY-MM-DD') AS diagnosis_date,
coalesce(zdorov_elem ->> 'kodZabol' &lt;&gt; 'null', false) as has_disease
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(zdorov_arr) AS zdorov_elem;</sql>
CROSS JOIN LATERAL jsonb_array_elements(COALESCE(
NULLIF(zdorov_arr, 'null'::jsonb),
'[]'::jsonb)) AS zdorov_elem;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -342,11 +582,35 @@ FROM filteredData fd
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1360</xloc>
<xloc>1664</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error 2</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>Update 2</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>health_state_upsert</source_transform>
<target_transform>Change job status on error</target_transform>

299
mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_repeat.hpl
View file

@ -7,7 +7,18 @@
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<pipeline_status>0</pipeline_status>
<parameters>
<parameter>
<name>IDM_ID</name>
<default_value>e08dd467-2d0d-4217-880d-9938dbb59afe</default_value>
<description/>
</parameter>
<parameter>
<name>JOB_NAME</name>
<default_value>health_state_job</default_value>
<description/>
</parameter>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
@ -40,21 +51,11 @@
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>health_state_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_upsert</from>
<to>Change job status on error</to>
@ -65,6 +66,46 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error 2</from>
<to>Abort 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update 2</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_upsert</from>
<to>Update 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_input</from>
<to>Filter rows 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows 2</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows 2</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -82,10 +123,52 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1168</xloc>
<xloc>1696</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>Abort 2</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>752</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1488</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -117,10 +200,45 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>944</xloc>
<xloc>1136</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error 2</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>752</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
@ -148,7 +266,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1168</xloc>
<xloc>1696</xloc>
<yloc>272</yloc>
</GUI>
</transform>
@ -197,7 +315,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>656</xloc>
<xloc>848</xloc>
<yloc>272</yloc>
</GUI>
</transform>
@ -233,7 +351,44 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1168</xloc>
<xloc>1696</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>Filter rows 2</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>has_disease</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_false_to>Update</send_false_to>
<send_true_to>Identify last row in a stream</send_true_to>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>448</yloc>
</GUI>
</transform>
@ -251,7 +406,81 @@ and recruitment_id = '${IDM_ID}';</sql>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>656</xloc>
<xloc>848</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_health_state_info</name>
<rename>has_health_state_info</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>Update 2</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_disease</name>
<rename>has_disease</rename>
</value>
<value>
<name>has_health_state_info</name>
<rename>has_health_state_info</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1296</xloc>
<yloc>448</yloc>
</GUI>
</transform>
@ -270,19 +499,25 @@ and recruitment_id = '${IDM_ID}';</sql>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved' as source_update_date,
ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov' as zdorov_arr
to_date(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov' as zdorov_arr,
coalesce(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'priznakNalichSved' =
'1', false) as has_health_state_info
FROM recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov') =
'array'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
WHERE ri.info -> 'svedFL' -> 'svedSostZdorov' ->> 'svedZdorov' &lt;&gt; 'null'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
SELECT fd.recruit_id,
has_health_state_info,
zdorov_elem ->> 'id' AS source_id,
source_update_date,
zdorov_elem ->> 'kodZabol' AS disease_code,
to_date(zdorov_elem ->> 'dataDiagn', 'YYYY-MM-DD') AS diagnosis_date
to_date(zdorov_elem ->> 'dataDiagn', 'YYYY-MM-DD') AS diagnosis_date,
coalesce(zdorov_elem ->> 'kodZabol' &lt;&gt; 'null', false) as has_disease
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(zdorov_arr) AS zdorov_elem;</sql>
CROSS JOIN LATERAL jsonb_array_elements(COALESCE(
NULLIF(zdorov_arr, 'null'::jsonb),
'[]'::jsonb)) AS zdorov_elem;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -314,12 +549,12 @@ FROM filteredData fd
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>Y</update>
<update>N</update>
</value>
<value>
<name>source_id</name>
<rename>source_id</rename>
<update>Y</update>
<update>N</update>
</value>
<value>
<name>source_update_date</name>
@ -340,11 +575,23 @@ FROM filteredData fd
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>944</xloc>
<xloc>1136</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error 2</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>health_state_upsert</source_transform>
<target_transform>Change job status on error</target_transform>

2
mappings/info_recruits/citizen_tables/medicine/health_state/recruitment_five_flow.hpl
View file

@ -57,7 +57,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>SELECT
idm_id

40
mappings/info_recruits/citizen_tables/medicine/health_state/recruitment_five_flow_delta.hpl
View file

@ -62,27 +62,31 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_source_update_date</lookup>
<sql>SELECT
r.idm_id,
? max_source_update_date
<sql>WITH mud AS (SELECT recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS', 'DELTA_SUCCESS')
GROUP BY recruitment_id)
SELECT r.idm_id
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = 'health_state_job'
JOIN recruits_info ri
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > (
SELECT MAX(execution_datetime)
FROM etl.job_execution
WHERE job_name = 'health_state_job'
AND recruitment_id = r.idm_id
)
where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
<variables_active>N</variables_active>
AND ri.updated_at > mud.max_upd_date
union
select r2.idm_id
from recruitment r2
join etl.job_execution je
on r2.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>816</xloc>
@ -103,7 +107,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(source_update_date)
<sql>select max(source_update_date) as max_source_update_date
from ervu_dashboard.health_state;</sql>
<variables_active>N</variables_active>
<attributes/>

6
mappings/info_recruits/citizen_tables/medicine/health_state/recruitment_five_flow_repeat.hpl
View file

@ -57,17 +57,17 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT r.idm_id AS recruitment_id
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
AND je.job_name = 'health_state_job'
AND je.job_name = '${JOB_NAME}'
WHERE je.id IS NULL
OR je.status IN ('ERROR', 'PROCESSING');</sql>
<variables_active>N</variables_active>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>400</xloc>

129
mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow.hpl
View file

@ -70,21 +70,21 @@
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Group by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Group by</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -276,76 +276,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Group by</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<aggregate>has_incapacity_info</aggregate>
<subject>has_incapacity_info</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>source_id</aggregate>
<subject>source_id</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>source_update_date</aggregate>
<subject>source_update_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>capacity_status</aggregate>
<subject>capacity_status</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>court_name</aggregate>
<subject>court_name</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>register_date</aggregate>
<subject>register_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>deregistration_date</aggregate>
<subject>deregistration_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>last_row</aggregate>
<subject>last_row</subject>
<type>FIRST_INCL_NULL</type>
</field>
</fields>
<give_back_row>N</give_back_row>
<group>
<field>
<name>recruit_id</name>
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
@ -364,6 +294,40 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
@ -417,7 +381,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob' as nedeesposob_arr
FROM recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob') = 'array'
-- AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
SELECT fd.recruit_id,
nedeesposob_elem ->> 'id' AS source_id,
@ -483,11 +447,14 @@ FROM filteredData fd
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>incapacity</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>

182
mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_delta.hpl
View file

@ -7,6 +7,7 @@
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<pipeline_status>0</pipeline_status>
<parameters>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
@ -60,21 +61,11 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Group by</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Group by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>incapacity_upsert</from>
<to>Block until transforms finish</to>
@ -85,6 +76,16 @@
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -106,6 +107,28 @@
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1744</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -257,76 +280,6 @@ and recruitment_id = '${IDM_ID}';</sql>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Group by</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<aggregate>has_incapacity_info</aggregate>
<subject>has_incapacity_info</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>source_id</aggregate>
<subject>source_id</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>source_update_date</aggregate>
<subject>source_update_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>capacity_status</aggregate>
<subject>capacity_status</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>court_name</aggregate>
<subject>court_name</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>register_date</aggregate>
<subject>register_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>deregistration_date</aggregate>
<subject>deregistration_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>last_row</aggregate>
<subject>last_row</subject>
<type>FIRST_INCL_NULL</type>
</field>
</fields>
<give_back_row>N</give_back_row>
<group>
<field>
<name>recruit_id</name>
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>1232</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
@ -345,6 +298,40 @@ and recruitment_id = '${IDM_ID}';</sql>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>1232</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
@ -404,10 +391,14 @@ SELECT fd.recruit_id,
nedeesposob_elem ->> 'naimStatus' AS capacity_status,
nedeesposob_elem ->> 'naimOrg' AS court_name,
to_date(nedeesposob_elem ->> 'dataPrisv', 'YYYY-MM-DD') AS register_date,
to_date(nedeesposob_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS deregistration_date
to_date(nedeesposob_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS deregistration_date,
true as has_incapacity_info
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(fd.nedeesposob_arr -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob') AS nedeesposob_elem
WHERE to_date(nedeesposob_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}';</sql>
CROSS JOIN LATERAL jsonb_array_elements(nedeesposob_arr) AS nedeesposob_elem
WHERE case when '${MAX_SOURCE_UPDATE_DATE}' &lt;&gt; ''
then to_date(nedeesposob_elem ->> 'dataSved', 'YYYY-MM-DD') >
to_date('${MAX_SOURCE_UPDATE_DATE}', 'YYYY-MM-DD')
else true end;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -433,9 +424,10 @@ WHERE to_date(nedeesposob_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPD
<condition>=</condition>
<field>source_id</field>
<name>source_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>disability</table>
<table>incapacity</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
@ -479,28 +471,6 @@ WHERE to_date(nedeesposob_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPD
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1744</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>

264
mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_repeat.hpl
View file

@ -60,21 +60,11 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Group by</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Group by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>incapacity_upsert</from>
<to>Block until transforms finish</to>
@ -85,6 +75,16 @@
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -106,6 +106,28 @@
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1280</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -275,6 +297,75 @@ and recruitment_id = '${IDM_ID}';</sql>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>832</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_incapacity_info</name>
<rename>has_incapacity_info</rename>
</value>
</lookup>
<skip_lookup>Y</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>incapacity_input</name>
<type>TableInput</type>
@ -331,7 +422,7 @@ FROM filteredData fd
<name>source_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>disability</table>
<table>incapacity</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
@ -375,146 +466,7 @@ FROM filteredData fd
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Group by</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<aggregate>has_incapacity_info</aggregate>
<subject>has_incapacity_info</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>source_id</aggregate>
<subject>source_id</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>source_update_date</aggregate>
<subject>source_update_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>capacity_status</aggregate>
<subject>capacity_status</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>court_name</aggregate>
<subject>court_name</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>register_date</aggregate>
<subject>register_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>deregistration_date</aggregate>
<subject>deregistration_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>last_row</aggregate>
<subject>last_row</subject>
<type>FIRST_INCL_NULL</type>
</field>
</fields>
<give_back_row>N</give_back_row>
<group>
<field>
<name>recruit_id</name>
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>832</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_incapacity_info</name>
<rename>has_incapacity_info</rename>
</value>
</lookup>
<skip_lookup>Y</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1280</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>incapacity_upsert</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
@ -527,6 +479,18 @@ FROM filteredData fd
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>incapacity_upsert</source_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
</transform_error_handling>
<attributes/>
</pipeline>

27
mappings/info_recruits/citizen_tables/medicine/incapacity/recruitment_five_flow.hpl
View file

@ -57,7 +57,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>SELECT
idm_id
@ -93,11 +93,6 @@ FROM ervu_dashboard.recruitment;</sql>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>incapacity_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -149,11 +144,6 @@ FROM ervu_dashboard.recruitment;</sql>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>incapacity_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -205,11 +195,6 @@ FROM ervu_dashboard.recruitment;</sql>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>incapacity_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -261,11 +246,6 @@ FROM ervu_dashboard.recruitment;</sql>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>incapacity_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -317,11 +297,6 @@ FROM ervu_dashboard.recruitment;</sql>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>incapacity_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>

68
mappings/info_recruits/citizen_tables/medicine/incapacity/recruitment_five_flow_delta.hpl
View file

@ -7,7 +7,13 @@
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<pipeline_status>0</pipeline_status>
<parameters>
<parameter>
<name>JOB_NAME</name>
<default_value>incapacity_job</default_value>
<description/>
</parameter>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
@ -62,28 +68,31 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_source_update_date</lookup>
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id,
? max_source_update_date
<sql>WITH mud AS (SELECT recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS', 'DELTA_SUCCESS')
GROUP BY recruitment_id)
SELECT r.idm_id
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
<variables_active>N</variables_active>
AND ri.updated_at > mud.max_upd_date
union
select r2.idm_id
from recruitment r2
join etl.job_execution je
on r2.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>704</xloc>
@ -104,7 +113,7 @@ FROM ervu_dashboard.recruitment r
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(source_update_date)
<sql>select max(source_update_date) as max_source_update_date
from ervu_dashboard.incapacity;</sql>
<variables_active>N</variables_active>
<attributes/>
@ -137,11 +146,6 @@ from ervu_dashboard.incapacity;</sql>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>incapacity_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
@ -198,11 +202,6 @@ from ervu_dashboard.incapacity;</sql>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>incapacity_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
@ -259,11 +258,6 @@ from ervu_dashboard.incapacity;</sql>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>incapacity_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
@ -320,11 +314,6 @@ from ervu_dashboard.incapacity;</sql>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>incapacity_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
@ -381,11 +370,6 @@ from ervu_dashboard.incapacity;</sql>
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>incapacity_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>

31
mappings/info_recruits/citizen_tables/medicine/incapacity/recruitment_five_flow_repeat.hpl
View file

@ -57,17 +57,17 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT r.idm_id AS recruitment_id
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
AND je.job_name = 'incapacity_job'
AND je.job_name = '${JOB_NAME}'
WHERE je.id IS NULL
OR je.status IN ('ERROR', 'PROCESSING');</sql>
<variables_active>N</variables_active>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>656</xloc>
@ -98,11 +98,6 @@ WHERE je.id IS NULL
<field>recruitment_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>incapacity_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -154,11 +149,6 @@ WHERE je.id IS NULL
<field>recruitment_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>incapacity_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -210,11 +200,6 @@ WHERE je.id IS NULL
<field>recruitment_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>incapacity_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -266,11 +251,6 @@ WHERE je.id IS NULL
<field>recruitment_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>incapacity_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>
@ -322,11 +302,6 @@ WHERE je.id IS NULL
<field>recruitment_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>incapacity_job</input>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
</parameters>
<execution_result_target_transform/>

321
mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow.hpl
View file

@ -21,8 +21,8 @@
</notepads>
<order>
<hop>
<from>Change job status on error</from>
<to>Abort</to>
<from>change_job_status_on_output_error</from>
<to>abort_on_output</to>
<enabled>Y</enabled>
</hop>
<hop>
@ -47,12 +47,7 @@
</hop>
<hop>
<from>vich_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>vich_output</from>
<to>Filter rows</to>
<to>change_job_status_on_output_error</to>
<enabled>Y</enabled>
</hop>
<hop>
@ -65,62 +60,37 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>vich_output</from>
<to>block_until_update_finished</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>block_until_update_finished</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>vich_input</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>change_job_status_on_update_error</from>
<to>abort_on_update_error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>change_job_status_on_update_error</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>992</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
@ -148,7 +118,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1216</xloc>
<xloc>1440</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -230,7 +200,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1216</xloc>
<xloc>1440</xloc>
<yloc>416</yloc>
</GUI>
</transform>
@ -252,11 +222,213 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>512</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_vich_info</name>
<rename>has_vich_info</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>512</xloc>
<yloc>704</yloc>
</GUI>
</transform>
<transform>
<name>abort_on_output</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>abort_on_update_error</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>720</xloc>
<yloc>864</yloc>
</GUI>
</transform>
<transform>
<name>block_until_update_finished</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<CopyNr/>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>change_job_status_on_output_error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>992</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>change_job_status_on_update_error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>512</xloc>
<yloc>864</yloc>
</GUI>
</transform>
<transform>
<name>vich_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -277,7 +449,8 @@ SELECT fd.recruit_id,
vich_elem ->> 'id' AS source_id,
fd.source_update_date,
to_date(vich_elem ->> 'dataSnyat', 'YYYY-MM-DD') AS register_date,
to_date(vich_elem ->> 'dataPostUchet', 'YYYY-MM-DD') AS deregistration_date
to_date(vich_elem ->> 'dataPostUchet', 'YYYY-MM-DD') AS deregistration_date,
true as has_vich_info
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(vich_arr) AS vich_elem;</sql>
<variables_active>Y</variables_active>
@ -342,9 +515,21 @@ FROM filteredData fd
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>change_job_status_on_update_error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>vich_output</source_transform>
<target_transform>Change job status on error</target_transform>
<target_transform>change_job_status_on_output_error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

211
mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_delta.hpl
View file

@ -56,13 +56,38 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>vich_upsert</from>
<to>Filter rows</to>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<from>Sort rows</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>change_job_status_on_update_error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>change_job_status_on_update_error</from>
<to>abort_on_update_error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>vich_input</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>vich_upsert</from>
<to>block_until_update_finished</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>block_until_update_finished</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
@ -82,7 +107,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1744</xloc>
<xloc>1904</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -148,7 +173,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1744</xloc>
<xloc>1904</xloc>
<yloc>208</yloc>
</GUI>
</transform>
@ -233,7 +258,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1744</xloc>
<xloc>1904</xloc>
<yloc>384</yloc>
</GUI>
</transform>
@ -256,8 +281,8 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>vich_input</name>
<type>TableInput</type>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -266,9 +291,132 @@ and recruitment_id = '${IDM_ID}';</sql>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>528</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>has_vich_info</name>
<rename>has_vich_info</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>656</yloc>
</GUI>
</transform>
<transform>
<name>abort_on_update_error</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1248</xloc>
<yloc>816</yloc>
</GUI>
</transform>
<transform>
<name>change_job_status_on_update_error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>816</yloc>
</GUI>
</transform>
<transform>
<name>vich_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
to_date(ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' ->> 'dataSved',
'YYYY-MM-DD') as source_update_date,
@ -282,9 +430,13 @@ SELECT fd.recruit_id,
vich_elem ->> 'id' AS source_id,
fd.source_update_date,
to_date(vich_elem ->> 'dataSnyat', 'YYYY-MM-DD') AS register_date,
to_date(vich_elem ->> 'dataPostUchet', 'YYYY-MM-DD') AS deregistration_date
to_date(vich_elem ->> 'dataPostUchet', 'YYYY-MM-DD') AS deregistration_date,
true as has_vich_info
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(vich_arr) AS vich_elem;</sql>
CROSS JOIN LATERAL jsonb_array_elements(vich_arr) AS vich_elem
WHERE case when '${MAX_SOURCE_UPDATE_DATE}' &lt;&gt; ''
then source_update_date > to_date('${MAX_SOURCE_UPDATE_DATE}', 'YYYY-MM-DD')
else true end;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -346,7 +498,42 @@ FROM filteredData fd
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>block_until_update_finished</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<CopyNr/>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1712</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>change_job_status_on_update_error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>vich_upsert</source_transform>
<target_transform>Change job status on error</target_transform>

3
mappings/info_recruits/citizen_tables/medicine/vich/recruitment_five_flow.hpl
View file

@ -57,8 +57,9 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>SELECT
idm_id
FROM ervu_dashboard.recruitment;</sql>

37
mappings/info_recruits/citizen_tables/medicine/vich/recruitment_five_flow_delta.hpl
View file

@ -62,28 +62,31 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_source_update_date</lookup>
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id,
? max_source_update_date
<sql>WITH mud AS (SELECT recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS', 'DELTA_SUCCESS')
GROUP BY recruitment_id)
SELECT r.idm_id
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
<variables_active>N</variables_active>
AND ri.updated_at > mud.max_upd_date
union
select r2.idm_id
from recruitment r2
join etl.job_execution je
on r2.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>704</xloc>
@ -104,7 +107,7 @@ FROM ervu_dashboard.recruitment r
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(source_update_date)
<sql>select max(source_update_date) as max_source_update_date
from ervu_dashboard.vich_registered;</sql>
<variables_active>N</variables_active>
<attributes/>

2
mappings/info_recruits/citizen_tables/medicine/vich/recruitment_five_flow_repeat.hpl
View file

@ -57,7 +57,7 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT r.idm_id AS recruitment_id

35
mappings/info_recruits/citizen_tables/sport/recruitment_five_flow_delta.hpl
View file

@ -62,27 +62,30 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_source_update_date</lookup>
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id,
? max_source_update_date
<sql>WITH mud AS (SELECT recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS', 'DELTA_SUCCESS')
GROUP BY recruitment_id)
SELECT r.idm_id
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
AND ri.updated_at > mud.max_upd_date
union
select r2.idm_id
from recruitment r2
join etl.job_execution je
on r2.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -106,7 +109,7 @@ FROM ervu_dashboard.recruitment r
<limit>0</limit>
<sql>WITH max_sport_elem_upd_date AS (
SELECT max(source_update_date) AS dt FROM ervu_dashboard.sports_rank
UNION ALL
UNION
SELECT max(source_update_date) FROM ervu_dashboard.sports_title
)
SELECT max(dt) AS max_source_update_date

42
mappings/info_recruits/citizen_tables/work/employer/recruitment_five_flow_delta.hpl
View file

@ -62,28 +62,31 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_source_update_date</lookup>
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id,
? max_source_update_date
<sql>WITH mud AS (SELECT recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS', 'DELTA_SUCCESS')
GROUP BY recruitment_id)
SELECT r.idm_id
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
<variables_active>N</variables_active>
AND ri.updated_at > mud.max_upd_date
union
select r2.idm_id
from recruitment r2
join etl.job_execution je
on r2.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>576</xloc>
@ -114,11 +117,6 @@ FROM ervu_dashboard.recruitment r
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>employer_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
@ -409,7 +407,7 @@ FROM ervu_dashboard.recruitment r
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(source_update_date)
<sql>select max(source_update_date) as max_source_update_date
from ervu_dashboard.employer;</sql>
<variables_active>N</variables_active>
<attributes/>

37
mappings/info_recruits/citizen_tables/work/individual_entrepreneur/recruitment_five_flow_delta.hpl
View file

@ -65,7 +65,7 @@
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(source_update_date)
<sql>select max(source_update_date) as max_source_update_date
from ervu_dashboard.individual_entrepreneur;</sql>
<variables_active>N</variables_active>
<attributes/>
@ -85,28 +85,31 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>Get_max_source_update_date</lookup>
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id,
? max_source_update_date
<sql>WITH mud AS (SELECT recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS', 'DELTA_SUCCESS')
GROUP BY recruitment_id)
SELECT r.idm_id
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
<variables_active>N</variables_active>
AND ri.updated_at > mud.max_upd_date
union
select r2.idm_id
from recruitment r2
join etl.job_execution je
on r2.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>544</xloc>

60
mappings/info_recruits/citizen_tables/work/self_employed/recruitment_five_flow_delta.hpl
View file

@ -65,7 +65,7 @@
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(source_update_date)
<sql>select max(source_update_date) as max_source_update_date
from ervu_dashboard.self_employed;</sql>
<variables_active>N</variables_active>
<attributes/>
@ -88,25 +88,28 @@
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>Get_max_source_update_date</lookup>
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id,
? max_source_update_date
<sql>WITH mud AS (SELECT recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS', 'DELTA_SUCCESS')
GROUP BY recruitment_id)
SELECT r.idm_id
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
<variables_active>N</variables_active>
AND ri.updated_at > mud.max_upd_date
union
select r2.idm_id
from recruitment r2
join etl.job_execution je
on r2.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>384</xloc>
@ -137,11 +140,6 @@ FROM ervu_dashboard.recruitment r
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>self_employed_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
@ -198,11 +196,6 @@ FROM ervu_dashboard.recruitment r
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>self_employed_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
@ -259,11 +252,6 @@ FROM ervu_dashboard.recruitment r
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>self_employed_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
@ -320,11 +308,6 @@ FROM ervu_dashboard.recruitment r
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>self_employed_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>
@ -381,11 +364,6 @@ FROM ervu_dashboard.recruitment r
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>self_employed_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>

42
mappings/info_recruits/citizen_tables/work/work_activity/recruitment_five_flow_delta.hpl
View file

@ -62,28 +62,31 @@
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard-test</connection>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<lookup>get_max_source_update_date</lookup>
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id,
? max_source_update_date
<sql>WITH mud AS (SELECT recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS', 'DELTA_SUCCESS')
GROUP BY recruitment_id)
SELECT r.idm_id
FROM ervu_dashboard.recruitment r
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
<variables_active>N</variables_active>
AND ri.updated_at > mud.max_upd_date
union
select r2.idm_id
from recruitment r2
join etl.job_execution je
on r2.idm_id = je.recruitment_id
where je.status in ('DELTA_ERROR', 'DELTA_PROCESSING')
and je.job_name = '${JOB_NAME}';</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>352</xloc>
@ -104,7 +107,7 @@ FROM ervu_dashboard.recruitment r
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select max(source_update_date)
<sql>select max(source_update_date) as max_source_update_date
FROM ervu_dashboard.citizen_work_activity;</sql>
<variables_active>N</variables_active>
<attributes/>
@ -137,11 +140,6 @@ FROM ervu_dashboard.recruitment r
<field>idm_id</field>
<input/>
</variable_mapping>
<variable_mapping>
<variable>JOB_NAME</variable>
<field/>
<input>work_activity_job</input>
</variable_mapping>
<variable_mapping>
<variable>MAX_SOURCE_UPDATE_DATE</variable>
<field>max_source_update_date</field>