micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

some fixes

Browse source
This commit is contained in:
Fusionshh 2025-10-21 18:05:17 +03:00
parent 44c15abe85
commit ce4f92e598
39 changed files with 1349 additions and 3452 deletions
Download patch file Download diff file Expand all files Collapse all files

108
mappings/info_recruits/citizen_tables/deputy/parallel/deputy_flow.hpl
View file

@ -35,49 +35,39 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>deputy_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_flag_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_flag_update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Block until transforms finish</to>
<from>deputy_input</from>
<to>deputy_flag_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>deputy_flag_update</to>
<from>deputy_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_input</from>
<to>deputy_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
@ -97,30 +87,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1504</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>deputy_output</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>416</yloc>
<xloc>1072</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
@ -155,8 +123,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>592</yloc>
<xloc>784</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
@ -187,7 +155,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1504</xloc>
<xloc>1072</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -233,7 +201,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>704</xloc>
<xloc>512</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -269,25 +237,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1504</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>704</xloc>
<xloc>1072</xloc>
<yloc>416</yloc>
</GUI>
</transform>
@ -322,15 +272,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>416</yloc>
<xloc>512</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>deputy_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -341,7 +291,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedDeputat' -> 'deputat' as deputy_elem
ri.info -> 'svedFL' -> 'svedDeputat' -> 'deputat' as deputy_elem,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedDeputat' ->> 'deputat' &lt;&gt; 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -355,7 +306,8 @@ SELECT fd.recruit_id,
to_date(deputy_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS end_date,
deputy_elem ->> 'prichinaDosrochKod' AS termination_reason_code,
to_date(deputy_elem ->> 'dataDosroch', 'YYYY-MM-DD') AS termination_date,
true as has_deputy_info
true as has_deputy_info,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -430,8 +382,8 @@ FROM filteredData fd;</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>592</yloc>
<xloc>784</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform_error_handling>

90
mappings/info_recruits/citizen_tables/deputy/parallel/deputy_flow_delta.hpl
View file

@ -40,43 +40,33 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_flag_update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>deputy_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<from>deputy_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_input</from>
<to>deputy_flag_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_upsert</from>
<to>Block until transforms finish</to>
<from>deputy_input</from>
<to>deputy_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<from>deputy_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
@ -97,32 +87,10 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1360</xloc>
<xloc>928</xloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>deputy_flag_update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1152</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -155,7 +123,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>944</xloc>
<xloc>688</xloc>
<yloc>688</yloc>
</GUI>
</transform>
@ -187,7 +155,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1360</xloc>
<xloc>928</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -236,7 +204,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>624</xloc>
<xloc>432</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -272,25 +240,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1360</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>624</xloc>
<xloc>928</xloc>
<yloc>416</yloc>
</GUI>
</transform>
@ -325,7 +275,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>624</xloc>
<xloc>432</xloc>
<yloc>688</yloc>
</GUI>
</transform>
@ -333,7 +283,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<name>deputy_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -344,7 +294,8 @@ and recruitment_id = '${IDM_ID}';</sql>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedDeputat' -> 'deputat' as deputy_elem
ri.info -> 'svedFL' -> 'svedDeputat' -> 'deputat' as deputy_elem,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedDeputat' ->> 'deputat' &lt;&gt; 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -362,7 +313,8 @@ SELECT fd.recruit_id,
to_date(deputy_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS end_date,
deputy_elem ->> 'prichinaDosrochKod' AS termination_reason_code,
to_date(deputy_elem ->> 'dataDosroch', 'YYYY-MM-DD') AS termination_date,
true as has_deputy_info
true as has_deputy_info,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd;
</sql>
<variables_active>Y</variables_active>
@ -442,7 +394,7 @@ FROM filteredData fd;
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>944</xloc>
<xloc>688</xloc>
<yloc>416</yloc>
</GUI>
</transform>

105
mappings/info_recruits/citizen_tables/deputy/parallel/deputy_flow_repeat.hpl
View file

@ -51,46 +51,36 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_flag_update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>deputy_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_upsert</from>
<to>Block until transforms finish</to>
<from>deputy_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<from>deputy_input</from>
<to>deputy_flag_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_input</from>
<to>deputy_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>deputy_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -108,30 +98,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1440</xloc>
<yloc>784</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>deputy_flag_update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1248</xloc>
<yloc>512</yloc>
<xloc>1072</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
@ -166,8 +134,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>784</yloc>
<xloc>816</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
@ -198,7 +166,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1440</xloc>
<xloc>1072</xloc>
<yloc>336</yloc>
</GUI>
</transform>
@ -253,7 +221,7 @@ ON CONFLICT (job_name, recruitment_id)
</partitioning>
<attributes/>
<GUI>
<xloc>784</xloc>
<xloc>576</xloc>
<yloc>336</yloc>
</GUI>
</transform>
@ -289,25 +257,7 @@ ON CONFLICT (job_name, recruitment_id)
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1440</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>784</xloc>
<xloc>1072</xloc>
<yloc>512</yloc>
</GUI>
</transform>
@ -342,15 +292,15 @@ ON CONFLICT (job_name, recruitment_id)
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>784</xloc>
<yloc>784</yloc>
<xloc>576</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
<name>deputy_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -359,9 +309,9 @@ ON CONFLICT (job_name, recruitment_id)
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedDeputat' -> 'deputat' as deputy_elem
ri.info -> 'svedFL' -> 'svedDeputat' -> 'deputat' as deputy_elem,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedDeputat' ->> 'deputat' &lt;&gt; 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -375,7 +325,8 @@ SELECT fd.recruit_id,
to_date(deputy_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS end_date,
deputy_elem ->> 'prichinaDosrochKod' AS termination_reason_code,
to_date(deputy_elem ->> 'dataDosroch', 'YYYY-MM-DD') AS termination_date,
true as has_deputy_info
true as has_deputy_info,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -454,7 +405,7 @@ FROM filteredData fd;</sql>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1072</xloc>
<xloc>816</xloc>
<yloc>512</yloc>
</GUI>
</transform>

212
mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow.hpl
View file

@ -61,21 +61,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_education_output</from>
<to>block_until_all_complete</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>block_until_all_complete</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>change_job_status_on_error_upsert</from>
<to>abort_2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sort_by_recruit_id</from>
<to>Update</to>
@ -96,29 +81,34 @@
<to>sort_by_source_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sort_by_source_id</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>citizen_education_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_education_output</from>
<to>change_job_status_on_error_main_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_leave_output</from>
<to>change_job_status_on_error_upsert</to>
<from>citizen_education_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sort_by_source_id</from>
<to>citizen_education_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>change_job_status_on_error_upsert</to>
<to>change_job_status_on_error_main_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_leave_output</from>
<to>change_job_status_on_error_main_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
</order>
@ -150,8 +140,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1824</xloc>
<yloc>256</yloc>
<xloc>1696</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
@ -196,8 +186,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>256</yloc>
<xloc>432</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
@ -232,8 +222,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1824</xloc>
<yloc>432</yloc>
<xloc>1696</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
@ -260,25 +250,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>block_until_main_finished</send_true_to>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>912</xloc>
<xloc>704</xloc>
<yloc>432</yloc>
</GUI>
</transform>
@ -313,8 +285,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1264</xloc>
<yloc>768</yloc>
<xloc>1120</xloc>
<yloc>544</yloc>
</GUI>
</transform>
<transform>
@ -333,53 +305,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1824</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>abort_2</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1824</xloc>
<yloc>656</yloc>
</GUI>
</transform>
<transform>
<name>block_until_all_complete</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>education_leave_output</name>
</transform>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1600</xloc>
<yloc>432</yloc>
<xloc>1696</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
@ -400,8 +327,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</transforms>
<attributes/>
<GUI>
<xloc>1056</xloc>
<yloc>576</yloc>
<xloc>928</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
@ -435,44 +362,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1600</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>change_job_status_on_error_upsert</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1600</xloc>
<yloc>656</yloc>
<xloc>1456</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
@ -689,8 +580,8 @@ and recruitment_id = '${IDM_ID}';
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1264</xloc>
<yloc>432</yloc>
<xloc>1120</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
@ -709,7 +600,8 @@ and recruitment_id = '${IDM_ID}';
<sql>WITH filteredData AS (SELECT ri.recruit_id,
education_elem,
academ_elem,
'Образовательная организация высшего образования' as educational_institution_type
'Образовательная организация высшего образования' as educational_institution_type,
ri.ctid
FROM ervu_dashboard.recruits_info ri
CROSS JOIN LATERAL jsonb_array_elements(
ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') AS education_elem
@ -728,7 +620,8 @@ and recruitment_id = '${IDM_ID}';
SELECT ri.recruit_id,
education_elem,
academ_elem,
'Профессиональная образовательная организация' as educational_institution_type
'Профессиональная образовательная организация' as educational_institution_type,
ri.ctid
FROM ervu_dashboard.recruits_info ri
CROSS JOIN LATERAL jsonb_array_elements(
ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') AS education_elem
@ -747,7 +640,8 @@ and recruitment_id = '${IDM_ID}';
SELECT ri.recruit_id,
education_elem,
academ_elem,
'Общеобразовательная организация' as educational_institution_type
'Общеобразовательная организация' as educational_institution_type,
ri.ctid
FROM ervu_dashboard.recruits_info ri
CROSS JOIN LATERAL jsonb_array_elements(
ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') AS education_elem
@ -773,7 +667,7 @@ SELECT education_elem ->> 'id'
education_elem -> 'svedOrg' ->> 'naimOrg' as educational_institution_name,
education_elem -> 'svedOrg' -> 'extend' ->> 'statusOrg' as educational_institution_status_code,
education_elem -> 'svedOrg' -> 'extend' ->> 'vidPodrazdOrg' as educational_organization_separate_unit_type,
education_elem -> 'svedOrg' -> 'extend' -&gt;&gt; 'adresPodrazdOrg' as educational_organization_separate_unit_address,
education_elem -> 'svedOrg' -> 'extend' ->> 'adresPodrazdOrg' as educational_organization_separate_unit_address,
education_elem ->> 'naimStatus' as student_status,
case
when educational_institution_type = 'Образовательная организация высшего образования'
@ -825,7 +719,8 @@ SELECT education_elem ->> 'id'
to_date(academ_elem ->> 'dataNach', 'YYYY-MM-DD') as academ_start_date,
to_date(academ_elem ->> 'dataOkonch', 'YYYY-MM-DD') as academ_end_date,
academ_elem -> 'naimPrichOtpusk' as academ_reason,
true as has_educational_institution_info
true as has_educational_institution_info,
fd.ctid = (SELECT max(ctid) FROM filteredData) AS last_row
FROM filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -869,22 +764,19 @@ FROM filteredData fd;</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>educational_leave</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1264</xloc>
<yloc>576</yloc>
<xloc>1120</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
@ -917,8 +809,8 @@ FROM filteredData fd;</sql>
</fields>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>768</yloc>
<xloc>704</xloc>
<yloc>544</yloc>
</GUI>
</transform>
<transform>
@ -951,14 +843,14 @@ FROM filteredData fd;</sql>
</fields>
<attributes/>
<GUI>
<xloc>672</xloc>
<yloc>432</yloc>
<xloc>704</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>change_job_status_on_error_upsert</target_transform>
<target_transform>change_job_status_on_error_main_upsert</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
@ -982,7 +874,7 @@ FROM filteredData fd;</sql>
</error>
<error>
<source_transform>education_leave_output</source_transform>
<target_transform>change_job_status_on_error_upsert</target_transform>
<target_transform>change_job_status_on_error_main_upsert</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

15
mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_delta.hpl
View file

@ -540,13 +540,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<condition>=</condition>
<field>educational_institution_learning_source_id</field>
<name>source_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>start_date</field>
<name>academ_start_date</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>educational_leave</table>
@ -850,9 +848,11 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz' as education_arr,
'Образовательная организация высшего образования' as educational_institution_type
'Образовательная организация высшего образования' as educational_institution_typ,
ri.ctid
FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -861,7 +861,8 @@ and recruitment_id = '${IDM_ID}';</sql>
SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh' as education_arr,
'Профессиональная образовательная организация' as educational_institution_type
'Профессиональная образовательная организация' as educational_institution_type,
ri.ctid
FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -870,7 +871,8 @@ and recruitment_id = '${IDM_ID}';</sql>
SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola' as education_arr,
'Общеобразовательная организация' as educational_institution_type
'Общеобразовательная организация' as educational_institution_type,
ri.ctid
FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -940,7 +942,8 @@ SELECT education_elem ->> 'id'
to_date(academ_elem ->> 'dataNach', 'YYYY-MM-DD') as academ_start_date,
to_date(academ_elem ->> 'dataOkonch', 'YYYY-MM-DD') as academ_end_date,
academ_elem -> 'naimPrichOtpusk' as academ_reason,
true as has_educational_institution_info
true as has_educational_institution_info,
fd.ctid = (SELECT max(ctid) FROM filteredData) AS last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(education_arr) AS education_elem
LEFT JOIN LATERAL jsonb_array_elements(

241
mappings/info_recruits/citizen_tables/education/educational_institution_learning/parallel/citizen_education_flow_repeat.hpl
View file

@ -51,36 +51,6 @@
<to>block_until_main_finished</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter_null_parental_leave_dates</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>sort_by_recruit_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>sort_by_source_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>block_until_all_complete</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sort_by_source_id</from>
<to>Group by</to>
@ -91,11 +61,6 @@
<to>upsert_edu_inst_info</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>upsert_edu_inst_info</from>
<to>block_until_all_complete</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>block_until_main_finished</from>
<to>upsert_educational_leave</to>
@ -106,24 +71,39 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error 2</from>
<to>Abort 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sort_by_recruit_id</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>upsert_edu_inst_info</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_input</from>
<to>sort_by_source_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_input</from>
<to>Filter_null_parental_leave_dates</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_input</from>
<to>sort_by_recruit_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>upsert_educational_leave</from>
<to>Change job status on error 2</to>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error 2</to>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
</order>
@ -143,28 +123,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1888</xloc>
<yloc>800</yloc>
</GUI>
</transform>
<transform>
<name>Abort 2</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>2048</xloc>
<yloc>1024</yloc>
<xloc>1936</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
@ -199,44 +159,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1664</xloc>
<yloc>800</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error 2</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1824</xloc>
<yloc>1024</yloc>
<xloc>1728</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
@ -267,8 +191,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>2032</xloc>
<yloc>464</yloc>
<xloc>1936</xloc>
<yloc>368</yloc>
</GUI>
</transform>
<transform>
@ -322,8 +246,8 @@ ON CONFLICT (job_name, recruitment_id)
</partitioning>
<attributes/>
<GUI>
<xloc>1056</xloc>
<yloc>464</yloc>
<xloc>864</xloc>
<yloc>368</yloc>
</GUI>
</transform>
<transform>
@ -358,8 +282,8 @@ ON CONFLICT (job_name, recruitment_id)
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>2032</xloc>
<yloc>640</yloc>
<xloc>1936</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
@ -386,8 +310,8 @@ ON CONFLICT (job_name, recruitment_id)
<send_true_to>block_until_main_finished</send_true_to>
<attributes/>
<GUI>
<xloc>1152</xloc>
<yloc>912</yloc>
<xloc>1120</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
@ -646,26 +570,8 @@ ON CONFLICT (job_name, recruitment_id)
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>1472</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1056</xloc>
<yloc>640</yloc>
<xloc>1312</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
@ -699,33 +605,8 @@ ON CONFLICT (job_name, recruitment_id)
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1600</xloc>
<yloc>1152</yloc>
</GUI>
</transform>
<transform>
<name>block_until_all_complete</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>upsert_educational_leave</name>
</transform>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1888</xloc>
<yloc>640</yloc>
<xloc>1504</xloc>
<yloc>784</yloc>
</GUI>
</transform>
<transform>
@ -746,15 +627,15 @@ ON CONFLICT (job_name, recruitment_id)
</transforms>
<attributes/>
<GUI>
<xloc>1408</xloc>
<yloc>912</yloc>
<xloc>1312</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>education_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -766,7 +647,8 @@ ON CONFLICT (job_name, recruitment_id)
<sql>WITH filteredData AS (SELECT ri.recruit_id,
education_elem,
academ_elem,
'Образовательная организация высшего образования' as educational_institution_type
'Образовательная организация высшего образования' as educational_institution_type,
ri.ctid
FROM ervu_dashboard.recruits_info ri
CROSS JOIN LATERAL jsonb_array_elements(
ri.info -> 'svedFL' -> 'svedVUZ' -> 'vuz') AS education_elem
@ -785,7 +667,8 @@ ON CONFLICT (job_name, recruitment_id)
SELECT ri.recruit_id,
education_elem,
academ_elem,
'Профессиональная образовательная организация' as educational_institution_type
'Профессиональная образовательная организация' as educational_institution_type,
ri.ctid
FROM ervu_dashboard.recruits_info ri
CROSS JOIN LATERAL jsonb_array_elements(
ri.info -> 'svedFL' -> 'svedKolledzh' -> 'kolledzh') AS education_elem
@ -804,7 +687,8 @@ ON CONFLICT (job_name, recruitment_id)
SELECT ri.recruit_id,
education_elem,
academ_elem,
'Общеобразовательная организация' as educational_institution_type
'Общеобразовательная организация' as educational_institution_type,
ri.ctid
FROM ervu_dashboard.recruits_info ri
CROSS JOIN LATERAL jsonb_array_elements(
ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') AS education_elem
@ -817,9 +701,8 @@ ON CONFLICT (job_name, recruitment_id)
) AS academ_elem ON true
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSHkola' -> 'shkola') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')))
SELECT fd.recruit_id,
true as has_educational_institution_info,
education_elem ->> 'id' AS source_id,
SELECT education_elem ->> 'id' AS source_id,
fd.recruit_id,
to_date(education_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
educational_institution_type,
education_elem -> 'svedOrg' ->> 'naimTipOrg' as org_type,
@ -828,6 +711,7 @@ SELECT fd.recruit_id,
coalesce(education_elem -> 'svedOrg' ->> 'ogrn', education_elem -> 'svedOrg' ->>
'ogrnip') as ogrn,
education_elem -> 'svedOrg' ->> 'adresOrg' as educational_institution_address,
education_elem -> 'svedOrg' ->> 'naimOrg' as educational_institution_name,
education_elem -> 'svedOrg' -> 'extend' ->> 'statusOrg' as educational_institution_status_code,
education_elem -> 'svedOrg' -> 'extend' ->> 'vidPodrazdOrg' as educational_organization_separate_unit_type,
education_elem -> 'svedOrg' -> 'extend' ->> 'adresPodrazdOrg' as educational_organization_separate_unit_address,
@ -881,7 +765,9 @@ SELECT fd.recruit_id,
'YYYY-MM-DD') as military_department_end_date,
to_date(academ_elem ->> 'dataNach', 'YYYY-MM-DD') as academ_start_date,
to_date(academ_elem ->> 'dataOkonch', 'YYYY-MM-DD') as academ_end_date,
academ_elem -> 'naimPrichOtpusk' as academ_reason
academ_elem -> 'naimPrichOtpusk' as academ_reason,
true as has_educational_institution_info,
fd.ctid = (SELECT max(ctid) FROM filteredData) AS last_row
FROM filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -920,8 +806,8 @@ FROM filteredData fd;</sql>
</fields>
<attributes/>
<GUI>
<xloc>1152</xloc>
<yloc>1152</yloc>
<xloc>1120</xloc>
<yloc>784</yloc>
</GUI>
</transform>
<transform>
@ -954,8 +840,8 @@ FROM filteredData fd;</sql>
</fields>
<attributes/>
<GUI>
<xloc>1280</xloc>
<yloc>640</yloc>
<xloc>1120</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
@ -1203,8 +1089,8 @@ FROM filteredData fd;</sql>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1664</xloc>
<yloc>640</yloc>
<xloc>1504</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
@ -1225,7 +1111,6 @@ FROM filteredData fd;</sql>
<condition>=</condition>
<field>educational_institution_learning_source_id</field>
<name>source_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>educational_leave</table>
@ -1253,14 +1138,14 @@ FROM filteredData fd;</sql>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1600</xloc>
<yloc>912</yloc>
<xloc>1504</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error 2</target_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -1284,7 +1169,7 @@ FROM filteredData fd;</sql>
</error>
<error>
<source_transform>upsert_educational_leave</source_transform>
<target_transform>Change job status on error 2</target_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

119
mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow.hpl
View file

@ -35,36 +35,16 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>citizen_education_diploma_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_education_diploma_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>foerign_diploma_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Update</to>
@ -76,12 +56,22 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_education_diploma_output</from>
<to>Block until transforms finish</to>
<from>foerign_diploma_input</from>
<to>citizen_education_diploma_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<from>foerign_diploma_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>foerign_diploma_input</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_education_diploma_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
@ -102,30 +92,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1488</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1280</xloc>
<yloc>432</yloc>
<xloc>1248</xloc>
<yloc>528</yloc>
</GUI>
</transform>
<transform>
@ -160,8 +128,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1264</xloc>
<yloc>608</yloc>
<xloc>1024</xloc>
<yloc>528</yloc>
</GUI>
</transform>
<transform>
@ -192,7 +160,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1488</xloc>
<xloc>1024</xloc>
<yloc>256</yloc>
</GUI>
</transform>
@ -238,7 +206,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>736</xloc>
<xloc>544</xloc>
<yloc>256</yloc>
</GUI>
</transform>
@ -274,25 +242,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1488</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>736</xloc>
<xloc>1024</xloc>
<yloc>432</yloc>
</GUI>
</transform>
@ -307,24 +257,27 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<compress>N</compress>
<directory>${java.io.tmpdir}</directory>
<prefix/>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<name>recruit_id</name>
<presorted>N</presorted>
</field>
</fields>
<sort_size>1000000</sort_size>
<unique_rows>Y</unique_rows>
<attributes/>
<GUI>
<xloc>736</xloc>
<yloc>608</yloc>
<xloc>544</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
@ -358,8 +311,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1024</xloc>
<yloc>608</yloc>
<xloc>768</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
@ -452,7 +405,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1024</xloc>
<xloc>768</xloc>
<yloc>432</yloc>
</GUI>
</transform>
@ -460,7 +413,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<name>foerign_diploma_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -471,7 +424,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr
ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -491,7 +445,8 @@ SELECT fd.recruit_id,
diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code,
diploma_elem ->> 'kodOKSM' as oksm_code,
coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec,
true as has_foreign_diploma
true as has_foreign_diploma,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem;
</sql>

114
mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_delta.hpl
View file

@ -35,21 +35,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>foreign_diploma_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>foreign_diploma_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>foreign_diploma_upsert</from>
<to>Change job status on error</to>
@ -71,20 +56,20 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<from>foreign_diploma_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>foreign_diploma_input</from>
<to>foreign_diploma_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>foreign_diploma_input</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>foreign_diploma_upsert</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -102,32 +87,10 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1472</xloc>
<xloc>1328</xloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1232</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -160,7 +123,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1280</xloc>
<xloc>1136</xloc>
<yloc>688</yloc>
</GUI>
</transform>
@ -192,7 +155,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1472</xloc>
<xloc>1136</xloc>
<yloc>320</yloc>
</GUI>
</transform>
@ -241,7 +204,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>768</xloc>
<xloc>576</xloc>
<yloc>320</yloc>
</GUI>
</transform>
@ -277,25 +240,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1472</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>768</xloc>
<xloc>1136</xloc>
<yloc>496</yloc>
</GUI>
</transform>
@ -310,23 +255,26 @@ and recruitment_id = '${IDM_ID}';</sql>
<method>none</method>
<schema_name/>
</partitioning>
<compress>N</compress>
<directory>${java.io.tmpdir}</directory>
<prefix/>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<name>recruit_id</name>
<presorted>N</presorted>
</field>
</fields>
<sort_size>1000000</sort_size>
<unique_rows>Y</unique_rows>
<attributes/>
<GUI>
<xloc>768</xloc>
<xloc>576</xloc>
<yloc>688</yloc>
</GUI>
</transform>
@ -361,7 +309,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>992</xloc>
<xloc>832</xloc>
<yloc>688</yloc>
</GUI>
</transform>
@ -369,7 +317,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<name>foreign_diploma_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -378,12 +326,13 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr
ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
)
SELECT fd.recruit_id,
diploma_elem ->> 'id' AS source_id,
to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
@ -399,7 +348,8 @@ SELECT fd.recruit_id,
diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code,
diploma_elem ->> 'kodOKSM' as oksm_code,
coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec,
true as has_foreign_diploma
true as has_foreign_diploma,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem
where case
@ -514,7 +464,7 @@ where case
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>992</xloc>
<xloc>832</xloc>
<yloc>496</yloc>
</GUI>
</transform>

116
mappings/info_recruits/citizen_tables/education/foreign_diploma/parallel/foreign_diploma_flow_repeat.hpl
View file

@ -35,21 +35,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>foerign_diploma_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>foreign_diploma_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>foreign_diploma_upsert</from>
<to>Change job status on error</to>
@ -65,23 +50,28 @@
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>foreign_diploma_upsert</from>
<to>Block until transforms finish</to>
<from>foerign_diploma_input</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<from>foerign_diploma_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>foerign_diploma_input</from>
<to>foreign_diploma_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>foreign_diploma_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
@ -102,32 +92,10 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1488</xloc>
<xloc>1216</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1248</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -160,7 +128,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1232</xloc>
<xloc>960</xloc>
<yloc>608</yloc>
</GUI>
</transform>
@ -192,7 +160,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1488</xloc>
<xloc>960</xloc>
<yloc>272</yloc>
</GUI>
</transform>
@ -247,7 +215,7 @@ ON CONFLICT (job_name, recruitment_id)
</partitioning>
<attributes/>
<GUI>
<xloc>736</xloc>
<xloc>544</xloc>
<yloc>272</yloc>
</GUI>
</transform>
@ -283,25 +251,7 @@ ON CONFLICT (job_name, recruitment_id)
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1488</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>736</xloc>
<xloc>960</xloc>
<yloc>448</yloc>
</GUI>
</transform>
@ -316,23 +266,26 @@ ON CONFLICT (job_name, recruitment_id)
<method>none</method>
<schema_name/>
</partitioning>
<compress>N</compress>
<directory>${java.io.tmpdir}</directory>
<prefix/>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<name>recruit_id</name>
<presorted>N</presorted>
</field>
</fields>
<sort_size>1000000</sort_size>
<unique_rows>Y</unique_rows>
<attributes/>
<GUI>
<xloc>736</xloc>
<xloc>544</xloc>
<yloc>608</yloc>
</GUI>
</transform>
@ -367,7 +320,7 @@ ON CONFLICT (job_name, recruitment_id)
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>960</xloc>
<xloc>720</xloc>
<yloc>608</yloc>
</GUI>
</transform>
@ -375,7 +328,7 @@ ON CONFLICT (job_name, recruitment_id)
<name>foerign_diploma_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -384,9 +337,9 @@ ON CONFLICT (job_name, recruitment_id)
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr
ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr' as diploma_arr,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedInObr' -> 'inObr') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -396,8 +349,8 @@ SELECT fd.recruit_id,
to_date(diploma_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
diploma_elem ->> 'kodUrObr' as education_level_code,
diploma_elem ->> 'naimUchZaved' as educational_institution_name,
diploma_elem ->> 'naimInoDok' as foreign_doc_name,
diploma_elem ->> 'serNomDok' as ser_num_doc,
diploma_elem ->> 'naimInoDok' as foreign_doc_name,
diploma_elem ->> 'serNomDok' as ser_num_doc,
diploma_elem ->> 'regNomDok' as reg_num,
diploma_elem ->> 'nomSved' as recognition_certificate_number,
to_date(diploma_elem ->> 'dataPriz', 'YYYY-MM-DD') as recognition_date,
@ -406,7 +359,8 @@ SELECT fd.recruit_id,
diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code,
diploma_elem ->> 'kodOKSM' as oksm_code,
coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec,
true as has_foreign_diploma
true as has_foreign_diploma,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem;
</sql>
@ -517,7 +471,7 @@ FROM filteredData fd
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>960</xloc>
<xloc>736</xloc>
<yloc>448</yloc>
</GUI>
</transform>

111
mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow.hpl
View file

@ -35,26 +35,11 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_education_diploma_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>citizen_education_diploma_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
@ -65,23 +50,28 @@
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_education_diploma_output</from>
<to>Block until transforms finish</to>
<from>education_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<from>education_input</from>
<to>citizen_education_diploma_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_input</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_education_diploma_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
@ -102,32 +92,10 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1328</xloc>
<xloc>1200</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1152</xloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -160,7 +128,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1088</xloc>
<xloc>960</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -192,7 +160,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1328</xloc>
<xloc>960</xloc>
<yloc>224</yloc>
</GUI>
</transform>
@ -238,7 +206,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>640</xloc>
<xloc>448</xloc>
<yloc>224</yloc>
</GUI>
</transform>
@ -274,25 +242,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1328</xloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>640</xloc>
<xloc>960</xloc>
<yloc>400</yloc>
</GUI>
</transform>
@ -307,23 +257,26 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<compress>N</compress>
<directory>${java.io.tmpdir}</directory>
<prefix/>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<name>recruit_id</name>
<presorted>N</presorted>
</field>
</fields>
<sort_size>1000000</sort_size>
<unique_rows>Y</unique_rows>
<attributes/>
<GUI>
<xloc>640</xloc>
<xloc>448</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -358,7 +311,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>832</xloc>
<xloc>720</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -456,7 +409,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>928</xloc>
<xloc>720</xloc>
<yloc>400</yloc>
</GUI>
</transform>
@ -464,7 +417,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<name>education_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -475,7 +428,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr
ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedRObr' -> 'robr') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -496,7 +450,8 @@ SELECT fd.recruit_id,
diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code,
diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code,
coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec,
true as has_russian_diploma
true as has_russian_diploma,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem;</sql>
<variables_active>Y</variables_active>

82
mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_delta.hpl
View file

@ -51,21 +51,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Insert / update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Change job status on error</to>
@ -81,11 +66,6 @@
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
@ -96,6 +76,21 @@
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_input</from>
<to>Insert / update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_input</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -113,7 +108,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1472</xloc>
<xloc>1328</xloc>
<yloc>656</yloc>
</GUI>
</transform>
@ -149,7 +144,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1248</xloc>
<xloc>1120</xloc>
<yloc>656</yloc>
</GUI>
</transform>
@ -181,7 +176,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1472</xloc>
<xloc>1120</xloc>
<yloc>288</yloc>
</GUI>
</transform>
@ -230,7 +225,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>880</xloc>
<xloc>688</xloc>
<yloc>288</yloc>
</GUI>
</transform>
@ -266,25 +261,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1472</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>880</xloc>
<xloc>1120</xloc>
<yloc>464</yloc>
</GUI>
</transform>
@ -393,7 +370,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1248</xloc>
<xloc>912</xloc>
<yloc>464</yloc>
</GUI>
</transform>
@ -427,7 +404,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</fields>
<attributes/>
<GUI>
<xloc>880</xloc>
<xloc>688</xloc>
<yloc>656</yloc>
</GUI>
</transform>
@ -462,7 +439,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1072</xloc>
<xloc>912</xloc>
<yloc>656</yloc>
</GUI>
</transform>
@ -470,7 +447,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<name>education_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -479,13 +456,13 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>
WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' as diploma_arr
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedRObr' -> 'robr') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
)
SELECT fd.recruit_id,
diploma_elem ->> 'id' AS source_id,
@ -503,7 +480,8 @@ SELECT fd.recruit_id,
diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code,
diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code,
coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec,
true as has_russian_diploma
true as has_russian_diploma,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem
where case

103
mappings/info_recruits/citizen_tables/education/russian_diploma/parallel/russian_diploma_flow_repeat.hpl
View file

@ -41,21 +41,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Insert / update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Change job status on error</to>
@ -71,23 +56,28 @@
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Block until transforms finish</to>
<from>education_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<from>education_input</from>
<to>Insert / update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>education_input</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
@ -108,32 +98,10 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1216</xloc>
<xloc>1184</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -166,7 +134,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>992</xloc>
<xloc>944</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -198,7 +166,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1216</xloc>
<xloc>944</xloc>
<yloc>208</yloc>
</GUI>
</transform>
@ -253,7 +221,7 @@ ON CONFLICT (job_name, recruitment_id)
</partitioning>
<attributes/>
<GUI>
<xloc>624</xloc>
<xloc>432</xloc>
<yloc>208</yloc>
</GUI>
</transform>
@ -289,25 +257,7 @@ ON CONFLICT (job_name, recruitment_id)
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>624</xloc>
<xloc>944</xloc>
<yloc>384</yloc>
</GUI>
</transform>
@ -416,7 +366,7 @@ ON CONFLICT (job_name, recruitment_id)
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>864</xloc>
<xloc>672</xloc>
<yloc>384</yloc>
</GUI>
</transform>
@ -450,7 +400,7 @@ ON CONFLICT (job_name, recruitment_id)
</fields>
<attributes/>
<GUI>
<xloc>624</xloc>
<xloc>432</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -468,13 +418,11 @@ ON CONFLICT (job_name, recruitment_id)
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
@ -487,7 +435,7 @@ ON CONFLICT (job_name, recruitment_id)
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>816</xloc>
<xloc>672</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -495,7 +443,7 @@ ON CONFLICT (job_name, recruitment_id)
<name>education_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -504,8 +452,10 @@ ON CONFLICT (job_name, recruitment_id)
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr
ri.info -> 'svedFL' -> 'svedRObr' -> 'robr' AS diploma_arr,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedRObr' -> 'robr') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -526,7 +476,8 @@ SELECT fd.recruit_id,
diploma_elem -> 'svedSpeczMinobr' ->> 'kodSpecz' as minobr_speciality_code,
diploma_elem -> 'svedSpeczMinprosv' ->> 'kodSpecz' as minprosv_speciality_code,
coalesce(diploma_elem ->> 'prVUSpecz' = '1', false) as vus_spec,
true as has_russian_diploma
true as has_russian_diploma,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(diploma_arr) AS diploma_elem;</sql>
<variables_active>Y</variables_active>

100
mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_delta.hpl
View file

@ -41,43 +41,33 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>election_candidate_flag_update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>election_candidate_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>election_candidate_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>election_candidate_upsert</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<from>election_candidate_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>election_candidate_input</from>
<to>election_candidate_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>election_candidate_input</from>
<to>election_candidate_flag_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>election_candidate_upsert</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
@ -98,30 +88,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1296</xloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>election_candidate_upsert</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>416</yloc>
<xloc>976</xloc>
<yloc>672</yloc>
</GUI>
</transform>
<transform>
@ -156,8 +124,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>896</xloc>
<yloc>688</yloc>
<xloc>704</xloc>
<yloc>672</yloc>
</GUI>
</transform>
<transform>
@ -188,7 +156,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1296</xloc>
<xloc>976</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -237,7 +205,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>624</xloc>
<xloc>432</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -273,25 +241,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1296</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>624</xloc>
<xloc>976</xloc>
<yloc>416</yloc>
</GUI>
</transform>
@ -331,15 +281,15 @@ and recruitment_id = '${IDM_ID}';</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>688</yloc>
<xloc>432</xloc>
<yloc>672</yloc>
</GUI>
</transform>
<transform>
<name>election_candidate_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -350,7 +300,8 @@ and recruitment_id = '${IDM_ID}';</sql>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem
ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedKandidat' ->> 'kandidat' &lt;&gt; 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -367,7 +318,8 @@ SELECT fd.recruit_id,
to_date(deputy_elem ->> 'dataReg', 'YYYY-MM-DD') AS registration_date,
deputy_elem ->> 'prichinaSnyatNaim' AS termination_reason,
to_date(deputy_elem ->> 'dataSnyatKandidat', 'YYYY-MM-DD') AS termination_date,
true as has_election_candidate_info
true as has_election_candidate_info,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -441,7 +393,7 @@ FROM filteredData fd;</sql>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>896</xloc>
<xloc>704</xloc>
<yloc>416</yloc>
</GUI>
</transform>

94
mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_flow.hpl
View file

@ -46,16 +46,6 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>election_candidate_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
@ -68,24 +58,24 @@
</hop>
<hop>
<from>election_candidate_input</from>
<to>Identify last row in a stream</to>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>election_candidate_input</from>
<to>election_candidate_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>election_candidate_input</from>
<to>election_candidate_flag_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>election_candidate_output</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>election_candidate_flag_update</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -103,32 +93,10 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1248</xloc>
<xloc>1072</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>election_candidate_flag_update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1248</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -161,7 +129,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>976</xloc>
<xloc>832</xloc>
<yloc>592</yloc>
</GUI>
</transform>
@ -193,7 +161,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1504</xloc>
<xloc>1072</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -239,7 +207,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>704</xloc>
<xloc>512</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -275,25 +243,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1504</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>704</xloc>
<xloc>1072</xloc>
<yloc>416</yloc>
</GUI>
</transform>
@ -328,7 +278,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>704</xloc>
<xloc>512</xloc>
<yloc>592</yloc>
</GUI>
</transform>
@ -336,7 +286,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<name>election_candidate_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -347,7 +297,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem
ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedKandidat' ->> 'kandidat' &lt;&gt; 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -360,7 +311,8 @@ SELECT fd.recruit_id,
to_date(deputy_elem ->> 'dataReg', 'YYYY-MM-DD') AS registration_date,
deputy_elem ->> 'prichinaSnyatNaim' AS termination_reason,
to_date(deputy_elem ->> 'dataSnyatKandidat', 'YYYY-MM-DD') AS termination_date,
true as has_election_candidate_info
true as has_election_candidate_info,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -431,7 +383,7 @@ FROM filteredData fd;</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>976</xloc>
<xloc>832</xloc>
<yloc>416</yloc>
</GUI>
</transform>

84
mappings/info_recruits/citizen_tables/election_candidate/parallel/election_candidate_repeat.hpl
View file

@ -46,21 +46,11 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>election_candidate_flag_update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>election_candidate_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>election_candidate_upsert</from>
<to>Change job status on error</to>
@ -73,22 +63,22 @@
</hop>
<hop>
<from>election_candidate_input</from>
<to>Identify last row in a stream</to>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>election_candidate_flag_update</to>
<from>election_candidate_input</from>
<to>election_candidate_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>election_candidate_upsert</from>
<to>Block until transforms finish</to>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<from>election_candidate_input</from>
<to>election_candidate_flag_update</to>
<enabled>Y</enabled>
</hop>
</order>
@ -108,32 +98,10 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1584</xloc>
<xloc>1056</xloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>election_candidate_flag_update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1408</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -166,7 +134,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1152</xloc>
<xloc>848</xloc>
<yloc>688</yloc>
</GUI>
</transform>
@ -198,7 +166,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1584</xloc>
<xloc>1056</xloc>
<yloc>336</yloc>
</GUI>
</transform>
@ -247,7 +215,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>784</xloc>
<xloc>576</xloc>
<yloc>336</yloc>
</GUI>
</transform>
@ -283,25 +251,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1584</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>784</xloc>
<xloc>1056</xloc>
<yloc>512</yloc>
</GUI>
</transform>
@ -336,7 +286,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>784</xloc>
<xloc>576</xloc>
<yloc>688</yloc>
</GUI>
</transform>
@ -344,7 +294,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<name>election_candidate_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -355,7 +305,8 @@ and recruitment_id = '${IDM_ID}';</sql>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem
ri.info -> 'svedFL' -> 'svedKandidat' -> 'kandidat' as deputy_elem,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedKandidat' ->> 'kandidat' &lt;&gt; 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -368,7 +319,8 @@ SELECT fd.recruit_id,
to_date(deputy_elem ->> 'dataReg', 'YYYY-MM-DD') AS registration_date,
deputy_elem ->> 'prichinaSnyatNaim' AS termination_reason,
to_date(deputy_elem ->> 'dataSnyatKandidat', 'YYYY-MM-DD') AS termination_date,
true as has_election_candidate_info
true as has_election_candidate_info,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -442,7 +394,7 @@ FROM filteredData fd;</sql>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1152</xloc>
<xloc>848</xloc>
<yloc>512</yloc>
</GUI>
</transform>

77
mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow.hpl
View file

@ -46,44 +46,39 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>disability_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>disability_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>disability_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>disability_output</from>
<from>has_disability_update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>disability_input</from>
<to>has_disability_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>has_disability_update</from>
<to>Filter rows</to>
<from>disability_input</from>
<to>disability_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>has_disability_update</from>
<to>Change job status on error</to>
<from>disability_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>disability_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
@ -103,7 +98,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1504</xloc>
<xloc>960</xloc>
<yloc>576</yloc>
</GUI>
</transform>
@ -139,7 +134,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1216</xloc>
<xloc>736</xloc>
<yloc>576</yloc>
</GUI>
</transform>
@ -171,7 +166,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1504</xloc>
<xloc>960</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -217,7 +212,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>704</xloc>
<xloc>512</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -253,25 +248,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1504</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>704</xloc>
<xloc>960</xloc>
<yloc>416</yloc>
</GUI>
</transform>
@ -279,7 +256,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<name>disability_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -290,7 +267,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' AS invalid_elem
ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' AS invalid_elem,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedInvalid' ->> 'invalid' &lt;&gt; 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -304,7 +282,8 @@ SELECT fd.recruit_id,
to_date(invalid_elem -> 'ustanInvalid' ->> 'dataPodtverzhdPlan', 'YYYY-MM-DD') as confirmation_date,
to_date(invalid_elem ->> 'dataSnyat', 'YYYY-MM-DD') as deregistration_date,
coalesce(invalid_elem -> 'ustanInvalid' ->> 'prBessrInvl' = '1', false) as permanent,
true as has_disability
true as has_disability,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -379,7 +358,7 @@ FROM filteredData fd;</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>992</xloc>
<xloc>736</xloc>
<yloc>416</yloc>
</GUI>
</transform>
@ -414,8 +393,8 @@ FROM filteredData fd;</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>416</yloc>
<xloc>512</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform_error_handling>

79
mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_delta.hpl
View file

@ -41,16 +41,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>disability_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>disability_upsert</from>
<to>Change job status on error</to>
@ -61,16 +51,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>disability_upsert</from>
<to>has_disability_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>has_disability_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>has_disability_update</from>
<to>Change job status on error</to>
@ -78,7 +58,22 @@
</hop>
<hop>
<from>disability_input</from>
<to>Identify last row in a stream</to>
<to>has_disability_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>disability_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>disability_input</from>
<to>disability_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>disability_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
@ -98,7 +93,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1968</xloc>
<xloc>1472</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -134,7 +129,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1744</xloc>
<xloc>1280</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -166,7 +161,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1968</xloc>
<xloc>1472</xloc>
<yloc>208</yloc>
</GUI>
</transform>
@ -215,7 +210,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>1264</xloc>
<xloc>1040</xloc>
<yloc>208</yloc>
</GUI>
</transform>
@ -251,25 +246,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1968</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1264</xloc>
<xloc>1472</xloc>
<yloc>384</yloc>
</GUI>
</transform>
@ -277,7 +254,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<name>disability_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -288,7 +265,8 @@ and recruitment_id = '${IDM_ID}';</sql>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' AS invalid_elem
ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' AS invalid_elem,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedInvalid' ->> 'invalid' &lt;&gt; 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -308,7 +286,8 @@ SELECT fd.recruit_id,
to_date(invalid_elem -> 'ustanInvalid' ->> 'dataPodtverzhdPlan', 'YYYY-MM-DD') as confirmation_date,
to_date(invalid_elem ->> 'dataSnyat', 'YYYY-MM-DD') as deregistration_date,
coalesce(invalid_elem -> 'ustanInvalid' ->> 'prBessrInvl' = '1', false) as permanent,
true as has_disability
true as has_disability,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -387,7 +366,7 @@ FROM filteredData fd;</sql>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1552</xloc>
<xloc>1280</xloc>
<yloc>384</yloc>
</GUI>
</transform>
@ -422,8 +401,8 @@ FROM filteredData fd;</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1744</xloc>
<yloc>384</yloc>
<xloc>1040</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform_error_handling>

80
mappings/info_recruits/citizen_tables/medicine/disability/parallel/disability_flow_repeat.hpl
View file

@ -35,21 +35,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>disability_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>disability_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>disability_upsert</from>
<to>Change job status on error</to>
@ -61,18 +46,28 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>disability_upsert</from>
<from>has_disability_update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>disability_input</from>
<to>has_disability_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>has_disability_update</from>
<to>Filter rows</to>
<from>disability_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>has_disability_update</from>
<to>Change job status on error</to>
<from>disability_input</from>
<to>disability_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>disability_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
@ -92,7 +87,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1504</xloc>
<xloc>1104</xloc>
<yloc>592</yloc>
</GUI>
</transform>
@ -128,7 +123,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1120</xloc>
<xloc>896</xloc>
<yloc>592</yloc>
</GUI>
</transform>
@ -160,7 +155,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1504</xloc>
<xloc>1104</xloc>
<yloc>256</yloc>
</GUI>
</transform>
@ -209,7 +204,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>832</xloc>
<xloc>640</xloc>
<yloc>256</yloc>
</GUI>
</transform>
@ -245,25 +240,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1504</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>832</xloc>
<xloc>1104</xloc>
<yloc>432</yloc>
</GUI>
</transform>
@ -271,7 +248,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<name>disability_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -282,9 +259,10 @@ and recruitment_id = '${IDM_ID}';</sql>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' AS invalid_elem
ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' AS invalid_elem,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedInvalid' -> 'invalid' &lt;&gt; 'null'
WHERE ri.info -> 'svedFL' -> 'svedInvalid' ->> 'invalid' &lt;&gt; 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
)
SELECT fd.recruit_id,
@ -295,7 +273,9 @@ SELECT fd.recruit_id,
to_date(invalid_elem -> 'ustanInvalid' ->> 'dataUstan', 'YYYY-MM-DD') as register_date,
to_date(invalid_elem -> 'ustanInvalid' ->> 'dataPodtverzhdPlan', 'YYYY-MM-DD') as confirmation_date,
to_date(invalid_elem ->> 'dataSnyat', 'YYYY-MM-DD') as deregistration_date,
coalesce(invalid_elem -> 'ustanInvalid' ->> 'prBessrInvl' = '1', false) as permanent
coalesce(invalid_elem -> 'ustanInvalid' ->> 'prBessrInvl' = '1', false) as permanent,
true as has_disability,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -374,7 +354,7 @@ FROM filteredData fd;</sql>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1120</xloc>
<xloc>896</xloc>
<yloc>432</yloc>
</GUI>
</transform>
@ -409,8 +389,8 @@ FROM filteredData fd;</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1312</xloc>
<yloc>432</yloc>
<xloc>640</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform_error_handling>

167
mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow.hpl
View file

@ -35,16 +35,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>health_state_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_output</from>
<to>Change job status on error</to>
@ -61,13 +51,13 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error 2</to>
<from>health_state_input</from>
<to>Filter rows 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update 2</from>
<to>Block until transforms finish</to>
<from>Filter rows 2</from>
<to>health_state_output</to>
<enabled>Y</enabled>
</hop>
<hop>
@ -76,23 +66,18 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update 2</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_input</from>
<to>Filter rows 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows 2</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error 2</from>
<to>Abort 2</to>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
</order>
@ -112,52 +97,10 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1472</xloc>
<xloc>1440</xloc>
<yloc>672</yloc>
</GUI>
</transform>
<transform>
<name>Abort 2</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>928</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1584</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -190,46 +133,10 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1248</xloc>
<xloc>1104</xloc>
<yloc>672</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error 2</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
@ -258,8 +165,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1744</xloc>
<yloc>288</yloc>
<xloc>1424</xloc>
<yloc>304</yloc>
</GUI>
</transform>
<transform>
@ -304,8 +211,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>960</xloc>
<yloc>288</yloc>
<xloc>560</xloc>
<yloc>304</yloc>
</GUI>
</transform>
<transform>
@ -340,7 +247,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1744</xloc>
<xloc>1424</xloc>
<yloc>464</yloc>
</GUI>
</transform>
@ -374,31 +281,13 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</condition>
</compare>
<send_false_to>Update</send_false_to>
<send_true_to>Identify last row in a stream</send_true_to>
<send_true_to>health_state_output</send_true_to>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>960</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
@ -431,7 +320,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>624</yloc>
<yloc>672</yloc>
</GUI>
</transform>
<transform>
@ -465,7 +354,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1392</xloc>
<xloc>1248</xloc>
<yloc>464</yloc>
</GUI>
</transform>
@ -473,7 +362,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<name>health_state_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -482,12 +371,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved' as source_update_date,
ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov' as zdorov_arr,
coalesce(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'priznakNalichSved' =
'1', false) as has_health_state_info
coalesce(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'priznakNalichSved' = '1', false) as has_health_state_info,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedSostZdorov' ->> 'svedZdorov' &lt;&gt; 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -498,7 +386,8 @@ SELECT fd.recruit_id,
source_update_date,
zdorov_elem ->> 'kodZabol' AS disease_code,
to_date(zdorov_elem ->> 'dataDiagn', 'YYYY-MM-DD') AS diagnosis_date,
coalesce(zdorov_elem ->> 'kodZabol' &lt;&gt; 'null', false) as has_disease
coalesce(zdorov_elem ->> 'kodZabol' &lt;&gt; 'null', false) as has_disease,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(COALESCE(
NULLIF(zdorov_arr, 'null'::jsonb),
@ -560,14 +449,14 @@ FROM filteredData fd
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1248</xloc>
<xloc>1104</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error 2</target_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

7
mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_delta.hpl
View file

@ -505,8 +505,8 @@ and recruitment_id = '${IDM_ID}';</sql>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved' as source_update_date,
ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov' as zdorov_arr,
coalesce(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'priznakNalichSved' =
'1', false) as has_health_state_info
coalesce(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'priznakNalichSved' = '1', false) as has_health_state_info,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedSostZdorov' ->> 'svedZdorov' &lt;&gt; 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -523,7 +523,8 @@ SELECT fd.recruit_id,
source_update_date,
zdorov_elem ->> 'kodZabol' AS disease_code,
to_date(zdorov_elem ->> 'dataDiagn', 'YYYY-MM-DD') AS diagnosis_date,
coalesce(zdorov_elem ->> 'kodZabol' &lt;&gt; 'null', false) as has_disease
coalesce(zdorov_elem ->> 'kodZabol' &lt;&gt; 'null', false) as has_disease,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(COALESCE(
NULLIF(zdorov_arr, 'null'::jsonb),

182
mappings/info_recruits/citizen_tables/medicine/health_state/parallel/health_state_flow_repeat.hpl
View file

@ -46,16 +46,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>health_state_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_upsert</from>
<to>Change job status on error</to>
@ -66,31 +56,11 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error 2</from>
<to>Abort 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update 2</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_upsert</from>
<to>Update 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_input</from>
<to>Filter rows 2</to>
@ -98,12 +68,27 @@
</hop>
<hop>
<from>Filter rows 2</from>
<to>Identify last row in a stream</to>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update 2</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows 2</from>
<to>Update</to>
<to>health_state_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>health_state_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
</order>
@ -123,50 +108,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1696</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>Abort 2</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>752</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1488</xloc>
<yloc>448</yloc>
<xloc>1184</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
@ -201,44 +144,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1136</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error 2</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>752</yloc>
<xloc>832</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
@ -269,8 +176,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1696</xloc>
<yloc>272</yloc>
<xloc>1184</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform>
@ -318,8 +225,8 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>272</yloc>
<xloc>464</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform>
@ -354,7 +261,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1696</xloc>
<xloc>1184</xloc>
<yloc>448</yloc>
</GUI>
</transform>
@ -388,31 +295,13 @@ and recruitment_id = '${IDM_ID}';</sql>
</condition>
</compare>
<send_false_to>Update</send_false_to>
<send_true_to>Identify last row in a stream</send_true_to>
<send_true_to>health_state_upsert</send_true_to>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
@ -483,7 +372,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1296</xloc>
<xloc>992</xloc>
<yloc>448</yloc>
</GUI>
</transform>
@ -491,7 +380,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<name>health_state_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -502,10 +391,10 @@ and recruitment_id = '${IDM_ID}';</sql>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
to_date(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved' as source_update_date,
ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov' as zdorov_arr,
coalesce(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'priznakNalichSved' =
'1', false) as has_health_state_info
coalesce(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'priznakNalichSved' = '1', false) as has_health_state_info,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedSostZdorov' ->> 'svedZdorov' &lt;&gt; 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -516,7 +405,8 @@ SELECT fd.recruit_id,
source_update_date,
zdorov_elem ->> 'kodZabol' AS disease_code,
to_date(zdorov_elem ->> 'dataDiagn', 'YYYY-MM-DD') AS diagnosis_date,
coalesce(zdorov_elem ->> 'kodZabol' &lt;&gt; 'null', false) as has_disease
coalesce(zdorov_elem ->> 'kodZabol' &lt;&gt; 'null', false) as has_disease,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(COALESCE(
NULLIF(zdorov_arr, 'null'::jsonb),
@ -578,14 +468,14 @@ FROM filteredData fd
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1136</xloc>
<xloc>832</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error 2</target_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

119
mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow.hpl
View file

@ -35,54 +35,44 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>incapacity_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>incapacity_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>incapacity_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>incapacity_output</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<from>Sort rows</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>incapacity_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>incapacity_input</from>
<to>incapacity_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>incapacity_input</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Update</to>
<from>incapacity_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
@ -102,30 +92,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1424</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1168</xloc>
<yloc>416</yloc>
<xloc>1152</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -160,8 +128,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1136</xloc>
<yloc>560</yloc>
<xloc>960</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -192,7 +160,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1424</xloc>
<xloc>1152</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -238,7 +206,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>704</xloc>
<xloc>512</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -274,25 +242,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1424</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>704</xloc>
<xloc>1152</xloc>
<yloc>416</yloc>
</GUI>
</transform>
@ -326,8 +276,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</fields>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>560</yloc>
<xloc>512</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -361,15 +311,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>912</xloc>
<yloc>560</yloc>
<xloc>720</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
<name>incapacity_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -378,10 +328,10 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (
SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob' as nedeesposob_arr
ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob' as nedeesposob_arr,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -393,7 +343,8 @@ SELECT fd.recruit_id,
nedeesposob_elem ->> 'naimOrg' AS court_name,
to_date(nedeesposob_elem ->> 'dataPrisv', 'YYYY-MM-DD') AS register_date,
to_date(nedeesposob_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS deregistration_date,
true as has_incapacity_info
true as has_incapacity_info,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(fd.nedeesposob_arr) AS nedeesposob_elem;</sql>
<variables_active>Y</variables_active>
@ -461,7 +412,7 @@ FROM filteredData fd
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>912</xloc>
<xloc>720</xloc>
<yloc>416</yloc>
</GUI>
</transform>

111
mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_delta.hpl
View file

@ -36,21 +36,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>incapacity_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>incapacity_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>incapacity_upsert</from>
<to>Change job status on error</to>
@ -67,23 +52,28 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>incapacity_upsert</from>
<to>Block until transforms finish</to>
<from>Sort rows</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<from>incapacity_input</from>
<to>incapacity_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<from>incapacity_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>incapacity_input</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Update</to>
<from>incapacity_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
@ -103,32 +93,10 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1888</xloc>
<xloc>1696</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1744</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -161,7 +129,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1568</xloc>
<xloc>1440</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -193,7 +161,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1888</xloc>
<xloc>1696</xloc>
<yloc>208</yloc>
</GUI>
</transform>
@ -242,7 +210,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>1232</xloc>
<xloc>1040</xloc>
<yloc>208</yloc>
</GUI>
</transform>
@ -278,25 +246,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1888</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1232</xloc>
<xloc>1696</xloc>
<yloc>384</yloc>
</GUI>
</transform>
@ -311,23 +261,26 @@ and recruitment_id = '${IDM_ID}';</sql>
<method>none</method>
<schema_name/>
</partitioning>
<compress>N</compress>
<directory>${java.io.tmpdir}</directory>
<prefix/>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<name>recruit_id</name>
<presorted>N</presorted>
</field>
</fields>
<sort_size>1000000</sort_size>
<unique_rows>Y</unique_rows>
<attributes/>
<GUI>
<xloc>1232</xloc>
<xloc>1040</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -362,7 +315,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1392</xloc>
<xloc>1200</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -370,7 +323,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<name>incapacity_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -381,19 +334,21 @@ and recruitment_id = '${IDM_ID}';</sql>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob' as nedeesposob_arr
ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob' as nedeesposob_arr,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
)
SELECT fd.recruit_id,
nedeesposob_elem -&gt;&gt; 'id' AS source_id,
nedeesposob_elem ->> 'id' AS source_id,
to_date(nedeesposob_elem ->> 'dataSved', 'YYYY-MM-DD') AS source_update_date,
nedeesposob_elem ->> 'naimStatus' AS capacity_status,
nedeesposob_elem ->> 'naimOrg' AS court_name,
to_date(nedeesposob_elem ->> 'dataPrisv', 'YYYY-MM-DD') AS register_date,
to_date(nedeesposob_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS deregistration_date,
true as has_incapacity_info
true as has_incapacity_info,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(nedeesposob_arr) AS nedeesposob_elem
WHERE case when '${MAX_SOURCE_UPDATE_DATE}' &lt;&gt; ''
@ -467,7 +422,7 @@ WHERE case when '${MAX_SOURCE_UPDATE_DATE}' &lt;&gt; ''
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1568</xloc>
<xloc>1440</xloc>
<yloc>384</yloc>
</GUI>
</transform>

114
mappings/info_recruits/citizen_tables/medicine/incapacity/parallel/incapacity_flow_repeat.hpl
View file

@ -35,21 +35,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>incapacity_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>incapacity_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>incapacity_upsert</from>
<to>Change job status on error</to>
@ -66,23 +51,28 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>incapacity_upsert</from>
<to>Block until transforms finish</to>
<from>Sort rows</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<from>incapacity_input</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Update</to>
<from>incapacity_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>incapacity_input</from>
<to>incapacity_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>incapacity_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
@ -101,31 +91,9 @@
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1424</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1280</xloc>
<yloc>432</yloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
@ -160,8 +128,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1120</xloc>
<yloc>640</yloc>
<xloc>1056</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
@ -192,7 +160,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1424</xloc>
<xloc>1280</xloc>
<yloc>256</yloc>
</GUI>
</transform>
@ -241,7 +209,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>832</xloc>
<xloc>640</xloc>
<yloc>256</yloc>
</GUI>
</transform>
@ -277,25 +245,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1424</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>832</xloc>
<xloc>1280</xloc>
<yloc>432</yloc>
</GUI>
</transform>
@ -329,8 +279,8 @@ and recruitment_id = '${IDM_ID}';</sql>
</fields>
<attributes/>
<GUI>
<xloc>832</xloc>
<yloc>640</yloc>
<xloc>640</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
@ -364,15 +314,15 @@ and recruitment_id = '${IDM_ID}';</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>640</yloc>
<xloc>832</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>incapacity_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -381,13 +331,13 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (
SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob' as nedeesposob_arr
ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob' as nedeesposob_arr,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedNedeesp' -> 'nedeesposob') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
)
SELECT fd.recruit_id,
nedeesposob_elem ->> 'id' AS source_id,
@ -395,7 +345,9 @@ SELECT fd.recruit_id,
nedeesposob_elem ->> 'naimStatus' AS capacity_status,
nedeesposob_elem ->> 'naimOrg' AS court_name,
to_date(nedeesposob_elem ->> 'dataPrisv', 'YYYY-MM-DD') AS register_date,
to_date(nedeesposob_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS deregistration_date
to_date(nedeesposob_elem ->> 'dataOkonch', 'YYYY-MM-DD') AS deregistration_date,
true as has_incapacity_info,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(fd.nedeesposob_arr) AS nedeesposob_elem;</sql>
<variables_active>Y</variables_active>
@ -465,7 +417,7 @@ FROM filteredData fd
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1120</xloc>
<xloc>1056</xloc>
<yloc>432</yloc>
</GUI>
</transform>

148
mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow.hpl
View file

@ -46,16 +46,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>vich_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>vich_output</from>
<to>change_job_status_on_output_error</to>
@ -86,16 +76,6 @@
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>change_job_status_on_update_error</from>
<to>abort_on_update_error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>change_job_status_on_update_error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>vich_input</from>
<to>Filter rows 2</to>
@ -103,7 +83,17 @@
</hop>
<hop>
<from>Filter rows 2</from>
<to>Identify last row in a stream</to>
<to>vich_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>vich_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>change_job_status_on_output_error</to>
<enabled>Y</enabled>
</hop>
</order>
@ -135,7 +125,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1664</xloc>
<xloc>1360</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -181,7 +171,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>928</xloc>
<xloc>512</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -217,7 +207,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1664</xloc>
<xloc>1360</xloc>
<yloc>416</yloc>
</GUI>
</transform>
@ -250,31 +240,13 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value>
</condition>
</compare>
<send_true_to>Identify last row in a stream</send_true_to>
<send_true_to>vich_output</send_true_to>
<attributes/>
<GUI>
<xloc>720</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>928</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
@ -286,24 +258,27 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<compress>N</compress>
<directory>${java.io.tmpdir}</directory>
<prefix/>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<name>recruit_id</name>
<presorted>N</presorted>
</field>
</fields>
<sort_size>1000000</sort_size>
<unique_rows>Y</unique_rows>
<attributes/>
<GUI>
<xloc>512</xloc>
<yloc>576</yloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
@ -341,8 +316,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>512</xloc>
<yloc>704</yloc>
<xloc>720</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
@ -361,30 +336,10 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1440</xloc>
<xloc>1136</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>abort_on_update_error</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>720</xloc>
<yloc>864</yloc>
</GUI>
</transform>
<transform>
<name>block_until_update_finished</name>
<type>BlockUntilTransformsFinish</type>
@ -403,7 +358,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</transforms>
<attributes/>
<GUI>
<xloc>1440</xloc>
<xloc>1136</xloc>
<yloc>416</yloc>
</GUI>
</transform>
@ -439,46 +394,10 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1216</xloc>
<xloc>912</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>change_job_status_on_update_error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>512</xloc>
<yloc>864</yloc>
</GUI>
</transform>
<transform>
<name>vich_input</name>
<type>TableInput</type>
@ -492,7 +411,6 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
to_date(ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' ->> 'dataSved',
'YYYY-MM-DD') as source_update_date,
@ -500,7 +418,8 @@ and recruitment_id = '${IDM_ID}';
coalesce(
ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' ->> 'priznakNalichSved' = '1',
false) as vich_registered,
true as has_vich_info
true as has_vich_info,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedUchetVICH' ->> 'prOtsSvedUchetVICH' IS DISTINCT FROM '1'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -511,7 +430,8 @@ SELECT fd.recruit_id,
to_date(vich_elem ->> 'dataSnyat', 'YYYY-MM-DD') AS register_date,
to_date(vich_elem ->> 'dataPostUchet', 'YYYY-MM-DD') AS deregistration_date,
vich_registered,
has_vich_info
has_vich_info,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(COALESCE(
NULLIF(vich_arr, 'null'::jsonb),
@ -573,14 +493,14 @@ FROM filteredData fd
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1216</xloc>
<xloc>912</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>change_job_status_on_update_error</target_transform>
<target_transform>change_job_status_on_output_error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

149
mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_delta.hpl
View file

@ -46,16 +46,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>vich_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>vich_upsert</from>
<to>Change job status on error</to>
@ -71,16 +61,6 @@
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>change_job_status_on_update_error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>change_job_status_on_update_error</from>
<to>abort_on_update_error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>vich_input</from>
<to>Sort rows</to>
@ -103,7 +83,17 @@
</hop>
<hop>
<from>Filter rows 2</from>
<to>Identify last row in a stream</to>
<to>vich_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>vich_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
</order>
@ -123,7 +113,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>2160</xloc>
<xloc>1856</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -159,7 +149,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1776</xloc>
<xloc>1472</xloc>
<yloc>560</yloc>
</GUI>
</transform>
@ -191,8 +181,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>2160</xloc>
<yloc>208</yloc>
<xloc>1856</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
@ -240,8 +230,8 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>1488</xloc>
<yloc>208</yloc>
<xloc>1040</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
@ -276,7 +266,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>2160</xloc>
<xloc>1856</xloc>
<yloc>384</yloc>
</GUI>
</transform>
@ -309,31 +299,13 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Identify last row in a stream</send_true_to>
<send_true_to>vich_upsert</send_true_to>
<attributes/>
<GUI>
<xloc>1248</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1488</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
@ -345,24 +317,27 @@ and recruitment_id = '${IDM_ID}';</sql>
<method>none</method>
<schema_name/>
</partitioning>
<compress>N</compress>
<directory>${java.io.tmpdir}</directory>
<prefix/>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<name>recruit_id</name>
<presorted>N</presorted>
</field>
</fields>
<sort_size>1000000</sort_size>
<unique_rows>Y</unique_rows>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>528</yloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
@ -395,29 +370,9 @@ and recruitment_id = '${IDM_ID}';</sql>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>656</yloc>
</GUI>
</transform>
<transform>
<name>abort_on_update_error</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1248</xloc>
<yloc>816</yloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
@ -438,46 +393,10 @@ and recruitment_id = '${IDM_ID}';</sql>
</transforms>
<attributes/>
<GUI>
<xloc>1968</xloc>
<xloc>1664</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>change_job_status_on_update_error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>816</yloc>
</GUI>
</transform>
<transform>
<name>vich_input</name>
<type>TableInput</type>
@ -499,7 +418,8 @@ and recruitment_id = '${IDM_ID}';
coalesce(
ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' ->> 'priznakNalichSved' = '1',
false) as vich_registered,
true as has_vich_info
true as has_vich_info,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedUchetVICH' ->> 'prOtsSvedUchetVICH' IS DISTINCT FROM '1'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -510,7 +430,8 @@ SELECT fd.recruit_id,
to_date(vich_elem ->> 'dataSnyat', 'YYYY-MM-DD') AS register_date,
to_date(vich_elem ->> 'dataPostUchet', 'YYYY-MM-DD') AS deregistration_date,
vich_registered,
has_vich_info
has_vich_info,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(COALESCE(
NULLIF(vich_arr, 'null'::jsonb),
@ -575,14 +496,14 @@ WHERE case when '${MAX_SOURCE_UPDATE_DATE}' &lt;&gt; ''
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1776</xloc>
<xloc>1472</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>change_job_status_on_update_error</target_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

181
mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_repeat.hpl
View file

@ -46,16 +46,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>vich_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>vich_upsert</from>
<to>Change job status on error</to>
@ -71,41 +61,36 @@
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>change_job_status_on_update_error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>change_job_status_on_update_error</from>
<to>abort_on_update_error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>vich_input</from>
<to>Filter rows 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows 2</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>vich_input</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>vich_upsert</from>
<to>block_until_update_finished</to>
<from>Filter rows 2</from>
<to>vich_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>block_until_update_finished</from>
<from>vich_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>vich_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -123,7 +108,7 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1744</xloc>
<xloc>1424</xloc>
<yloc>608</yloc>
</GUI>
</transform>
@ -159,7 +144,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1328</xloc>
<xloc>1008</xloc>
<yloc>608</yloc>
</GUI>
</transform>
@ -191,8 +176,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1744</xloc>
<yloc>256</yloc>
<xloc>1424</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform>
@ -240,8 +225,8 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>256</yloc>
<xloc>640</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform>
@ -276,7 +261,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1744</xloc>
<xloc>1424</xloc>
<yloc>432</yloc>
</GUI>
</transform>
@ -309,31 +294,13 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Identify last row in a stream</send_true_to>
<send_true_to>vich_upsert</send_true_to>
<attributes/>
<GUI>
<xloc>816</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
@ -345,24 +312,27 @@ and recruitment_id = '${IDM_ID}';</sql>
<method>none</method>
<schema_name/>
</partitioning>
<compress>N</compress>
<directory>${java.io.tmpdir}</directory>
<prefix/>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<name>recruit_id</name>
<presorted>N</presorted>
</field>
</fields>
<sort_size>1000000</sort_size>
<unique_rows>Y</unique_rows>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>576</yloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
@ -400,86 +370,8 @@ and recruitment_id = '${IDM_ID}';</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>704</yloc>
</GUI>
</transform>
<transform>
<name>abort_on_update_error</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>864</yloc>
</GUI>
</transform>
<transform>
<name>block_until_update_finished</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1536</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>change_job_status_on_update_error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>864</yloc>
<xloc>816</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
@ -495,7 +387,6 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
to_date(ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' ->> 'dataSved',
'YYYY-MM-DD') as source_update_date,
@ -503,7 +394,8 @@ and recruitment_id = '${IDM_ID}';
coalesce(
ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' ->> 'priznakNalichSved' = '1',
false) as vich_registered,
true as has_vich_info
true as has_vich_info,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedUchetVICH' ->> 'prOtsSvedUchetVICH' IS DISTINCT FROM '1'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -514,7 +406,8 @@ SELECT fd.recruit_id,
to_date(vich_elem ->> 'dataSnyat', 'YYYY-MM-DD') AS register_date,
to_date(vich_elem ->> 'dataPostUchet', 'YYYY-MM-DD') AS deregistration_date,
vich_registered,
has_vich_info
has_vich_info,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(COALESCE(
NULLIF(vich_arr, 'null'::jsonb),
@ -576,14 +469,14 @@ FROM filteredData fd
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1328</xloc>
<xloc>1008</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>
<target_transform>change_job_status_on_update_error</target_transform>
<target_transform>Change job status on error</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

138
mappings/info_recruits/citizen_tables/sport/parallel/sport_flow.hpl
View file

@ -46,21 +46,11 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_rank_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
@ -76,31 +66,6 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>sport_flags_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_title_output</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_rank_output</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter rows 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows 2</from>
<to>sport_title_output</to>
@ -111,6 +76,31 @@
<to>sport_rank_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_input</from>
<to>sport_flags_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_input</from>
<to>Filter rows 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_title_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_rank_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -128,30 +118,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1504</xloc>
<yloc>672</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>sport_flags_update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1264</xloc>
<yloc>432</yloc>
<xloc>1296</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -185,8 +153,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>672</yloc>
<xloc>1120</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -217,8 +185,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1504</xloc>
<yloc>240</yloc>
<xloc>1296</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform>
@ -263,8 +231,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>240</yloc>
<xloc>512</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform>
@ -299,7 +267,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1504</xloc>
<xloc>1296</xloc>
<yloc>432</yloc>
</GUI>
</transform>
@ -336,25 +304,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>sport_title_output</send_true_to>
<attributes/>
<GUI>
<xloc>880</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>704</xloc>
<xloc>688</xloc>
<yloc>416</yloc>
</GUI>
</transform>
@ -393,15 +343,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>672</yloc>
<xloc>512</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
<name>sport_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -412,7 +362,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedSport' -> 'sport' AS sport_arr
ri.info -> 'svedFL' -> 'svedSport' -> 'sport' AS sport_arr,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSport' -> 'sport') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -431,7 +382,8 @@ SELECT fd.recruit_id,
sport_elem -> 'sportZvan' ->> 'sbornaya' as title_national_team,
sport_elem -> 'sportZvan' ->> 'zvanieNaim' as title,
to_date(sport_elem -> 'sportZvan' ->> 'dataPrisv', 'YYYY-MM-DD') as title_assigment_date,
coalesce(sport_elem ->> 'sportZvan' &lt;&gt; 'null', false) as has_sport_title
coalesce(sport_elem ->> 'sportZvan' &lt;&gt; 'null', false) as has_sport_title,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(sport_arr) as sport_elem;</sql>
<variables_active>Y</variables_active>
@ -507,8 +459,8 @@ FROM filteredData fd
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1056</xloc>
<yloc>480</yloc>
<xloc>928</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
@ -569,7 +521,7 @@ FROM filteredData fd
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1056</xloc>
<xloc>928</xloc>
<yloc>368</yloc>
</GUI>
</transform>

135
mappings/info_recruits/citizen_tables/sport/parallel/sport_flow_delta.hpl
View file

@ -40,21 +40,11 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_rank_flag_update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_rank_upsert</from>
<to>Change job status on error</to>
@ -65,31 +55,6 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>sport_rank_flag_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_rank_upsert</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sports_title_upsert</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter rows 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows 2</from>
<to>sport_rank_upsert</to>
@ -100,6 +65,31 @@
<to>sports_title_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_input</from>
<to>Filter rows 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_input</from>
<to>sport_rank_flag_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_rank_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sports_title_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -117,30 +107,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1552</xloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>sport_rank_flag_update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1264</xloc>
<yloc>416</yloc>
<xloc>1120</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
@ -175,8 +143,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1136</xloc>
<yloc>688</yloc>
<xloc>896</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
@ -207,8 +175,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1552</xloc>
<yloc>240</yloc>
<xloc>1120</xloc>
<yloc>256</yloc>
</GUI>
</transform>
<transform>
@ -256,8 +224,8 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>240</yloc>
<xloc>432</xloc>
<yloc>256</yloc>
</GUI>
</transform>
<transform>
@ -292,7 +260,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1552</xloc>
<xloc>1120</xloc>
<yloc>416</yloc>
</GUI>
</transform>
@ -328,24 +296,6 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_false_to>sports_title_upsert</send_false_to>
<send_true_to>sport_rank_upsert</send_true_to>
<attributes/>
<GUI>
<xloc>784</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>416</yloc>
@ -355,7 +305,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<name>sport_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -364,9 +314,9 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedSport' -> 'sport' AS sport_arr
ri.info -> 'svedFL' -> 'svedSport' -> 'sport' AS sport_arr,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSport' -> 'sport') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -385,7 +335,8 @@ SELECT fd.recruit_id,
sport_elem -> 'sportZvan' ->> 'sbornaya' as title_national_team,
sport_elem -> 'sportZvan' ->> 'zvanieNaim' as title,
to_date(sport_elem -> 'sportZvan' ->> 'dataPrisv', 'YYYY-MM-DD') as title_assigment_date,
coalesce(sport_elem ->> 'sportZvan' &lt;&gt; 'null', false) as has_sport_title
coalesce(sport_elem ->> 'sportZvan' &lt;&gt; 'null', false) as has_sport_title,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(sport_arr) as sport_elem
where case
@ -436,8 +387,8 @@ where case
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>624</xloc>
<yloc>688</yloc>
<xloc>432</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
@ -510,7 +461,7 @@ where case
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>944</xloc>
<xloc>784</xloc>
<yloc>368</yloc>
</GUI>
</transform>
@ -574,7 +525,7 @@ where case
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>944</xloc>
<xloc>784</xloc>
<yloc>512</yloc>
</GUI>
</transform>

137
mappings/info_recruits/citizen_tables/sport/parallel/sport_flow_repeat.hpl
View file

@ -40,21 +40,11 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_flags_update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sports_title_upsert</from>
<to>Change job status on error</to>
@ -65,31 +55,6 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>sport_flags_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sports_title_upsert</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_rank_upsert</from>
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter rows 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows 2</from>
<to>sport_rank_upsert</to>
@ -100,6 +65,31 @@
<to>sports_title_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_rank_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sports_title_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_input</from>
<to>Filter rows 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sport_input</from>
<to>sport_flags_update</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -117,30 +107,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1584</xloc>
<yloc>784</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>sport_flags_update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1392</xloc>
<yloc>512</yloc>
<xloc>1280</xloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
@ -175,8 +143,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1296</xloc>
<yloc>784</yloc>
<xloc>1072</xloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
@ -207,7 +175,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1584</xloc>
<xloc>1280</xloc>
<yloc>336</yloc>
</GUI>
</transform>
@ -256,7 +224,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>784</xloc>
<xloc>592</xloc>
<yloc>336</yloc>
</GUI>
</transform>
@ -292,7 +260,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1584</xloc>
<xloc>1280</xloc>
<yloc>512</yloc>
</GUI>
</transform>
@ -329,25 +297,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>sport_rank_upsert</send_true_to>
<attributes/>
<GUI>
<xloc>944</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>784</xloc>
<xloc>752</xloc>
<yloc>512</yloc>
</GUI>
</transform>
@ -386,15 +336,15 @@ and recruitment_id = '${IDM_ID}';</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>784</xloc>
<yloc>784</yloc>
<xloc>592</xloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
<name>sport_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -403,9 +353,9 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedSport' -> 'sport' AS sport_arr
ri.info -> 'svedFL' -> 'svedSport' -> 'sport' AS sport_arr,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedSport' -> 'sport') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -424,7 +374,8 @@ SELECT fd.recruit_id,
sport_elem -> 'sportZvan' ->> 'sbornaya' as title_national_team,
sport_elem -> 'sportZvan' ->> 'zvanieNaim' as title,
to_date(sport_elem -> 'sportZvan' ->> 'dataPrisv', 'YYYY-MM-DD') as title_assigment_date,
coalesce(sport_elem ->> 'sportZvan' &lt;&gt; 'null', false) as has_sport_title
coalesce(sport_elem ->> 'sportZvan' &lt;&gt; 'null', false) as has_sport_title,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(sport_arr) as sport_elem;</sql>
<variables_active>Y</variables_active>
@ -504,8 +455,8 @@ FROM filteredData fd
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>448</yloc>
<xloc>912</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
@ -568,8 +519,8 @@ FROM filteredData fd
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>592</yloc>
<xloc>912</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform_error_handling>

136
mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow.hpl
View file

@ -35,21 +35,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>individual_entrepreneur_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
@ -75,16 +60,6 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>filter_null_npd_dates</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>filter_non_actual_ip</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>filter_non_actual_ip</from>
<to>Update</to>
@ -96,12 +71,27 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_output</from>
<to>Block until transforms finish</to>
<from>individual_entrepreneur_input</from>
<to>filter_null_npd_dates</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<from>individual_entrepreneur_input</from>
<to>filter_non_actual_ip</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_input</from>
<to>individual_entrepreneur_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
@ -121,34 +111,9 @@
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1632</xloc>
<yloc>704</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
<transform>
<name>individual_entrepreneur_npd_output</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1424</xloc>
<yloc>448</yloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
@ -183,8 +148,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1424</xloc>
<yloc>704</yloc>
<xloc>1232</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
@ -215,7 +180,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1600</xloc>
<xloc>1232</xloc>
<yloc>288</yloc>
</GUI>
</transform>
@ -261,7 +226,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>800</xloc>
<xloc>592</xloc>
<yloc>288</yloc>
</GUI>
</transform>
@ -297,25 +262,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1600</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>800</xloc>
<xloc>1232</xloc>
<yloc>448</yloc>
</GUI>
</transform>
@ -350,8 +297,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1184</xloc>
<yloc>576</yloc>
<xloc>992</xloc>
<yloc>544</yloc>
</GUI>
</transform>
<transform>
@ -386,8 +333,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Update</send_true_to>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>576</yloc>
<xloc>784</xloc>
<yloc>544</yloc>
</GUI>
</transform>
<transform>
@ -414,15 +361,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>unique_by_rec_id</send_true_to>
<attributes/>
<GUI>
<xloc>800</xloc>
<yloc>704</yloc>
<xloc>592</xloc>
<yloc>672</yloc>
</GUI>
</transform>
<transform>
<name>individual_entrepreneur_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -439,7 +386,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
'YYYY-MM-DD') as reg_npd_date,
to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataSnUchNPD',
'YYYY-MM-DD') as dereg_npd_date,
true as actual
true as actual,
ri.ctid
from ervu_dashboard.recruits_info ri
where ri.info -> 'svedFL' -> 'svedRegIP' ->> 'regIP' != 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -452,7 +400,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
'YYYY-MM-DD') as reg_npd_date,
to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataSnUchNPD',
'YYYY-MM-DD') as dereg_npd_date,
false as actual
false as actual,
ri.ctid
from ervu_dashboard.recruits_info ri
where ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'predRegIP' != 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -462,10 +411,11 @@ select fd.recruit_id,
ip_elem ->> 'ogrnip' as ogrnip,
to_date(ip_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
to_date(ip_elem ->> 'dataRegIP', 'YYYY-MM-DD') as registration_date,
to_date(ip_elem -&gt;&gt; 'dataINNNed', 'YYYY-MM-DD') as deregistration_date,
to_date(ip_elem ->> 'dataINNNed', 'YYYY-MM-DD') as deregistration_date,
reg_npd_date,
dereg_npd_date,
fd.actual as actual
fd.actual as actual,
fd.ctid = (select max(ctid) from filteredData) as last_row
from filtered_data fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -520,8 +470,8 @@ from filtered_data fd;</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1184</xloc>
<yloc>704</yloc>
<xloc>976</xloc>
<yloc>672</yloc>
</GUI>
</transform>
<transform>
@ -582,7 +532,7 @@ from filtered_data fd;</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1184</xloc>
<xloc>992</xloc>
<yloc>448</yloc>
</GUI>
</transform>
@ -607,8 +557,8 @@ from filtered_data fd;</sql>
</fields>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>704</yloc>
<xloc>768</xloc>
<yloc>672</yloc>
</GUI>
</transform>
<transform_error_handling>

61
mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow_delta.hpl
View file

@ -20,11 +20,6 @@
<notepads>
</notepads>
<order>
<hop>
<from>Block until transforms finish 2</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<to>Abort</to>
@ -50,11 +45,6 @@
<to>upsert_pred</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Block until transforms finish 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>delete_akt_become_null</from>
<to>Change job status on error</to>
@ -155,6 +145,11 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -172,36 +167,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1872</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish 2</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>update_akt_flag_on_delete</name>
</transform>
<transform>
<name>update_akt_flag</name>
</transform>
<transform>
<name>upsert_pred</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1872</xloc>
<yloc>400</yloc>
<xloc>1856</xloc>
<yloc>656</yloc>
</GUI>
</transform>
<transform>
@ -236,8 +203,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1696</xloc>
<yloc>576</yloc>
<xloc>1648</xloc>
<yloc>656</yloc>
</GUI>
</transform>
<transform>
@ -268,7 +235,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>2096</xloc>
<xloc>1920</xloc>
<yloc>400</yloc>
</GUI>
</transform>
@ -384,7 +351,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Block until transforms finish 2</send_true_to>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1696</xloc>
@ -630,7 +597,8 @@ and recruitment_id = '${IDM_ID}';</sql>
to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataUchNPD',
'YYYY-MM-DD') as reg_npd_date,
to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataSnUchNPD',
'YYYY-MM-DD') as dereg_npd_date
'YYYY-MM-DD') as dereg_npd_date,
ctid
from ervu_dashboard.recruits_info ri
where ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'aktRegIP' != 'null'
and (to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'aktRegIP' ->> 'dataSved',
@ -652,7 +620,8 @@ select fd.recruit_id,
to_date(pred_ip_elem ->> 'dataSved', 'YYYY-MM-DD') as pred_source_update_date,
to_date(pred_ip_elem ->> 'dataRegIP', 'YYYY-MM-DD') as pred_registration_date,
to_date(pred_ip_elem ->> 'dataINNNed', 'YYYY-MM-DD') as pred_deregistration_date,
to_date(fd.pred_ip_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' as pred_need_update
to_date(fd.pred_ip_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' as pred_need_update,
fd.ctid = (select max(ctid) from filteredData) as last_row
from filtered_data fd;
</sql>
<variables_active>Y</variables_active>

144
mappings/info_recruits/citizen_tables/work/individual_entrepreneur/parallel/individual_entrepreneur_flow_repeat.hpl
View file

@ -36,31 +36,16 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>filter_null_npd_dates</from>
<to>unique_by_rec_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>individual_entrepreneur_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_upsert</from>
<to>Change job status on error</to>
@ -71,11 +56,6 @@
<to>individual_entrepreneur_npd_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>filter_null_npd_dates</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_npd_upsert</from>
<to>Change job status on error</to>
@ -86,23 +66,33 @@
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>filter_non_actual_ip</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_upsert</from>
<to>Block until transforms finish</to>
<from>individual_entrepreneur_input</from>
<to>filter_null_npd_dates</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<from>individual_entrepreneur_input</from>
<to>filter_non_actual_ip</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_input</from>
<to>individual_entrepreneur_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>individual_entrepreneur_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
@ -123,33 +113,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>2032</xloc>
<yloc>736</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
<transform>
<name>individual_entrepreneur_npd_upsert</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1808</xloc>
<yloc>464</yloc>
<xloc>1632</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
@ -184,8 +149,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1568</xloc>
<yloc>736</yloc>
<xloc>1456</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
@ -216,8 +181,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>2032</xloc>
<yloc>304</yloc>
<xloc>1440</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform>
@ -265,8 +230,8 @@ SET status = 'PROCESSING',
</partitioning>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>304</yloc>
<xloc>736</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform>
@ -301,25 +266,7 @@ SET status = 'PROCESSING',
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>2032</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>976</xloc>
<xloc>1440</xloc>
<yloc>464</yloc>
</GUI>
</transform>
@ -354,8 +301,8 @@ SET status = 'PROCESSING',
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1328</xloc>
<yloc>592</yloc>
<xloc>1088</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -390,8 +337,8 @@ SET status = 'PROCESSING',
<send_true_to>Update</send_true_to>
<attributes/>
<GUI>
<xloc>1136</xloc>
<yloc>592</yloc>
<xloc>896</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
@ -418,15 +365,15 @@ SET status = 'PROCESSING',
<send_true_to>unique_by_rec_id</send_true_to>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>736</yloc>
<xloc>736</xloc>
<yloc>720</yloc>
</GUI>
</transform>
<transform>
<name>individual_entrepreneur_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -443,9 +390,10 @@ SET status = 'PROCESSING',
'YYYY-MM-DD') as reg_npd_date,
to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataSnUchNPD',
'YYYY-MM-DD') as dereg_npd_date,
true as actual
true as actual,
ri.ctid
from ervu_dashboard.recruits_info ri
where ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'aktRegIP' != 'null'
where ri.info -> 'svedFL' -> 'svedRegIP' ->> 'regIP' != 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
union all
@ -456,10 +404,11 @@ SET status = 'PROCESSING',
'YYYY-MM-DD') as reg_npd_date,
to_date(ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' -> 'extend' ->> 'dataSnUchNPD',
'YYYY-MM-DD') as dereg_npd_date,
false as actual
false as actual,
ri.ctid
from ervu_dashboard.recruits_info ri
where ri.info -> 'svedFL' -> 'svedRegIP' -> 'regIP' ->> 'predRegIP' != 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
)
select fd.recruit_id,
fd.source_id,
@ -469,7 +418,8 @@ select fd.recruit_id,
to_date(ip_elem ->> 'dataINNNed', 'YYYY-MM-DD') as deregistration_date,
reg_npd_date,
dereg_npd_date,
fd.actual as actual
fd.actual as actual,
fd.ctid = (select max(ctid) from filteredData) as last_row
from filtered_data fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -523,8 +473,8 @@ from filtered_data fd;</sql>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1328</xloc>
<yloc>736</yloc>
<xloc>1088</xloc>
<yloc>720</yloc>
</GUI>
</transform>
<transform>
@ -587,7 +537,7 @@ from filtered_data fd;</sql>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1568</xloc>
<xloc>1088</xloc>
<yloc>464</yloc>
</GUI>
</transform>
@ -612,8 +562,8 @@ from filtered_data fd;</sql>
</fields>
<attributes/>
<GUI>
<xloc>1136</xloc>
<yloc>736</yloc>
<xloc>896</xloc>
<yloc>720</yloc>
</GUI>
</transform>
<transform_error_handling>

103
mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow.hpl
View file

@ -30,16 +30,6 @@
<to>self_employed_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>self_employed_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>self_employed_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
@ -50,21 +40,11 @@
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>filter_non_actual_self_employed</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>filter_non_actual_self_employed</from>
<to>Update</to>
@ -76,15 +56,25 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>self_employed_output</from>
<to>Block until transforms finish</to>
<from>self_employed_input</from>
<to>self_employed_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<from>self_employed_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>self_employed_input</from>
<to>filter_non_actual_self_employed</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>self_employed_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -102,32 +92,10 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1264</xloc>
<xloc>832</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1088</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -160,7 +128,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>896</xloc>
<xloc>656</xloc>
<yloc>496</yloc>
</GUI>
</transform>
@ -192,7 +160,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1264</xloc>
<xloc>832</xloc>
<yloc>48</yloc>
</GUI>
</transform>
@ -238,7 +206,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>624</xloc>
<xloc>432</xloc>
<yloc>48</yloc>
</GUI>
</transform>
@ -274,25 +242,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1264</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>624</xloc>
<xloc>832</xloc>
<yloc>224</yloc>
</GUI>
</transform>
@ -327,7 +277,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>624</xloc>
<xloc>432</xloc>
<yloc>496</yloc>
</GUI>
</transform>
@ -363,7 +313,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Update</send_true_to>
<attributes/>
<GUI>
<xloc>624</xloc>
<xloc>432</xloc>
<yloc>368</yloc>
</GUI>
</transform>
@ -371,7 +321,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<name>self_employed_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -383,7 +333,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<limit>0</limit>
<sql>with filtered_data as (select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' as npd_elem,
true as actual
true as actual,
ctid
from ervu_dashboard.recruits_info ri
where ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' ->> 'aktNPD' != 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -392,7 +343,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' as npd_elem,
false as actual
false as actual,
ctid
from ervu_dashboard.recruits_info ri
where ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' ->> 'predNPD' != 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -402,7 +354,8 @@ select fd.recruit_id,
to_date(npd_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
to_date(npd_elem ->> 'dataUchNPD', 'YYYY-MM-DD') as register_date,
to_date(npd_elem ->> 'dataSnUchNPD', 'YYYY-MM-DD') as deregistration_date,
actual
actual,
fd.ctid = (select max(ctid) from filteredData) as last_row
from filtered_data fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -465,7 +418,7 @@ from filtered_data fd;</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>896</xloc>
<xloc>656</xloc>
<yloc>224</yloc>
</GUI>
</transform>

55
mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_delta.hpl
View file

@ -20,11 +20,6 @@
<notepads>
</notepads>
<order>
<hop>
<from>Block until transforms finish 2</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<to>Abort</to>
@ -50,11 +45,6 @@
<to>upsert_pred</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Block until transforms finish 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>delete_akt_become_null</from>
<to>Change job status on error</to>
@ -135,6 +125,11 @@
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -156,34 +151,6 @@
<yloc>736</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish 2</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>update_akt_flag_on_delete</name>
</transform>
<transform>
<name>update_akt_flag</name>
</transform>
<transform>
<name>upsert_pred</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1872</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -246,7 +213,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>2096</xloc>
<xloc>1856</xloc>
<yloc>384</yloc>
</GUI>
</transform>
@ -362,7 +329,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Block until transforms finish 2</send_true_to>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1696</xloc>
@ -526,10 +493,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<limit/>
<sql>with filtered_data as (select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' as akt_npd_elem,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' as pred_npd_elem
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' as pred_npd_elem,
ctid
from ervu_dashboard.recruits_info ri
where ri.info -> 'svedFL' -> 'svedNPD' ->> 'npd' != 'null'
and (to_date(ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' ->> 'dataSved', 'YYYY-MM-DD') &gt; '${MAX_SOURCE_UPDATE_DATE}'
and (to_date(ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'
or to_date(ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}')
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
)
@ -545,7 +513,8 @@ select fd.recruit_id,
to_date(pred_npd_elem ->> 'dataSved', 'YYYY-MM-DD') as pred_source_update_date,
to_date(pred_npd_elem ->> 'dataUchNPD', 'YYYY-MM-DD') as pred_register_date,
to_date(pred_npd_elem ->> 'dataSnUchNPD', 'YYYY-MM-DD') as pred_deregistration_date,
to_date(pred_npd_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' as pred_need_update
to_date(pred_npd_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' as pred_need_update,
fd.ctid = (select max(ctid) from filteredData) as last_row
from filtered_data fd;</sql>
<variables_active>Y</variables_active>
<attributes/>

103
mappings/info_recruits/citizen_tables/work/self_employed/parallel/self_employed_flow_repeat.hpl
View file

@ -35,26 +35,11 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>self_employed_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>self_employed_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>self_employed_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
@ -65,23 +50,28 @@
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>filter_non_actual_self_employed</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>self_employed_output</from>
<to>Block until transforms finish</to>
<from>self_employed_input</from>
<to>filter_non_actual_self_employed</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<from>self_employed_input</from>
<to>self_employed_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>self_employed_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>self_employed_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
@ -102,32 +92,10 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1296</xloc>
<xloc>880</xloc>
<yloc>656</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -159,7 +127,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>864</xloc>
<xloc>688</xloc>
<yloc>656</yloc>
</GUI>
</transform>
@ -190,7 +158,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1296</xloc>
<xloc>880</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -239,7 +207,7 @@ SET status = 'PROCESSING',
</partitioning>
<attributes/>
<GUI>
<xloc>656</xloc>
<xloc>464</xloc>
<yloc>240</yloc>
</GUI>
</transform>
@ -275,25 +243,7 @@ SET status = 'PROCESSING',
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1296</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>656</xloc>
<xloc>880</xloc>
<yloc>416</yloc>
</GUI>
</transform>
@ -328,7 +278,7 @@ SET status = 'PROCESSING',
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>656</xloc>
<xloc>464</xloc>
<yloc>656</yloc>
</GUI>
</transform>
@ -364,7 +314,7 @@ SET status = 'PROCESSING',
<send_true_to>Update</send_true_to>
<attributes/>
<GUI>
<xloc>656</xloc>
<xloc>464</xloc>
<yloc>544</yloc>
</GUI>
</transform>
@ -372,7 +322,7 @@ SET status = 'PROCESSING',
<name>self_employed_input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
@ -384,7 +334,8 @@ SET status = 'PROCESSING',
<limit>0</limit>
<sql>with filtered_data as (select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'aktNPD' as npd_elem,
true as actual
true as actual,
ctid
from ervu_dashboard.recruits_info ri
where ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' ->> 'aktNPD' != 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -393,7 +344,8 @@ SET status = 'PROCESSING',
select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' -> 'predNPD' as npd_elem,
false as actual
false as actual,
ctid
from ervu_dashboard.recruits_info ri
where ri.info -> 'svedFL' -> 'svedNPD' -> 'npd' ->> 'predNPD' != 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -403,7 +355,8 @@ select fd.recruit_id,
to_date(npd_elem ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
to_date(npd_elem ->> 'dataUchNPD', 'YYYY-MM-DD') as register_date,
to_date(npd_elem ->> 'dataSnUchNPD', 'YYYY-MM-DD') as deregistration_date,
actual
actual,
fd.ctid = (select max(ctid) from filteredData) as last_row
from filtered_data fd;</sql>
<variables_active>Y</variables_active>
<attributes/>
@ -462,7 +415,7 @@ from filtered_data fd;</sql>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>864</xloc>
<xloc>688</xloc>
<yloc>416</yloc>
</GUI>
</transform>

224
mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow.hpl
View file

@ -41,11 +41,6 @@
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>employee_parental_leave_output</from>
<to>Change job status on error</to>
@ -71,39 +66,34 @@
<to>Filter_null_parental_leave_dates</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_input</from>
<to>Group by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Group by</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>work_activity_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>work_activity_output</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>work_activity_output</from>
<to>Block until transforms finish 2</to>
<from>Create job execution record</from>
<to>citizen_work_activity_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish 2</from>
<from>work_activity_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Create job execution record</from>
<to>citizen_work_activity_input</to>
<from>citizen_work_activity_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_input</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>work_activity_output</to>
<enabled>Y</enabled>
</hop>
</order>
@ -123,8 +113,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1728</xloc>
<yloc>624</yloc>
<xloc>1456</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
@ -145,30 +135,8 @@
</transforms>
<attributes/>
<GUI>
<xloc>928</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish 2</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>employee_parental_leave_output</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1392</xloc>
<yloc>320</yloc>
<xloc>720</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
@ -203,8 +171,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1504</xloc>
<yloc>624</yloc>
<xloc>1152</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
@ -235,7 +203,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1584</xloc>
<xloc>1152</xloc>
<yloc>144</yloc>
</GUI>
</transform>
@ -281,7 +249,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<attributes/>
<GUI>
<xloc>928</xloc>
<xloc>496</xloc>
<yloc>144</yloc>
</GUI>
</transform>
@ -317,7 +285,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1584</xloc>
<xloc>1152</xloc>
<yloc>320</yloc>
</GUI>
</transform>
@ -346,101 +314,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<send_true_to>Block until transforms finish</send_true_to>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
<name>Group by</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<aggregate>recruit_id</aggregate>
<subject>recruit_id</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>employer_name</aggregate>
<subject>employer_name</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>position</aggregate>
<subject>position</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>personnel_event_date</aggregate>
<subject>personnel_event_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>personnel_event_type</aggregate>
<subject>personnel_event_type</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>source_update_date</aggregate>
<subject>source_update_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>actual_employer</aggregate>
<subject>actual_employer</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>parental_leave_start_date</aggregate>
<subject>parental_leave_start_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>parental_leave_end_date</aggregate>
<subject>parental_leave_end_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
</fields>
<give_back_row>N</give_back_row>
<group>
<field>
<name>source_id</name>
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>736</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>928</xloc>
<yloc>320</yloc>
<xloc>496</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
@ -456,9 +331,9 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' as trud_arr
ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' as trud_arr,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -472,7 +347,8 @@ SELECT fd.recruit_id,
to_date(trud_elem ->> 'dataSved', 'YYYY-MM-DD') AS source_update_date,
coalesce((trud_elem ->> 'prAktMestRab') = '1', false) AS actual_employer,
to_date(uhod_elem ->> 'dataNachUhodReb', 'YYYY-MM-DD') AS parental_leave_start_date,
to_date(uhod_elem ->> 'dataKonUhodReb', 'YYYY-MM-DD') AS parental_leave_end_date
to_date(uhod_elem ->> 'dataKonUhodReb', 'YYYY-MM-DD') AS parental_leave_end_date,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(trud_arr) AS trud_elem
LEFT JOIN LATERAL jsonb_array_elements(
@ -531,8 +407,8 @@ FROM filteredData fd
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>624</yloc>
<xloc>944</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
@ -597,7 +473,41 @@ FROM filteredData fd
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1200</xloc>
<xloc>896</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>source_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>320</yloc>
</GUI>
</transform>

229
mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow_delta.hpl
View file

@ -45,11 +45,6 @@
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
@ -67,30 +62,50 @@
</hop>
<hop>
<from>citizen_work_activity_input</from>
<to>Group by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Group by</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>citizen_work_activity_upsert</to>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_upsert</from>
<to>Block until transforms finish 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish 2</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_input</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>citizen_work_activity_upsert</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
<type>Abort</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<abort_option>ABORT_WITH_ERROR</abort_option>
<always_log_rows>Y</always_log_rows>
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1344</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
@ -109,32 +124,10 @@
</transforms>
<attributes/>
<GUI>
<xloc>880</xloc>
<xloc>672</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish 2</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>parental_leave_upsert</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1360</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
@ -170,7 +163,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1552</xloc>
<xloc>1168</xloc>
<yloc>432</yloc>
</GUI>
</transform>
@ -202,7 +195,7 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1552</xloc>
<xloc>1184</xloc>
<yloc>48</yloc>
</GUI>
</transform>
@ -251,7 +244,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>832</xloc>
<xloc>432</xloc>
<yloc>48</yloc>
</GUI>
</transform>
@ -287,7 +280,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1552</xloc>
<xloc>1184</xloc>
<yloc>224</yloc>
</GUI>
</transform>
@ -316,103 +309,10 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Block until transforms finish</send_true_to>
<attributes/>
<GUI>
<xloc>640</xloc>
<xloc>432</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Group by</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<aggregate>recruit_id</aggregate>
<subject>recruit_id</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>employer_name</aggregate>
<subject>employer_name</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>position</aggregate>
<subject>position</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>personnel_event_date</aggregate>
<subject>personnel_event_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>personnel_event_type</aggregate>
<subject>personnel_event_type</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>source_update_date</aggregate>
<subject>source_update_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>actual_employer</aggregate>
<subject>actual_employer</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>parental_leave_start_date</aggregate>
<subject>parental_leave_start_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>parental_leave_end_date</aggregate>
<subject>parental_leave_end_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
</fields>
<give_back_row>N</give_back_row>
<group>
<field>
<name>source_id</name>
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>832</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>citizen_work_activity_input</name>
<type>TableInput</type>
@ -426,9 +326,9 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' as trud_arr
ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' as trud_arr,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -442,7 +342,8 @@ SELECT fd.recruit_id,
to_date(trud_elem ->> 'dataSved', 'YYYY-MM-DD') AS source_update_date,
coalesce((trud_elem ->> 'prAktMestRab') = '1', false) AS actual_employer,
to_date(uhod_elem ->> 'dataNachUhodReb', 'YYYY-MM-DD') AS parental_leave_start_date,
to_date(uhod_elem ->> 'dataKonUhodReb', 'YYYY-MM-DD') AS parental_leave_end_date
to_date(uhod_elem ->> 'dataKonUhodReb', 'YYYY-MM-DD') AS parental_leave_end_date,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(trud_arr) AS trud_elem
LEFT JOIN LATERAL jsonb_array_elements(
@ -526,7 +427,7 @@ where to_date(trud_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DAT
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1136</xloc>
<xloc>928</xloc>
<yloc>224</yloc>
</GUI>
</transform>
@ -570,10 +471,44 @@ where to_date(trud_elem ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DAT
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1136</xloc>
<xloc>928</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>source_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>672</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>citizen_work_activity_upsert</source_transform>

228
mappings/info_recruits/citizen_tables/work/work_activity/parallel/work_activity_flow_repeat.hpl
View file

@ -36,11 +36,6 @@
<to>Block until transforms finish</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>parental_leave_upsert</to>
@ -56,21 +51,6 @@
<to>Filter_null_parental_leave_dates</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_input</from>
<to>Group by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Group by</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>citizen_work_activity_upsert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_upsert</from>
<to>Change job status on error</to>
@ -82,18 +62,28 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_upsert</from>
<to>Block until transforms finish 2</to>
<from>Change job status on error</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish 2</from>
<from>citizen_work_activity_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_work_activity_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<to>Abort</to>
<from>citizen_work_activity_input</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>citizen_work_activity_upsert</to>
<enabled>Y</enabled>
</hop>
</order>
@ -113,8 +103,8 @@
<row_threshold>0</row_threshold>
<attributes/>
<GUI>
<xloc>1968</xloc>
<yloc>704</yloc>
<xloc>1552</xloc>
<yloc>720</yloc>
</GUI>
</transform>
<transform>
@ -135,30 +125,8 @@
</transforms>
<attributes/>
<GUI>
<xloc>1088</xloc>
<yloc>704</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish 2</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>parental_leave_upsert</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1600</xloc>
<yloc>496</yloc>
<xloc>896</xloc>
<yloc>720</yloc>
</GUI>
</transform>
<transform>
@ -193,8 +161,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1776</xloc>
<yloc>704</yloc>
<xloc>1344</xloc>
<yloc>720</yloc>
</GUI>
</transform>
<transform>
@ -225,8 +193,8 @@ and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1776</xloc>
<yloc>320</yloc>
<xloc>1344</xloc>
<yloc>336</yloc>
</GUI>
</transform>
<transform>
@ -274,8 +242,8 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<attributes/>
<GUI>
<xloc>1056</xloc>
<yloc>320</yloc>
<xloc>640</xloc>
<yloc>336</yloc>
</GUI>
</transform>
<transform>
@ -310,7 +278,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1776</xloc>
<xloc>1344</xloc>
<yloc>496</yloc>
</GUI>
</transform>
@ -339,101 +307,8 @@ and recruitment_id = '${IDM_ID}';</sql>
<send_true_to>Block until transforms finish</send_true_to>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>704</yloc>
</GUI>
</transform>
<transform>
<name>Group by</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<aggregate>recruit_id</aggregate>
<subject>recruit_id</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>employer_name</aggregate>
<subject>employer_name</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>position</aggregate>
<subject>position</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>personnel_event_date</aggregate>
<subject>personnel_event_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>personnel_event_type</aggregate>
<subject>personnel_event_type</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>source_update_date</aggregate>
<subject>source_update_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>actual_employer</aggregate>
<subject>actual_employer</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>parental_leave_start_date</aggregate>
<subject>parental_leave_start_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
<field>
<aggregate>parental_leave_end_date</aggregate>
<subject>parental_leave_end_date</subject>
<type>FIRST_INCL_NULL</type>
</field>
</fields>
<give_back_row>N</give_back_row>
<group>
<field>
<name>source_id</name>
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>832</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1056</xloc>
<yloc>496</yloc>
<xloc>640</xloc>
<yloc>720</yloc>
</GUI>
</transform>
<transform>
@ -450,7 +325,8 @@ and recruitment_id = '${IDM_ID}';</sql>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' as trud_arr
ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat' as trud_arr,
ctid
FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedTrud' -> 'trudDeyat') = 'array'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
@ -464,7 +340,8 @@ SELECT fd.recruit_id,
to_date(trud_elem ->> 'dataSved', 'YYYY-MM-DD') AS source_update_date,
coalesce((trud_elem ->> 'prAktMestRab') = '1', false) AS actual_employer,
to_date(uhod_elem ->> 'dataNachUhodReb', 'YYYY-MM-DD') AS parental_leave_start_date,
to_date(uhod_elem ->> 'dataKonUhodReb', 'YYYY-MM-DD') AS parental_leave_end_date
to_date(uhod_elem ->> 'dataKonUhodReb', 'YYYY-MM-DD') AS parental_leave_end_date,
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(trud_arr) AS trud_elem
LEFT JOIN LATERAL jsonb_array_elements(
@ -546,7 +423,7 @@ FROM filteredData fd
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1360</xloc>
<xloc>1136</xloc>
<yloc>496</yloc>
</GUI>
</transform>
@ -568,7 +445,6 @@ FROM filteredData fd
<condition>=</condition>
<field>citizen_work_activity_source_id</field>
<name>source_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>educational_leave</table>
@ -596,8 +472,42 @@ FROM filteredData fd
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1360</xloc>
<yloc>704</yloc>
<xloc>1136</xloc>
<yloc>720</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>source_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>896</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform_error_handling>