micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fixes

Browse source
This commit is contained in:
adel.ka 2025-11-12 17:38:09 +03:00
parent 9f22b17418
commit e89c18bd20
24 changed files with 1755 additions and 2116 deletions
Download patch file Download diff file Expand all files Collapse all files

300
mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl
View file

@ -45,11 +45,6 @@
<to>child_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>child_input </from>
<to>child_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>child_output</from>
<to>child_error_job_status</to>
@ -65,11 +60,6 @@
<to>User defined Java expression 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>child_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>child_input </from>
<to>Detect empty stream</to>
@ -77,12 +67,22 @@
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<to>child_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>child_success_job_status</to>
<from>child_output</from>
<to>Identify last row in a stream </to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream </from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>child_input </from>
<to>child_output</to>
<enabled>Y</enabled>
</hop>
</order>
@ -106,83 +106,6 @@
<yloc>1024</yloc>
</GUI>
</transform>
<transform>
<name>child_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>800</yloc>
</GUI>
</transform>
<transform>
<name>child_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
error_description = null,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>576</xloc>
<yloc>1040</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record </name>
<type>ExecSql</type>
@ -209,7 +132,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<attributes/>
<GUI>
<xloc>112</xloc>
<yloc>800</yloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
@ -258,13 +181,90 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>child_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>896</xloc>
<yloc>1040</yloc>
</GUI>
</transform>
<transform>
<name>User defined Java expression 2</name>
<type>Janino</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<formula>
<field_name>minors_count</field_name>
<formula_string>minors_count_raw == 1 ? "1" : minors_count_raw == 2 ? "2" : minors_count_raw >= 3 ? "3 и более" : "0"</formula_string>
<value_type>String</value_type>
<value_length>10</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<formula>
<field_name>has_minor_child</field_name>
<formula_string>has_minor_child_raw == 1 </formula_string>
<value_type>Boolean</value_type>
<value_length>-1</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<formula>
<field_name>has_dead_child</field_name>
<formula_string>has_dead_child_raw == 1 </formula_string>
<value_type>Boolean</value_type>
<value_length>-1</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<attributes/>
<GUI>
<xloc>688</xloc>
<yloc>480</yloc>
</GUI>
</transform>
<transform>
<name>child_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
<name>child_group_by</name>
<type>GroupBy</type>
@ -284,19 +284,16 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<aggregate>minors_count_raw</aggregate>
<subject>is_minor</subject>
<type>SUM</type>
<valuefield/>
</field>
<field>
<aggregate>has_minor_child_raw</aggregate>
<subject>is_minor</subject>
<type>MAX</type>
<valuefield/>
</field>
<field>
<aggregate>has_dead_child_raw</aggregate>
<subject>is_dead</subject>
<type>MAX</type>
<valuefield/>
</field>
</fields>
<give_back_row>N</give_back_row>
@ -306,7 +303,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<linenr_fieldname/>
<prefix>grp</prefix>
<attributes/>
<GUI>
@ -328,8 +324,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH children_data AS (
SELECT
<sql>SELECT
ri.recruit_id AS recruit_id,
(ch->>'id')::uuid AS child_external_id,
NULLIF(ch->>'idERN', '') AS child_id_ern,
@ -355,8 +350,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
WHEN dates.birth_date IS NULL THEN 0
WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1
ELSE 0
END AS is_minor,
ri.ctid as source_ctid
END AS is_minor
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch
@ -377,16 +371,12 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM children_data) as last_row
FROM children_data;</sql>
ORDER BY recruit_id;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>304</xloc>
<yloc>800</yloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
@ -456,14 +446,11 @@ FROM children_data;</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>citizen_child</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -471,7 +458,40 @@ FROM children_data;</sql>
<attributes/>
<GUI>
<xloc>896</xloc>
<yloc>800</yloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
<name>child_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
error_description = null,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>592</xloc>
<yloc>1040</yloc>
</GUI>
</transform>
<transform>
@ -488,13 +508,11 @@ FROM children_data;</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
@ -520,8 +538,8 @@ FROM children_data;</sql>
</GUI>
</transform>
<transform>
<name>User defined Java expression 2</name>
<type>Janino</type>
<name>Identify last row in a stream </name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -530,59 +548,11 @@ FROM children_data;</sql>
<method>none</method>
<schema_name/>
</partitioning>
<formula>
<field_name>minors_count</field_name>
<formula_string>minors_count_raw == 1 ? "1" : minors_count_raw == 2 ? "2" : minors_count_raw >= 3 ? "3 и более" : "0"</formula_string>
<value_type>String</value_type>
<value_length>10</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<formula>
<field_name>has_minor_child</field_name>
<formula_string>has_minor_child_raw == 1 </formula_string>
<value_type>Boolean</value_type>
<value_length>-1</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<formula>
<field_name>has_dead_child</field_name>
<formula_string>has_dead_child_raw == 1 </formula_string>
<value_type>Boolean</value_type>
<value_length>-1</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>688</xloc>
<yloc>480</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>736</xloc>
<yloc>1040</yloc>
<xloc>896</xloc>
<yloc>912</yloc>
</GUI>
</transform>
<transform_error_handling>

123
mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl
View file

@ -65,23 +65,23 @@
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>child_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>child_input</from>
<to>child_group_by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<from>child_insert_or_update</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>child_success_job_status</to>
<enabled>Y</enabled>
</hop>
@ -103,7 +103,7 @@
<attributes/>
<GUI>
<xloc>1376</xloc>
<yloc>400</yloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
@ -142,43 +142,6 @@ and recruitment_id = '${IDM_ID}';
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>child_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
error_description = null,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>784</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
@ -224,7 +187,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>496</yloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
@ -256,16 +219,16 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>child_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>496</yloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -274,18 +237,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>960</xloc>
<yloc>496</yloc>
<xloc>1072</xloc>
<yloc>368</yloc>
</GUI>
</transform>
<transform>
@ -387,8 +343,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH children_data AS (
SELECT
<sql>SELECT
ri.recruit_id AS recruit_id,
(ch->>'id')::uuid AS child_external_id,
NULLIF(ch->>'idERN', '') AS child_id_ern,
@ -414,8 +369,7 @@ and recruitment_id = '${IDM_ID}';</sql>
WHEN dates.birth_date IS NULL THEN 0
WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1
ELSE 0
END AS is_minor,
ri.ctid as source_ctid
END AS is_minor
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch
@ -437,11 +391,7 @@ and recruitment_id = '${IDM_ID}';</sql>
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM children_data) as last_row
FROM children_data;</sql>
ORDER BY recruit_id;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -543,6 +493,39 @@ FROM children_data;</sql>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>child_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
error_description = null,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>784</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
<name>child_update_flags</name>
<type>Update</type>

84
mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl
View file

@ -65,23 +65,23 @@
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>child_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>child_input</from>
<to>child_group_by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<from>child_insert_or_update</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>child_success_job_status</to>
<enabled>Y</enabled>
</hop>
@ -149,7 +149,7 @@ DO UPDATE SET
<attributes/>
<GUI>
<xloc>0</xloc>
<yloc>432</yloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform>
@ -198,38 +198,13 @@ DO UPDATE SET
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>child_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>608</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>User defined Java expression</name>
<type>Janino</type>
@ -304,7 +279,7 @@ and recruitment_id = '${IDM_ID}';
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>432</yloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform>
@ -365,8 +340,8 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH children_data AS (
SELECT
<limit/>
<sql>SELECT
ri.recruit_id AS recruit_id,
(ch->>'id')::uuid AS child_external_id,
NULLIF(ch->>'idERN', '') AS child_id_ern,
@ -392,8 +367,7 @@ and recruitment_id = '${IDM_ID}';
WHEN dates.birth_date IS NULL THEN 0
WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1
ELSE 0
END AS is_minor,
ri.ctid as source_ctid
END AS is_minor
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch
@ -414,16 +388,12 @@ and recruitment_id = '${IDM_ID}';
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM children_data) as last_row
FROM children_data;</sql>
ORDER BY recruit_id;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>224</xloc>
<yloc>432</yloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform>
@ -444,13 +414,11 @@ FROM children_data;</sql>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>child_external_id</field>
<name>child_external_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen_child</table>
@ -519,7 +487,7 @@ FROM children_data;</sql>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>432</yloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform>
@ -599,6 +567,24 @@ and status = 'PROCESSING';
<yloc>176</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>child_insert_or_update</source_transform>

195
mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl
View file

@ -40,11 +40,6 @@
<to>citizen_address_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_address_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_address_input</from>
<to>citizen_address_output</to>
@ -56,12 +51,17 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<from>citizen_address_output</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>citizen_address_success_job_status</to>
<enabled>Y</enabled>
</hop>
@ -86,83 +86,6 @@
<yloc>704</yloc>
</GUI>
</transform>
<transform>
<name>citizen_address_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>928</xloc>
<yloc>704</yloc>
</GUI>
</transform>
<transform>
<name>citizen_address_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1328</xloc>
<yloc>96</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
@ -238,7 +161,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>citizen_address_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>1328</xloc>
@ -246,8 +169,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<name>citizen_address_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -256,18 +179,29 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1328</xloc>
<yloc>208</yloc>
<xloc>928</xloc>
<yloc>704</yloc>
</GUI>
</transform>
<transform>
@ -284,10 +218,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH address_data AS (
SELECT
<sql>SELECT
ri.recruit_id::uuid,
ri.ctid as source_ctid,
MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address,
MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date,
MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date,
@ -308,12 +240,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
GROUP BY ri.recruit_id, ri.ctid
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM address_data) as last_row
FROM address_data;</sql>
GROUP BY ri.recruit_id</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -384,14 +311,11 @@ FROM address_data;</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>citizen_address</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -402,6 +326,57 @@ FROM address_data;</sql>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>citizen_address_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>1328</xloc>
<yloc>96</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1136</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>citizen_address_output</source_transform>

274
mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl
View file

@ -52,16 +52,16 @@
</hop>
<hop>
<from>citizen_address_insert_or_update</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>citizen_address_success_job_status</to>
<enabled>Y</enabled>
</hop>
@ -86,83 +86,6 @@
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>citizen_address_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>citizen_address_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1120</xloc>
<yloc>16</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
@ -241,10 +164,108 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>citizen_address_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>1120</xloc>
<xloc>1184</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1024</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>citizen_address_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>citizen_address_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>SELECT
ri.recruit_id::uuid,
MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address,
MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date,
MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date,
MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address,
MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date,
MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date,
CASE
WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица'
WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы'
END AS place_stay_registration_type,
MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address,
MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date,
MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
GROUP BY ri.recruit_id</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>224</yloc>
</GUI>
</transform>
@ -266,7 +287,6 @@ and recruitment_id = '${IDM_ID}';</sql>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen_address</table>
@ -334,60 +354,8 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>citizen_address_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH address_data AS (
SELECT
ri.recruit_id::uuid,
ri.ctid as source_ctid,
MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address,
MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date,
MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date,
MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address,
MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date,
MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date,
CASE
WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица'
WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы'
END AS place_stay_registration_type,
MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address,
MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date,
MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
GROUP BY ri.recruit_id, ri.ctid
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM address_data) as last_row
FROM address_data;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<name>citizen_address_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -396,18 +364,26 @@ FROM address_data;</sql>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>1120</xloc>
<yloc>112</yloc>
<xloc>1184</xloc>
<yloc>16</yloc>
</GUI>
</transform>
<transform_error_handling>

72
mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl
View file

@ -52,16 +52,16 @@
</hop>
<hop>
<from>citizen_address_insert_or_update</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>citizen_address_success_job_status</to>
<enabled>Y</enabled>
</hop>
@ -178,38 +178,13 @@ DO UPDATE SET
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>citizen_address_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>1120</xloc>
<xloc>1200</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>1120</xloc>
<yloc>144</yloc>
</GUI>
</transform>
<transform>
<name>citizen_address_error_job_status</name>
<type>ExecSql</type>
@ -259,10 +234,9 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH address_data AS (
SELECT
<limit/>
<sql>SELECT
ri.recruit_id::uuid,
ri.ctid as source_ctid,
MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address,
MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date,
MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date,
@ -283,12 +257,7 @@ and recruitment_id = '${IDM_ID}';
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
GROUP BY ri.recruit_id, ri.ctid
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM address_data) as last_row
FROM address_data;</sql>
GROUP BY ri.recruit_id</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -314,7 +283,6 @@ FROM address_data;</sql>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen_address</table>
@ -410,10 +378,28 @@ and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>1120</xloc>
<xloc>1200</xloc>
<yloc>16</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>citizen_address_insert_or_update</source_transform>

89
mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl
View file

@ -60,23 +60,23 @@
<to>update_guardianship_flags</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_guardianship_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_guardianship_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<from>citizen_guardianship_output</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>citizen_guardianship_success_job_status</to>
<enabled>Y</enabled>
</hop>
@ -144,7 +144,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>560</yloc>
<yloc>720</yloc>
</GUI>
</transform>
<transform>
@ -176,36 +176,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>citizen_guardianship_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>816</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>816</xloc>
<yloc>576</yloc>
<xloc>880</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
@ -225,11 +200,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
@ -241,7 +213,6 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1280</xloc>
@ -261,8 +232,8 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH guardianship_data AS (
SELECT
<limit/>
<sql>SELECT
ri.recruit_id AS recruit_id,
(ch->>'id')::uuid AS guardianship_external_id,
NULLIF(ch->>'idERN', '') AS guardian_id_ern,
@ -280,20 +251,14 @@ and recruitment_id = '${IDM_ID}';
NULLIF(ch->'dataRozhdDok'->>'den', '')::int
) AS birth_date,
REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils,
true AS is_guardian,
ri.ctid as source_ctid
true AS is_guardian
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM guardianship_data) as last_row
FROM guardianship_data;</sql>
AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array'</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -367,7 +332,7 @@ FROM guardianship_data;</sql>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>816</xloc>
<xloc>880</xloc>
<yloc>320</yloc>
</GUI>
</transform>
@ -434,7 +399,7 @@ and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>816</xloc>
<xloc>880</xloc>
<yloc>720</yloc>
</GUI>
</transform>
@ -478,6 +443,24 @@ and status = 'PROCESSING';
<yloc>0</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>880</xloc>
<yloc>480</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>citizen_guardianship_output</source_transform>

42
mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl
View file

@ -67,16 +67,16 @@
</hop>
<hop>
<from>citizen_guardianship_insert_or_update</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>citizen_guardianship_success_job_status</to>
<enabled>Y</enabled>
</hop>
@ -178,16 +178,16 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>citizen_guardianship_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>880</xloc>
<yloc>384</yloc>
<yloc>480</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -196,18 +196,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>880</xloc>
<yloc>496</yloc>
<yloc>352</yloc>
</GUI>
</transform>
<transform>
@ -262,8 +255,7 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH guardianship_data AS (
SELECT
<sql>SELECT
ri.recruit_id AS recruit_id,
(ch->>'id')::uuid AS guardianship_external_id,
NULLIF(ch->>'idERN', '') AS guardian_id_ern,
@ -281,8 +273,7 @@ and recruitment_id = '${IDM_ID}';
NULLIF(ch->'dataRozhdDok'->>'den', '')::int
) AS birth_date,
REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils,
true AS is_guardian,
ri.ctid as source_ctid
true AS is_guardian
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch
@ -290,12 +281,7 @@ and recruitment_id = '${IDM_ID}';
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM guardianship_data) as last_row
FROM guardianship_data;</sql>
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>

73
mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl
View file

@ -67,16 +67,16 @@
</hop>
<hop>
<from>citizen_guardianship_insert_update</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>citizen_guardianship_success_job_status</to>
<enabled>Y</enabled>
</hop>
@ -193,36 +193,11 @@ DO UPDATE SET
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>citizen_guardianship_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>496</yloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
@ -278,8 +253,8 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH guardianship_data AS (
SELECT
<limit/>
<sql>SELECT
ri.recruit_id AS recruit_id,
(ch->>'id')::uuid AS guardianship_external_id,
NULLIF(ch->>'idERN', '') AS guardian_id_ern,
@ -297,20 +272,14 @@ and recruitment_id = '${IDM_ID}';
NULLIF(ch->'dataRozhdDok'->>'den', '')::int
) AS birth_date,
REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils,
true AS is_guardian,
ri.ctid as source_ctid
true AS is_guardian
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM guardianship_data) as last_row
FROM guardianship_data;</sql>
AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array'</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -336,13 +305,11 @@ FROM guardianship_data;</sql>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>guardianship_external_id</field>
<name>guardianship_external_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen_guardianship</table>
@ -506,6 +473,24 @@ and status = 'PROCESSING';
<yloc>48</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>352</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>citizen_guardianship_insert_update</source_transform>

204
mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl
View file

@ -62,16 +62,16 @@
</hop>
<hop>
<from>citizen_information_search_output</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>citizen_information_success_job_status</to>
<enabled>Y</enabled>
</hop>
@ -93,84 +93,7 @@
<attributes/>
<GUI>
<xloc>880</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
<name>citizen_information_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>720</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
<name>citizen_information_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>160</yloc>
<yloc>720</yloc>
</GUI>
</transform>
<transform>
@ -199,7 +122,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<attributes/>
<GUI>
<xloc>80</xloc>
<yloc>320</yloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
@ -248,16 +171,16 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>citizen_information_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>320</yloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -266,18 +189,47 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>240</yloc>
<xloc>960</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>citizen_information_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>720</xloc>
<yloc>720</yloc>
</GUI>
</transform>
<transform>
@ -293,8 +245,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH wanted_data AS (
SELECT
<limit/>
<sql>SELECT
ri.recruit_id,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code,
@ -302,23 +254,17 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search,
true AS is_wanted,
ri.ctid as source_ctid
true AS is_wanted
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
WHERE '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM wanted_data) as last_row
FROM wanted_data;</sql>
AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object'</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>336</xloc>
<yloc>320</yloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
@ -368,14 +314,11 @@ FROM wanted_data;</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>citizen_information_search</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -383,7 +326,40 @@ FROM wanted_data;</sql>
<attributes/>
<GUI>
<xloc>720</xloc>
<yloc>320</yloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>citizen_information_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>160</yloc>
</GUI>
</transform>
<transform>
@ -423,7 +399,7 @@ FROM wanted_data;</sql>
<attributes/>
<GUI>
<xloc>336</xloc>
<yloc>624</yloc>
<yloc>720</yloc>
</GUI>
</transform>
<transform_error_handling>

197
mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl
View file

@ -55,23 +55,23 @@
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_information_search_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_information_search_input</from>
<to>update_citizen_information_search_flags</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<from>citizen_information_search_insert_or_update</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>citizen_information_success_job_status</to>
<enabled>Y</enabled>
</hop>
@ -96,83 +96,6 @@
<yloc>480</yloc>
</GUI>
</transform>
<transform>
<name>citizen_information_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>480</yloc>
</GUI>
</transform>
<transform>
<name>citizen_information_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>32</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
@ -251,16 +174,16 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>citizen_information_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>1008</xloc>
<xloc>1296</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -269,18 +192,47 @@ and recruitment_id = '${IDM_ID}';</sql>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>112</yloc>
<xloc>1120</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>citizen_information_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>480</yloc>
</GUI>
</transform>
<transform>
@ -296,8 +248,8 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH wanted_data AS (
SELECT
<limit/>
<sql>SELECT
ri.recruit_id,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code,
@ -305,19 +257,13 @@ and recruitment_id = '${IDM_ID}';</sql>
TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search,
true AS is_wanted,
ri.ctid as source_ctid
true AS is_wanted
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
WHERE '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM wanted_data) as last_row
FROM wanted_data;</sql>
AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object'</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -389,6 +335,39 @@ FROM wanted_data;</sql>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>citizen_information_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>1296</xloc>
<yloc>32</yloc>
</GUI>
</transform>
<transform>
<name>update_citizen_information_search_flags</name>
<type>Update</type>

79
mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl
View file

@ -50,11 +50,6 @@
<to>citizen_information_search_insert_or_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_information_search_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_information_search_input</from>
<to>update_citizen_information_search_flags</to>
@ -66,12 +61,17 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<from>citizen_information_search_insert_or_update</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>citizen_information_success_job_status</to>
<enabled>Y</enabled>
</hop>
@ -188,38 +188,13 @@ DO UPDATE SET
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>citizen_information_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>1024</xloc>
<xloc>1216</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>1024</xloc>
<yloc>160</yloc>
</GUI>
</transform>
<transform>
<name>citizen_information_error_job_status</name>
<type>ExecSql</type>
@ -269,8 +244,8 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH wanted_data AS (
SELECT
<limit/>
<sql>SELECT
ri.recruit_id,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code,
@ -278,18 +253,13 @@ and recruitment_id = '${IDM_ID}';
TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search,
true AS is_wanted,
ri.ctid as source_ctid
true AS is_wanted
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
WHERE '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM wanted_data) as last_row
FROM wanted_data;</sql>
</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -315,7 +285,6 @@ FROM wanted_data;</sql>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen_information_search</table>
@ -391,7 +360,7 @@ and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>1024</xloc>
<xloc>1216</xloc>
<yloc>80</yloc>
</GUI>
</transform>
@ -435,6 +404,24 @@ and status = 'PROCESSING';
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>citizen_information_search_insert_or_update</source_transform>

93
mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl
View file

@ -65,21 +65,6 @@
<to>spouse_detect_empty_stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_spouse_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>spouse_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>spouse_sort_rows</from>
<to>spouse_update_flags</to>
@ -90,6 +75,21 @@
<to>spouse_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>citizen_spouse_output</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>spouse_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -169,32 +169,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>800</xloc>
<yloc>912</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<send_true_to>spouse_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>592</xloc>
@ -403,8 +378,8 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH spouse_data AS (
SELECT
<limit/>
<sql>SELECT
ri.recruit_id AS recruit_id,
(supr->>'id')::uuid AS spouse_external_id,
NULLIF(supr->>'idERN', '') AS spouse_id_ern,
@ -432,8 +407,7 @@ and recruitment_id = '${IDM_ID}';
CASE
WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true
ELSE false
END AS information_excluded,
ri.ctid as source_ctid
END AS information_excluded
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
CROSS JOIN LATERAL (
@ -443,13 +417,7 @@ and recruitment_id = '${IDM_ID}';
) AS supr(supr)
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM spouse_data) as last_row
FROM spouse_data;
</sql>
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -538,19 +506,16 @@ and status = 'PROCESSING';
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>marital_status</field>
<name>marital_status</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
@ -567,6 +532,24 @@ and status = 'PROCESSING';
<yloc>32</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>800</xloc>
<yloc>912</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>citizen_spouse_output</source_transform>

91
mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl
View file

@ -60,26 +60,11 @@
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>spouse_update_or_insert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>marriage_registry_update_or_insert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>spouse_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>spouse_update_flags</to>
@ -90,6 +75,21 @@
<to>spouse_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>spouse_update_or_insert</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>spouse_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -188,11 +188,11 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>spouse_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>496</yloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
@ -229,31 +229,6 @@ and recruitment_id = '${IDM_ID}';</sql>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>marriage_registry_update_or_insert</name>
<type>InsertUpdate</type>
@ -361,8 +336,7 @@ and recruitment_id = '${IDM_ID}';
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH spouse_data AS (
SELECT
<sql>SELECT
ri.recruit_id AS recruit_id,
(supr->>'id')::uuid AS spouse_external_id,
NULLIF(supr->>'idERN', '') AS spouse_id_ern,
@ -390,8 +364,7 @@ and recruitment_id = '${IDM_ID}';
CASE
WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true
ELSE false
END AS information_excluded,
ri.ctid as source_ctid
END AS information_excluded
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
CROSS JOIN LATERAL (
@ -402,13 +375,7 @@ and recruitment_id = '${IDM_ID}';
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.update_date >= '${M_R_UP_DATE}'::timestamp
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM spouse_data) as last_row
FROM spouse_data;
</sql>
AND ri.update_date >= '${M_R_UP_DATE}'::timestamp</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -583,6 +550,24 @@ WHERE job_name = '${JOB_NAME}'
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>marriage_registry_update_or_insert</source_transform>

170
mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl
View file

@ -87,26 +87,11 @@ M_R_CR_DATE
<to>marriage_registry_update_or_insert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>spouse_update_or_insert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>spouse_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>spouse_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>spouse_update_flags</to>
@ -117,6 +102,21 @@ M_R_CR_DATE
<to>spouse_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>spouse_update_or_insert</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>spouse_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -230,11 +230,11 @@ DO UPDATE SET
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>spouse_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>944</xloc>
<yloc>528</yloc>
<yloc>784</yloc>
</GUI>
</transform>
<transform>
@ -271,31 +271,6 @@ DO UPDATE SET
<yloc>368</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>944</xloc>
<yloc>784</yloc>
</GUI>
</transform>
<transform>
<name>marriage_registry_update_or_insert</name>
<type>InsertUpdate</type>
@ -402,8 +377,8 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH spouse_data AS (
SELECT
<limit/>
<sql>SELECT
ri.recruit_id AS recruit_id,
(supr->>'id')::uuid AS spouse_external_id,
NULLIF(supr->>'idERN', '') AS spouse_id_ern,
@ -431,8 +406,7 @@ and recruitment_id = '${IDM_ID}';
CASE
WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true
ELSE false
END AS information_excluded,
ri.ctid as source_ctid
END AS information_excluded
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
CROSS JOIN LATERAL (
@ -442,13 +416,7 @@ and recruitment_id = '${IDM_ID}';
) AS supr(supr)
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM spouse_data) as last_row
FROM spouse_data;
</sql>
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -491,6 +459,46 @@ and status = 'PROCESSING';
<yloc>784</yloc>
</GUI>
</transform>
<transform>
<name>spouse_update_flags</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<key>
<condition>&lt;></condition>
<field>marital_status</field>
<name>marital_status</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>marital_status</name>
<rename>marital_status</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>512</xloc>
<yloc>368</yloc>
</GUI>
</transform>
<transform>
<name>spouse_update_or_insert</name>
<type>InsertUpdate</type>
@ -586,8 +594,8 @@ and status = 'PROCESSING';
</GUI>
</transform>
<transform>
<name>spouse_update_flags</name>
<type>Update</type>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -596,33 +604,11 @@ and status = 'PROCESSING';
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<key>
<condition>&lt;></condition>
<field>marital_status</field>
<name>marital_status</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
<value>
<name>marital_status</name>
<rename>marital_status</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>512</xloc>
<yloc>368</yloc>
<xloc>944</xloc>
<yloc>528</yloc>
</GUI>
</transform>
<transform_error_handling>
@ -638,18 +624,6 @@ and status = 'PROCESSING';
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>spouse_update_or_insert</source_transform>
<target_transform>spouse_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>spouse_update_flags</source_transform>
<target_transform>spouse_error_job_status</target_transform>
@ -662,6 +636,18 @@ and status = 'PROCESSING';
<max_pct_errors/>
<min_pct_rows/>
</error>
<error>
<source_transform>spouse_update_or_insert</source_transform>
<target_transform>spouse_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
<fields_valuename/>
<codes_valuename/>
<max_errors/>
<max_pct_errors/>
<min_pct_rows/>
</error>
</transform_error_handling>
<attributes/>
</pipeline>

206
mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl
View file

@ -35,11 +35,6 @@
<to>passport_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>passport_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>passport_error_job_status</from>
<to>Abort</to>
@ -56,12 +51,17 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<from>passport_output</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>passport_success_job_status</to>
<enabled>Y</enabled>
</hop>
@ -83,84 +83,7 @@
<attributes/>
<GUI>
<xloc>1264</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>passport_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>896</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>passport_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1264</xloc>
<yloc>112</yloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
@ -189,7 +112,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<attributes/>
<GUI>
<xloc>288</xloc>
<yloc>320</yloc>
<yloc>352</yloc>
</GUI>
</transform>
<transform>
@ -238,11 +161,47 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>passport_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>1264</xloc>
<yloc>320</yloc>
<yloc>352</yloc>
</GUI>
</transform>
<transform>
<name>passport_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>896</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
@ -258,9 +217,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH passport_data AS (
SELECT
<sql>SELECT
ri.recruit_id,
NULLIF(p.passport_data->>'nomDok', '') AS passport_number,
NULLIF(p.passport_data->>'serDok', '') AS passport_series,
@ -270,8 +227,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
CASE
WHEN NULLIF(p.passport_data->>'kodStatus', '') = '1' THEN TRUE
ELSE FALSE
END AS actual,
ri.ctid as source_ctid
END AS actual
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r
ON r.recruit_id = ri.recruit_id
@ -286,17 +242,12 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
WHERE passport_data IS NOT NULL AND passport_data::text &lt;&gt; 'null'
) AS p ON TRUE
WHERE LENGTH(REGEXP_REPLACE(p.passport_data->>'serDok', '[^0-9]', '', 'g')) = 4
AND LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM passport_data) as last_row
FROM passport_data;</sql>
AND LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>320</yloc>
<yloc>352</yloc>
</GUI>
</transform>
<transform>
@ -346,14 +297,11 @@ FROM passport_data;</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>passport</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -361,12 +309,12 @@ FROM passport_data;</sql>
<attributes/>
<GUI>
<xloc>896</xloc>
<yloc>320</yloc>
<yloc>352</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<name>passport_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -375,18 +323,44 @@ FROM passport_data;</sql>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>1264</xloc>
<yloc>208</yloc>
<yloc>112</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>352</yloc>
</GUI>
</transform>
<transform_error_handling>

292
mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl
View file

@ -35,11 +35,6 @@
<to>passport_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>passport_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>passport_error_job_status</from>
<to>Abort</to>
@ -56,12 +51,17 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<from>passport_insert_or_update</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>passport_success_job_status</to>
<enabled>Y</enabled>
</hop>
@ -86,83 +86,6 @@
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>passport_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>passport_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1120</xloc>
<yloc>16</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
@ -241,10 +164,116 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>passport_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>1120</xloc>
<xloc>1248</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>960</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>passport_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>464</yloc>
</GUI>
</transform>
<transform>
<name>passport_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql> SELECT
ri.recruit_id,
NULLIF(p.passport_data->>'nomDok', '') AS passport_number,
NULLIF(p.passport_data->>'serDok', '') AS passport_series,
NULLIF(p.passport_data->>'vydDok', '') AS organization_name,
NULLIF(p.passport_data->>'kodVydDok', '') AS unit_code,
TO_DATE(NULLIF(p.passport_data->>'dataDok', ''), 'YYYY-MM-DD') AS issue_date,
CASE
WHEN NULLIF(p.passport_data->>'kodStatus', '') = '1' THEN TRUE
ELSE FALSE
END AS actual
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r
ON r.recruit_id = ri.recruit_id
AND '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND r.update_date >= '${M_R_UP_DATE}'::timestamp
JOIN LATERAL (
SELECT passport_data
FROM UNNEST(ARRAY[
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF',
ri.info->'svedFL'->'svedBS'->'pasportRF'->'predPasportRF'
]) AS passport_data
WHERE passport_data IS NOT NULL AND passport_data::text &lt;&gt; 'null'
) AS p ON TRUE
WHERE LENGTH(REGEXP_REPLACE(p.passport_data->>'serDok', '[^0-9]', '', 'g')) = 4
AND LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6
</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>224</yloc>
</GUI>
</transform>
@ -266,13 +295,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>actual</field>
<name>actual</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>passport</table>
@ -320,63 +347,8 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>passport_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH passport_data AS (
SELECT
ri.recruit_id,
NULLIF(p.passport_data->>'nomDok', '') AS passport_number,
NULLIF(p.passport_data->>'serDok', '') AS passport_series,
NULLIF(p.passport_data->>'vydDok', '') AS organization_name,
NULLIF(p.passport_data->>'kodVydDok', '') AS unit_code,
TO_DATE(NULLIF(p.passport_data->>'dataDok', ''), 'YYYY-MM-DD') AS issue_date,
CASE
WHEN NULLIF(p.passport_data->>'kodStatus', '') = '1' THEN TRUE
ELSE FALSE
END AS actual,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r
ON r.recruit_id = ri.recruit_id
AND '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND r.update_date >= '${M_R_UP_DATE}'::timestamp
JOIN LATERAL (
SELECT passport_data
FROM UNNEST(ARRAY[
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF',
ri.info->'svedFL'->'svedBS'->'pasportRF'->'predPasportRF'
]) AS passport_data
WHERE passport_data IS NOT NULL AND passport_data::text &lt;&gt; 'null'
) AS p ON TRUE
WHERE LENGTH(REGEXP_REPLACE(p.passport_data->>'serDok', '[^0-9]', '', 'g')) = 4
AND LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM passport_data) as last_row
FROM passport_data;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<name>passport_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -385,18 +357,26 @@ FROM passport_data;</sql>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>1120</xloc>
<yloc>112</yloc>
<xloc>1248</xloc>
<yloc>16</yloc>
</GUI>
</transform>
<transform_error_handling>

54
mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl
View file

@ -35,11 +35,6 @@
<to>passport_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>passport_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>passport_error_job_status</from>
<to>Abort</to>
@ -56,12 +51,17 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<from>passport_insert_or_update</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>passport_success_job_status</to>
<enabled>Y</enabled>
</hop>
@ -178,16 +178,16 @@ DO UPDATE SET
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>passport_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>1040</xloc>
<xloc>1312</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -196,18 +196,11 @@ DO UPDATE SET
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>144</yloc>
<xloc>1088</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform>
@ -259,8 +252,7 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH passport_data AS (
SELECT
<sql> SELECT
ri.recruit_id,
NULLIF(p.passport_data->>'nomDok', '') AS passport_number,
NULLIF(p.passport_data->>'serDok', '') AS passport_series,
@ -270,8 +262,7 @@ and recruitment_id = '${IDM_ID}';
CASE
WHEN NULLIF(p.passport_data->>'kodStatus', '') = '1' THEN TRUE
ELSE FALSE
END AS actual,
ri.ctid as source_ctid
END AS actual
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r
ON r.recruit_id = ri.recruit_id
@ -286,12 +277,7 @@ and recruitment_id = '${IDM_ID}';
WHERE passport_data IS NOT NULL AND passport_data::text &lt;&gt; 'null'
) AS p ON TRUE
WHERE LENGTH(REGEXP_REPLACE(p.passport_data->>'serDok', '[^0-9]', '', 'g')) = 4
AND LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM passport_data) as last_row
FROM passport_data;</sql>
AND LENGTH(REGEXP_REPLACE(p.passport_data->>'nomDok', '[^0-9]', '', 'g')) = 6</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -317,13 +303,11 @@ FROM passport_data;</sql>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>actual</field>
<name>actual</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>passport</table>
@ -399,7 +383,7 @@ and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>1040</xloc>
<xloc>1312</xloc>
<yloc>16</yloc>
</GUI>
</transform>

60
mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl
View file

@ -65,19 +65,19 @@
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>prosecution_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<to>prosecution_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>prosecution_success_job_status</to>
<from>prosecution_output</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
</order>
@ -127,7 +127,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<attributes/>
<GUI>
<xloc>272</xloc>
<yloc>464</yloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
@ -144,7 +144,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<attributes/>
<GUI>
<xloc>480</xloc>
<yloc>640</yloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
@ -176,16 +176,16 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>prosecution_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>464</yloc>
<xloc>1248</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -194,18 +194,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>544</yloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
@ -257,11 +250,11 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH ugolov_presl AS (
SELECT
jsonb_array_elements(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') AS up,
ri.recruit_id,
ri.ctid as source_ctid
ri.recruit_id
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
AND '${IDM_ID}' != ''
@ -286,7 +279,7 @@ SELECT
NULLIF(up->'sledSud'->'svedSnyat'->>'osnSnyatSud', '') AS conviction_removal_basis,
TO_DATE(up->'sledSud'->'svedSnyat'->>'dataSnyatSud', 'YYYY-MM-DD') AS conviction_removal_date,
TO_DATE(up->'sledSud'->>'dataOsuzhd', 'YYYY-MM-DD') AS conviction_date,
TO_DATE(up->'sledSud'->>'dataVozbUD', 'YYYY-MM-DD') AS create_date,
TO_DATE(up->'sledSud'-&gt;&gt;'dataVozbUD', 'YYYY-MM-DD') AS create_date,
NULLIF(up->'sledSud'->>'osnPrekrUD', '') AS termination_basis,
NULLIF(up->'sledSud'->>'statusNaim', '') AS conviction_status,
TO_DATE(up->'sledSud'->>'dataPrekrUD', 'YYYY-MM-DD') AS close_date,
@ -297,17 +290,16 @@ SELECT
WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде'
ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей'
END AS type_info,
true AS has_criminal_prosecution,
up.source_ctid = (SELECT MAX(source_ctid) FROM ugolov_presl) as last_row
true AS has_criminal_prosecution
FROM ervu_dashboard.citizen r
JOIN ugolov_presl up ON r.recruit_id = up.recruit_id
LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true
GROUP BY r.recruit_id, up.up, up.source_ctid;</sql>
GROUP BY r.recruit_id, up.up;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>480</xloc>
<yloc>464</yloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
@ -425,7 +417,7 @@ GROUP BY r.recruit_id, up.up, up.source_ctid;</sql>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>464</yloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
@ -491,8 +483,8 @@ and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>640</yloc>
<xloc>1248</xloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>

115
mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl
View file

@ -60,23 +60,23 @@
<to>prosecution_sort_rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>prosecution_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>prosecution_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<from>prosecution_insert_or_update</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>prosecution_success_job_status</to>
<enabled>Y</enabled>
</hop>
@ -179,16 +179,16 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>prosecution_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>896</xloc>
<xloc>1008</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
<name>prosecution_sort_rows</name>
<type>SortRows</type>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -197,52 +197,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>384</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>896</xloc>
<yloc>864</yloc>
<xloc>880</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
@ -297,8 +256,7 @@ and recruitment_id = '${IDM_ID}';
<sql>WITH ugolov_presl AS (
SELECT
jsonb_array_elements(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') AS up,
ri.recruit_id,
ri.ctid as source_ctid
ri.recruit_id
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
AND '${IDM_ID}' != ''
@ -335,12 +293,11 @@ SELECT
WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде'
ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей'
END AS type_info,
true AS has_criminal_prosecution,
up.source_ctid = (SELECT MAX(source_ctid) FROM ugolov_presl) as last_row
true AS has_criminal_prosecution
FROM ervu_dashboard.citizen r
JOIN ugolov_presl up ON r.recruit_id = up.recruit_id
LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true
GROUP BY r.recruit_id, up.up, up.source_ctid;</sql>
GROUP BY r.recruit_id, up.up;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -487,6 +444,40 @@ GROUP BY r.recruit_id, up.up, up.source_ctid;</sql>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
<name>prosecution_sort_rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>384</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>prosecution_success_job_status</name>
<type>ExecSql</type>
@ -516,7 +507,7 @@ and status = 'DELTA_PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>896</xloc>
<xloc>1008</xloc>
<yloc>944</yloc>
</GUI>
</transform>

37
mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl
View file

@ -72,6 +72,11 @@
</hop>
<hop>
<from>Insert / update</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
@ -190,6 +195,24 @@ DO UPDATE SET
</compare>
<send_true_to>prosecution_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>944</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>928</xloc>
<yloc>944</yloc>
@ -213,13 +236,11 @@ DO UPDATE SET
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>prosecution_external_id</field>
<name>prosecution_external_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>prosecution</table>
@ -352,8 +373,7 @@ DO UPDATE SET
<sql>WITH ugolov_presl AS (
SELECT
jsonb_array_elements(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') AS up,
ri.recruit_id,
ri.ctid as source_ctid
ri.recruit_id
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
AND '${IDM_ID}' != ''
@ -378,7 +398,7 @@ SELECT
NULLIF(up->'sledSud'->'svedSnyat'->>'osnSnyatSud', '') AS conviction_removal_basis,
TO_DATE(up->'sledSud'->'svedSnyat'->>'dataSnyatSud', 'YYYY-MM-DD') AS conviction_removal_date,
TO_DATE(up->'sledSud'->>'dataOsuzhd', 'YYYY-MM-DD') AS conviction_date,
TO_DATE(up->'sledSud'->>'dataVozbUD', 'YYYY-MM-DD') AS create_date,
TO_DATE(up->'sledSud'-&gt;&gt;'dataVozbUD', 'YYYY-MM-DD') AS create_date,
NULLIF(up->'sledSud'->>'osnPrekrUD', '') AS termination_basis,
NULLIF(up->'sledSud'->>'statusNaim', '') AS conviction_status,
TO_DATE(up->'sledSud'->>'dataPrekrUD', 'YYYY-MM-DD') AS close_date,
@ -389,12 +409,11 @@ SELECT
WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде'
ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей'
END AS type_info,
true AS has_criminal_prosecution,
up.source_ctid = (SELECT MAX(source_ctid) FROM ugolov_presl) as last_row
true AS has_criminal_prosecution
FROM ervu_dashboard.citizen r
JOIN ugolov_presl up ON r.recruit_id = up.recruit_id
LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true
GROUP BY r.recruit_id, up.up, up.source_ctid;</sql>
GROUP BY r.recruit_id, up.up;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -501,7 +520,7 @@ and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>928</xloc>
<xloc>1072</xloc>
<yloc>1088</yloc>
</GUI>
</transform>

108
mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl
View file

@ -67,16 +67,16 @@
</hop>
<hop>
<from>punishment_output</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>punishment_success_job_status</to>
<enabled>Y</enabled>
</hop>
@ -176,7 +176,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>punishment_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>720</xloc>
@ -184,8 +184,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI>
</transform>
<transform>
<name>punishment_sort_rows</name>
<type>SortRows</type>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -194,52 +194,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>336</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>144</yloc>
<xloc>720</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
@ -291,11 +250,11 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH punishment_data AS (
SELECT
ri.recruit_id,
jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info,
ri.ctid as source_ctid
jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
AND '${IDM_ID}' != ''
@ -332,11 +291,10 @@ SELECT recruit_id,
punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения
TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения
true AS has_punishment_execution_info,
true AS has_sentence_info,
source_ctid = (SELECT MAX(source_ctid) FROM punishment_data) as last_row
true AS has_sentence_info
FROM punishment_data
LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true
GROUP BY recruit_id, punishment_info,source_ctid;</sql>
GROUP BY recruit_id, punishment_info;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -454,6 +412,40 @@ GROUP BY recruit_id, punishment_info,source_ctid;</sql>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>punishment_sort_rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>336</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
<name>punishment_success_job_status</name>
<type>ExecSql</type>
@ -484,7 +476,7 @@ and status = 'PROCESSING';
<attributes/>
<GUI>
<xloc>512</xloc>
<yloc>256</yloc>
<yloc>144</yloc>
</GUI>
</transform>
<transform>

113
mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl
View file

@ -86,16 +86,16 @@
</hop>
<hop>
<from>punishment_insert_or_update</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>punishment_success_job_status</to>
<enabled>Y</enabled>
</hop>
@ -166,7 +166,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>32</yloc>
<yloc>0</yloc>
</GUI>
</transform>
<transform>
@ -198,16 +198,16 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>punishment_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>784</xloc>
<yloc>32</yloc>
<yloc>0</yloc>
</GUI>
</transform>
<transform>
<name>punishment_sort_rows</name>
<type>SortRows</type>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -216,52 +216,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>32</yloc>
<xloc>784</xloc>
<yloc>96</yloc>
</GUI>
</transform>
<transform>
@ -316,8 +275,7 @@ and recruitment_id = '${IDM_ID}';
<sql>WITH punishment_data AS (
SELECT
ri.recruit_id,
jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info,
ri.ctid as source_ctid
jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
AND '${IDM_ID}' != ''
@ -355,11 +313,10 @@ SELECT recruit_id,
punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения
TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения
true AS has_punishment_execution_info,
true AS has_sentence_info,
source_ctid = (SELECT MAX(source_ctid) FROM punishment_data) as last_row
true AS has_sentence_info
FROM punishment_data
LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true
GROUP BY recruit_id, punishment_info,source_ctid;</sql>
GROUP BY recruit_id, punishment_info;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -496,6 +453,40 @@ GROUP BY recruit_id, punishment_info,source_ctid;</sql>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>punishment_sort_rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>punishment_success_job_status</name>
<type>ExecSql</type>
@ -525,8 +516,8 @@ and status = 'DELTA_PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>592</xloc>
<yloc>160</yloc>
<xloc>576</xloc>
<yloc>0</yloc>
</GUI>
</transform>
<transform>

50
mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl
View file

@ -67,16 +67,16 @@
</hop>
<hop>
<from>punishment_insert_or_update</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>punishment_success_job_status</to>
<enabled>Y</enabled>
</hop>
@ -161,7 +161,7 @@ DO UPDATE SET
<attributes/>
<GUI>
<xloc>416</xloc>
<yloc>112</yloc>
<yloc>64</yloc>
</GUI>
</transform>
<transform>
@ -193,16 +193,16 @@ DO UPDATE SET
</value>
</condition>
</compare>
<send_true_to>Unique rows</send_true_to>
<send_true_to>punishment_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>816</xloc>
<yloc>112</yloc>
<xloc>800</xloc>
<yloc>64</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -211,18 +211,11 @@ DO UPDATE SET
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>736</xloc>
<yloc>112</yloc>
<xloc>800</xloc>
<yloc>144</yloc>
</GUI>
</transform>
<transform>
@ -274,11 +267,11 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH punishment_data AS (
SELECT
ri.recruit_id,
jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info,
ri.ctid as source_ctid
jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
AND '${IDM_ID}' != ''
@ -315,11 +308,10 @@ SELECT recruit_id,
punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения
TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения
true AS has_punishment_execution_info,
true AS has_sentence_info,
source_ctid = (SELECT MAX(source_ctid) FROM punishment_data) as last_row
true AS has_sentence_info
FROM punishment_data
LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true
GROUP BY recruit_id, punishment_info,source_ctid;</sql>
GROUP BY recruit_id, punishment_info;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -345,13 +337,11 @@ GROUP BY recruit_id, punishment_info,source_ctid;</sql>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>punishment_external_id</field>
<name>punishment_external_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>punishment</table>
@ -521,8 +511,8 @@ and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>192</yloc>
<xloc>608</xloc>
<yloc>64</yloc>
</GUI>
</transform>
<transform>