micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

фикс

Browse source
This commit is contained in:
adel.ka 2025-10-22 14:48:42 +03:00
parent a55ca97ac6
commit d9412cfe6e
24 changed files with 3287 additions and 2505 deletions
Download patch file Download diff file Expand all files Collapse all files

110
mappings/info_recruits/citizen_tables/child/parallel/child_flow.hpl
View file

@ -22,64 +22,69 @@
<order>
<hop>
<from>Create job execution record </from>
<to>Table input </to>
<to>child_input </to>
<enabled>Y</enabled>
</hop>
<hop>
<from>User defined Java expression 2</from>
<to>Update 2</to>
<to>child_update_flags</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update 2</from>
<to>Change job status on error </to>
<from>child_update_flags</from>
<to>child_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error </from>
<from>child_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success </to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success </to>
<to>child_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input </from>
<to>Table output</to>
<from>child_input </from>
<to>child_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<to>Change job status on error </to>
<from>child_output</from>
<to>child_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input </from>
<to>Group by 2</to>
<from>child_input </from>
<to>child_group_by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Group by 2</from>
<from>child_group_by</from>
<to>User defined Java expression 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<from>child_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input </from>
<from>child_input </from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>child_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -102,7 +107,7 @@
</GUI>
</transform>
<transform>
<name>Change job status on error </name>
<name>child_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -118,8 +123,11 @@
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
@ -131,6 +139,7 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1216</xloc>
@ -138,7 +147,7 @@ and recruitment_id = '${IDM_ID}';
</GUI>
</transform>
<transform>
<name>Change job status on success </name>
<name>child_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -151,8 +160,11 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
@ -164,6 +176,7 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>576</xloc>
@ -245,7 +258,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value>
</condition>
</compare>
<send_true_to>Change job status on success </send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>896</xloc>
@ -253,7 +266,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI>
</transform>
<transform>
<name>Group by 2</name>
<name>child_group_by</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
@ -271,16 +284,19 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<aggregate>minors_count_raw</aggregate>
<subject>is_minor</subject>
<type>SUM</type>
<valuefield/>
</field>
<field>
<aggregate>has_minor_child_raw</aggregate>
<subject>is_minor</subject>
<type>MAX</type>
<valuefield/>
</field>
<field>
<aggregate>has_dead_child_raw</aggregate>
<subject>is_dead</subject>
<type>MAX</type>
<valuefield/>
</field>
</fields>
<give_back_row>N</give_back_row>
@ -290,6 +306,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<linenr_fieldname/>
<prefix>grp</prefix>
<attributes/>
<GUI>
@ -298,7 +315,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI>
</transform>
<transform>
<name>Table input </name>
<name>child_input </name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
@ -310,6 +327,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH children_data AS (
SELECT
ri.recruit_id AS recruit_id,
@ -372,7 +390,7 @@ FROM children_data;</sql>
</GUI>
</transform>
<transform>
<name>Table output</name>
<name>child_output</name>
<type>TableOutput</type>
<description/>
<distribute>N</distribute>
@ -438,11 +456,14 @@ FROM children_data;</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>citizen_child</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -454,7 +475,7 @@ FROM children_data;</sql>
</GUI>
</transform>
<transform>
<name>Update 2</name>
<name>child_update_flags</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
@ -539,10 +560,35 @@ FROM children_data;</sql>
<yloc>480</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>736</xloc>
<yloc>1040</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Create job execution record </source_transform>
<target_transform>Change job status on error </target_transform>
<target_transform>child_error_job_status</target_transform>
<is_enabled>N</is_enabled>
<nr_valuename/>
<descriptions_valuename/>
@ -553,8 +599,8 @@ FROM children_data;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>Table output</source_transform>
<target_transform>Change job status on error </target_transform>
<source_transform>child_output</source_transform>
<target_transform>child_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -565,8 +611,8 @@ FROM children_data;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>Update 2</source_transform>
<target_transform>Change job status on error </target_transform>
<source_transform>child_update_flags</source_transform>
<target_transform>child_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

316
mappings/info_recruits/citizen_tables/child/parallel/child_flow_delta.hpl
View file

@ -21,33 +21,28 @@
</notepads>
<order>
<hop>
<from>Insert / update</from>
<from>child_insert_or_update</from>
<to>Change job status on error </to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update </from>
<from>child_update_flags</from>
<to>Change job status on error </to>
<enabled>Y</enabled>
</hop>
<hop>
<from>User defined Java expression</from>
<to>Update </to>
<to>child_update_flags</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Create job execution record</from>
<to>Table input 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success </to>
<to>child_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success </to>
<to>child_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
@ -56,28 +51,38 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input 2</from>
<to>Insert / update</to>
<from>child_input</from>
<to>child_insert_or_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Group by</from>
<from>child_group_by</from>
<to>User defined Java expression</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input 2</from>
<from>child_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<from>child_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input 2</from>
<to>Group by</to>
<from>child_input</from>
<to>child_group_by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>child_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
@ -138,7 +143,7 @@ and recruitment_id = '${IDM_ID}';
</GUI>
</transform>
<transform>
<name>Change job status on success </name>
<name>child_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -151,8 +156,11 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
@ -164,6 +172,7 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>784</xloc>
@ -247,7 +256,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Change job status on success </send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>1072</xloc>
@ -255,7 +264,73 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>Group by</name>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>960</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>User defined Java expression</name>
<type>Janino</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<formula>
<field_name>minors_count</field_name>
<formula_string>minors_count_raw == 1 ? "1" : minors_count_raw == 2 ? "2" : minors_count_raw >= 3 ? "3 и более" : "0"</formula_string>
<value_type>String</value_type>
<value_length>10</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<formula>
<field_name>has_minor_child</field_name>
<formula_string>has_minor_child_raw == 1 </formula_string>
<value_type>Boolean</value_type>
<value_length>-1</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<formula>
<field_name>has_dead_child</field_name>
<formula_string>has_dead_child_raw == 1 </formula_string>
<value_type>Boolean</value_type>
<value_length>-1</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<attributes/>
<GUI>
<xloc>784</xloc>
<yloc>0</yloc>
</GUI>
</transform>
<transform>
<name>child_group_by</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
@ -300,7 +375,82 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<name>child_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH children_data AS (
SELECT
ri.recruit_id AS recruit_id,
(ch->>'id')::uuid AS child_external_id,
NULLIF(ch->>'idERN', '') AS child_id_ern,
CASE NULLIF(ch->>'rodstvSvyazReb', '')::int
WHEN 1 THEN 3
WHEN 2 THEN 4
END AS kinship_type,
ch->'svedFLBS'->'fio'->>'familiya' AS last_name,
ch->'svedFLBS'->'fio'->>'imya' AS first_name,
ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name,
CONCAT_WS(' ',
ch->'svedFLBS'->'fio'->>'familiya',
ch->'svedFLBS'->'fio'->>'imya',
ch->'svedFLBS'->'fio'->>'otchestvo'
) AS full_name,
dates.birth_date,
dates.death_date,
ch->'svedSmert'->>'nomerZapis' AS death_az_number,
ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number,
CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead,
CASE
WHEN dates.death_date IS NOT NULL THEN 0
WHEN dates.birth_date IS NULL THEN 0
WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1
ELSE 0
END AS is_minor,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch
CROSS JOIN LATERAL (
SELECT
MAKE_DATE(
NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int,
NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int,
NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int
) AS birth_date,
MAKE_DATE(
NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int,
NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int,
NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int
) AS death_date
) AS dates
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM children_data) as last_row
FROM children_data;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>child_insert_or_update</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
@ -394,82 +544,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>Table input 2</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH children_data AS (
SELECT
ri.recruit_id AS recruit_id,
(ch->>'id')::uuid AS child_external_id,
NULLIF(ch->>'idERN', '') AS child_id_ern,
CASE NULLIF(ch->>'rodstvSvyazReb', '')::int
WHEN 1 THEN 3
WHEN 2 THEN 4
END AS kinship_type,
ch->'svedFLBS'->'fio'->>'familiya' AS last_name,
ch->'svedFLBS'->'fio'->>'imya' AS first_name,
ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name,
CONCAT_WS(' ',
ch->'svedFLBS'->'fio'->>'familiya',
ch->'svedFLBS'->'fio'->>'imya',
ch->'svedFLBS'->'fio'->>'otchestvo'
) AS full_name,
dates.birth_date,
dates.death_date,
ch->'svedSmert'->>'nomerZapis' AS death_az_number,
ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number,
CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead,
CASE
WHEN dates.death_date IS NOT NULL THEN 0
WHEN dates.birth_date IS NULL THEN 0
WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1
ELSE 0
END AS is_minor,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch
CROSS JOIN LATERAL (
SELECT
MAKE_DATE(
NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int,
NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int,
NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int
) AS birth_date,
MAKE_DATE(
NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int,
NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int,
NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int
) AS death_date
) AS dates
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM children_data) as last_row
FROM children_data;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Update </name>
<name>child_update_flags</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
@ -482,13 +557,11 @@ FROM children_data;</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
@ -513,50 +586,9 @@ FROM children_data;</sql>
<yloc>0</yloc>
</GUI>
</transform>
<transform>
<name>User defined Java expression</name>
<type>Janino</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<formula>
<field_name>minors_count</field_name>
<formula_string>minors_count_raw == 1 ? "1" : minors_count_raw == 2 ? "2" : minors_count_raw >= 3 ? "3 и более" : "0"</formula_string>
<value_type>String</value_type>
<value_length>10</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<formula>
<field_name>has_minor_child</field_name>
<formula_string>has_minor_child_raw == 1 </formula_string>
<value_type>Boolean</value_type>
<value_length>-1</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<formula>
<field_name>has_dead_child</field_name>
<formula_string>has_dead_child_raw == 1 </formula_string>
<value_type>Boolean</value_type>
<value_length>-1</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<attributes/>
<GUI>
<xloc>784</xloc>
<yloc>0</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Insert / update</source_transform>
<source_transform>child_insert_or_update</source_transform>
<target_transform>Change job status on error </target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
@ -568,7 +600,7 @@ FROM children_data;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>Update </source_transform>
<source_transform>child_update_flags</source_transform>
<target_transform>Change job status on error </target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>

368
mappings/info_recruits/citizen_tables/child/parallel/child_flow_repeat.hpl
View file

@ -22,62 +22,67 @@
<order>
<hop>
<from>User defined Java expression</from>
<to>Update</to>
<to>child_update_flags</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Create job execution record 2</from>
<to>Table input 2</to>
<to>child_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error 2</to>
<from>child_update_flags</from>
<to>child_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Change job status on error 2</to>
<from>child_insert_or_update</from>
<to>child_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>child_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error 2</from>
<from>child_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input 2</from>
<to>Insert / update</to>
<from>child_input</from>
<to>child_insert_or_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Group by</from>
<from>child_group_by</from>
<to>User defined Java expression</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input 2</from>
<from>child_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<from>child_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input 2</from>
<to>Group by</to>
<from>child_input</from>
<to>child_group_by</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>child_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
@ -102,7 +107,7 @@
</GUI>
</transform>
<transform>
<name>Change job status on error 2</name>
<name>child_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -118,8 +123,11 @@
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
@ -131,46 +139,13 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>976</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
error_description = null,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>416</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record 2</name>
<type>ExecSql</type>
@ -263,7 +238,32 @@ DO UPDATE SET
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>752</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>608</xloc>
@ -271,7 +271,48 @@ DO UPDATE SET
</GUI>
</transform>
<transform>
<name>Group by</name>
<name>User defined Java expression</name>
<type>Janino</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<formula>
<field_name>minors_count</field_name>
<formula_string>minors_count_raw == 1 ? "1" : minors_count_raw == 2 ? "2" : minors_count_raw >= 3 ? "3 и более" : "0"</formula_string>
<value_type>String</value_type>
<value_length>10</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<formula>
<field_name>has_minor_child</field_name>
<formula_string>has_minor_child_raw == 1 </formula_string>
<value_type>Boolean</value_type>
<value_length>-1</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<formula>
<field_name>has_dead_child</field_name>
<formula_string>has_dead_child_raw == 1 </formula_string>
<value_type>Boolean</value_type>
<value_length>-1</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<attributes/>
<GUI>
<xloc>608</xloc>
<yloc>176</yloc>
</GUI>
</transform>
<transform>
<name>child_group_by</name>
<type>GroupBy</type>
<description/>
<distribute>N</distribute>
@ -316,7 +357,81 @@ DO UPDATE SET
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<name>child_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH children_data AS (
SELECT
ri.recruit_id AS recruit_id,
(ch->>'id')::uuid AS child_external_id,
NULLIF(ch->>'idERN', '') AS child_id_ern,
CASE NULLIF(ch->>'rodstvSvyazReb', '')::int
WHEN 1 THEN 3
WHEN 2 THEN 4
END AS kinship_type,
ch->'svedFLBS'->'fio'->>'familiya' AS last_name,
ch->'svedFLBS'->'fio'->>'imya' AS first_name,
ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name,
CONCAT_WS(' ',
ch->'svedFLBS'->'fio'->>'familiya',
ch->'svedFLBS'->'fio'->>'imya',
ch->'svedFLBS'->'fio'->>'otchestvo'
) AS full_name,
dates.birth_date,
dates.death_date,
ch->'svedSmert'->>'nomerZapis' AS death_az_number,
ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number,
CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead,
CASE
WHEN dates.death_date IS NOT NULL THEN 0
WHEN dates.birth_date IS NULL THEN 0
WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1
ELSE 0
END AS is_minor,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch
CROSS JOIN LATERAL (
SELECT
MAKE_DATE(
NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int,
NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int,
NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int
) AS birth_date,
MAKE_DATE(
NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int,
NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int,
NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int
) AS death_date
) AS dates
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM children_data) as last_row
FROM children_data;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>224</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>child_insert_or_update</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
@ -405,86 +520,46 @@ DO UPDATE SET
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>608</xloc>
<xloc>752</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Table input 2</name>
<type>TableInput</type>
<name>child_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>N</distribute>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH children_data AS (
SELECT
ri.recruit_id AS recruit_id,
(ch->>'id')::uuid AS child_external_id,
NULLIF(ch->>'idERN', '') AS child_id_ern,
CASE NULLIF(ch->>'rodstvSvyazReb', '')::int
WHEN 1 THEN 3
WHEN 2 THEN 4
END AS kinship_type,
ch->'svedFLBS'->'fio'->>'familiya' AS last_name,
ch->'svedFLBS'->'fio'->>'imya' AS first_name,
ch->'svedFLBS'->'fio'->>'otchestvo' AS middle_name,
CONCAT_WS(' ',
ch->'svedFLBS'->'fio'->>'familiya',
ch->'svedFLBS'->'fio'->>'imya',
ch->'svedFLBS'->'fio'->>'otchestvo'
) AS full_name,
dates.birth_date,
dates.death_date,
ch->'svedSmert'->>'nomerZapis' AS death_az_number,
ch->'svedAZRozhd'->>'nomerZapis' AS birth_az_number,
CASE WHEN dates.death_date IS NOT NULL THEN 1 ELSE 0 END AS is_dead,
CASE
WHEN dates.death_date IS NOT NULL THEN 0
WHEN dates.birth_date IS NULL THEN 0
WHEN dates.birth_date > CURRENT_DATE - INTERVAL '18 years' THEN 1
ELSE 0
END AS is_minor,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedDeti'->'rebenok') AS ch
CROSS JOIN LATERAL (
SELECT
MAKE_DATE(
NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'god', '')::int,
NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int,
NULLIF(ch->'svedFLBS'->'dataRozhdDok'->>'den', '')::int
) AS birth_date,
MAKE_DATE(
NULLIF(ch->'svedSmert'->'dataSmert'->>'god', '')::int,
NULLIF(ch->'svedSmert'->'dataSmert'->>'mesyacz', '')::int,
NULLIF(ch->'svedSmert'->'dataSmert'->>'den', '')::int
) AS death_date
) AS dates
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND jsonb_typeof(ri.info->'svedFL'->'svedDeti'->'rebenok') = 'array'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM children_data) as last_row
FROM children_data;</sql>
<variables_active>Y</variables_active>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
error_description = null,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>224</xloc>
<yloc>432</yloc>
<xloc>448</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<name>child_update_flags</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
@ -497,13 +572,11 @@ FROM children_data;</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
@ -528,51 +601,10 @@ FROM children_data;</sql>
<yloc>176</yloc>
</GUI>
</transform>
<transform>
<name>User defined Java expression</name>
<type>Janino</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<formula>
<field_name>minors_count</field_name>
<formula_string>minors_count_raw == 1 ? "1" : minors_count_raw == 2 ? "2" : minors_count_raw >= 3 ? "3 и более" : "0"</formula_string>
<value_type>String</value_type>
<value_length>10</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<formula>
<field_name>has_minor_child</field_name>
<formula_string>has_minor_child_raw == 1 </formula_string>
<value_type>Boolean</value_type>
<value_length>-1</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<formula>
<field_name>has_dead_child</field_name>
<formula_string>has_dead_child_raw == 1 </formula_string>
<value_type>Boolean</value_type>
<value_length>-1</value_length>
<value_precision>-1</value_precision>
<replace_field/>
</formula>
<attributes/>
<GUI>
<xloc>608</xloc>
<yloc>176</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Insert / update</source_transform>
<target_transform>Change job status on error 2</target_transform>
<source_transform>child_insert_or_update</source_transform>
<target_transform>child_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -583,8 +615,8 @@ FROM children_data;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error 2</target_transform>
<source_transform>child_update_flags</source_transform>
<target_transform>child_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

84
mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow.hpl
View file

@ -22,44 +22,49 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>citizen_address_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<to>Change job status on error</to>
<from>citizen_address_output</from>
<to>citizen_address_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>citizen_address_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>citizen_address_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<from>citizen_address_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Table output</to>
<from>citizen_address_input</from>
<to>citizen_address_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>citizen_address_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>citizen_address_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -82,7 +87,7 @@
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<name>citizen_address_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -98,8 +103,11 @@
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
@ -111,6 +119,7 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>928</xloc>
@ -118,7 +127,7 @@ and recruitment_id = '${IDM_ID}';
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<name>citizen_address_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -131,8 +140,11 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
@ -144,6 +156,7 @@ and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1328</xloc>
@ -225,7 +238,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>1328</xloc>
@ -233,7 +246,32 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI>
</transform>
<transform>
<name>Table input</name>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>1328</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
<name>citizen_address_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
@ -245,6 +283,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH address_data AS (
SELECT
ri.recruit_id::uuid,
@ -283,7 +322,7 @@ FROM address_data;</sql>
</GUI>
</transform>
<transform>
<name>Table output</name>
<name>citizen_address_output</name>
<type>TableOutput</type>
<description/>
<distribute>N</distribute>
@ -345,11 +384,14 @@ FROM address_data;</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>citizen_address</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -362,8 +404,8 @@ FROM address_data;</sql>
</transform>
<transform_error_handling>
<error>
<source_transform>Table output</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>citizen_address_output</source_transform>
<target_transform>citizen_address_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

81
mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_delta.hpl
View file

@ -22,44 +22,49 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>citizen_address_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Change job status on error</to>
<from>citizen_address_insert_or_update</from>
<to>citizen_address_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>citizen_address_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>citizen_address_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Insert / update</to>
<from>citizen_address_input</from>
<to>citizen_address_insert_or_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>citizen_address_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<from>citizen_address_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>citizen_address_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -82,7 +87,7 @@
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<name>citizen_address_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -98,8 +103,11 @@
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
@ -111,6 +119,7 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>752</xloc>
@ -118,7 +127,7 @@ and recruitment_id = '${IDM_ID}';
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<name>citizen_address_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -131,8 +140,11 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
@ -144,6 +156,7 @@ and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1120</xloc>
@ -228,7 +241,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>1120</xloc>
@ -236,7 +249,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<name>citizen_address_insert_or_update</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
@ -253,6 +266,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen_address</table>
@ -320,7 +334,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>Table input</name>
<name>citizen_address_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
@ -371,10 +385,35 @@ FROM address_data;</sql>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>1120</xloc>
<yloc>112</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Insert / update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>citizen_address_insert_or_update</source_transform>
<target_transform>citizen_address_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

178
mappings/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow_repeat.hpl
View file

@ -22,44 +22,49 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>citizen_address_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Change job status on error</to>
<from>citizen_address_insert_or_update</from>
<to>citizen_address_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>citizen_address_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>citizen_address_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Insert / update</to>
<from>citizen_address_input</from>
<to>citizen_address_insert_or_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>citizen_address_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<from>citizen_address_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>citizen_address_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -82,7 +87,7 @@
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<name>citizen_address_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -98,8 +103,11 @@
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
@ -111,6 +119,7 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>768</xloc>
@ -118,7 +127,7 @@ and recruitment_id = '${IDM_ID}';
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<name>citizen_address_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -131,8 +140,11 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
@ -144,6 +156,7 @@ and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1120</xloc>
@ -242,7 +255,7 @@ DO UPDATE SET
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>1120</xloc>
@ -250,7 +263,82 @@ DO UPDATE SET
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>1120</xloc>
<yloc>144</yloc>
</GUI>
</transform>
<transform>
<name>citizen_address_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH address_data AS (
SELECT
ri.recruit_id::uuid,
ri.ctid as source_ctid,
MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address,
MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date,
MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date,
MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address,
MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date,
MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date,
CASE
WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица'
WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы'
END AS place_stay_registration_type,
MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address,
MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date,
MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
GROUP BY ri.recruit_id, ri.ctid
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM address_data) as last_row
FROM address_data;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>480</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform>
<name>citizen_address_insert_or_update</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
@ -333,60 +421,10 @@ DO UPDATE SET
<yloc>272</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH address_data AS (
SELECT
ri.recruit_id::uuid,
ri.ctid as source_ctid,
MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN addr->>'reg' END) AS residence_address,
MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateStart')::date END) AS residence_registration_date,
MAX(CASE WHEN addr->>'type' = 'actualAddress' THEN (addr->>'regDateEnd')::date END) AS residence_deregistration_date,
MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'reg' END) AS place_stay_address,
MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateStart')::date END) AS place_stay_registration_date,
MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN (addr->>'regDateEnd')::date END) AS place_stay_deregistration_date,
CASE
WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '1' THEN 'В жилом помещении, не являющимся местом жительства физического лица'
WHEN MAX(CASE WHEN addr->>'type' = 'residenceAddress' THEN addr->>'kodTipReg' END) = '2' THEN 'В учреждении уголовно-исполнительной системы'
END AS place_stay_registration_type,
MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN addr->>'reg' END) AS factual_address,
MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateStart')::date END) AS factual_info_received_date,
MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
GROUP BY ri.recruit_id, ri.ctid
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM address_data) as last_row
FROM address_data;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>480</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Insert / update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>citizen_address_insert_or_update</source_transform>
<target_transform>citizen_address_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

263
mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow.hpl
View file

@ -21,60 +21,65 @@
</notepads>
<order>
<hop>
<from>Table input</from>
<to>Table output</to>
<from>citizen_guardianship_input</from>
<to>citizen_guardianship_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>citizen_guardianship_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<to>Change job status on error</to>
<from>citizen_guardianship_output</from>
<to>citizen_guardianship_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<from>update_guardianship_flags</from>
<to>citizen_guardianship_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>citizen_guardianship_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success 2</to>
<to>citizen_guardianship_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Sort rows</to>
<from>citizen_guardianship_input</from>
<to>citizen_guardianship_sort_rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Update</to>
<from>citizen_guardianship_sort_rows</from>
<to>update_guardianship_flags</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<from>citizen_guardianship_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>citizen_guardianship_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>citizen_guardianship_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -96,75 +101,6 @@
<yloc>544</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1280</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success 2</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>816</xloc>
<yloc>720</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
@ -240,7 +176,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value>
</condition>
</compare>
<send_true_to>Change job status on success 2</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>816</xloc>
@ -248,8 +184,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -258,31 +194,62 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>0</yloc>
<xloc>816</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<name>citizen_guardianship_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1280</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>citizen_guardianship_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
@ -335,7 +302,7 @@ FROM guardianship_data;</sql>
</GUI>
</transform>
<transform>
<name>Table output</name>
<name>citizen_guardianship_output</name>
<type>TableOutput</type>
<description/>
<distribute>N</distribute>
@ -405,7 +372,74 @@ FROM guardianship_data;</sql>
</GUI>
</transform>
<transform>
<name>Update</name>
<name>citizen_guardianship_sort_rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>0</yloc>
</GUI>
</transform>
<transform>
<name>citizen_guardianship_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>816</xloc>
<yloc>720</yloc>
</GUI>
</transform>
<transform>
<name>update_guardianship_flags</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
@ -418,19 +452,16 @@ FROM guardianship_data;</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>is_guardian</field>
<name>is_guardian</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
@ -449,8 +480,8 @@ FROM guardianship_data;</sql>
</transform>
<transform_error_handling>
<error>
<source_transform>Table output</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>citizen_guardianship_output</source_transform>
<target_transform>citizen_guardianship_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -461,8 +492,8 @@ FROM guardianship_data;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>update_guardianship_flags</source_transform>
<target_transform>citizen_guardianship_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

317
mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_delta.hpl
View file

@ -22,59 +22,64 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>citizen_guardianship_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Change job status on error</to>
<from>citizen_guardianship_insert_or_update</from>
<to>citizen_guardianship_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Insert / update</to>
<from>citizen_guardianship_input</from>
<to>citizen_guardianship_insert_or_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<from>update_guardianship_flags</from>
<to>citizen_guardianship_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>citizen_guardianship_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>citizen_guardianship_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>citizen_guardianship_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Sort rows</to>
<from>citizen_guardianship_input</from>
<to>citizen_guardianship_sort_rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Update</to>
<from>citizen_guardianship_sort_rows</from>
<to>update_guardianship_flags</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<from>citizen_guardianship_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>citizen_guardianship_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -96,78 +101,6 @@
<yloc>480</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
<argument>
<name>error_code</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
error_description = null,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>880</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
@ -245,7 +178,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>880</xloc>
@ -253,7 +186,125 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>880</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>citizen_guardianship_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
<argument>
<name>error_code</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1200</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>citizen_guardianship_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH guardianship_data AS (
SELECT
ri.recruit_id AS recruit_id,
(ch->>'id')::uuid AS guardianship_external_id,
NULLIF(ch->>'idERN', '') AS guardian_id_ern,
ch->'fioOpek'->>'familiya' AS last_name,
ch->'fioOpek'->>'imya' AS first_name,
ch->'fioOpek'->>'otchestvo' AS middle_name,
CONCAT_WS(' ',
ch->'fioOpek'->>'familiya',
ch->'fioOpek'->>'imya',
ch->'fioOpek'->>'otchestvo'
) AS full_name,
MAKE_DATE(
NULLIF(ch->'dataRozhdDok'->>'god', '')::int,
NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int,
NULLIF(ch->'dataRozhdDok'->>'den', '')::int
) AS birth_date,
REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils,
true AS is_guardian,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM guardianship_data) as last_row
FROM guardianship_data;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>544</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>citizen_guardianship_insert_or_update</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
@ -332,7 +383,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<name>citizen_guardianship_sort_rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
@ -344,7 +395,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
@ -366,61 +417,40 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
<name>citizen_guardianship_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>N</distribute>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH guardianship_data AS (
SELECT
ri.recruit_id AS recruit_id,
(ch->>'id')::uuid AS guardianship_external_id,
NULLIF(ch->>'idERN', '') AS guardian_id_ern,
ch->'fioOpek'->>'familiya' AS last_name,
ch->'fioOpek'->>'imya' AS first_name,
ch->'fioOpek'->>'otchestvo' AS middle_name,
CONCAT_WS(' ',
ch->'fioOpek'->>'familiya',
ch->'fioOpek'->>'imya',
ch->'fioOpek'->>'otchestvo'
) AS full_name,
MAKE_DATE(
NULLIF(ch->'dataRozhdDok'->>'god', '')::int,
NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int,
NULLIF(ch->'dataRozhdDok'->>'den', '')::int
) AS birth_date,
REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils,
true AS is_guardian,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM guardianship_data) as last_row
FROM guardianship_data;</sql>
<variables_active>Y</variables_active>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
error_description = null,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>544</xloc>
<yloc>224</yloc>
<xloc>880</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<name>update_guardianship_flags</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
@ -433,19 +463,16 @@ FROM guardianship_data;</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>is_guardian</field>
<name>is_guardian</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
@ -464,8 +491,8 @@ FROM guardianship_data;</sql>
</transform>
<transform_error_handling>
<error>
<source_transform>Insert / update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>citizen_guardianship_insert_or_update</source_transform>
<target_transform>citizen_guardianship_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -476,8 +503,8 @@ FROM guardianship_data;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>update_guardianship_flags</source_transform>
<target_transform>citizen_guardianship_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

379
mappings/info_recruits/citizen_tables/citizen_guardianship/parallel/citizen_guardianship_flow_repeat.hpl
View file

@ -22,59 +22,64 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>citizen_guardianship_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Insert / update</to>
<from>citizen_guardianship_input</from>
<to>citizen_guardianship_insert_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Change job status on error</to>
<from>citizen_guardianship_insert_update</from>
<to>citizen_guardianship_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<from>update_guardianship_flags</from>
<to>citizen_guardianship_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>citizen_guardianship_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>citizen_guardianship_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Update</to>
<from>citizen_guardianship_sort_rows</from>
<to>update_guardianship_flags</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Sort rows</to>
<from>citizen_guardianship_input</from>
<to>citizen_guardianship_sort_rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>citizen_guardianship_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<from>citizen_guardianship_insert_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>citizen_guardianship_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -96,79 +101,6 @@
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
<argument>
<name>error_code</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1232</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
@ -261,7 +193,7 @@ DO UPDATE SET
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>704</xloc>
@ -269,7 +201,159 @@ DO UPDATE SET
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<name>citizen_guardianship_sort_rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>416</xloc>
<yloc>48</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>citizen_guardianship_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
<argument>
<name>error_code</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1232</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
<name>citizen_guardianship_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH guardianship_data AS (
SELECT
ri.recruit_id AS recruit_id,
(ch->>'id')::uuid AS guardianship_external_id,
NULLIF(ch->>'idERN', '') AS guardian_id_ern,
ch->'fioOpek'->>'familiya' AS last_name,
ch->'fioOpek'->>'imya' AS first_name,
ch->'fioOpek'->>'otchestvo' AS middle_name,
CONCAT_WS(' ',
ch->'fioOpek'->>'familiya',
ch->'fioOpek'->>'imya',
ch->'fioOpek'->>'otchestvo'
) AS full_name,
MAKE_DATE(
NULLIF(ch->'dataRozhdDok'->>'god', '')::int,
NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int,
NULLIF(ch->'dataRozhdDok'->>'den', '')::int
) AS birth_date,
REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils,
true AS is_guardian,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM guardianship_data) as last_row
FROM guardianship_data;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>416</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
<name>citizen_guardianship_insert_update</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
@ -348,8 +432,8 @@ DO UPDATE SET
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<name>citizen_guardianship_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -358,84 +442,30 @@ DO UPDATE SET
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>416</xloc>
<yloc>48</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH guardianship_data AS (
SELECT
ri.recruit_id AS recruit_id,
(ch->>'id')::uuid AS guardianship_external_id,
NULLIF(ch->>'idERN', '') AS guardian_id_ern,
ch->'fioOpek'->>'familiya' AS last_name,
ch->'fioOpek'->>'imya' AS first_name,
ch->'fioOpek'->>'otchestvo' AS middle_name,
CONCAT_WS(' ',
ch->'fioOpek'->>'familiya',
ch->'fioOpek'->>'imya',
ch->'fioOpek'->>'otchestvo'
) AS full_name,
MAKE_DATE(
NULLIF(ch->'dataRozhdDok'->>'god', '')::int,
NULLIF(ch->'dataRozhdDok'->>'mesyacz', '')::int,
NULLIF(ch->'dataRozhdDok'->>'den', '')::int
) AS birth_date,
REGEXP_REPLACE(ch->>'snils', '[-\s]', '', 'g') AS snils,
true AS is_guardian,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
CROSS JOIN LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') AS ch
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND jsonb_typeof(ri.info->'svedFL'->'svedOpekun'->'svedSoczPod') = 'array'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM guardianship_data) as last_row
FROM guardianship_data;</sql>
<variables_active>Y</variables_active>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>416</xloc>
<yloc>208</yloc>
<xloc>704</xloc>
<yloc>624</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<name>update_guardianship_flags</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
@ -448,19 +478,16 @@ FROM guardianship_data;</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>is_guardian</field>
<name>is_guardian</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
@ -479,8 +506,8 @@ FROM guardianship_data;</sql>
</transform>
<transform_error_handling>
<error>
<source_transform>Insert / update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>citizen_guardianship_insert_update</source_transform>
<target_transform>citizen_guardianship_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -491,8 +518,8 @@ FROM guardianship_data;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>update_guardianship_flags</source_transform>
<target_transform>citizen_guardianship_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

100
mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow.hpl
View file

@ -22,54 +22,59 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>citizen_information_search_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<to>Change job status on error</to>
<from>citizen_information_search_output</from>
<to>citizen_information_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>citizen_information_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>citizen_information_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Table output</to>
<from>citizen_information_search_input</from>
<to>citizen_information_search_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<from>update_citizen_information_search_flags</from>
<to>citizen_information_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Update</to>
<from>citizen_information_search_input</from>
<to>update_citizen_information_search_flags</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>citizen_information_search_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<from>citizen_information_search_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>citizen_information_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -92,7 +97,7 @@
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<name>citizen_information_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -108,8 +113,11 @@
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
@ -121,6 +129,7 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>720</xloc>
@ -128,7 +137,7 @@ and recruitment_id = '${IDM_ID}';
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<name>citizen_information_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -141,8 +150,11 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
@ -154,6 +166,7 @@ and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1104</xloc>
@ -235,7 +248,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>1104</xloc>
@ -243,7 +256,32 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI>
</transform>
<transform>
<name>Table input</name>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>240</yloc>
</GUI>
</transform>
<transform>
<name>citizen_information_search_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
@ -284,7 +322,7 @@ FROM wanted_data;</sql>
</GUI>
</transform>
<transform>
<name>Table output</name>
<name>citizen_information_search_output</name>
<type>TableOutput</type>
<description/>
<distribute>N</distribute>
@ -330,11 +368,14 @@ FROM wanted_data;</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>citizen_information_search</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -346,7 +387,7 @@ FROM wanted_data;</sql>
</GUI>
</transform>
<transform>
<name>Update</name>
<name>update_citizen_information_search_flags</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
@ -359,19 +400,16 @@ FROM wanted_data;</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>is_wanted</field>
<name>is_wanted</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
@ -390,8 +428,8 @@ FROM wanted_data;</sql>
</transform>
<transform_error_handling>
<error>
<source_transform>Table output</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>citizen_information_search_output</source_transform>
<target_transform>citizen_information_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -402,8 +440,8 @@ FROM wanted_data;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>update_citizen_information_search_flags</source_transform>
<target_transform>citizen_information_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_decription</descriptions_valuename>

179
mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_delta.hpl
View file

@ -22,52 +22,57 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>citizen_information_search_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Change job status on error</to>
<from>citizen_information_search_insert_or_update</from>
<to>citizen_information_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>citizen_information_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>citizen_information_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Insert / update</to>
<from>citizen_information_search_input</from>
<to>citizen_information_search_insert_or_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<from>update_citizen_information_search_flags</from>
<to>citizen_information_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>citizen_information_search_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<from>citizen_information_search_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Update</to>
<from>citizen_information_search_input</from>
<to>update_citizen_information_search_flags</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>citizen_information_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
@ -92,7 +97,7 @@
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<name>citizen_information_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -108,8 +113,11 @@
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
@ -121,6 +129,7 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>752</xloc>
@ -128,7 +137,7 @@ and recruitment_id = '${IDM_ID}';
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<name>citizen_information_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -141,8 +150,11 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
@ -154,6 +166,7 @@ and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1008</xloc>
@ -238,7 +251,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>1008</xloc>
@ -246,7 +259,74 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>112</yloc>
</GUI>
</transform>
<transform>
<name>citizen_information_search_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH wanted_data AS (
SELECT
ri.recruit_id,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code,
TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date,
TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search,
true AS is_wanted,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
WHERE '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM wanted_data) as last_row
FROM wanted_data;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>citizen_information_search_insert_or_update</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
@ -310,49 +390,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH wanted_data AS (
SELECT
ri.recruit_id,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code,
TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date,
TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search,
true AS is_wanted,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
WHERE '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM wanted_data) as last_row
FROM wanted_data;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<name>update_citizen_information_search_flags</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
@ -365,19 +403,16 @@ FROM wanted_data;</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>is_wanted</field>
<name>is_wanted</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
@ -396,8 +431,8 @@ FROM wanted_data;</sql>
</transform>
<transform_error_handling>
<error>
<source_transform>Insert / update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>citizen_information_search_insert_or_update</source_transform>
<target_transform>citizen_information_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -408,8 +443,8 @@ FROM wanted_data;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>update_citizen_information_search_flags</source_transform>
<target_transform>citizen_information_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

177
mappings/info_recruits/citizen_tables/citizen_information_search/parallel/citizen_information_search_flow_repeat.hpl
View file

@ -22,54 +22,59 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>citizen_information_search_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>citizen_information_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>citizen_information_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<from>update_citizen_information_search_flags</from>
<to>citizen_information_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<from>citizen_information_search_insert_or_update</from>
<to>citizen_information_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Change job status on error</to>
<from>citizen_information_search_input</from>
<to>citizen_information_search_insert_or_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Insert / update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<from>citizen_information_search_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Update</to>
<from>citizen_information_search_input</from>
<to>update_citizen_information_search_flags</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>citizen_information_search_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>citizen_information_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -92,7 +97,7 @@
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<name>citizen_information_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -108,8 +113,11 @@
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
@ -121,6 +129,7 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>688</xloc>
@ -128,7 +137,7 @@ and recruitment_id = '${IDM_ID}';
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<name>citizen_information_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -141,8 +150,11 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
@ -154,6 +166,7 @@ and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1024</xloc>
@ -252,7 +265,7 @@ DO UPDATE SET
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>1024</xloc>
@ -260,7 +273,73 @@ DO UPDATE SET
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>1024</xloc>
<yloc>160</yloc>
</GUI>
</transform>
<transform>
<name>citizen_information_search_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH wanted_data AS (
SELECT
ri.recruit_id,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code,
TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date,
TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search,
true AS is_wanted,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
WHERE '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM wanted_data) as last_row
FROM wanted_data;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>416</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform>
<name>citizen_information_search_insert_or_update</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
@ -324,48 +403,7 @@ DO UPDATE SET
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH wanted_data AS (
SELECT
ri.recruit_id,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrg' AS name_organ_start_search,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'ugolovZakon' AS article_criminal_code,
TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataNachRozisk', 'YYYY-MM-DD') AS wanted_start_date,
TO_DATE(ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'dataPrekrRozysk', 'YYYY-MM-DD') AS wanted_end_date,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'osnovPrekrRozysk' AS grounds_stopping_search,
ri.info -> 'svedFL' -> 'extend' -> 'rozysk' ->> 'naimOrgPrekrRozysk' AS name_organ_end_search,
true AS is_wanted,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
WHERE '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND jsonb_typeof(ri.info->'svedFL'->'extend'->'rozysk') = 'object'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM wanted_data) as last_row
FROM wanted_data;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>416</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<name>update_citizen_information_search_flags</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
@ -378,19 +416,16 @@ FROM wanted_data;</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>is_wanted</field>
<name>is_wanted</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
@ -409,8 +444,8 @@ FROM wanted_data;</sql>
</transform>
<transform_error_handling>
<error>
<source_transform>Insert / update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>citizen_information_search_insert_or_update</source_transform>
<target_transform>citizen_information_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -421,8 +456,8 @@ FROM wanted_data;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>update_citizen_information_search_flags</source_transform>
<target_transform>citizen_information_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

127
mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow.hpl
View file

@ -22,59 +22,64 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>spouse_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<to>Change job status on error</to>
<from>citizen_spouse_output</from>
<to>spouse_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<from>spouse_detect_empty_stream</from>
<to>spouse_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>spouse_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output 2</from>
<to>Change job status on error</to>
<from>marriage_registry_output </from>
<to>spouse_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Table output</to>
<from>spouse_input</from>
<to>citizen_spouse_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Sort rows</to>
<from>spouse_input</from>
<to>spouse_sort_rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Table output 2</to>
<from>spouse_sort_rows</from>
<to>marriage_registry_output </to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Detect empty stream</to>
<from>spouse_input</from>
<to>spouse_detect_empty_stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<from>citizen_spouse_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>spouse_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -97,7 +102,7 @@
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<name>spouse_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -113,8 +118,11 @@
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
@ -126,14 +134,15 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>736</xloc>
<xloc>800</xloc>
<yloc>192</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<name>spouse_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -146,8 +155,11 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
@ -159,10 +171,11 @@ and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>576</xloc>
<yloc>912</yloc>
<xloc>592</xloc>
<yloc>720</yloc>
</GUI>
</transform>
<transform>
@ -195,7 +208,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI>
</transform>
<transform>
<name>Detect empty stream</name>
<name>spouse_detect_empty_stream</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>Y</distribute>
@ -208,7 +221,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<attributes/>
<GUI>
<xloc>352</xloc>
<yloc>912</yloc>
<yloc>720</yloc>
</GUI>
</transform>
<transform>
@ -240,15 +253,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>736</xloc>
<xloc>800</xloc>
<yloc>912</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<name>spouse_sort_rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
@ -260,7 +273,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
@ -282,7 +295,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI>
</transform>
<transform>
<name>Table input</name>
<name>spouse_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
@ -294,6 +307,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH spouse_data AS (
SELECT
ri.recruit_id AS recruit_id,
@ -348,7 +362,7 @@ FROM spouse_data;
</GUI>
</transform>
<transform>
<name>Table output</name>
<name>citizen_spouse_output</name>
<type>TableOutput</type>
<description/>
<distribute>N</distribute>
@ -414,23 +428,26 @@ FROM spouse_data;
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>citizen_spouse</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>736</xloc>
<xloc>800</xloc>
<yloc>576</yloc>
</GUI>
</transform>
<transform>
<name>Table output 2</name>
<name>marriage_registry_output </name>
<type>TableOutput</type>
<description/>
<distribute>N</distribute>
@ -468,11 +485,14 @@ FROM spouse_data;
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>marriage_registry</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -483,10 +503,35 @@ FROM spouse_data;
<yloc>192</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>592</xloc>
<yloc>912</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Table output</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>citizen_spouse_output</source_transform>
<target_transform>spouse_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -497,8 +542,8 @@ FROM spouse_data;
<min_pct_rows/>
</error>
<error>
<source_transform>Table output 2</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>marriage_registry_output </source_transform>
<target_transform>spouse_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

533
mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_delta.hpl
View file

@ -22,57 +22,62 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>spouse_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Change job status on error</to>
<from>spouse_update_or_insert</from>
<to>spouse_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>spouse_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>spouse_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update 2</from>
<to>Change job status on error</to>
<from>marriage_registry_update_or_insert</from>
<to>spouse_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Insert / update</to>
<from>spouse_input</from>
<to>spouse_update_or_insert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>spouse_input</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>spouse_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<from>spouse_update_or_insert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Insert / update 2</to>
<to>marriage_registry_update_or_insert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>spouse_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
@ -96,78 +101,6 @@
<yloc>176</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
<argument>
<name>error_code</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>768</xloc>
<yloc>176</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
error_description = null,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
@ -245,7 +178,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>1072</xloc>
@ -253,7 +186,259 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>marriage_registry_update_or_insert</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>marriage_registry</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>marriage_az_number</name>
<rename>marriage_az_number</rename>
<update>Y</update>
</value>
<value>
<name>marriage_date</name>
<rename>marriage_date</rename>
<update>Y</update>
</value>
<value>
<name>divorce_date</name>
<rename>divorce_date</rename>
<update>Y</update>
</value>
<value>
<name>divorce_az_number</name>
<rename>divorce_az_number</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>176</yloc>
</GUI>
</transform>
<transform>
<name>spouse_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
<argument>
<name>error_code</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>768</xloc>
<yloc>176</yloc>
</GUI>
</transform>
<transform>
<name>spouse_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH spouse_data AS (
SELECT
ri.recruit_id AS recruit_id,
(supr->>'id')::uuid AS spouse_external_id,
NULLIF(supr->>'idERN', '') AS spouse_id_ern,
NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type,
supr->'svedFLBS'->'fio'->>'familiya' AS last_name,
supr->'svedFLBS'->'fio'->>'imya' AS first_name,
supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name,
CONCAT_WS(' ',
supr->'svedFLBS'->'fio'->>'familiya',
supr->'svedFLBS'->'fio'->>'imya',
supr->'svedFLBS'->'fio'->>'otchestvo'
) AS full_name,
MAKE_DATE(
NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int,
NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int,
NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int
) AS birth_date,
(supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date,
supr->'svedSmert'->>'nomerZapis' AS death_az_number,
ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number,
(ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date,
ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number,
(ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date,
CASE
WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true
ELSE false
END AS information_excluded,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
CROSS JOIN LATERAL (
SELECT supri
FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri
WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array'
) AS supr(supr)
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.update_date >= '${M_R_UP_DATE}'::timestamp
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM spouse_data) as last_row
FROM spouse_data;
</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>spouse_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
error_description = null,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>688</yloc>
</GUI>
</transform>
<transform>
<name>spouse_update_or_insert</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
@ -346,166 +531,10 @@ and recruitment_id = '${IDM_ID}';</sql>
<yloc>496</yloc>
</GUI>
</transform>
<transform>
<name>Insert / update 2</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>marriage_registry</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>marriage_az_number</name>
<rename>marriage_az_number</rename>
<update>Y</update>
</value>
<value>
<name>marriage_date</name>
<rename>marriage_date</rename>
<update>Y</update>
</value>
<value>
<name>divorce_date</name>
<rename>divorce_date</rename>
<update>Y</update>
</value>
<value>
<name>divorce_az_number</name>
<rename>divorce_az_number</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>176</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH spouse_data AS (
SELECT
ri.recruit_id AS recruit_id,
(supr->>'id')::uuid AS spouse_external_id,
NULLIF(supr->>'idERN', '') AS spouse_id_ern,
NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type,
supr->'svedFLBS'->'fio'->>'familiya' AS last_name,
supr->'svedFLBS'->'fio'->>'imya' AS first_name,
supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name,
CONCAT_WS(' ',
supr->'svedFLBS'->'fio'->>'familiya',
supr->'svedFLBS'->'fio'->>'imya',
supr->'svedFLBS'->'fio'->>'otchestvo'
) AS full_name,
MAKE_DATE(
NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int,
NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int,
NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int
) AS birth_date,
(supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date,
supr->'svedSmert'->>'nomerZapis' AS death_az_number,
ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number,
(ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date,
ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number,
(ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date,
CASE
WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true
ELSE false
END AS information_excluded,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
CROSS JOIN LATERAL (
SELECT supri
FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri
WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array'
) AS supr(supr)
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.update_date >= '${M_R_UP_DATE}'::timestamp
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM spouse_data) as last_row
FROM spouse_data;
</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>496</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Insert / update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>marriage_registry_update_or_insert</source_transform>
<target_transform>spouse_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -516,8 +545,8 @@ FROM spouse_data;
<min_pct_rows/>
</error>
<error>
<source_transform>Insert / update 2</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>spouse_update_or_insert</source_transform>
<target_transform>spouse_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

465
mappings/info_recruits/citizen_tables/citizen_spouse/parallel/citizen_spouse_flow_repeat.hpl
View file

@ -49,59 +49,64 @@ M_R_CR_DATE
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>spouse_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Change job status on error</to>
<from>spouse_update_or_insert</from>
<to>spouse_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>spouse_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>spouse_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update 2</from>
<to>Change job status on error</to>
<from>marriage_registry_update_or_insert</from>
<to>spouse_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Insert / update</to>
<from>spouse_input</from>
<to>spouse_update_or_insert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>spouse_input</from>
<to>Sort rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Insert / update 2</to>
<to>marriage_registry_update_or_insert</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<from>spouse_update_or_insert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>spouse_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>spouse_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -124,7 +129,7 @@ M_R_CR_DATE
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<name>spouse_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -143,8 +148,11 @@ M_R_CR_DATE
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
@ -156,47 +164,13 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
error_description = null,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>944</xloc>
<yloc>752</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
@ -257,7 +231,7 @@ DO UPDATE SET
<attributes/>
<GUI>
<xloc>352</xloc>
<yloc>752</yloc>
<yloc>784</yloc>
</GUI>
</transform>
<transform>
@ -289,7 +263,7 @@ DO UPDATE SET
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>944</xloc>
@ -297,7 +271,221 @@ DO UPDATE SET
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>352</xloc>
<yloc>368</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>944</xloc>
<yloc>784</yloc>
</GUI>
</transform>
<transform>
<name>marriage_registry_update_or_insert</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>marriage_registry</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>marriage_az_number</name>
<rename>marriage_az_number</rename>
<update>Y</update>
</value>
<value>
<name>marriage_date</name>
<rename>marriage_date</rename>
<update>Y</update>
</value>
<value>
<name>divorce_date</name>
<rename>divorce_date</rename>
<update>Y</update>
</value>
<value>
<name>divorce_az_number</name>
<rename>divorce_az_number</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>352</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>spouse_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH spouse_data AS (
SELECT
ri.recruit_id AS recruit_id,
(supr->>'id')::uuid AS spouse_external_id,
NULLIF(supr->>'idERN', '') AS spouse_id_ern,
NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type,
supr->'svedFLBS'->'fio'->>'familiya' AS last_name,
supr->'svedFLBS'->'fio'->>'imya' AS first_name,
supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name,
CONCAT_WS(' ',
supr->'svedFLBS'->'fio'->>'familiya',
supr->'svedFLBS'->'fio'->>'imya',
supr->'svedFLBS'->'fio'->>'otchestvo'
) AS full_name,
MAKE_DATE(
NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int,
NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int,
NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int
) AS birth_date,
(supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date,
supr->'svedSmert'->>'nomerZapis' AS death_az_number,
ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number,
(ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date,
ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number,
(ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date,
CASE
WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true
ELSE false
END AS information_excluded,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
CROSS JOIN LATERAL (
SELECT supri
FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri
WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array'
) AS supr(supr)
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM spouse_data) as last_row
FROM spouse_data;
</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>352</xloc>
<yloc>528</yloc>
</GUI>
</transform>
<transform>
<name>spouse_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
error_description = null,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>672</xloc>
<yloc>784</yloc>
</GUI>
</transform>
<transform>
<name>spouse_update_or_insert</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
@ -390,165 +578,10 @@ DO UPDATE SET
<yloc>528</yloc>
</GUI>
</transform>
<transform>
<name>Insert / update 2</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>marriage_registry</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>marriage_az_number</name>
<rename>marriage_az_number</rename>
<update>Y</update>
</value>
<value>
<name>marriage_date</name>
<rename>marriage_date</rename>
<update>Y</update>
</value>
<value>
<name>divorce_date</name>
<rename>divorce_date</rename>
<update>Y</update>
</value>
<value>
<name>divorce_az_number</name>
<rename>divorce_az_number</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>352</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>352</xloc>
<yloc>368</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH spouse_data AS (
SELECT
ri.recruit_id AS recruit_id,
(supr->>'id')::uuid AS spouse_external_id,
NULLIF(supr->>'idERN', '') AS spouse_id_ern,
NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type,
supr->'svedFLBS'->'fio'->>'familiya' AS last_name,
supr->'svedFLBS'->'fio'->>'imya' AS first_name,
supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name,
CONCAT_WS(' ',
supr->'svedFLBS'->'fio'->>'familiya',
supr->'svedFLBS'->'fio'->>'imya',
supr->'svedFLBS'->'fio'->>'otchestvo'
) AS full_name,
MAKE_DATE(
NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int,
NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int,
NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int
) AS birth_date,
(supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date,
supr->'svedSmert'->>'nomerZapis' AS death_az_number,
ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number,
(ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date,
ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number,
(ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date,
CASE
WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true
ELSE false
END AS information_excluded,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
CROSS JOIN LATERAL (
SELECT supri
FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri
WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array'
) AS supr(supr)
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
SELECT
*,
source_ctid = (SELECT MAX(source_ctid) FROM spouse_data) as last_row
FROM spouse_data;
</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>352</xloc>
<yloc>528</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Insert / update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>marriage_registry_update_or_insert</source_transform>
<target_transform>spouse_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -559,8 +592,8 @@ FROM spouse_data;
<min_pct_rows/>
</error>
<error>
<source_transform>Insert / update 2</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>spouse_update_or_insert</source_transform>
<target_transform>spouse_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

87
mappings/info_recruits/citizen_tables/passport/parallel/passport_flow.hpl
View file

@ -22,44 +22,49 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>passport_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<to>Change job status on error</to>
<from>passport_output</from>
<to>passport_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>passport_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<from>passport_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>passport_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Table output</to>
<from>passport_input</from>
<to>passport_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>passport_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>passport_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -82,7 +87,7 @@
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<name>passport_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -98,8 +103,11 @@
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
@ -111,6 +119,7 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>896</xloc>
@ -118,7 +127,7 @@ and recruitment_id = '${IDM_ID}';
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<name>passport_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -131,8 +140,11 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
@ -144,10 +156,11 @@ and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1264</xloc>
<yloc>176</yloc>
<yloc>112</yloc>
</GUI>
</transform>
<transform>
@ -193,7 +206,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>176</yloc>
<yloc>112</yloc>
</GUI>
</transform>
<transform>
@ -225,7 +238,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>1264</xloc>
@ -233,7 +246,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI>
</transform>
<transform>
<name>Table input</name>
<name>passport_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
@ -287,7 +300,7 @@ FROM passport_data;</sql>
</GUI>
</transform>
<transform>
<name>Table output</name>
<name>passport_output</name>
<type>TableOutput</type>
<description/>
<distribute>N</distribute>
@ -333,11 +346,14 @@ FROM passport_data;</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>passport</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -348,10 +364,35 @@ FROM passport_data;</sql>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>1264</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Table output</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>passport_output</source_transform>
<target_transform>passport_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

86
mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_delta.hpl
View file

@ -22,42 +22,47 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>passport_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Change job status on error</to>
<from>passport_insert_or_update</from>
<to>passport_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>passport_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<from>passport_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>passport_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>passport_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Insert / update</to>
<from>passport_input</from>
<to>passport_insert_or_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>passport_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
@ -82,7 +87,7 @@
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<name>passport_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -98,8 +103,11 @@
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
@ -111,6 +119,7 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>752</xloc>
@ -118,7 +127,7 @@ and recruitment_id = '${IDM_ID}';
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<name>passport_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -131,8 +140,11 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
@ -144,10 +156,11 @@ and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1120</xloc>
<yloc>80</yloc>
<yloc>16</yloc>
</GUI>
</transform>
<transform>
@ -196,7 +209,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>80</yloc>
<yloc>16</yloc>
</GUI>
</transform>
<transform>
@ -228,7 +241,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>1120</xloc>
@ -236,7 +249,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<name>passport_insert_or_update</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
@ -253,11 +266,13 @@ and recruitment_id = '${IDM_ID}';</sql>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>actual</field>
<name>actual</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>passport</table>
@ -305,7 +320,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>Table input</name>
<name>passport_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
@ -359,10 +374,35 @@ FROM passport_data;</sql>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>1120</xloc>
<yloc>112</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Insert / update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>passport_insert_or_update</source_transform>
<target_transform>passport_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

82
mappings/info_recruits/citizen_tables/passport/parallel/passport_flow_repeat.hpl
View file

@ -22,42 +22,47 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>passport_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Change job status on error</to>
<from>passport_insert_or_update</from>
<to>passport_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>passport_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<from>passport_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>passport_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>passport_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Insert / update</to>
<from>passport_input</from>
<to>passport_insert_or_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>passport_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
@ -82,7 +87,7 @@
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<name>passport_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -98,8 +103,11 @@
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
@ -111,6 +119,7 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>768</xloc>
@ -118,7 +127,7 @@ and recruitment_id = '${IDM_ID}';
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<name>passport_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
@ -131,8 +140,11 @@ and recruitment_id = '${IDM_ID}';
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
@ -144,6 +156,7 @@ and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<update_field/>
<attributes/>
<GUI>
<xloc>1040</xloc>
@ -242,7 +255,7 @@ DO UPDATE SET
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>1040</xloc>
@ -250,7 +263,7 @@ DO UPDATE SET
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<name>passport_insert_or_update</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
@ -267,11 +280,13 @@ DO UPDATE SET
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>actual</field>
<name>actual</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>passport</table>
@ -319,7 +334,7 @@ DO UPDATE SET
</GUI>
</transform>
<transform>
<name>Table input</name>
<name>passport_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
@ -372,10 +387,35 @@ FROM passport_data;</sql>
<yloc>272</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>144</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Insert / update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>passport_insert_or_update</source_transform>
<target_transform>passport_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

256
mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow.hpl
View file

@ -22,59 +22,64 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>prosecution_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<to>Change job status on error</to>
<from>prosecution_output</from>
<to>prosecution_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>prosecution_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>prosecution_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Table output</to>
<from>prosecution_input</from>
<to>prosecution_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<from>prosecution_update_flags</from>
<to>prosecution_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Sort rows</to>
<from>prosecution_input</from>
<to>prosecution_sort_rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Update</to>
<from>prosecution_sort_rows</from>
<to>prosecution_update_flags</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>prosecution_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<from>prosecution_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>prosecution_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -96,75 +101,6 @@
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
@ -240,7 +176,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>1072</xloc>
@ -248,8 +184,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -258,31 +194,58 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>480</xloc>
<yloc>32</yloc>
<xloc>1072</xloc>
<yloc>544</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<name>prosecution_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>848</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>prosecution_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
@ -348,7 +311,7 @@ GROUP BY r.recruit_id, up.up, up.source_ctid;</sql>
</GUI>
</transform>
<transform>
<name>Table output</name>
<name>prosecution_output</name>
<type>TableOutput</type>
<description/>
<distribute>N</distribute>
@ -466,7 +429,74 @@ GROUP BY r.recruit_id, up.up, up.source_ctid;</sql>
</GUI>
</transform>
<transform>
<name>Update</name>
<name>prosecution_sort_rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>480</xloc>
<yloc>32</yloc>
</GUI>
</transform>
<transform>
<name>prosecution_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>1072</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>prosecution_update_flags</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
@ -507,8 +537,8 @@ GROUP BY r.recruit_id, up.up, up.source_ctid;</sql>
</transform>
<transform_error_handling>
<error>
<source_transform>Table output</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>prosecution_output</source_transform>
<target_transform>prosecution_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -519,8 +549,8 @@ GROUP BY r.recruit_id, up.up, up.source_ctid;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>prosecution_update_flags</source_transform>
<target_transform>prosecution_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

399
mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_delta.hpl
View file

@ -22,59 +22,64 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>prosecution_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Change job status on error</to>
<from>prosecution_insert_or_update</from>
<to>prosecution_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>prosecution_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>prosecution_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<from>prosecution_update_flags</from>
<to>prosecution_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Insert / update</to>
<from>prosecution_input</from>
<to>prosecution_insert_or_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Update</to>
<from>prosecution_sort_rows</from>
<to>prosecution_update_flags</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Sort rows</to>
<from>prosecution_input</from>
<to>prosecution_sort_rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<from>prosecution_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>prosecution_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>prosecution_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -96,75 +101,6 @@
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>736</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>896</xloc>
<yloc>944</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
@ -243,7 +179,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>896</xloc>
@ -251,7 +187,169 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<name>prosecution_sort_rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>384</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>896</xloc>
<yloc>864</yloc>
</GUI>
</transform>
<transform>
<name>prosecution_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>736</xloc>
<yloc>592</yloc>
</GUI>
</transform>
<transform>
<name>prosecution_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH ugolov_presl AS (
SELECT
jsonb_array_elements(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') AS up,
ri.recruit_id,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
AND '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
WHERE jsonb_typeof(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') = 'array'
)
SELECT
r.recruit_id AS recruit_id,
(up->>'id')::uuid AS prosecution_external_id,
NULLIF(up->'sledSud'->>'uk', '') AS uk_name,
NULLIF(up->'sledSud'->'extend'->>'nomerUDPredv', '') AS case_number,
NULLIF(up->'sledSud'->'extend'->>'svedPerekvalif', '') AS reclassification,
NULLIF(up->'sledSud'->'extend'->>'naimVidUgolovPresl', '') AS kind,
CASE
WHEN up->'sledSud'->'extend'->>'priznakPeredachSud' = '1' THEN true
ELSE false -- нет или нет инфы
END AS transfer_to_court,
NULLIF(up->'sledSud'->>'naimSud','') AS court_name,
NULLIF(up->'sledSud'->>'nomerUD','') AS case_number_court,
STRING_AGG(TRIM(BOTH ' "' FROM law.f), ', ') AS law_point,
NULLIF(up->'sledSud'->>'naimOrgan', '') AS authority_name,
NULLIF(up->'sledSud'->'svedSnyat'->>'osnSnyatSud', '') AS conviction_removal_basis,
TO_DATE(up->'sledSud'->'svedSnyat'->>'dataSnyatSud', 'YYYY-MM-DD') AS conviction_removal_date,
TO_DATE(up->'sledSud'->>'dataOsuzhd', 'YYYY-MM-DD') AS conviction_date,
TO_DATE(up->'sledSud'->>'dataVozbUD', 'YYYY-MM-DD') AS create_date,
NULLIF(up->'sledSud'->>'osnPrekrUD', '') AS termination_basis,
NULLIF(up->'sledSud'->>'statusNaim', '') AS conviction_status,
TO_DATE(up->'sledSud'->>'dataPrekrUD', 'YYYY-MM-DD') AS close_date,
NULLIF(up->'sledSudTekst'->>'svedTekst' , '') AS sled_sud_tekst_sved_tekst,
NULLIF(up->'sledSudTekst'->>'statusNaim', '') AS sled_sud_tekst_status_naim,
CASE
WHEN (up->>'sledSud') IS NOT NULL AND up->>'sledSud' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в структурированном формате'
WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде'
ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей'
END AS type_info,
true AS has_criminal_prosecution,
up.source_ctid = (SELECT MAX(source_ctid) FROM ugolov_presl) as last_row
FROM ervu_dashboard.citizen r
JOIN ugolov_presl up ON r.recruit_id = up.recruit_id
LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true
GROUP BY r.recruit_id, up.up, up.source_ctid;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>384</xloc>
<yloc>768</yloc>
</GUI>
</transform>
<transform>
<name>prosecution_insert_or_update</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
@ -390,8 +488,8 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<name>prosecution_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -400,98 +498,30 @@ and recruitment_id = '${IDM_ID}';</sql>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>384</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH ugolov_presl AS (
SELECT
jsonb_array_elements(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') AS up,
ri.recruit_id,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
AND '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
WHERE jsonb_typeof(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') = 'array'
)
SELECT
r.recruit_id AS recruit_id,
(up->>'id')::uuid AS prosecution_external_id,
NULLIF(up->'sledSud'->>'uk', '') AS uk_name,
NULLIF(up->'sledSud'->'extend'->>'nomerUDPredv', '') AS case_number,
NULLIF(up->'sledSud'->'extend'->>'svedPerekvalif', '') AS reclassification,
NULLIF(up->'sledSud'->'extend'->>'naimVidUgolovPresl', '') AS kind,
CASE
WHEN up->'sledSud'->'extend'->>'priznakPeredachSud' = '1' THEN true
ELSE false -- нет или нет инфы
END AS transfer_to_court,
NULLIF(up->'sledSud'->>'naimSud','') AS court_name,
NULLIF(up->'sledSud'->>'nomerUD','') AS case_number_court,
STRING_AGG(TRIM(BOTH ' "' FROM law.f), ', ') AS law_point,
NULLIF(up->'sledSud'->>'naimOrgan', '') AS authority_name,
NULLIF(up->'sledSud'->'svedSnyat'->>'osnSnyatSud', '') AS conviction_removal_basis,
TO_DATE(up->'sledSud'->'svedSnyat'->>'dataSnyatSud', 'YYYY-MM-DD') AS conviction_removal_date,
TO_DATE(up->'sledSud'->>'dataOsuzhd', 'YYYY-MM-DD') AS conviction_date,
TO_DATE(up->'sledSud'->>'dataVozbUD', 'YYYY-MM-DD') AS create_date,
NULLIF(up->'sledSud'->>'osnPrekrUD', '') AS termination_basis,
NULLIF(up->'sledSud'->>'statusNaim', '') AS conviction_status,
TO_DATE(up->'sledSud'->>'dataPrekrUD', 'YYYY-MM-DD') AS close_date,
NULLIF(up->'sledSudTekst'->>'svedTekst' , '') AS sled_sud_tekst_sved_tekst,
NULLIF(up->'sledSudTekst'->>'statusNaim', '') AS sled_sud_tekst_status_naim,
CASE
WHEN (up->>'sledSud') IS NOT NULL AND up->>'sledSud' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в структурированном формате'
WHEN (up->>'sledSudTekst') IS NOT NULL AND up->>'sledSudTekst' != 'null' THEN 'Сведения о фактах уголовного преследования или судимости в текстовом виде'
ELSE 'Есть сведения о фактах уголовного преследования или судимости без указания подробностей'
END AS type_info,
true AS has_criminal_prosecution,
up.source_ctid = (SELECT MAX(source_ctid) FROM ugolov_presl) as last_row
FROM ervu_dashboard.citizen r
JOIN ugolov_presl up ON r.recruit_id = up.recruit_id
LEFT JOIN LATERAL jsonb_array_elements_text(up->'sledSud'->'statiUK') AS law(f) ON true
GROUP BY r.recruit_id, up.up, up.source_ctid;</sql>
<variables_active>Y</variables_active>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>384</xloc>
<yloc>768</yloc>
<xloc>896</xloc>
<yloc>944</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<name>prosecution_update_flags</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
@ -504,19 +534,16 @@ GROUP BY r.recruit_id, up.up, up.source_ctid;</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_criminal_prosecution</field>
<name>has_criminal_prosecution</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
@ -535,8 +562,8 @@ GROUP BY r.recruit_id, up.up, up.source_ctid;</sql>
</transform>
<transform_error_handling>
<error>
<source_transform>Insert / update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>prosecution_insert_or_update</source_transform>
<target_transform>prosecution_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -547,8 +574,8 @@ GROUP BY r.recruit_id, up.up, up.source_ctid;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>prosecution_update_flags</source_transform>
<target_transform>prosecution_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

173
mappings/info_recruits/citizen_tables/prosecution/parallel/prosecution_flow_repeat.hpl
View file

@ -27,21 +27,21 @@
</hop>
<hop>
<from>Insert / update</from>
<to>Change job status on error</to>
<to>prosecution_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>prosecution_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<to>prosecution_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>prosecution_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
@ -51,18 +51,18 @@
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<from>prosecution_update_flags</from>
<to>prosecution_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Update</to>
<from>prosecution_sort_rows</from>
<to>prosecution_update_flags</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Sort rows</to>
<to>prosecution_sort_rows</to>
<enabled>Y</enabled>
</hop>
<hop>
@ -96,75 +96,6 @@
<yloc>736</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>768</xloc>
<yloc>736</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>928</xloc>
<yloc>1088</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
@ -257,7 +188,7 @@ DO UPDATE SET
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>prosecution_success_job_status</send_true_to>
<attributes/>
<GUI>
<xloc>928</xloc>
@ -404,7 +335,7 @@ DO UPDATE SET
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<name>prosecution_sort_rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
@ -416,7 +347,7 @@ DO UPDATE SET
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
@ -504,7 +435,76 @@ GROUP BY r.recruit_id, up.up, up.source_ctid;</sql>
</GUI>
</transform>
<transform>
<name>Update</name>
<name>prosecution_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>768</xloc>
<yloc>736</yloc>
</GUI>
</transform>
<transform>
<name>prosecution_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>928</xloc>
<yloc>1088</yloc>
</GUI>
</transform>
<transform>
<name>prosecution_update_flags</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
@ -517,19 +517,16 @@ GROUP BY r.recruit_id, up.up, up.source_ctid;</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_criminal_prosecution</field>
<name>has_criminal_prosecution</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
@ -549,7 +546,7 @@ GROUP BY r.recruit_id, up.up, up.source_ctid;</sql>
<transform_error_handling>
<error>
<source_transform>Insert / update</source_transform>
<target_transform>Change job status on error</target_transform>
<target_transform>prosecution_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -560,8 +557,8 @@ GROUP BY r.recruit_id, up.up, up.source_ctid;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>prosecution_update_flags</source_transform>
<target_transform>prosecution_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

232
mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow.hpl
View file

@ -22,59 +22,64 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>punishment_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<to>Change job status on error</to>
<from>punishment_output</from>
<to>punishment_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>punishment_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>punishment_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Table output</to>
<from>punishment_input</from>
<to>punishment_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<from>punishment_update_flags</from>
<to>punishment_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Update</to>
<from>punishment_sort_rows</from>
<to>punishment_update_flags</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Sort rows</to>
<from>punishment_input</from>
<to>punishment_sort_rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>punishment_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<from>punishment_output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>punishment_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -96,75 +101,6 @@
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>544</xloc>
<yloc>144</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
@ -240,7 +176,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>720</xloc>
@ -248,7 +184,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<name>punishment_sort_rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
@ -260,7 +196,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
@ -282,7 +218,68 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI>
</transform>
<transform>
<name>Table input</name>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>144</yloc>
</GUI>
</transform>
<transform>
<name>punishment_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>punishment_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
@ -348,7 +345,7 @@ GROUP BY recruit_id, punishment_info,source_ctid;</sql>
</GUI>
</transform>
<transform>
<name>Table output</name>
<name>punishment_output</name>
<type>TableOutput</type>
<description/>
<distribute>N</distribute>
@ -458,7 +455,40 @@ GROUP BY recruit_id, punishment_info,source_ctid;</sql>
</GUI>
</transform>
<transform>
<name>Update</name>
<name>punishment_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>512</xloc>
<yloc>256</yloc>
</GUI>
</transform>
<transform>
<name>punishment_update_flags</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
@ -471,25 +501,21 @@ GROUP BY recruit_id, punishment_info,source_ctid;</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_punishment_execution_info</field>
<name>has_punishment_execution_info</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_sentence_info</field>
<name>has_sentence_info</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
@ -512,8 +538,8 @@ GROUP BY recruit_id, punishment_info,source_ctid;</sql>
</transform>
<transform_error_handling>
<error>
<source_transform>Table output</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>punishment_output</source_transform>
<target_transform>punishment_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -524,8 +550,8 @@ GROUP BY recruit_id, punishment_info,source_ctid;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>punishment_update_flags</source_transform>
<target_transform>punishment_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_decription</descriptions_valuename>

400
mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_delta.hpl
View file

@ -41,59 +41,64 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>punishment_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Change job status on error</to>
<from>punishment_insert_or_update</from>
<to>punishment_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>punishment_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>punishment_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Insert / update</to>
<from>punishment_input</from>
<to>punishment_insert_or_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<from>punishment_update_flags</from>
<to>punishment_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Sort rows</to>
<from>punishment_input</from>
<to>punishment_sort_rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>punishment_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Update</to>
<from>punishment_sort_rows</from>
<to>punishment_update_flags</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<from>punishment_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>punishment_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -115,75 +120,6 @@
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>960</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>608</xloc>
<yloc>32</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
@ -262,7 +198,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>784</xloc>
@ -270,7 +206,169 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<name>punishment_sort_rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>32</yloc>
</GUI>
</transform>
<transform>
<name>punishment_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>960</xloc>
<yloc>384</yloc>
</GUI>
</transform>
<transform>
<name>punishment_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH punishment_data AS (
SELECT
ri.recruit_id,
jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
AND '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null'
)
SELECT recruit_id,
(punishment_info ->> 'id')::uuid AS punishment_external_id,
-- Блок "Сведения о вынесенном приговоре"
punishment_info ->> 'vidNakaz' AS punishment_type, -- Вид наказания
punishment_info ->> 'kodVidNakaz' AS punishment_code, -- Код вида наказания
TO_DATE(punishment_info ->> 'dataOsuzhd', 'YYYY-MM-DD') AS sentense_date, -- Дата осуждения
punishment_info ->> 'naimSud' AS authority_name, -- Наименование суда
punishment_info ->> 'srokNakaz' AS sentence, -- Срок наказания
CASE
WHEN punishment_info ->> 'kodIzmSrokNakaz' = '1' THEN 'Изменений срока наказания не было'
WHEN punishment_info ->> 'kodIzmSrokNakaz' = '2' THEN 'Были изменения срока наказания'
ELSE punishment_info ->> 'kodIzmSrokNakaz'
END AS sentence_changed, -- Признак изменения срока
STRING_AGG(TRIM(BOTH ' "' FROM law_articles), ', ') AS law_point, -- Статьи УК
-- Блок "Сведения об отбывании наказания"
punishment_info ->> 'katUcheta' AS registration_category, -- Категория учета
punishment_info ->> 'naimSubekt' AS region_name, -- Наименование субъекта
punishment_info ->> 'naimOrg' AS facility_name, -- Название учреждения
punishment_info ->> 'adresOrg' AS facility_address, -- Адрес учреждения
punishment_info ->> 'nomerUD' AS criminal_case_number, -- Номер уголовного дела
TO_DATE(punishment_info ->> 'dataNachSrok', 'YYYY-MM-DD') AS start_date, -- Дата начала срока
TO_DATE(punishment_info ->> 'dataOkonchSrok', 'YYYY-MM-DD') AS planned_date, -- Планируемая дата освобождения
CASE
WHEN punishment_info ->> 'kodPerekvalif' = '1' THEN 'Изменений статьи не было'
WHEN punishment_info ->> 'kodPerekvalif' = '2' THEN 'Были изменения статьи'
ELSE punishment_info ->> 'kodPerekvalif'
END AS article_modification_flag, -- Признак переквалификации
punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения
TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения
true AS has_punishment_execution_info,
true AS has_sentence_info,
source_ctid = (SELECT MAX(source_ctid) FROM punishment_data) as last_row
FROM punishment_data
LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true
GROUP BY recruit_id, punishment_info,source_ctid;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>punishment_insert_or_update</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
@ -399,8 +497,8 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<name>punishment_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -409,98 +507,30 @@ and recruitment_id = '${IDM_ID}';</sql>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH punishment_data AS (
SELECT
ri.recruit_id,
jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
AND '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null'
)
SELECT recruit_id,
(punishment_info ->> 'id')::uuid AS punishment_external_id,
-- Блок "Сведения о вынесенном приговоре"
punishment_info ->> 'vidNakaz' AS punishment_type, -- Вид наказания
punishment_info ->> 'kodVidNakaz' AS punishment_code, -- Код вида наказания
TO_DATE(punishment_info ->> 'dataOsuzhd', 'YYYY-MM-DD') AS sentense_date, -- Дата осуждения
punishment_info ->> 'naimSud' AS authority_name, -- Наименование суда
punishment_info ->> 'srokNakaz' AS sentence, -- Срок наказания
CASE
WHEN punishment_info ->> 'kodIzmSrokNakaz' = '1' THEN 'Изменений срока наказания не было'
WHEN punishment_info ->> 'kodIzmSrokNakaz' = '2' THEN 'Были изменения срока наказания'
ELSE punishment_info ->> 'kodIzmSrokNakaz'
END AS sentence_changed, -- Признак изменения срока
STRING_AGG(TRIM(BOTH ' "' FROM law_articles), ', ') AS law_point, -- Статьи УК
-- Блок "Сведения об отбывании наказания"
punishment_info ->> 'katUcheta' AS registration_category, -- Категория учета
punishment_info ->> 'naimSubekt' AS region_name, -- Наименование субъекта
punishment_info ->> 'naimOrg' AS facility_name, -- Название учреждения
punishment_info ->> 'adresOrg' AS facility_address, -- Адрес учреждения
punishment_info ->> 'nomerUD' AS criminal_case_number, -- Номер уголовного дела
TO_DATE(punishment_info ->> 'dataNachSrok', 'YYYY-MM-DD') AS start_date, -- Дата начала срока
TO_DATE(punishment_info ->> 'dataOkonchSrok', 'YYYY-MM-DD') AS planned_date, -- Планируемая дата освобождения
CASE
WHEN punishment_info ->> 'kodPerekvalif' = '1' THEN 'Изменений статьи не было'
WHEN punishment_info ->> 'kodPerekvalif' = '2' THEN 'Были изменения статьи'
ELSE punishment_info ->> 'kodPerekvalif'
END AS article_modification_flag, -- Признак переквалификации
punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения
TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения
true AS has_punishment_execution_info,
true AS has_sentence_info,
source_ctid = (SELECT MAX(source_ctid) FROM punishment_data) as last_row
FROM punishment_data
LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true
GROUP BY recruit_id, punishment_info,source_ctid;</sql>
<variables_active>Y</variables_active>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'DELTA_PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>400</xloc>
<yloc>224</yloc>
<xloc>592</xloc>
<yloc>160</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<name>punishment_update_flags</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
@ -513,25 +543,21 @@ GROUP BY recruit_id, punishment_info,source_ctid;</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_punishment_execution_info</field>
<name>has_punishment_execution_info</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_sentence_info</field>
<name>has_sentence_info</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
@ -554,8 +580,8 @@ GROUP BY recruit_id, punishment_info,source_ctid;</sql>
</transform>
<transform_error_handling>
<error>
<source_transform>Insert / update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>punishment_insert_or_update</source_transform>
<target_transform>punishment_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -566,8 +592,8 @@ GROUP BY recruit_id, punishment_info,source_ctid;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>punishment_update_flags</source_transform>
<target_transform>punishment_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>

400
mappings/info_recruits/citizen_tables/punishment/parallel/punishment_flow_repeat.hpl
View file

@ -22,59 +22,64 @@
<order>
<hop>
<from>Create job execution record</from>
<to>Table input</to>
<to>punishment_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Change job status on error</from>
<from>punishment_error_job_status</from>
<to>Abort</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<to>punishment_success_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Change job status on success</to>
<from>punishment_update_flags</from>
<to>punishment_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Update</from>
<to>Change job status on error</to>
<from>punishment_insert_or_update</from>
<to>punishment_error_job_status</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<to>Change job status on error</to>
<from>punishment_input</from>
<to>punishment_insert_or_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Insert / update</to>
<from>punishment_input</from>
<to>punishment_sort_rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Sort rows</to>
<from>punishment_sort_rows</from>
<to>punishment_update_flags</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Sort rows</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<from>punishment_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update</from>
<from>punishment_insert_or_update</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Filter rows</from>
<to>Unique rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Unique rows</from>
<to>punishment_success_job_status</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Abort</name>
@ -96,75 +101,6 @@
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>992</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>112</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
@ -257,7 +193,7 @@ DO UPDATE SET
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<send_true_to>Unique rows</send_true_to>
<attributes/>
<GUI>
<xloc>816</xloc>
@ -265,7 +201,168 @@ DO UPDATE SET
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<name>punishment_sort_rows</name>
<type>SortRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>100000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>416</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>Unique rows</name>
<type>Unique</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<count_rows>N</count_rows>
<fields>
<field>
<case_insensitive>N</case_insensitive>
<name>recruit_id</name>
</field>
</fields>
<reject_duplicate_row>N</reject_duplicate_row>
<attributes/>
<GUI>
<xloc>736</xloc>
<yloc>112</yloc>
</GUI>
</transform>
<transform>
<name>punishment_error_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>992</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>punishment_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH punishment_data AS (
SELECT
ri.recruit_id,
jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
AND '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null'
)
SELECT recruit_id,
(punishment_info ->> 'id')::uuid AS punishment_external_id,
-- Блок "Сведения о вынесенном приговоре"
punishment_info ->> 'vidNakaz' AS punishment_type, -- Вид наказания
punishment_info ->> 'kodVidNakaz' AS punishment_code, -- Код вида наказания
TO_DATE(punishment_info ->> 'dataOsuzhd', 'YYYY-MM-DD') AS sentense_date, -- Дата осуждения
punishment_info ->> 'naimSud' AS authority_name, -- Наименование суда
punishment_info ->> 'srokNakaz' AS sentence, -- Срок наказания
CASE
WHEN punishment_info ->> 'kodIzmSrokNakaz' = '1' THEN 'Изменений срока наказания не было'
WHEN punishment_info ->> 'kodIzmSrokNakaz' = '2' THEN 'Были изменения срока наказания'
ELSE punishment_info ->> 'kodIzmSrokNakaz'
END AS sentence_changed, -- Признак изменения срока
STRING_AGG(TRIM(BOTH ' "' FROM law_articles), ', ') AS law_point, -- Статьи УК
-- Блок "Сведения об отбывании наказания"
punishment_info ->> 'katUcheta' AS registration_category, -- Категория учета
punishment_info ->> 'naimSubekt' AS region_name, -- Наименование субъекта
punishment_info ->> 'naimOrg' AS facility_name, -- Название учреждения
punishment_info ->> 'adresOrg' AS facility_address, -- Адрес учреждения
punishment_info ->> 'nomerUD' AS criminal_case_number, -- Номер уголовного дела
TO_DATE(punishment_info ->> 'dataNachSrok', 'YYYY-MM-DD') AS start_date, -- Дата начала срока
TO_DATE(punishment_info ->> 'dataOkonchSrok', 'YYYY-MM-DD') AS planned_date, -- Планируемая дата освобождения
CASE
WHEN punishment_info ->> 'kodPerekvalif' = '1' THEN 'Изменений статьи не было'
WHEN punishment_info ->> 'kodPerekvalif' = '2' THEN 'Были изменения статьи'
ELSE punishment_info ->> 'kodPerekvalif'
END AS article_modification_flag, -- Признак переквалификации
punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения
TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения
true AS has_punishment_execution_info,
true AS has_sentence_info,
source_ctid = (SELECT MAX(source_ctid) FROM punishment_data) as last_row
FROM punishment_data
LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true
GROUP BY recruit_id, punishment_info,source_ctid;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>416</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform>
<name>punishment_insert_or_update</name>
<type>InsertUpdate</type>
<description/>
<distribute>N</distribute>
@ -389,13 +486,13 @@ DO UPDATE SET
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>816</xloc>
<xloc>800</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform>
<name>Sort rows</name>
<type>SortRows</type>
<name>punishment_success_job_status</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -404,97 +501,30 @@ DO UPDATE SET
<method>none</method>
<schema_name/>
</partitioning>
<directory>${java.io.tmpdir}</directory>
<prefix>out</prefix>
<sort_size>1000000</sort_size>
<free_memory/>
<compress>N</compress>
<compress_variable/>
<unique_rows>Y</unique_rows>
<fields>
<field>
<name>recruit_id</name>
<ascending>Y</ascending>
<case_sensitive>N</case_sensitive>
<collator_enabled>N</collator_enabled>
<collator_strength>0</collator_strength>
<presorted>N</presorted>
</field>
</fields>
<attributes/>
<GUI>
<xloc>416</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH punishment_data AS (
SELECT
ri.recruit_id,
jsonb_array_elements(ri.info -> 'svedFL' -> 'svedNakaz' -> 'nakaz') AS punishment_info,
ri.ctid as source_ctid
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
AND '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
WHERE ri.info -> 'svedFL' -> 'svedNakaz' ->> 'nakaz' != 'null'
)
SELECT recruit_id,
(punishment_info ->> 'id')::uuid AS punishment_external_id,
-- Блок "Сведения о вынесенном приговоре"
punishment_info ->> 'vidNakaz' AS punishment_type, -- Вид наказания
punishment_info ->> 'kodVidNakaz' AS punishment_code, -- Код вида наказания
TO_DATE(punishment_info ->> 'dataOsuzhd', 'YYYY-MM-DD') AS sentense_date, -- Дата осуждения
punishment_info ->> 'naimSud' AS authority_name, -- Наименование суда
punishment_info ->> 'srokNakaz' AS sentence, -- Срок наказания
CASE
WHEN punishment_info ->> 'kodIzmSrokNakaz' = '1' THEN 'Изменений срока наказания не было'
WHEN punishment_info ->> 'kodIzmSrokNakaz' = '2' THEN 'Были изменения срока наказания'
ELSE punishment_info ->> 'kodIzmSrokNakaz'
END AS sentence_changed, -- Признак изменения срока
STRING_AGG(TRIM(BOTH ' "' FROM law_articles), ', ') AS law_point, -- Статьи УК
-- Блок "Сведения об отбывании наказания"
punishment_info ->> 'katUcheta' AS registration_category, -- Категория учета
punishment_info ->> 'naimSubekt' AS region_name, -- Наименование субъекта
punishment_info ->> 'naimOrg' AS facility_name, -- Название учреждения
punishment_info ->> 'adresOrg' AS facility_address, -- Адрес учреждения
punishment_info ->> 'nomerUD' AS criminal_case_number, -- Номер уголовного дела
TO_DATE(punishment_info ->> 'dataNachSrok', 'YYYY-MM-DD') AS start_date, -- Дата начала срока
TO_DATE(punishment_info ->> 'dataOkonchSrok', 'YYYY-MM-DD') AS planned_date, -- Планируемая дата освобождения
CASE
WHEN punishment_info ->> 'kodPerekvalif' = '1' THEN 'Изменений статьи не было'
WHEN punishment_info ->> 'kodPerekvalif' = '2' THEN 'Были изменения статьи'
ELSE punishment_info ->> 'kodPerekvalif'
END AS article_modification_flag, -- Признак переквалификации
punishment_info ->> 'osnOsvob' AS release_reason, -- Основание освобождения
TO_DATE(punishment_info ->> 'dataOsvob', 'YYYY-MM-DD') AS release_date, -- Фактическая дата освобождения
true AS has_punishment_execution_info,
true AS has_sentence_info,
source_ctid = (SELECT MAX(source_ctid) FROM punishment_data) as last_row
FROM punishment_data
LEFT JOIN LATERAL jsonb_array_elements_text(punishment_info -> 'statiUK') AS law_articles ON true
GROUP BY recruit_id, punishment_info,source_ctid;</sql>
<variables_active>Y</variables_active>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';
</sql>
<attributes/>
<GUI>
<xloc>416</xloc>
<yloc>272</yloc>
<xloc>640</xloc>
<yloc>192</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<name>punishment_update_flags</name>
<type>Update</type>
<description/>
<distribute>N</distribute>
@ -507,25 +537,21 @@ GROUP BY recruit_id, punishment_info,source_ctid;</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_punishment_execution_info</field>
<name>has_punishment_execution_info</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_sentence_info</field>
<name>has_sentence_info</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
@ -548,8 +574,8 @@ GROUP BY recruit_id, punishment_info,source_ctid;</sql>
</transform>
<transform_error_handling>
<error>
<source_transform>Insert / update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>punishment_insert_or_update</source_transform>
<target_transform>punishment_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>
@ -560,8 +586,8 @@ GROUP BY recruit_id, punishment_info,source_ctid;</sql>
<min_pct_rows/>
</error>
<error>
<source_transform>Update</source_transform>
<target_transform>Change job status on error</target_transform>
<source_transform>punishment_update_flags</source_transform>
<target_transform>punishment_error_job_status</target_transform>
<is_enabled>Y</is_enabled>
<nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename>