micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/develop' into develop

Browse source
This commit is contained in:
Fusionshh 2025-09-29 14:53:26 +03:00
commit ff73504d79
20 changed files with 448 additions and 188 deletions
Download patch file Download diff file Expand all files Collapse all files

21
mappings/info_recruits/citizen_tables/border_crossing/border_crossing_job.hwf
View file

@ -228,7 +228,6 @@
</field>
</fields>
<file_variable_type>CURRENT_WORKFLOW</file_variable_type>
<filename/>
<replacevars>N</replacevars>
<parallel>N</parallel>
<xloc>224</xloc>
@ -380,6 +379,26 @@ M_R_CR_DATE
3001-01-01 00:00:00</note>
<width>117</width>
</notepad>
<notepad>
<backgroundcolorblue>251</backgroundcolorblue>
<backgroundcolorgreen>232</backgroundcolorgreen>
<backgroundcolorred>201</backgroundcolorred>
<bordercolorblue>90</bordercolorblue>
<bordercolorgreen>58</bordercolorgreen>
<bordercolorred>14</bordercolorred>
<fontbold>N</fontbold>
<fontcolorblue>90</fontcolorblue>
<fontcolorgreen>58</fontcolorgreen>
<fontcolorred>14</fontcolorred>
<fontitalic>N</fontitalic>
<fontname>Segoe UI</fontname>
<fontsize>9</fontsize>
<height>26</height>
<xloc>631</xloc>
<yloc>124</yloc>
<note>dataSved - не предусмотрена</note>
<width>171</width>
</notepad>
</notepads>
<attributes/>
</workflow>

11
mappings/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow.hpl
View file

@ -320,15 +320,17 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>select
ri.recruit_id,
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataSved', 'YYYY-MM-DD') AS source_update_date, -- дата обновления сведений
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'osnPriob' acquiring_basis, -- основание приобретения гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' acquiring_number, -- номер решения о приобретении гражданства РФ +
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshPriob', 'YYYY-MM-DD') acquiring_date, -- дата решения о приобретении гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'organReshPriob' acquiring_authority, -- наименование органа, принявшего решение о приобретении гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'osnVyh' renunciation_basis, -- основание выхода гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshVyh' renunciation_number, -- номер решения о выходе из гражданства РФ +
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshVyh', 'YYYY-MM-DD') renunciation_date, -- дата решения о выходе из гражданства РФ +
to_date(ri.info->'svedFL'-&gt;'svedGrazhdanRF'->'grazhdanRF'->>'dataReshVyh', 'YYYY-MM-DD') renunciation_date, -- дата решения о выходе из гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'organReshVyh' renunciation_authority, -- наименование органа, принявшего решение о выходе из гражданства РФ +
CASE
WHEN ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' is not null THEN 1
@ -397,6 +399,10 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and
<column_name>renunciation_authority</column_name>
<stream_name>renunciation_authority</stream_name>
</field>
<field>
<column_name>source_update_date</column_name>
<stream_name>source_update_date</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
@ -434,19 +440,16 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>acquired_citizenship</field>
<name>acquired_citizenship</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

18
mappings/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow_delta.hpl
View file

@ -328,19 +328,16 @@ and recruitment_id = '${IDM_ID}';</sql>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>acquiring_number</field>
<name>acquiring_number</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>renunciation_number</field>
<name>renunciation_number</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizenship</table>
@ -389,6 +386,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<rename>renunciation_authority</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -410,15 +412,17 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>select
ri.recruit_id,
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataSved', 'YYYY-MM-DD') AS source_update_date, -- дата обновления сведений
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'osnPriob' acquiring_basis, -- основание приобретения гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' acquiring_number, -- номер решения о приобретении гражданства РФ +
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshPriob', 'YYYY-MM-DD') acquiring_date, -- дата решения о приобретении гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'organReshPriob' acquiring_authority, -- наименование органа, принявшего решение о приобретении гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'osnVyh' renunciation_basis, -- основание выхода гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshVyh' renunciation_number, -- номер решения о выходе из гражданства РФ +
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshVyh', 'YYYY-MM-DD') renunciation_date, -- дата решения о выходе из гражданства РФ +
to_date(ri.info->'svedFL'-&gt;'svedGrazhdanRF'->'grazhdanRF'->>'dataReshVyh', 'YYYY-MM-DD') renunciation_date, -- дата решения о выходе из гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'organReshVyh' renunciation_authority, -- наименование органа, принявшего решение о выходе из гражданства РФ +
CASE
WHEN ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' is not null THEN 1
@ -427,7 +431,8 @@ and recruitment_id = '${IDM_ID}';</sql>
FROM ervu_dashboard.recruits_info ri
join ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
and to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'
--AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and
(ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' is not null or ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'osnVyh' is not null)</sql>
<variables_active>Y</variables_active>
@ -451,19 +456,16 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>acquired_citizenship</field>
<name>acquired_citizenship</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

12
mappings/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow_repeat.hpl
View file

@ -404,6 +404,11 @@ DO UPDATE SET
<rename>renunciation_authority</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -425,15 +430,17 @@ DO UPDATE SET
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>select
ri.recruit_id,
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataSved', 'YYYY-MM-DD') AS source_update_date, -- дата обновления сведений
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'osnPriob' acquiring_basis, -- основание приобретения гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' acquiring_number, -- номер решения о приобретении гражданства РФ +
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshPriob', 'YYYY-MM-DD') acquiring_date, -- дата решения о приобретении гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'organReshPriob' acquiring_authority, -- наименование органа, принявшего решение о приобретении гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'osnVyh' renunciation_basis, -- основание выхода гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshVyh' renunciation_number, -- номер решения о выходе из гражданства РФ +
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshVyh', 'YYYY-MM-DD') renunciation_date, -- дата решения о выходе из гражданства РФ +
to_date(ri.info->'svedFL'-&gt;'svedGrazhdanRF'->'grazhdanRF'->>'dataReshVyh', 'YYYY-MM-DD') renunciation_date, -- дата решения о выходе из гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'organReshVyh' renunciation_authority, -- наименование органа, принявшего решение о выходе из гражданства РФ +
CASE
WHEN ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' is not null THEN 1
@ -466,19 +473,16 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>acquired_citizenship</field>
<name>acquired_citizenship</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

59
mappings/info_recruits/citizen_tables/citizenship/recruitment_five_flow_delta.hpl
View file

@ -45,6 +45,11 @@
<to>citizenship_flow_delta.hpl 5</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>get_max_source_update_date</from>
<to>Table input</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Table input</name>
@ -60,13 +65,22 @@
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT
r.idm_id as recruitment_id
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = 'citizenship_job'
where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
@ -95,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -146,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -197,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -248,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -299,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -329,6 +343,29 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>get_max_source_update_date</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT max(source_update_date) AS max_source_update_date
FROM ervu_dashboard.citizenship;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>144</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>

10
mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow.hpl
View file

@ -325,6 +325,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH base AS (
SELECT
ri.recruit_id,
@ -354,6 +355,7 @@ citizen AS (
)
SELECT
recruit_id,
to_date(data->>'dataSved', 'YYYY-MM-DD') AS source_update_date, -- дата обновления сведений
CASE
WHEN type='residense' THEN data->>'osnovPriobrPP'
END::text AS residense_right_basis,
@ -426,6 +428,10 @@ FROM citizen</sql>
<column_name>type</column_name>
<stream_name>type</stream_name>
</field>
<field>
<column_name>source_update_date</column_name>
<stream_name>source_update_date</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
@ -463,25 +469,21 @@ FROM citizen</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_foreign_residence</field>
<name>has_foreign_residence</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_foreign_citizenship</field>
<name>has_foreign_citizenship</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

42
mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl
View file

@ -333,13 +333,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>type</field>
<name>type</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizenship_foreign</table>
@ -378,6 +376,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<rename>type</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -399,35 +402,44 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH base AS (
SELECT
ri.recruit_id,
ri.info
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
WHERE '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
WHERE jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedIG') = 'array'
OR jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedPP') = 'array'
AND (
jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedIG') = 'array'
OR jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedPP') = 'array'
)
),
citizen AS (
filtered_data AS (
SELECT
ri.recruit_id,
jsonb_array_elements(ri.info->'svedFL'->'svedIGPP'->'svedIG') AS data,
recruit_id,
elem AS data,
'citizenship' AS type
FROM base ri
FROM base ri,
LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedIGPP'->'svedIG') AS elem
WHERE jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedIG') = 'array'
AND to_date(elem->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date
UNION ALL
SELECT
ri.recruit_id,
jsonb_array_elements(ri.info->'svedFL'->'svedIGPP'->'svedPP') AS data,
recruit_id,
elem AS data,
'residense' AS type
FROM base ri WHERE jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedPP') = 'array'
FROM base ri,
LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedIGPP'->'svedPP') AS elem
WHERE jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedPP') = 'array'
AND to_date(elem->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date
)
SELECT
recruit_id,
to_date(data->>'dataSved', 'YYYY-MM-DD') AS source_update_date, -- дата обновления сведений
CASE
WHEN type='residense' THEN data->>'osnovPriobrPP'
END::text AS residense_right_basis,
@ -472,25 +484,21 @@ FROM citizen</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_foreign_residence</field>
<name>has_foreign_residence</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_foreign_citizenship</field>
<name>has_foreign_citizenship</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

11
mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_repeat.hpl
View file

@ -393,6 +393,11 @@ DO UPDATE SET
<rename>type</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -414,6 +419,7 @@ DO UPDATE SET
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH base AS (
SELECT
ri.recruit_id,
@ -443,6 +449,7 @@ citizen AS (
)
SELECT
recruit_id,
to_date(data->>'dataSved', 'YYYY-MM-DD') AS source_update_date, -- дата обновления сведений
CASE
WHEN type='residense' THEN data->>'osnovPriobrPP'
END::text AS residense_right_basis,
@ -487,25 +494,21 @@ FROM citizen</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_foreign_residence</field>
<name>has_foreign_residence</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_foreign_citizenship</field>
<name>has_foreign_citizenship</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

61
mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_delta.hpl
View file

@ -45,6 +45,11 @@
<to>citizenship_foreign_flow_delta.hpl 5</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>get_max_source_update_date</from>
<to>Table input</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Table input</name>
@ -60,14 +65,23 @@
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT
r.idm_id as recruitment_id
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = 'citizenship_foreign_job'
where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
<variables_active>N</variables_active>
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>352</xloc>
@ -95,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -146,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -197,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -248,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -299,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -329,6 +343,29 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>get_max_source_update_date</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT max(source_update_date) AS max_source_update_date
FROM ervu_dashboard.citizenship_foreign;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>160</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>

16
mappings/info_recruits/citizen_tables/drivers_licence/parallel/drivers_licence_flow.hpl
View file

@ -325,6 +325,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH
base AS (
SELECT
@ -341,6 +342,7 @@ base AS (
all_data AS (
SELECT
ri.recruit_id,
to_date(ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->>'kodStatus' AS status,-- статус удостоверения // 1 - действителен, 2 - недействителен
ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->>'serNomDok' AS licence_series, -- серия и номер
NULL AS licence_number, -- ?серия и номер в одном поле?
@ -360,7 +362,8 @@ all_data AS (
SELECT
ri.recruit_id,
'1' AS status, -- статус удостоверения тракториста // ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'kodStatusUTM'
to_date(ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
'1' AS status, -- статус удостоверения тракториста // ri.info-&gt;'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'kodStatusUTM'
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'serUTM' AS licence_series, -- серия удостоверения тракториста
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'nomUTM' AS licence_number, -- номер удостоверения тракториста
(SELECT array_to_string(ARRAY(SELECT jsonb_array_elements_text(
@ -377,7 +380,8 @@ all_data AS (
SELECT
ri.recruit_id,
'0' AS status, -- статус удостоверения тракториста // ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'kodStatusUTM'
to_date(ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
'0' AS status, -- статус удостоверения тракториста // ri.info->'svedFL'->'svedUdostTraktMash'-&gt;'udostTraktMash'->'predUTM'->>'kodStatusUTM'
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'serUTM' AS licence_series, -- серия удостоверения тракториста
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'nomUTM' AS licence_number, -- номер удостоверения тракториста
(SELECT array_to_string(ARRAY(SELECT jsonb_array_elements_text(
@ -462,6 +466,10 @@ FROM all_data</sql>
<column_name>tractor_driver</column_name>
<stream_name>tractor_driver</stream_name>
</field>
<field>
<column_name>source_update_date</column_name>
<stream_name>source_update_date</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
@ -499,25 +507,21 @@ FROM all_data</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>tractor_license</field>
<name>tractor_license</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_driver_license</field>
<name>has_driver_license</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

37
mappings/info_recruits/citizen_tables/drivers_licence/parallel/drivers_licence_flow_delta.hpl
View file

@ -333,19 +333,16 @@ and recruitment_id = '${IDM_ID}';</sql>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>tractor_driver</field>
<name>tractor_driver</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>driver</field>
<name>driver</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>drivers_licence</table>
@ -399,6 +396,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<rename>tractor_driver</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -420,15 +422,28 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH
base AS (
SELECT
ri.recruit_id recruit_id,
ri.info
ri.info,
CASE
WHEN jsonb_typeof(ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->'svedKat') = 'array'
THEN to_date(ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->>'dataSved', 'YYYY-MM-DD')
END as driver_update_date,
CASE
WHEN ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM' IS NOT NULL
THEN to_date(ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'dataSved', 'YYYY-MM-DD')
END as akt_update_date,
CASE
WHEN ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM' IS NOT NULL
THEN to_date(ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'dataSved', 'YYYY-MM-DD')
END as pred_update_date
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
--AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
WHERE jsonb_typeof(ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->'svedKat') = 'array'
OR ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM' IS NOT NULL
OR ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM' IS NOT NULL
@ -436,6 +451,7 @@ base AS (
all_data AS (
SELECT
ri.recruit_id,
ri.driver_update_date AS source_update_date,
ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->>'kodStatus' AS status,-- статус удостоверения // 1 - действителен, 2 - недействителен
ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->>'serNomDok' AS licence_series, -- серия и номер
NULL AS licence_number, -- ?серия и номер в одном поле?
@ -449,12 +465,14 @@ all_data AS (
NULL AS return_date,
false AS tractor_driver
FROM base ri
WHERE jsonb_typeof(ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->'svedKat') = 'array'
WHERE jsonb_typeof(ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->'svedKat') = 'array'
AND ri.driver_update_date > '${MAX_SOURCE_UPDATE_DATE}'::date
UNION ALL
SELECT
ri.recruit_id,
ri.akt_update_date AS source_update_date,
'1' AS status, -- статус удостоверения тракториста // ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'kodStatusUTM'
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'serUTM' AS licence_series, -- серия удостоверения тракториста
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'nomUTM' AS licence_number, -- номер удостоверения тракториста
@ -467,11 +485,13 @@ all_data AS (
true AS tractor_driver
FROM base ri
WHERE ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM' IS NOT NULL
AND ri.akt_update_date > '${MAX_SOURCE_UPDATE_DATE}'::date
UNION ALL
SELECT
ri.recruit_id,
ri.pred_update_date AS source_update_date,
'0' AS status, -- статус удостоверения тракториста // ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'kodStatusUTM'
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'serUTM' AS licence_series, -- серия удостоверения тракториста
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'nomUTM' AS licence_number, -- номер удостоверения тракториста
@ -484,6 +504,7 @@ all_data AS (
true AS tractor_driver
FROM base ri
WHERE ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM' IS NOT NULL
AND ri.pred_update_date > '${MAX_SOURCE_UPDATE_DATE}'::date
)
SELECT
*,
@ -517,25 +538,21 @@ FROM all_data</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>tractor_license</field>
<name>tractor_license</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_driver_license</field>
<name>has_driver_license</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

17
mappings/info_recruits/citizen_tables/drivers_licence/parallel/drivers_licence_flow_repeat.hpl
View file

@ -414,6 +414,11 @@ DO UPDATE SET
<rename>tractor_driver</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -435,6 +440,7 @@ DO UPDATE SET
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH
base AS (
SELECT
@ -451,6 +457,7 @@ base AS (
all_data AS (
SELECT
ri.recruit_id,
to_date(ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->>'kodStatus' AS status,-- статус удостоверения // 1 - действителен, 2 - недействителен
ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->>'serNomDok' AS licence_series, -- серия и номер
NULL AS licence_number, -- ?серия и номер в одном поле?
@ -470,7 +477,8 @@ all_data AS (
SELECT
ri.recruit_id,
'1' AS status, -- статус удостоверения тракториста // ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'kodStatusUTM'
to_date(ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
'1' AS status, -- статус удостоверения тракториста // ri.info-&gt;'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'kodStatusUTM'
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'serUTM' AS licence_series, -- серия удостоверения тракториста
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'nomUTM' AS licence_number, -- номер удостоверения тракториста
(SELECT array_to_string(ARRAY(SELECT jsonb_array_elements_text(
@ -487,7 +495,8 @@ all_data AS (
SELECT
ri.recruit_id,
'0' AS status, -- статус удостоверения тракториста // ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'kodStatusUTM'
to_date(ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
'0' AS status, -- статус удостоверения тракториста // ri.info->'svedFL'->'svedUdostTraktMash'-&gt;'udostTraktMash'->'predUTM'->>'kodStatusUTM'
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'serUTM' AS licence_series, -- серия удостоверения тракториста
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'nomUTM' AS licence_number, -- номер удостоверения тракториста
(SELECT array_to_string(ARRAY(SELECT jsonb_array_elements_text(
@ -532,25 +541,21 @@ FROM all_data</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>tractor_license</field>
<name>tractor_license</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_driver_license</field>
<name>has_driver_license</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

61
mappings/info_recruits/citizen_tables/drivers_licence/recruitment_five_flow_on_error.hpl
View file

@ -45,6 +45,11 @@
<to>drivers_licence_flow_repeat.hpl 5</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>get_max_source_update_date</from>
<to>Table input</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Table input</name>
@ -60,14 +65,23 @@
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT
r.idm_id as recruitment_id
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = 'drivers_licence_job'
where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<variables_active>N</variables_active>
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>272</xloc>
@ -95,7 +109,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -146,7 +160,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -197,7 +211,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -248,7 +262,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -299,7 +313,7 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -329,6 +343,29 @@ where je.status is null or je.status in('ERROR', 'PROCESSING');</sql>
<yloc>528</yloc>
</GUI>
</transform>
<transform>
<name>get_max_source_update_date</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT max(source_update_date) AS max_source_update_date
FROM ervu_dashboard.drivers_licence;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>64</xloc>
<yloc>368</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>

38
mappings/info_recruits/citizen_tables/property/parallel/property_flow.hpl
View file

@ -340,6 +340,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH
base as(
SELECT
@ -354,7 +355,7 @@ prop AS (
SELECT
base.recruit_id,
'realty' AS property_type,
ry.value AS property_data
property_data.value AS property_data
FROM base
LEFT JOIN LATERAL jsonb_array_elements(
CASE
@ -362,7 +363,7 @@ prop AS (
THEN base.info->'svedFL'->'svedON'->'on'
ELSE '[]'::jsonb
END
) AS ry ON true
) AS property_data ON true
WHERE jsonb_typeof(base.info->'svedFL'->'svedON'->'on') = 'array'
UNION ALL
@ -417,9 +418,10 @@ all_data as(
-- Недвижимость
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
property_data->>'naimVidPrav' AS kind_right, -- Исправлено название поля
property_data->'adrObSob'->>'adrObSobTekst' AS address,
property_data->>'kodVidPrav' AS vehicle_type,
property_data->>'kodVidPrav' AS property_code,
NULL AS issue_year,
NULL AS brand_model,
NULL AS reg_plate,
@ -438,9 +440,10 @@ all_data as(
-- Наземный транспорт
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
NULL AS kind_right,
NULL AS address,
property_data->>'vidTS' AS vehicle_type,
property_data->>'vidTS' AS property_code,
property_data->>'godVyp' AS issue_year,
property_data->>'markaTS' AS brand_model,
property_data->>'gosRegZn' AS reg_plate,
@ -459,9 +462,10 @@ all_data as(
-- Воздушный транспорт
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
NULL AS kind_right,
NULL AS address,
property_data->>'naimVid' AS vehicle_type,
property_data->>'naimVid' AS property_code,
NULL AS issue_year,
NULL AS brand_model,
NULL AS reg_plate,
@ -480,9 +484,10 @@ all_data as(
-- Водный транспорт
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
NULL AS kind_right,
NULL AS address,
property_data->>'naimVid' AS vehicle_type,
property_data->>'naimVid' AS property_code,
NULL AS issue_year,
NULL AS brand_model,
NULL AS reg_plate,
@ -498,9 +503,10 @@ all_data as(
)
select
recruit_id,
source_update_date,
kind_right,
address,
vehicle_type,
property_code,
issue_year::int,
brand_model,
reg_plate,
@ -561,8 +567,8 @@ from all_data</sql>
<stream_name>address</stream_name>
</field>
<field>
<column_name>vehicle_type</column_name>
<stream_name>vehicle_type</stream_name>
<column_name>property_code</column_name>
<stream_name>property_code</stream_name>
</field>
<field>
<column_name>issue_year</column_name>
@ -596,19 +602,20 @@ from all_data</sql>
<column_name>type_property</column_name>
<stream_name>type_property</stream_name>
</field>
<field>
<column_name>source_update_date</column_name>
<stream_name>source_update_date</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>property</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -633,43 +640,36 @@ from all_data</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_property</field>
<name>has_property</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_ground_transport</field>
<name>has_ground_transport</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_air_transport</field>
<name>has_air_transport</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_water_transport</field>
<name>has_water_transport</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_possibility_reg_residence</field>
<name>has_possibility_reg_residence</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

85
mappings/info_recruits/citizen_tables/property/parallel/property_flow_delta.hpl
View file

@ -348,19 +348,16 @@ and recruitment_id = '${IDM_ID}';</sql>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>start_date</field>
<name>start_date</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>type_property</field>
<name>type_property</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>property</table>
@ -380,8 +377,8 @@ and recruitment_id = '${IDM_ID}';</sql>
<update>Y</update>
</value>
<value>
<name>vehicle_type</name>
<rename>vehicle_type</rename>
<name>property_code</name>
<rename>property_code</rename>
<update>Y</update>
</value>
<value>
@ -424,6 +421,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<rename>type_property</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -445,22 +447,21 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH
base as(
SELECT
ri.recruit_id,
ri.info
FROM ervu_dashboard.recruits_info ri
join ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
--AND '${M_R_CR_DATE}'::timestamp >= ri.created_at
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
),
prop AS (
SELECT
base.recruit_id,
'realty' AS property_type,
ry.value AS property_data
property_data.value AS property_data
FROM base
LEFT JOIN LATERAL jsonb_array_elements(
CASE
@ -468,8 +469,9 @@ prop AS (
THEN base.info->'svedFL'->'svedON'->'on'
ELSE '[]'::jsonb
END
) AS ry ON true
) AS property_data ON true
WHERE jsonb_typeof(base.info->'svedFL'->'svedON'->'on') = 'array'
AND to_date(property_data->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date
UNION ALL
@ -486,6 +488,7 @@ prop AS (
END
) AS property_data ON true
WHERE jsonb_typeof(base.info->'svedFL'->'svedNazTS'->'nazTS') = 'array'
AND to_date(property_data->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date
UNION ALL
@ -502,6 +505,7 @@ prop AS (
END
) AS property_data ON true
WHERE jsonb_typeof(base.info->'svedFL'->'svedVozTS'->'vozTS') = 'array'
AND to_date(property_data->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date
UNION ALL
@ -518,14 +522,16 @@ prop AS (
END
) AS property_data ON true
WHERE jsonb_typeof(base.info->'svedFL'->'svedVodTS'->'vodTS') = 'array'
AND to_date(property_data->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date
),
all_data as(
-- Недвижимость
SELECT
base.recruit_id,
property_data->>'naimVidPrav' AS kind_right, -- Исправлено название поля
prop.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
property_data->>'naimVidPrav' AS kind_right,
property_data->'adrObSob'->>'adrObSobTekst' AS address,
property_data->>'kodVidPrav' AS vehicle_type,
property_data->>'kodVidPrav' AS property_code,
NULL AS issue_year,
NULL AS brand_model,
NULL AS reg_plate,
@ -535,18 +541,18 @@ all_data as(
to_date(property_data->>'dataPrekPrav', 'YYYY-MM-DD') AS close_date,
'realty' AS type_property,
1 AS is_possibility_reg_residence
FROM base
JOIN prop ON base.recruit_id = prop.recruit_id
FROM prop
WHERE prop.property_type = 'realty'
UNION ALL
-- Наземный транспорт
SELECT
base.recruit_id,
prop.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
NULL AS kind_right,
NULL AS address,
property_data->>'vidTS' AS vehicle_type,
property_data->>'vidTS' AS property_code,
property_data->>'godVyp' AS issue_year,
property_data->>'markaTS' AS brand_model,
property_data->>'gosRegZn' AS reg_plate,
@ -556,18 +562,18 @@ all_data as(
to_date(property_data->>'dataPrekPr', 'YYYY-MM-DD') AS close_date,
'ground_transportation' AS type_property,
0 AS is_possibility_reg_residence
FROM base
JOIN prop ON base.recruit_id = prop.recruit_id
FROM prop
WHERE prop.property_type = 'ground_transportation'
UNION ALL
-- Воздушный транспорт
SELECT
base.recruit_id,
prop.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
NULL AS kind_right,
NULL AS address,
property_data->>'naimVid' AS vehicle_type,
property_data->>'naimVid' AS property_code,
NULL AS issue_year,
NULL AS brand_model,
NULL AS reg_plate,
@ -577,18 +583,18 @@ all_data as(
to_date(property_data->>'dataPrekPr', 'YYYY-MM-DD') AS close_date,
'air_vehicles' AS type_property,
0 AS is_possibility_reg_residence
FROM base
JOIN prop ON base.recruit_id = prop.recruit_id
FROM prop
WHERE prop.property_type = 'air_vehicles'
UNION ALL
-- Водный транспорт
SELECT
base.recruit_id,
prop.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
NULL AS kind_right,
NULL AS address,
property_data->>'naimVid' AS vehicle_type,
property_data->>'naimVid' AS property_code,
NULL AS issue_year,
NULL AS brand_model,
NULL AS reg_plate,
@ -598,15 +604,15 @@ all_data as(
to_date(property_data->>'dataPrekPr', 'YYYY-MM-DD') AS close_date,
'water_vehicles' AS type_property,
0 AS is_possibility_reg_residence
FROM base
JOIN prop ON base.recruit_id = prop.recruit_id
FROM prop
WHERE prop.property_type = 'water_vehicles'
)
select
SELECT
recruit_id,
source_update_date,
kind_right,
address,
vehicle_type,
property_code,
issue_year::int,
brand_model,
reg_plate,
@ -619,20 +625,20 @@ select
CASE
WHEN type_property = 'realty' THEN 1
ELSE 0
END AS is_property, -- недвиж//has_property
END AS is_property,
CASE
WHEN type_property = 'ground_transportation' THEN 1
ELSE 0
END AS is_ground_transport, -- наземный транспорт//has_ground_transport
END AS is_ground_transport,
CASE
WHEN type_property = 'air_vehicles' THEN 1
ELSE 0
END AS is_air_transport, -- воздушный транспорт//has_air_transport
END AS is_air_transport,
CASE
WHEN type_property = 'water_vehicles' THEN 1
ELSE 0
END AS is_water_transport -- водный транспорт//has_water_transport
from all_data</sql>
END AS is_water_transport
FROM all_data</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -654,43 +660,36 @@ from all_data</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_property</field>
<name>has_property</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_ground_transport</field>
<name>has_ground_transport</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_air_transport</field>
<name>has_air_transport</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_water_transport</field>
<name>has_water_transport</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_possibility_reg_residence</field>
<name>has_possibility_reg_residence</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

41
mappings/info_recruits/citizen_tables/property/parallel/property_flow_repeat.hpl
View file

@ -363,19 +363,16 @@ DO UPDATE SET
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>start_date</field>
<name>start_date</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>type_property</field>
<name>type_property</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>property</table>
@ -395,8 +392,8 @@ DO UPDATE SET
<update>Y</update>
</value>
<value>
<name>vehicle_type</name>
<rename>vehicle_type</rename>
<name>property_code</name>
<rename>property_code</rename>
<update>Y</update>
</value>
<value>
@ -439,6 +436,11 @@ DO UPDATE SET
<rename>type_property</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -460,6 +462,7 @@ DO UPDATE SET
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH
base as(
SELECT
@ -474,7 +477,7 @@ prop AS (
SELECT
base.recruit_id,
'realty' AS property_type,
ry.value AS property_data
property_data.value AS property_data
FROM base
LEFT JOIN LATERAL jsonb_array_elements(
CASE
@ -482,7 +485,7 @@ prop AS (
THEN base.info->'svedFL'->'svedON'->'on'
ELSE '[]'::jsonb
END
) AS ry ON true
) AS property_data ON true
WHERE jsonb_typeof(base.info->'svedFL'->'svedON'->'on') = 'array'
UNION ALL
@ -537,9 +540,10 @@ all_data as(
-- Недвижимость
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
property_data->>'naimVidPrav' AS kind_right, -- Исправлено название поля
property_data->'adrObSob'->>'adrObSobTekst' AS address,
property_data->>'kodVidPrav' AS vehicle_type,
property_data->>'kodVidPrav' AS property_code,
NULL AS issue_year,
NULL AS brand_model,
NULL AS reg_plate,
@ -558,9 +562,10 @@ all_data as(
-- Наземный транспорт
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
NULL AS kind_right,
NULL AS address,
property_data->>'vidTS' AS vehicle_type,
property_data->>'vidTS' AS property_code,
property_data->>'godVyp' AS issue_year,
property_data->>'markaTS' AS brand_model,
property_data->>'gosRegZn' AS reg_plate,
@ -579,9 +584,10 @@ all_data as(
-- Воздушный транспорт
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
NULL AS kind_right,
NULL AS address,
property_data->>'naimVid' AS vehicle_type,
property_data->>'naimVid' AS property_code,
NULL AS issue_year,
NULL AS brand_model,
NULL AS reg_plate,
@ -600,9 +606,10 @@ all_data as(
-- Водный транспорт
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
NULL AS kind_right,
NULL AS address,
property_data->>'naimVid' AS vehicle_type,
property_data->>'naimVid' AS property_code,
NULL AS issue_year,
NULL AS brand_model,
NULL AS reg_plate,
@ -618,9 +625,10 @@ all_data as(
)
select
recruit_id,
source_update_date,
kind_right,
address,
vehicle_type,
property_code,
issue_year::int,
brand_model,
reg_plate,
@ -635,7 +643,7 @@ select
ELSE 0
END AS is_property, -- недвиж//has_property
CASE
WHEN type_property = 'ground_transportation' THEN 1
WHEN type_property = 'ground_transportation'THEN 1
ELSE 0
END AS is_ground_transport, -- наземный транспорт//has_ground_transport
CASE
@ -668,43 +676,36 @@ from all_data</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_property</field>
<name>has_property</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_ground_transport</field>
<name>has_ground_transport</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_air_transport</field>
<name>has_air_transport</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_water_transport</field>
<name>has_water_transport</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_possibility_reg_residence</field>
<name>has_possibility_reg_residence</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

51
mappings/info_recruits/citizen_tables/property/recruitment_five_flow_delta.hpl
View file

@ -45,6 +45,11 @@
<to>property_flow_delta.hpl 5</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>get_max_source_update_date</from>
<to>Table input</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Table input</name>
@ -60,14 +65,23 @@
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT
r.idm_id as recruitment_id
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = 'property_job'
where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
<variables_active>N</variables_active>
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>352</xloc>
@ -329,6 +343,29 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>get_max_source_update_date</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT max(source_update_date) AS max_source_update_date
FROM ervu_dashboard.property;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>144</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>

15
mappings/info_recruits/citizen_tables/snils_inn/parallel/snils_inn_flow.hpl
View file

@ -300,6 +300,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -351,6 +356,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -372,6 +382,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH
base as (
SELECT
@ -384,6 +395,7 @@ base as (
)
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
null as inn,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'snils', '[^0-9]', '', 'g') AS snils,
true AS actual
@ -394,6 +406,7 @@ UNION ALL
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
null as inn,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'snils', '[^0-9]', '', 'g') AS snils,
false AS actual
@ -404,6 +417,7 @@ UNION ALL
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'inn', '[^0-9]', '', 'g') AS inn,
null as snils,
true as actual
@ -414,6 +428,7 @@ UNION ALL
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'inn', '[^0-9]', '', 'g') AS inn,
null as snils,
false as actual

15
mappings/info_recruits/citizen_tables/snils_inn/parallel/snils_inn_flow_delta.hpl
View file

@ -303,6 +303,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -354,6 +359,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -375,6 +385,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH
base as (
SELECT
@ -388,6 +399,7 @@ base as (
)
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
null as inn,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'snils', '[^0-9]', '', 'g') AS snils,
true AS actual
@ -398,6 +410,7 @@ UNION ALL
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
null as inn,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'snils', '[^0-9]', '', 'g') AS snils,
false AS actual
@ -408,6 +421,7 @@ UNION ALL
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'inn', '[^0-9]', '', 'g') AS inn,
null as snils,
true as actual
@ -418,6 +432,7 @@ UNION ALL
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'inn', '[^0-9]', '', 'g') AS inn,
null as snils,
false as actual

15
mappings/info_recruits/citizen_tables/snils_inn/parallel/snils_inn_flow_repeat.hpl
View file

@ -318,6 +318,11 @@ DO UPDATE SET
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -369,6 +374,11 @@ DO UPDATE SET
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -390,6 +400,7 @@ DO UPDATE SET
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH
base as (
SELECT
@ -402,6 +413,7 @@ base as (
)
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
null as inn,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'snils', '[^0-9]', '', 'g') AS snils,
true AS actual
@ -412,6 +424,7 @@ UNION ALL
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
null as inn,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'snils', '[^0-9]', '', 'g') AS snils,
false AS actual
@ -422,6 +435,7 @@ UNION ALL
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'inn', '[^0-9]', '', 'g') AS inn,
null as snils,
true as actual
@ -432,6 +446,7 @@ UNION ALL
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'inn', '[^0-9]', '', 'g') AS inn,
null as snils,
false as actual