micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix

Browse source
This commit is contained in:
Ruslan 2025-09-26 12:18:28 +05:00
parent c953ae5029
commit 8e30d7436b
18 changed files with 293 additions and 110 deletions
Download patch file Download diff file Expand all files Collapse all files

21
mappings/info_recruits/citizen_tables/border_crossing/border_crossing_job.hwf
View file

@ -228,7 +228,6 @@
</field>
</fields>
<file_variable_type>CURRENT_WORKFLOW</file_variable_type>
<filename/>
<replacevars>N</replacevars>
<parallel>N</parallel>
<xloc>224</xloc>
@ -380,6 +379,26 @@ M_R_CR_DATE
3001-01-01 00:00:00</note>
<width>117</width>
</notepad>
<notepad>
<backgroundcolorblue>251</backgroundcolorblue>
<backgroundcolorgreen>232</backgroundcolorgreen>
<backgroundcolorred>201</backgroundcolorred>
<bordercolorblue>90</bordercolorblue>
<bordercolorgreen>58</bordercolorgreen>
<bordercolorred>14</bordercolorred>
<fontbold>N</fontbold>
<fontcolorblue>90</fontcolorblue>
<fontcolorgreen>58</fontcolorgreen>
<fontcolorred>14</fontcolorred>
<fontitalic>N</fontitalic>
<fontname>Segoe UI</fontname>
<fontsize>9</fontsize>
<height>26</height>
<xloc>631</xloc>
<yloc>124</yloc>
<note>dataSved - не предусмотрена</note>
<width>171</width>
</notepad>
</notepads>
<attributes/>
</workflow>

11
mappings/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow.hpl
View file

@ -320,15 +320,17 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>select
ri.recruit_id,
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataSved', 'YYYY-MM-DD') AS source_update_date, -- дата обновления сведений
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'osnPriob' acquiring_basis, -- основание приобретения гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' acquiring_number, -- номер решения о приобретении гражданства РФ +
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshPriob', 'YYYY-MM-DD') acquiring_date, -- дата решения о приобретении гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'organReshPriob' acquiring_authority, -- наименование органа, принявшего решение о приобретении гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'osnVyh' renunciation_basis, -- основание выхода гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshVyh' renunciation_number, -- номер решения о выходе из гражданства РФ +
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshVyh', 'YYYY-MM-DD') renunciation_date, -- дата решения о выходе из гражданства РФ +
to_date(ri.info->'svedFL'-&gt;'svedGrazhdanRF'->'grazhdanRF'->>'dataReshVyh', 'YYYY-MM-DD') renunciation_date, -- дата решения о выходе из гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'organReshVyh' renunciation_authority, -- наименование органа, принявшего решение о выходе из гражданства РФ +
CASE
WHEN ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' is not null THEN 1
@ -397,6 +399,10 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and
<column_name>renunciation_authority</column_name>
<stream_name>renunciation_authority</stream_name>
</field>
<field>
<column_name>source_update_date</column_name>
<stream_name>source_update_date</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
@ -434,19 +440,16 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>acquired_citizenship</field>
<name>acquired_citizenship</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

18
mappings/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow_delta.hpl
View file

@ -328,19 +328,16 @@ and recruitment_id = '${IDM_ID}';</sql>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>acquiring_number</field>
<name>acquiring_number</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>renunciation_number</field>
<name>renunciation_number</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizenship</table>
@ -389,6 +386,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<rename>renunciation_authority</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -410,15 +412,17 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>select
ri.recruit_id,
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataSved', 'YYYY-MM-DD') AS source_update_date, -- дата обновления сведений
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'osnPriob' acquiring_basis, -- основание приобретения гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' acquiring_number, -- номер решения о приобретении гражданства РФ +
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshPriob', 'YYYY-MM-DD') acquiring_date, -- дата решения о приобретении гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'organReshPriob' acquiring_authority, -- наименование органа, принявшего решение о приобретении гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'osnVyh' renunciation_basis, -- основание выхода гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshVyh' renunciation_number, -- номер решения о выходе из гражданства РФ +
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshVyh', 'YYYY-MM-DD') renunciation_date, -- дата решения о выходе из гражданства РФ +
to_date(ri.info->'svedFL'-&gt;'svedGrazhdanRF'->'grazhdanRF'->>'dataReshVyh', 'YYYY-MM-DD') renunciation_date, -- дата решения о выходе из гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'organReshVyh' renunciation_authority, -- наименование органа, принявшего решение о выходе из гражданства РФ +
CASE
WHEN ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' is not null THEN 1
@ -427,7 +431,8 @@ and recruitment_id = '${IDM_ID}';</sql>
FROM ervu_dashboard.recruits_info ri
join ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
and to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'
--AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and
(ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' is not null or ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'osnVyh' is not null)</sql>
<variables_active>Y</variables_active>
@ -451,19 +456,16 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>acquired_citizenship</field>
<name>acquired_citizenship</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

12
mappings/info_recruits/citizen_tables/citizenship/parallel/citizenship_flow_repeat.hpl
View file

@ -404,6 +404,11 @@ DO UPDATE SET
<rename>renunciation_authority</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -425,15 +430,17 @@ DO UPDATE SET
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>select
ri.recruit_id,
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataSved', 'YYYY-MM-DD') AS source_update_date, -- дата обновления сведений
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'osnPriob' acquiring_basis, -- основание приобретения гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' acquiring_number, -- номер решения о приобретении гражданства РФ +
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshPriob', 'YYYY-MM-DD') acquiring_date, -- дата решения о приобретении гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'organReshPriob' acquiring_authority, -- наименование органа, принявшего решение о приобретении гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'osnVyh' renunciation_basis, -- основание выхода гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshVyh' renunciation_number, -- номер решения о выходе из гражданства РФ +
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshVyh', 'YYYY-MM-DD') renunciation_date, -- дата решения о выходе из гражданства РФ +
to_date(ri.info->'svedFL'-&gt;'svedGrazhdanRF'->'grazhdanRF'->>'dataReshVyh', 'YYYY-MM-DD') renunciation_date, -- дата решения о выходе из гражданства РФ +
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'organReshVyh' renunciation_authority, -- наименование органа, принявшего решение о выходе из гражданства РФ +
CASE
WHEN ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' is not null THEN 1
@ -466,19 +473,16 @@ where --ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>acquired_citizenship</field>
<name>acquired_citizenship</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

59
mappings/info_recruits/citizen_tables/citizenship/recruitment_five_flow_delta.hpl
View file

@ -45,6 +45,11 @@
<to>citizenship_flow_delta.hpl 5</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>get_max_source_update_date</from>
<to>Table input</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Table input</name>
@ -60,13 +65,22 @@
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT
r.idm_id as recruitment_id
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = 'citizenship_job'
where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
@ -95,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -146,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -197,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -248,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -299,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -329,6 +343,29 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>get_max_source_update_date</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT max(source_update_date) AS max_source_update_date
FROM citizenship;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>144</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>

10
mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow.hpl
View file

@ -325,6 +325,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH base AS (
SELECT
ri.recruit_id,
@ -354,6 +355,7 @@ citizen AS (
)
SELECT
recruit_id,
to_date(data->>'dataSved', 'YYYY-MM-DD') AS source_update_date, -- дата обновления сведений
CASE
WHEN type='residense' THEN data->>'osnovPriobrPP'
END::text AS residense_right_basis,
@ -426,6 +428,10 @@ FROM citizen</sql>
<column_name>type</column_name>
<stream_name>type</stream_name>
</field>
<field>
<column_name>source_update_date</column_name>
<stream_name>source_update_date</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
@ -463,25 +469,21 @@ FROM citizen</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_foreign_residence</field>
<name>has_foreign_residence</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_foreign_citizenship</field>
<name>has_foreign_citizenship</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

42
mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_delta.hpl
View file

@ -333,13 +333,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>type</field>
<name>type</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizenship_foreign</table>
@ -378,6 +376,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<rename>type</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -399,35 +402,44 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH base AS (
SELECT
ri.recruit_id,
ri.info
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
WHERE '${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND ri.updated_at >= '${M_R_UP_DATE}'::timestamp
WHERE jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedIG') = 'array'
OR jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedPP') = 'array'
AND (
jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedIG') = 'array'
OR jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedPP') = 'array'
)
),
citizen AS (
filtered_data AS (
SELECT
ri.recruit_id,
jsonb_array_elements(ri.info->'svedFL'->'svedIGPP'->'svedIG') AS data,
recruit_id,
elem AS data,
'citizenship' AS type
FROM base ri
FROM base ri,
LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedIGPP'->'svedIG') AS elem
WHERE jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedIG') = 'array'
AND to_date(elem->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date
UNION ALL
SELECT
ri.recruit_id,
jsonb_array_elements(ri.info->'svedFL'->'svedIGPP'->'svedPP') AS data,
recruit_id,
elem AS data,
'residense' AS type
FROM base ri WHERE jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedPP') = 'array'
FROM base ri,
LATERAL jsonb_array_elements(ri.info->'svedFL'->'svedIGPP'->'svedPP') AS elem
WHERE jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedPP') = 'array'
AND to_date(elem->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date
)
SELECT
recruit_id,
to_date(data->>'dataSved', 'YYYY-MM-DD') AS source_update_date, -- дата обновления сведений
CASE
WHEN type='residense' THEN data->>'osnovPriobrPP'
END::text AS residense_right_basis,
@ -472,25 +484,21 @@ FROM citizen</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_foreign_residence</field>
<name>has_foreign_residence</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_foreign_citizenship</field>
<name>has_foreign_citizenship</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

11
mappings/info_recruits/citizen_tables/citizenship_foreign/parallel/citizenship_foreign_flow_repeat.hpl
View file

@ -393,6 +393,11 @@ DO UPDATE SET
<rename>type</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -414,6 +419,7 @@ DO UPDATE SET
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH base AS (
SELECT
ri.recruit_id,
@ -443,6 +449,7 @@ citizen AS (
)
SELECT
recruit_id,
to_date(data->>'dataSved', 'YYYY-MM-DD') AS source_update_date, -- дата обновления сведений
CASE
WHEN type='residense' THEN data->>'osnovPriobrPP'
END::text AS residense_right_basis,
@ -487,25 +494,21 @@ FROM citizen</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_foreign_residence</field>
<name>has_foreign_residence</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_foreign_citizenship</field>
<name>has_foreign_citizenship</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

59
mappings/info_recruits/citizen_tables/citizenship_foreign/recruitment_five_flow_delta.hpl
View file

@ -45,6 +45,11 @@
<to>citizenship_foreign_flow_delta.hpl 5</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>get_max_source_update_date</from>
<to>Table input</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Table input</name>
@ -60,13 +65,22 @@
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT
r.idm_id as recruitment_id
<sql>WITH mud AS (
SELECT
recruitment_id,
MAX(execution_datetime) AS max_upd_date
FROM etl.job_execution
WHERE job_name = '${JOB_NAME}'
AND status IN ('SUCCESS','DELTA_ERROR','DELTA_SUCCESS','DELTA_PROCESSING')
GROUP BY recruitment_id
)
SELECT
r.idm_id
FROM ervu_dashboard.recruitment r
LEFT JOIN etl.job_execution je
ON r.idm_id = je.recruitment_id
and job_name = 'citizenship_foreign_job'
where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING');</sql>
JOIN mud ON mud.recruitment_id = r.idm_id
JOIN recruits_info ri
ON COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = r.idm_id
AND ri.updated_at > mud.max_upd_date;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
@ -95,7 +109,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -146,7 +160,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -197,7 +211,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -248,7 +262,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -299,7 +313,7 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<parameters>
<variable_mapping>
<variable>IDM_ID</variable>
<field>recruitment_id</field>
<field>idm_id</field>
<input/>
</variable_mapping>
<inherit_all_vars>Y</inherit_all_vars>
@ -329,6 +343,29 @@ where je.status in ('SUCCESS', 'DELTA_ERROR', 'DELTA_SUCCESS', 'DELTA_PROCESSING
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>get_max_source_update_date</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT max(source_update_date) AS max_source_update_date
FROM citizenship_foreign;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>144</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>

16
mappings/info_recruits/citizen_tables/drivers_licence/parallel/drivers_licence_flow.hpl
View file

@ -325,6 +325,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH
base AS (
SELECT
@ -341,6 +342,7 @@ base AS (
all_data AS (
SELECT
ri.recruit_id,
to_date(ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->>'kodStatus' AS status,-- статус удостоверения // 1 - действителен, 2 - недействителен
ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->>'serNomDok' AS licence_series, -- серия и номер
NULL AS licence_number, -- ?серия и номер в одном поле?
@ -360,7 +362,8 @@ all_data AS (
SELECT
ri.recruit_id,
'1' AS status, -- статус удостоверения тракториста // ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'kodStatusUTM'
to_date(ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
'1' AS status, -- статус удостоверения тракториста // ri.info-&gt;'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'kodStatusUTM'
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'serUTM' AS licence_series, -- серия удостоверения тракториста
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'nomUTM' AS licence_number, -- номер удостоверения тракториста
(SELECT array_to_string(ARRAY(SELECT jsonb_array_elements_text(
@ -377,7 +380,8 @@ all_data AS (
SELECT
ri.recruit_id,
'0' AS status, -- статус удостоверения тракториста // ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'kodStatusUTM'
to_date(ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
'0' AS status, -- статус удостоверения тракториста // ri.info->'svedFL'->'svedUdostTraktMash'-&gt;'udostTraktMash'->'predUTM'->>'kodStatusUTM'
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'serUTM' AS licence_series, -- серия удостоверения тракториста
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'nomUTM' AS licence_number, -- номер удостоверения тракториста
(SELECT array_to_string(ARRAY(SELECT jsonb_array_elements_text(
@ -462,6 +466,10 @@ FROM all_data</sql>
<column_name>tractor_driver</column_name>
<stream_name>tractor_driver</stream_name>
</field>
<field>
<column_name>source_update_date</column_name>
<stream_name>source_update_date</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
@ -499,25 +507,21 @@ FROM all_data</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>tractor_license</field>
<name>tractor_license</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_driver_license</field>
<name>has_driver_license</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

17
mappings/info_recruits/citizen_tables/drivers_licence/parallel/drivers_licence_flow_delta.hpl
View file

@ -399,6 +399,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<rename>tractor_driver</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -420,6 +425,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH
base AS (
SELECT
@ -436,6 +442,7 @@ base AS (
all_data AS (
SELECT
ri.recruit_id,
to_date(ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->>'kodStatus' AS status,-- статус удостоверения // 1 - действителен, 2 - недействителен
ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->>'serNomDok' AS licence_series, -- серия и номер
NULL AS licence_number, -- ?серия и номер в одном поле?
@ -455,7 +462,8 @@ all_data AS (
SELECT
ri.recruit_id,
'1' AS status, -- статус удостоверения тракториста // ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'kodStatusUTM'
to_date(ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
'1' AS status, -- статус удостоверения тракториста // ri.info-&gt;'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'kodStatusUTM'
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'serUTM' AS licence_series, -- серия удостоверения тракториста
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'nomUTM' AS licence_number, -- номер удостоверения тракториста
(SELECT array_to_string(ARRAY(SELECT jsonb_array_elements_text(
@ -472,7 +480,8 @@ all_data AS (
SELECT
ri.recruit_id,
'0' AS status, -- статус удостоверения тракториста // ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'kodStatusUTM'
to_date(ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
'0' AS status, -- статус удостоверения тракториста // ri.info->'svedFL'->'svedUdostTraktMash'-&gt;'udostTraktMash'->'predUTM'->>'kodStatusUTM'
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'serUTM' AS licence_series, -- серия удостоверения тракториста
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'nomUTM' AS licence_number, -- номер удостоверения тракториста
(SELECT array_to_string(ARRAY(SELECT jsonb_array_elements_text(
@ -517,25 +526,21 @@ FROM all_data</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>tractor_license</field>
<name>tractor_license</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_driver_license</field>
<name>has_driver_license</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

17
mappings/info_recruits/citizen_tables/drivers_licence/parallel/drivers_licence_flow_repeat.hpl
View file

@ -414,6 +414,11 @@ DO UPDATE SET
<rename>tractor_driver</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -435,6 +440,7 @@ DO UPDATE SET
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH
base AS (
SELECT
@ -451,6 +457,7 @@ base AS (
all_data AS (
SELECT
ri.recruit_id,
to_date(ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->>'kodStatus' AS status,-- статус удостоверения // 1 - действителен, 2 - недействителен
ri.info->'svedFL'->'svedVoditUdost'->'voditUdost'->>'serNomDok' AS licence_series, -- серия и номер
NULL AS licence_number, -- ?серия и номер в одном поле?
@ -470,7 +477,8 @@ all_data AS (
SELECT
ri.recruit_id,
'1' AS status, -- статус удостоверения тракториста // ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'kodStatusUTM'
to_date(ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
'1' AS status, -- статус удостоверения тракториста // ri.info-&gt;'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'kodStatusUTM'
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'serUTM' AS licence_series, -- серия удостоверения тракториста
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'aktUTM'->>'nomUTM' AS licence_number, -- номер удостоверения тракториста
(SELECT array_to_string(ARRAY(SELECT jsonb_array_elements_text(
@ -487,7 +495,8 @@ all_data AS (
SELECT
ri.recruit_id,
'0' AS status, -- статус удостоверения тракториста // ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'kodStatusUTM'
to_date(ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
'0' AS status, -- статус удостоверения тракториста // ri.info->'svedFL'->'svedUdostTraktMash'-&gt;'udostTraktMash'->'predUTM'->>'kodStatusUTM'
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'serUTM' AS licence_series, -- серия удостоверения тракториста
ri.info->'svedFL'->'svedUdostTraktMash'->'udostTraktMash'->'predUTM'->>'nomUTM' AS licence_number, -- номер удостоверения тракториста
(SELECT array_to_string(ARRAY(SELECT jsonb_array_elements_text(
@ -532,25 +541,21 @@ FROM all_data</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>tractor_license</field>
<name>tractor_license</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_driver_license</field>
<name>has_driver_license</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

21
mappings/info_recruits/citizen_tables/property/parallel/property_flow.hpl
View file

@ -340,6 +340,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH
base as(
SELECT
@ -354,7 +355,7 @@ prop AS (
SELECT
base.recruit_id,
'realty' AS property_type,
ry.value AS property_data
property_data.value AS property_data
FROM base
LEFT JOIN LATERAL jsonb_array_elements(
CASE
@ -362,7 +363,7 @@ prop AS (
THEN base.info->'svedFL'->'svedON'->'on'
ELSE '[]'::jsonb
END
) AS ry ON true
) AS property_data ON true
WHERE jsonb_typeof(base.info->'svedFL'->'svedON'->'on') = 'array'
UNION ALL
@ -417,6 +418,7 @@ all_data as(
-- Недвижимость
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
property_data->>'naimVidPrav' AS kind_right, -- Исправлено название поля
property_data->'adrObSob'->>'adrObSobTekst' AS address,
property_data->>'kodVidPrav' AS vehicle_type,
@ -438,6 +440,7 @@ all_data as(
-- Наземный транспорт
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
NULL AS kind_right,
NULL AS address,
property_data->>'vidTS' AS vehicle_type,
@ -459,6 +462,7 @@ all_data as(
-- Воздушный транспорт
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
NULL AS kind_right,
NULL AS address,
property_data->>'naimVid' AS vehicle_type,
@ -480,6 +484,7 @@ all_data as(
-- Водный транспорт
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
NULL AS kind_right,
NULL AS address,
property_data->>'naimVid' AS vehicle_type,
@ -498,6 +503,7 @@ all_data as(
)
select
recruit_id,
source_update_date,
kind_right,
address,
vehicle_type,
@ -596,6 +602,10 @@ from all_data</sql>
<column_name>type_property</column_name>
<stream_name>type_property</stream_name>
</field>
<field>
<column_name>source_update_date</column_name>
<stream_name>source_update_date</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
@ -633,43 +643,36 @@ from all_data</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_property</field>
<name>has_property</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_ground_transport</field>
<name>has_ground_transport</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_air_transport</field>
<name>has_air_transport</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_water_transport</field>
<name>has_water_transport</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_possibility_reg_residence</field>
<name>has_possibility_reg_residence</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

21
mappings/info_recruits/citizen_tables/property/parallel/property_flow_delta.hpl
View file

@ -424,6 +424,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<rename>type_property</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -460,7 +465,7 @@ prop AS (
SELECT
base.recruit_id,
'realty' AS property_type,
ry.value AS property_data
property_data.value AS property_data
FROM base
LEFT JOIN LATERAL jsonb_array_elements(
CASE
@ -468,7 +473,7 @@ prop AS (
THEN base.info->'svedFL'->'svedON'->'on'
ELSE '[]'::jsonb
END
) AS ry ON true
) AS property_data ON true
WHERE jsonb_typeof(base.info->'svedFL'->'svedON'->'on') = 'array'
UNION ALL
@ -523,6 +528,7 @@ all_data as(
-- Недвижимость
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
property_data->>'naimVidPrav' AS kind_right, -- Исправлено название поля
property_data->'adrObSob'->>'adrObSobTekst' AS address,
property_data->>'kodVidPrav' AS vehicle_type,
@ -544,6 +550,7 @@ all_data as(
-- Наземный транспорт
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
NULL AS kind_right,
NULL AS address,
property_data->>'vidTS' AS vehicle_type,
@ -565,6 +572,7 @@ all_data as(
-- Воздушный транспорт
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
NULL AS kind_right,
NULL AS address,
property_data->>'naimVid' AS vehicle_type,
@ -586,6 +594,7 @@ all_data as(
-- Водный транспорт
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
NULL AS kind_right,
NULL AS address,
property_data->>'naimVid' AS vehicle_type,
@ -604,6 +613,7 @@ all_data as(
)
select
recruit_id,
source_update_date,
kind_right,
address,
vehicle_type,
@ -654,43 +664,36 @@ from all_data</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_property</field>
<name>has_property</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_ground_transport</field>
<name>has_ground_transport</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_air_transport</field>
<name>has_air_transport</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_water_transport</field>
<name>has_water_transport</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_possibility_reg_residence</field>
<name>has_possibility_reg_residence</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

23
mappings/info_recruits/citizen_tables/property/parallel/property_flow_repeat.hpl
View file

@ -439,6 +439,11 @@ DO UPDATE SET
<rename>type_property</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -474,7 +479,7 @@ prop AS (
SELECT
base.recruit_id,
'realty' AS property_type,
ry.value AS property_data
property_data.value AS property_data
FROM base
LEFT JOIN LATERAL jsonb_array_elements(
CASE
@ -482,7 +487,7 @@ prop AS (
THEN base.info->'svedFL'->'svedON'->'on'
ELSE '[]'::jsonb
END
) AS ry ON true
) AS property_data ON true
WHERE jsonb_typeof(base.info->'svedFL'->'svedON'->'on') = 'array'
UNION ALL
@ -537,6 +542,7 @@ all_data as(
-- Недвижимость
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
property_data->>'naimVidPrav' AS kind_right, -- Исправлено название поля
property_data->'adrObSob'->>'adrObSobTekst' AS address,
property_data->>'kodVidPrav' AS vehicle_type,
@ -558,6 +564,7 @@ all_data as(
-- Наземный транспорт
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
NULL AS kind_right,
NULL AS address,
property_data->>'vidTS' AS vehicle_type,
@ -579,6 +586,7 @@ all_data as(
-- Воздушный транспорт
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
NULL AS kind_right,
NULL AS address,
property_data->>'naimVid' AS vehicle_type,
@ -600,6 +608,7 @@ all_data as(
-- Водный транспорт
SELECT
base.recruit_id,
to_date(property_data->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
NULL AS kind_right,
NULL AS address,
property_data->>'naimVid' AS vehicle_type,
@ -618,6 +627,7 @@ all_data as(
)
select
recruit_id,
source_update_date,
kind_right,
address,
vehicle_type,
@ -635,7 +645,7 @@ select
ELSE 0
END AS is_property, -- недвиж//has_property
CASE
WHEN type_property = 'ground_transportation' THEN 1
WHEN type_property = 'ground_transportation'THEN 1
ELSE 0
END AS is_ground_transport, -- наземный транспорт//has_ground_transport
CASE
@ -668,43 +678,36 @@ from all_data</sql>
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_property</field>
<name>has_property</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_ground_transport</field>
<name>has_ground_transport</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_air_transport</field>
<name>has_air_transport</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_water_transport</field>
<name>has_water_transport</name>
<name2/>
</key>
<key>
<condition>&lt;></condition>
<field>has_possibility_reg_residence</field>
<name>has_possibility_reg_residence</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>

15
mappings/info_recruits/citizen_tables/snils_inn/parallel/snils_inn_flow.hpl
View file

@ -300,6 +300,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -351,6 +356,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -372,6 +382,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH
base as (
SELECT
@ -384,6 +395,7 @@ base as (
)
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
null as inn,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'snils', '[^0-9]', '', 'g') AS snils,
true AS actual
@ -394,6 +406,7 @@ UNION ALL
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
null as inn,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'snils', '[^0-9]', '', 'g') AS snils,
false AS actual
@ -404,6 +417,7 @@ UNION ALL
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'inn', '[^0-9]', '', 'g') AS inn,
null as snils,
true as actual
@ -414,6 +428,7 @@ UNION ALL
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'inn', '[^0-9]', '', 'g') AS inn,
null as snils,
false as actual

15
mappings/info_recruits/citizen_tables/snils_inn/parallel/snils_inn_flow_delta.hpl
View file

@ -303,6 +303,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -354,6 +359,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -375,6 +385,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH
base as (
SELECT
@ -388,6 +399,7 @@ base as (
)
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
null as inn,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'snils', '[^0-9]', '', 'g') AS snils,
true AS actual
@ -398,6 +410,7 @@ UNION ALL
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
null as inn,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'snils', '[^0-9]', '', 'g') AS snils,
false AS actual
@ -408,6 +421,7 @@ UNION ALL
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'inn', '[^0-9]', '', 'g') AS inn,
null as snils,
true as actual
@ -418,6 +432,7 @@ UNION ALL
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'inn', '[^0-9]', '', 'g') AS inn,
null as snils,
false as actual

15
mappings/info_recruits/citizen_tables/snils_inn/parallel/snils_inn_flow_repeat.hpl
View file

@ -318,6 +318,11 @@ DO UPDATE SET
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -369,6 +374,11 @@ DO UPDATE SET
<rename>actual</rename>
<update>Y</update>
</value>
<value>
<name>source_update_date</name>
<rename>source_update_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
@ -390,6 +400,7 @@ DO UPDATE SET
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH
base as (
SELECT
@ -402,6 +413,7 @@ base as (
)
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
null as inn,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'snils', '[^0-9]', '', 'g') AS snils,
true AS actual
@ -412,6 +424,7 @@ UNION ALL
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
null as inn,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'snils', '[^0-9]', '', 'g') AS snils,
false AS actual
@ -422,6 +435,7 @@ UNION ALL
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'inn', '[^0-9]', '', 'g') AS inn,
null as snils,
true as actual
@ -432,6 +446,7 @@ UNION ALL
SELECT
recruit_id,
to_date(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'inn', '[^0-9]', '', 'g') AS inn,
null as snils,
false as actual