micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ERVU-506: insert/update

Browse source
This commit is contained in:
adel.ka 2025-07-30 14:08:12 +03:00
parent bba694939d
commit ebe2ca2f27
8 changed files with 148 additions and 212 deletions
Download patch file Download diff file Expand all files Collapse all files

4
mappings_new/info_recruits/citizen_tables/citizen_address/checkpoints/state_job_citizen_address.json
View file

@ -1,4 +0,0 @@
{
"status": "ERROR",
"pipeline": "job_citizen_address"
}

60
mappings_new/info_recruits/citizen_tables/citizen_address/delta_citizen_address.hpl
View file

@ -23,7 +23,7 @@
<hop>
<from>Table input</from>
<to>Insert / update</to>
<enabled>N</enabled>
<enabled>Y</enabled>
</hop>
</order>
<transform>
@ -45,79 +45,69 @@
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<key>
<condition>=</condition>
<field>child_external_id</field>
<name>child_external_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen_child</table>
<table>citizen_address</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>birth_az_number</name>
<rename>birth_az_number</rename>
<name>factual_address</name>
<rename>factual_address</rename>
<update>Y</update>
</value>
<value>
<name>birth_date</name>
<rename>birth_date</rename>
<name>factual_info_expiry_date</name>
<rename>factual_info_expiry_date</rename>
<update>Y</update>
</value>
<value>
<name>death_az_number</name>
<rename>death_az_number</rename>
<name>factual_info_received_date</name>
<rename>factual_info_received_date</rename>
<update>Y</update>
</value>
<value>
<name>death_date</name>
<rename>death_date</rename>
<name>place_stay_address</name>
<rename>place_stay_address</rename>
<update>Y</update>
</value>
<value>
<name>full_name</name>
<rename>full_name</rename>
<name>place_stay_deregistration_date</name>
<rename>place_stay_deregistration_date</rename>
<update>Y</update>
</value>
<value>
<name>kinship_type</name>
<rename>kinship_type</rename>
<name>place_stay_registration_date</name>
<rename>place_stay_registration_date</rename>
<update>Y</update>
</value>
<value>
<name>child_external_id</name>
<rename>child_external_id</rename>
<update>N</update>
</value>
<value>
<name>first_name</name>
<rename>first_name</rename>
<name>place_stay_registration_type</name>
<rename>place_stay_registration_type</rename>
<update>Y</update>
</value>
<value>
<name>middle_name</name>
<rename>middle_name</rename>
<name>residence_address</name>
<rename>residence_address</rename>
<update>Y</update>
</value>
<value>
<name>last_name</name>
<rename>last_name</rename>
<name>residence_deregistration_date</name>
<rename>residence_deregistration_date</rename>
<update>Y</update>
</value>
<value>
<name>child_id_ern</name>
<rename>child_id_ern</rename>
<name>residence_registration_date</name>
<rename>residence_registration_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1168</xloc>
<yloc>352</yloc>
<xloc>1040</xloc>
<yloc>336</yloc>
</GUI>
</transform>
<transform>
@ -153,7 +143,7 @@
MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
LEFT JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr ON true
CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr
WHERE
--'${ID_F1}' != ''
-- AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${ID_F1}'

8
mappings_new/info_recruits/citizen_tables/citizen_address/job_citizen_address.hwf
View file

@ -1229,28 +1229,28 @@
<hop>
<from>recruitment_rows_five_flow_citizen_address.hpl</from>
<to>citizen_address_flow2.hpl</to>
<enabled>N</enabled>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>recruitment_rows_five_flow_citizen_address.hpl</from>
<to>citizen_address_flow3.hpl</to>
<enabled>N</enabled>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>recruitment_rows_five_flow_citizen_address.hpl</from>
<to>citizen_address_flow4.hpl</to>
<enabled>N</enabled>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>recruitment_rows_five_flow_citizen_address.hpl</from>
<to>citizen_address_flow5.hpl</to>
<enabled>N</enabled>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>

54
mappings_new/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow1.hpl
View file

@ -83,71 +83,61 @@
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<key>
<condition>=</condition>
<field>child_external_id</field>
<name>child_external_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen_child</table>
<table>citizen_address</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>birth_az_number</name>
<rename>birth_az_number</rename>
<name>factual_address</name>
<rename>factual_address</rename>
<update>Y</update>
</value>
<value>
<name>birth_date</name>
<rename>birth_date</rename>
<name>factual_info_expiry_date</name>
<rename>factual_info_expiry_date</rename>
<update>Y</update>
</value>
<value>
<name>death_az_number</name>
<rename>death_az_number</rename>
<name>factual_info_received_date</name>
<rename>factual_info_received_date</rename>
<update>Y</update>
</value>
<value>
<name>death_date</name>
<rename>death_date</rename>
<name>place_stay_address</name>
<rename>place_stay_address</rename>
<update>Y</update>
</value>
<value>
<name>full_name</name>
<rename>full_name</rename>
<name>place_stay_deregistration_date</name>
<rename>place_stay_deregistration_date</rename>
<update>Y</update>
</value>
<value>
<name>kinship_type</name>
<rename>kinship_type</rename>
<name>place_stay_registration_date</name>
<rename>place_stay_registration_date</rename>
<update>Y</update>
</value>
<value>
<name>child_external_id</name>
<rename>child_external_id</rename>
<update>N</update>
</value>
<value>
<name>first_name</name>
<rename>first_name</rename>
<name>place_stay_registration_type</name>
<rename>place_stay_registration_type</rename>
<update>Y</update>
</value>
<value>
<name>middle_name</name>
<rename>middle_name</rename>
<name>residence_address</name>
<rename>residence_address</rename>
<update>Y</update>
</value>
<value>
<name>last_name</name>
<rename>last_name</rename>
<name>residence_deregistration_date</name>
<rename>residence_deregistration_date</rename>
<update>Y</update>
</value>
<value>
<name>child_id_ern</name>
<rename>child_id_ern</rename>
<name>residence_registration_date</name>
<rename>residence_registration_date</rename>
<update>Y</update>
</value>
</lookup>
@ -228,7 +218,7 @@
MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
LEFT JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr ON true
CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr
WHERE
'${ID_F1}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${ID_F1}'

58
mappings_new/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow2.hpl
View file

@ -23,7 +23,7 @@
<hop>
<from>Table input</from>
<to>Insert / update</to>
<enabled>N</enabled>
<enabled>Y</enabled>
</hop>
</order>
<transform>
@ -45,78 +45,68 @@
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<key>
<condition>=</condition>
<field>child_external_id</field>
<name>child_external_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen_child</table>
<table>citizen_address</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>birth_az_number</name>
<rename>birth_az_number</rename>
<name>factual_address</name>
<rename>factual_address</rename>
<update>Y</update>
</value>
<value>
<name>birth_date</name>
<rename>birth_date</rename>
<name>factual_info_expiry_date</name>
<rename>factual_info_expiry_date</rename>
<update>Y</update>
</value>
<value>
<name>death_az_number</name>
<rename>death_az_number</rename>
<name>factual_info_received_date</name>
<rename>factual_info_received_date</rename>
<update>Y</update>
</value>
<value>
<name>death_date</name>
<rename>death_date</rename>
<name>place_stay_address</name>
<rename>place_stay_address</rename>
<update>Y</update>
</value>
<value>
<name>full_name</name>
<rename>full_name</rename>
<name>place_stay_deregistration_date</name>
<rename>place_stay_deregistration_date</rename>
<update>Y</update>
</value>
<value>
<name>kinship_type</name>
<rename>kinship_type</rename>
<name>place_stay_registration_date</name>
<rename>place_stay_registration_date</rename>
<update>Y</update>
</value>
<value>
<name>child_external_id</name>
<rename>child_external_id</rename>
<update>N</update>
</value>
<value>
<name>first_name</name>
<rename>first_name</rename>
<name>place_stay_registration_type</name>
<rename>place_stay_registration_type</rename>
<update>Y</update>
</value>
<value>
<name>middle_name</name>
<rename>middle_name</rename>
<name>residence_address</name>
<rename>residence_address</rename>
<update>Y</update>
</value>
<value>
<name>last_name</name>
<rename>last_name</rename>
<name>residence_deregistration_date</name>
<rename>residence_deregistration_date</rename>
<update>Y</update>
</value>
<value>
<name>child_id_ern</name>
<rename>child_id_ern</rename>
<name>residence_registration_date</name>
<rename>residence_registration_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1088</xloc>
<xloc>960</xloc>
<yloc>288</yloc>
</GUI>
</transform>
@ -153,7 +143,7 @@
MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
LEFT JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr ON true
CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr
WHERE
'${ID_F2}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${ID_F2}'

60
mappings_new/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow3.hpl
View file

@ -23,7 +23,7 @@
<hop>
<from>Table input</from>
<to>Insert / update</to>
<enabled>N</enabled>
<enabled>Y</enabled>
</hop>
</order>
<transform>
@ -45,79 +45,69 @@
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<key>
<condition>=</condition>
<field>child_external_id</field>
<name>child_external_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen_child</table>
<table>citizen_address</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>birth_az_number</name>
<rename>birth_az_number</rename>
<name>factual_address</name>
<rename>factual_address</rename>
<update>Y</update>
</value>
<value>
<name>birth_date</name>
<rename>birth_date</rename>
<name>factual_info_expiry_date</name>
<rename>factual_info_expiry_date</rename>
<update>Y</update>
</value>
<value>
<name>death_az_number</name>
<rename>death_az_number</rename>
<name>factual_info_received_date</name>
<rename>factual_info_received_date</rename>
<update>Y</update>
</value>
<value>
<name>death_date</name>
<rename>death_date</rename>
<name>place_stay_address</name>
<rename>place_stay_address</rename>
<update>Y</update>
</value>
<value>
<name>full_name</name>
<rename>full_name</rename>
<name>place_stay_deregistration_date</name>
<rename>place_stay_deregistration_date</rename>
<update>Y</update>
</value>
<value>
<name>kinship_type</name>
<rename>kinship_type</rename>
<name>place_stay_registration_date</name>
<rename>place_stay_registration_date</rename>
<update>Y</update>
</value>
<value>
<name>child_external_id</name>
<rename>child_external_id</rename>
<update>N</update>
</value>
<value>
<name>first_name</name>
<rename>first_name</rename>
<name>place_stay_registration_type</name>
<rename>place_stay_registration_type</rename>
<update>Y</update>
</value>
<value>
<name>middle_name</name>
<rename>middle_name</rename>
<name>residence_address</name>
<rename>residence_address</rename>
<update>Y</update>
</value>
<value>
<name>last_name</name>
<rename>last_name</rename>
<name>residence_deregistration_date</name>
<rename>residence_deregistration_date</rename>
<update>Y</update>
</value>
<value>
<name>child_id_ern</name>
<rename>child_id_ern</rename>
<name>residence_registration_date</name>
<rename>residence_registration_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1216</xloc>
<yloc>352</yloc>
<xloc>1104</xloc>
<yloc>368</yloc>
</GUI>
</transform>
<transform>
@ -153,7 +143,7 @@
MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
LEFT JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr ON true
CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr
WHERE
'${ID_F3}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${ID_F3}'

58
mappings_new/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow4.hpl
View file

@ -23,7 +23,7 @@
<hop>
<from>Table input</from>
<to>Insert / update</to>
<enabled>N</enabled>
<enabled>Y</enabled>
</hop>
</order>
<transform>
@ -45,78 +45,68 @@
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<key>
<condition>=</condition>
<field>child_external_id</field>
<name>child_external_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen_child</table>
<table>citizen_address</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>birth_az_number</name>
<rename>birth_az_number</rename>
<name>factual_address</name>
<rename>factual_address</rename>
<update>Y</update>
</value>
<value>
<name>birth_date</name>
<rename>birth_date</rename>
<name>factual_info_expiry_date</name>
<rename>factual_info_expiry_date</rename>
<update>Y</update>
</value>
<value>
<name>death_az_number</name>
<rename>death_az_number</rename>
<name>factual_info_received_date</name>
<rename>factual_info_received_date</rename>
<update>Y</update>
</value>
<value>
<name>death_date</name>
<rename>death_date</rename>
<name>place_stay_address</name>
<rename>place_stay_address</rename>
<update>Y</update>
</value>
<value>
<name>full_name</name>
<rename>full_name</rename>
<name>place_stay_deregistration_date</name>
<rename>place_stay_deregistration_date</rename>
<update>Y</update>
</value>
<value>
<name>kinship_type</name>
<rename>kinship_type</rename>
<name>place_stay_registration_date</name>
<rename>place_stay_registration_date</rename>
<update>Y</update>
</value>
<value>
<name>child_external_id</name>
<rename>child_external_id</rename>
<update>N</update>
</value>
<value>
<name>first_name</name>
<rename>first_name</rename>
<name>place_stay_registration_type</name>
<rename>place_stay_registration_type</rename>
<update>Y</update>
</value>
<value>
<name>middle_name</name>
<rename>middle_name</rename>
<name>residence_address</name>
<rename>residence_address</rename>
<update>Y</update>
</value>
<value>
<name>last_name</name>
<rename>last_name</rename>
<name>residence_deregistration_date</name>
<rename>residence_deregistration_date</rename>
<update>Y</update>
</value>
<value>
<name>child_id_ern</name>
<rename>child_id_ern</rename>
<name>residence_registration_date</name>
<rename>residence_registration_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1072</xloc>
<xloc>992</xloc>
<yloc>320</yloc>
</GUI>
</transform>
@ -153,7 +143,7 @@
MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
LEFT JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr ON true
CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr
WHERE
'${ID_F4}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${ID_F4}'

58
mappings_new/info_recruits/citizen_tables/citizen_address/parallel/citizen_address_flow5.hpl
View file

@ -23,7 +23,7 @@
<hop>
<from>Table input</from>
<to>Insert / update</to>
<enabled>N</enabled>
<enabled>Y</enabled>
</hop>
</order>
<transform>
@ -45,78 +45,68 @@
<field>recruit_id</field>
<name>recruit_id</name>
</key>
<key>
<condition>=</condition>
<field>child_external_id</field>
<name>child_external_id</name>
</key>
<schema>ervu_dashboard</schema>
<table>citizen_child</table>
<table>citizen_address</table>
<value>
<name>recruit_id</name>
<rename>recruit_id</rename>
<update>N</update>
</value>
<value>
<name>birth_az_number</name>
<rename>birth_az_number</rename>
<name>factual_address</name>
<rename>factual_address</rename>
<update>Y</update>
</value>
<value>
<name>birth_date</name>
<rename>birth_date</rename>
<name>factual_info_expiry_date</name>
<rename>factual_info_expiry_date</rename>
<update>Y</update>
</value>
<value>
<name>death_az_number</name>
<rename>death_az_number</rename>
<name>factual_info_received_date</name>
<rename>factual_info_received_date</rename>
<update>Y</update>
</value>
<value>
<name>death_date</name>
<rename>death_date</rename>
<name>place_stay_address</name>
<rename>place_stay_address</rename>
<update>Y</update>
</value>
<value>
<name>full_name</name>
<rename>full_name</rename>
<name>place_stay_deregistration_date</name>
<rename>place_stay_deregistration_date</rename>
<update>Y</update>
</value>
<value>
<name>kinship_type</name>
<rename>kinship_type</rename>
<name>place_stay_registration_date</name>
<rename>place_stay_registration_date</rename>
<update>Y</update>
</value>
<value>
<name>child_external_id</name>
<rename>child_external_id</rename>
<update>N</update>
</value>
<value>
<name>first_name</name>
<rename>first_name</rename>
<name>place_stay_registration_type</name>
<rename>place_stay_registration_type</rename>
<update>Y</update>
</value>
<value>
<name>middle_name</name>
<rename>middle_name</rename>
<name>residence_address</name>
<rename>residence_address</rename>
<update>Y</update>
</value>
<value>
<name>last_name</name>
<rename>last_name</rename>
<name>residence_deregistration_date</name>
<rename>residence_deregistration_date</rename>
<update>Y</update>
</value>
<value>
<name>child_id_ern</name>
<rename>child_id_ern</rename>
<name>residence_registration_date</name>
<rename>residence_registration_date</rename>
<update>Y</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1040</xloc>
<xloc>1024</xloc>
<yloc>368</yloc>
</GUI>
</transform>
@ -153,7 +143,7 @@
MAX(CASE WHEN addr->>'type' = 'residentialAddress' THEN (addr->>'regDateEnd')::date END) AS factual_info_expiry_date
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen c ON c.recruit_id = ri.recruit_id
LEFT JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr ON true
CROSS JOIN LATERAL jsonb_array_elements(ri.addresses) AS addr
WHERE
'${ID_F5}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${ID_F5}'