micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

delta

Browse source
This commit is contained in:
Ruslan 2025-02-05 14:16:44 +03:00
parent c59b982f99
commit b027efce98
3 changed files with 128 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

19
v1_apache-hop dash mapping/info_recruits/info_recruits.hpl
View file

@ -320,15 +320,18 @@ type_place_stay</note>
jsonb_array_elements_text(ri.info->'svedDeti'->'rebenok') AS child
FROM
public.recruits_info ri
JOIN
public.recruits r ON ri.recruit_id = r.id
WHERE
jsonb_typeof(ri.info->'svedDeti'->'rebenok') = 'array'
AND r.system_update_date >= '${UP_D}'
),
children_birth_dates AS (
SELECT
recruit_id,
(child::jsonb->'svedFLBS'->'dataRozhdDok'->>'den')::int AS day,
(child::jsonb->'svedFLBS'->'dataRozhdDok'->>'mesyacz')::int AS month,
(child::jsonb->'svedFLBS'->'dataRozhdDok'-&gt;&gt;'god')::int AS year
(child::jsonb->'svedFLBS'->'dataRozhdDok'->>'god')::int AS year
FROM
extracted_children
),
@ -349,6 +352,7 @@ registration_reasons AS (
FROM public.recruits AS r
JOIN public.military_registration_reason AS mrr
ON r.registration_reasons::jsonb @> to_jsonb(mrr.code::text)
WHERE r.system_update_date >= '${UP_D}'
),
driver_license AS (
@ -360,9 +364,12 @@ driver_license AS (
) AS driver_license
FROM
public.recruits_info ri
JOIN
public.recruits r ON ri.recruit_id = r.id
LEFT JOIN LATERAL
jsonb_array_elements(ri.info->'svedVoditUdost'->'voditUdost'->'svedKat') AS sved_vodit
ON true
WHERE r.system_update_date >= '${UP_D}'
GROUP BY ri.recruit_id
),
@ -439,6 +446,8 @@ sports_category AS ( -- проверка на наличие спортивно
ELSE null -- Неопределенное значение для всех других случаев
END AS sports_category
FROM public.recruits_info ri
JOIN public.recruits r ON ri.recruit_id = r.id
WHERE r.system_update_date >= '${UP_D}'
),
disability_group AS (
@ -454,6 +463,8 @@ disability_group AS (
ELSE 'Нет информации об инвалидности'
END AS disability_group
FROM public.recruits_info ri
JOIN public.recruits r ON ri.recruit_id = r.id
WHERE r.system_update_date >= '${UP_D}'
),
tractor_license AS (
@ -470,6 +481,8 @@ tractor_license AS (
ELSE null
END AS tractor_license
FROM public.recruits_info ri
JOIN public.recruits r ON ri.recruit_id = r.id
WHERE r.system_update_date >= '${UP_D}'
),
is_wanted AS (
@ -507,6 +520,8 @@ is_wanted AS (
ELSE NULL
END AS criminal_prosecution
FROM public.recruits_info ri
JOIN public.recruits r ON ri.recruit_id = r.id
WHERE r.system_update_date >= '${UP_D}'
),
employment AS (
@ -568,6 +583,8 @@ employment AS (
END
) AS employment_status
FROM public.recruits_info ri
JOIN public.recruits r ON ri.recruit_id = r.id
WHERE r.system_update_date >= '${UP_D}'
)
SELECT

57
v1_apache-hop dash mapping/info_recruits/job_info_recruits.hwf
View file

@ -32,8 +32,8 @@
<schedulerType>0</schedulerType>
<weekDay>1</weekDay>
<parallel>N</parallel>
<xloc>592</xloc>
<yloc>1536</yloc>
<xloc>496</xloc>
<yloc>1936</yloc>
<attributes_hac/>
</action>
<action>
@ -89,8 +89,8 @@
<sqlfromfile>N</sqlfromfile>
<useVariableSubstitution>N</useVariableSubstitution>
<parallel>N</parallel>
<xloc>880</xloc>
<yloc>2752</yloc>
<xloc>864</xloc>
<yloc>1872</yloc>
<attributes_hac/>
</action>
<action>
@ -118,20 +118,24 @@
<clear_files>N</clear_files>
<clear_rows>Y</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>N</exec_per_row>
<exec_per_row>Y</exec_per_row>
<filename>${Internal.Entry.Current.Folder}/opekun.hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<parameter>
<name>UP_D</name>
<stream_name>UPDATE_DATE</stream_name>
</parameter>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<params_from_previous>N</params_from_previous>
<params_from_previous>Y</params_from_previous>
<run_configuration>local</run_configuration>
<set_append_logfile>N</set_append_logfile>
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>928</xloc>
<yloc>2256</yloc>
<xloc>992</xloc>
<yloc>1664</yloc>
<attributes_hac/>
</action>
<action>
@ -156,8 +160,8 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>976</xloc>
<yloc>2752</yloc>
<xloc>992</xloc>
<yloc>1776</yloc>
<attributes_hac/>
</action>
<action>
@ -741,8 +745,8 @@
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>864</xloc>
<yloc>1536</yloc>
<xloc>832</xloc>
<yloc>1664</yloc>
<attributes_hac/>
</action>
</actions>
@ -757,7 +761,7 @@
<hop>
<from>SQL.spouse</from>
<to>spouse</to>
<enabled>Y</enabled>
<enabled>N</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
@ -778,14 +782,14 @@
<hop>
<from>spouse</from>
<to>Success</to>
<enabled>N</enabled>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>opekun</from>
<to>Success</to>
<enabled>N</enabled>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
@ -1076,17 +1080,10 @@
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>Start</from>
<to>opekun</to>
<enabled>N</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>select_delta_time.hpl</from>
<to>info_recruits</to>
<enabled>Y</enabled>
<enabled>N</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
@ -1097,6 +1094,20 @@
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>select_delta_time.hpl</from>
<to>opekun</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>select_delta_time.hpl</from>
<to>spouse</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
</hops>
<notepads>
<notepad>

148
v1_apache-hop dash mapping/info_recruits/opekun.hpl
View file

@ -31,77 +31,6 @@
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu_person_registry</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with person as
(select jsonb_array_elements(ri.info->'svedOpekun'->'svedSoczPod') ch,
ri.recruit_id
from public.recruits_info ri
where jsonb_typeof(ri.info->'svedOpekun'->'svedSoczPod') = 'array')
select
r.id recruit_id,
concat_ws(' ', ch->'fioOpek'->>'familiya',
ch->'fioOpek'->>'imya',
ch->'fioOpek'->>'otchestvo') full_name,
make_date(NULLIF(ch->'dataRozhdDok'->>'god', '')::int,
NULLIF(ch->'dataRozhdDok'->>'mesyacz','')::int,
NULLIF(ch->'dataRozhdDok'->&gt;'den','')::int) birth_date,
ch->>'snils' snils
from public.recruits r
join person on r.id = person.recruit_id</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>288</xloc>
<yloc>240</yloc>
</GUI>
</transform>
<transform>
<name>Table output</name>
<type>TableOutput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>1000</commit>
<connection>ervu-dashboard</connection>
<fields>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_monthly>Y</partitioning_monthly>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>N</specify_fields>
<table>citizen_guardianship</table>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>480</xloc>
<yloc>112</yloc>
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<type>InsertUpdate</type>
@ -120,7 +49,6 @@ join person on r.id = person.recruit_id</sql>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen_guardianship</table>
@ -152,6 +80,82 @@ join person on r.id = person.recruit_id</sql>
<yloc>240</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu_person_registry</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with person as
(select jsonb_array_elements(ri.info->'svedOpekun'->'svedSoczPod') ch,
ri.recruit_id
from public.recruits_info ri
where jsonb_typeof(ri.info->'svedOpekun'->'svedSoczPod') = 'array')
select
r.id recruit_id,
concat_ws(' ', ch->'fioOpek'->>'familiya',
ch->'fioOpek'->>'imya',
ch->'fioOpek'->>'otchestvo') full_name,
make_date(NULLIF(ch->'dataRozhdDok'->>'god', '')::int,
NULLIF(ch->'dataRozhdDok'->>'mesyacz','')::int,
NULLIF(ch->'dataRozhdDok'->&gt;'den','')::int) birth_date,
ch->>'snils' snils
from public.recruits r
join person on r.id = person.recruit_id
where
r.system_update_date >= '${UP_D}'</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>288</xloc>
<yloc>240</yloc>
</GUI>
</transform>
<transform>
<name>Table output</name>
<type>TableOutput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>1000</commit>
<connection>ervu-dashboard</connection>
<fields>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>N</specify_fields>
<table>citizen_guardianship</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>112</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>