micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix

Browse source
This commit is contained in:
Ruslan 2025-04-02 13:10:14 +03:00
parent 035005515b
commit 46113d6066
8 changed files with 76 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

11
v1_apache-hop dash mapping/job_general.hwf
View file

@ -135,7 +135,7 @@
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>928</xloc>
<yloc>240</yloc>
<yloc>128</yloc>
<attributes_hac/>
</action>
</actions>
@ -164,13 +164,20 @@
<hop>
<from>job_info_recruits.hwf</from>
<to>job_last_recording_date</to>
<enabled>Y</enabled>
<enabled>N</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>job_recruitments_all</from>
<to>job_info_recruits.hwf</to>
<enabled>N</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>job_recruitments_all</from>
<to>job_last_recording_date</to>
<enabled>Y</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>

14
v1_apache-hop dash mapping/jobs_once_a_day/citizen_foreign_once.hpl
View file

@ -46,10 +46,10 @@
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with citizen as
(select jsonb_array_elements(ri.info->'svedIGPP'->'svedIG') cit,
(select jsonb_array_elements(ri.info->'svedFL'->'svedIGPP'->'svedIG') cit,
ri.recruit_id
from public.recruits_info ri
where jsonb_typeof(ri.info->'svedIGPP'->'svedIG') = 'array')
where jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedIG') = 'array')
select
r.id recruit_id,
cit->'oksm'->>'naimOKSM' country_name,
@ -79,10 +79,10 @@ join citizen on r.id = citizen.recruit_id;</sql>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with citizen as
(select jsonb_array_elements(ri.info->'svedIGPP'->'svedPP') cit,
(select jsonb_array_elements(ri.info->'svedFL'->'svedIGPP'->'svedPP') cit,
ri.recruit_id
from public.recruits_info ri
where jsonb_typeof(ri.info->'svedIGPP'->'svedPP') = 'array')
where jsonb_typeof(ri.info->'svedFL'->'svedIGPP'->'svedPP') = 'array')
select
r.id recruit_id,
cit->'oksm'->>'naimOKSM' country_name,
@ -116,14 +116,11 @@ join citizen on r.id = citizen.recruit_id;</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>N</specify_fields>
<table>citizenship_foreign</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -153,14 +150,11 @@ join citizen on r.id = citizen.recruit_id;</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>N</specify_fields>
<table>citizenship_foreign</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>

23
v1_apache-hop dash mapping/jobs_once_a_day/citizenship_once.hpl
View file

@ -41,18 +41,18 @@
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select r.id recruit_id,
ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'osnPriob' acquiring_basis,
ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' acquiring_number,
to_date(ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshPriob', 'YYYY-MM-DD') acquiring_date,
ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'organReshPriob' acquiring_authority,
ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'osnVyh' renunciation_basis,
ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshVyh' renunciation_number,
to_date(ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshVyh', 'YYYY-MM-DD') renunciation_date,
ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'organReshVyh' renunciation_authority
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'osnPriob' acquiring_basis,
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' acquiring_number,
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshPriob', 'YYYY-MM-DD') acquiring_date,
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'organReshPriob' acquiring_authority,
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'osnVyh' renunciation_basis,
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshVyh' renunciation_number,
to_date(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'dataReshVyh', 'YYYY-MM-DD') renunciation_date,
ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'organReshVyh' renunciation_authority
from public.recruits_info ri
join public.recruits r ON ri.recruit_id = r.id
where ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and
(ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' is not null or ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'osnVyh' is not null)</sql>
where ri.info->'svedFL'->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and
(ri.info->'svedFL'->'svedGrazhdanRF'->'grazhdanRF'->>'nomReshPriob' is not null or ri.info->'svedGrazhdanRF'->'grazhdanRF'->>'osnVyh' is not null)</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
@ -79,14 +79,11 @@ where ri.info->'svedGrazhdanRF'->>'prOtsGrazhdanRF' is null and
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>N</specify_fields>
<table>citizenship</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>

20
v1_apache-hop dash mapping/jobs_once_a_day/deputy_once.hpl
View file

@ -100,12 +100,12 @@
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select r.id recruit_id,
ri.info->'svedDeputat'->'deputat'->>'srokiPolnomochii' period_office,
to_date(ri.info->'svedDeputat'->'deputat'->>'dataDosrochPrekr', 'YYYY-MM-DD') end_date,
ri.info->'svedDeputat'->'deputat'->>'svedIzbrDeputatom' information
ri.info->'svedFL'->'svedDeputat'->'deputat'->>'srokiPolnomochii' period_office,
to_date(ri.info->'svedFL'->'svedDeputat'->'deputat'->>'dataDosrochPrekr', 'YYYY-MM-DD') end_date,
ri.info->'svedFL'->'svedDeputat'->'deputat'->>'svedIzbrDeputatom' information
from public.recruits_info ri
join public.recruits r ON ri.recruit_id = r.id
where ri.info->'svedDeputat'->'deputat'->>'svedIzbrDeputatom' is not null;</sql>
where ri.info->'svedFL'->'svedDeputat'->'deputat'->>'svedIzbrDeputatom' is not null;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
@ -128,12 +128,12 @@ where ri.info->'svedDeputat'->'deputat'->>'svedIzbrDeputatom' is not null;</sql>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select r.id recruit_id,
to_date(ri.info->'svedKandidat'->'kandidat'->>'dataSnyatiyaKandidatury', 'YYYY-MM-DD') end_date,
ri.info->'svedKandidat'->'kandidat'->>'svedRegKandidata' information,
to_date(ri.info->'svedFL'->'svedKandidat'->'kandidat'->>'dataSnyatKandidat', 'YYYY-MM-DD') end_date,
ri.info->'svedFL'->'svedKandidat'->'kandidat'->>'svedRegKandidata' information,
false deputy
from public.recruits_info ri
join public.recruits r ON ri.recruit_id = r.id
where ri.info->'svedKandidat'->'kandidat'->>'svedRegKandidata' is not null;</sql>
where ri.info->'svedFL'->'svedKandidat'->>'kandidat' is not null</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
@ -160,14 +160,11 @@ where ri.info->'svedKandidat'->'kandidat'->>'svedRegKandidata' is not null;</sql
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>N</specify_fields>
<table>deputy</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -197,14 +194,11 @@ where ri.info->'svedKandidat'->'kandidat'->>'svedRegKandidata' is not null;</sql
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>N</specify_fields>
<table>deputy</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>

29
v1_apache-hop dash mapping/jobs_once_a_day/passport_once.hpl
View file

@ -45,19 +45,22 @@
<connection>ervu_person_registry</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>select r.id recruit_id,
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'->>'nomDok' passport_number,
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'->>'serDok' passport_series,
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'->>'vydDok' organization_name,
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'->>'kodVydDok' unit_code,
to_date(ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'->>'dataDok', 'YYYY-MM-DD') issue_date,
case when ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'->>'kodStatus'='1' then true
else false
end actual
from public.recruits_info ri
join public.recruits r ON ri.recruit_id = r.id
where ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'->>'nomDok' is not null
</sql>
<sql>SELECT
r.id AS recruit_id,
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'->>'nomDok' AS passport_number,
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'->>'serDok' AS passport_series,
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'->>'vydDok' AS organization_name,
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'->>'kodVydDok' AS unit_code,
TO_DATE(ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'->>'dataDok', 'YYYY-MM-DD') AS issue_date,
CASE
WHEN ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'->>'kodStatus' = '1'
THEN TRUE ELSE FALSE
END AS actual
FROM public.recruits_info ri
JOIN public.recruits r ON ri.recruit_id = r.id
WHERE ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'->>'nomDok' IS NOT NULL
AND LENGTH(ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'->>'serDok') = 4
AND LENGTH(ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'->>'nomDok') = 6;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>

31
v1_apache-hop dash mapping/jobs_once_a_day/property_once.hpl
View file

@ -56,10 +56,10 @@
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with prop as
(select jsonb_array_elements(ri.info->'svedON'->'on') u,
(select jsonb_array_elements(ri.info->'svedFL'->'svedON'->'on') u,
ri.recruit_id
from public.recruits_info ri
where jsonb_typeof(ri.info->'svedON'->'on') = 'array')
where jsonb_typeof(ri.info->'svedFL'->'svedON'->'on') = 'array')
select
r.id recruit_id,
u->>'naimVidPrav' kind_right,
@ -91,10 +91,10 @@ join prop on r.id = prop.recruit_id;</sql>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with prop as
(select jsonb_array_elements(ri.info->'svedNazTS'->'nazTS') u,
(select jsonb_array_elements(ri.info->'svedFL'->'svedNazTS'->'nazTS') u,
ri.recruit_id
from public.recruits_info ri
where jsonb_typeof(ri.info->'svedNazTS'->'nazTS') = 'array')
where jsonb_typeof(ri.info->'svedFL'->'svedNazTS'->'nazTS') = 'array')
select
r.id recruit_id,
u->>'vidTS' vehicle_type,
@ -129,10 +129,10 @@ join prop on r.id = prop.recruit_id;</sql>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with prop as
(select jsonb_array_elements(ri.info->'svedVozTS'->'vozTS') u,
(select jsonb_array_elements(ri.info->'svedFL'->'svedVozTS'->'vozTS') u,
ri.recruit_id
from public.recruits_info ri
where jsonb_typeof(ri.info->'svedVozTS'->'vozTS') = 'array')
where jsonb_typeof(ri.info->'svedFL'->'svedVozTS'->'vozTS') = 'array')
select
r.id recruit_id,
u->>'naimVid' vehicle_type,
@ -164,10 +164,10 @@ join prop on r.id = prop.recruit_id;</sql>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with prop as
(select jsonb_array_elements(ri.info->'svedVodTS'->'vodTS') u,
(select jsonb_array_elements(ri.info->'svedFL'->'svedVodTS'->'vodTS') u,
ri.recruit_id
from public.recruits_info ri
where jsonb_typeof(ri.info->'svedVodTS'->'vodTS') = 'array')
where jsonb_typeof(ri.info->'svedFL'->'svedVodTS'->'vodTS') = 'array')
select
r.id recruit_id,
u->>'naimVid' vehicle_type,
@ -176,8 +176,7 @@ select
to_date(u->>'dataPrekPrav', 'YYYY-MM-DD') close_date,
'water_vehicles' type_property
from public.recruits r
join prop on r.id = prop.recruit_id
</sql>
join prop on r.id = prop.recruit_id</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
@ -204,14 +203,11 @@ join prop on r.id = prop.recruit_id
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>N</specify_fields>
<table>property</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -241,14 +237,11 @@ join prop on r.id = prop.recruit_id
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>N</specify_fields>
<table>property</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -278,14 +271,11 @@ join prop on r.id = prop.recruit_id
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>N</specify_fields>
<table>property</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
@ -315,14 +305,11 @@ join prop on r.id = prop.recruit_id
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>N</specify_fields>
<table>property</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>

37
v1_apache-hop dash mapping/jobs_once_a_day/prosecution_once.hpl
View file

@ -40,21 +40,25 @@
<connection>ervu_person_registry</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with ugol as
(select jsonb_array_elements(ri.info->'svedUgolovPresl'->'faktyUgolovPresl') u,
ri.recruit_id
from public.recruits_info ri
where jsonb_typeof(ri.info->'svedUgolovPresl'->'faktyUgolovPresl') = 'array')
select
r.id recruit_id,
u->>'vidPreslNaim' kind,
to_date(u->>'dataSved', 'YYYY-MM-DD') create_date,
to_date(u->>'dataPrekrashh', 'YYYY-MM-DD') close_date,
u->>'naimOrgan' authority_name,
u->>'ugolovZakon' law_point,
u->>'osnovPrekrashh' termination_basis
from public.recruits r
join ugol on r.id = ugol.recruit_id;</sql>
<sql>WITH ugol AS (
SELECT
jsonb_array_elements(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') AS u,
ri.recruit_id
FROM public.recruits_info ri
WHERE jsonb_typeof(ri.info->'svedFL'->'svedUgolovPresl'->'faktyUgolovPresl') = 'array'
)
SELECT
r.id AS recruit_id,
u->'sledSud'->>'naimSud' AS kind,
TO_DATE(u->'sledSud'->>'dataVozbUD', 'YYYY-MM-DD') AS create_date,
TO_DATE(u->'sledSud'->'svedSnyat'->>'dataSnyatSud', 'YYYY-MM-DD') AS close_date,
u->'sledSud'->&gt;'naimOrgan' AS authority_name,
STRING_AGG(TRIM(BOTH ' "' FROM law.f), ', ') AS law_point,
u->'sledSud'->'svedSnyat'->>'osnSnyatSud' AS termination_basis
FROM public.recruits r
JOIN ugol ON r.id = ugol.recruit_id
LEFT JOIN LATERAL jsonb_array_elements_text(u->'sledSud'->'statiUK') AS law(f) ON true
GROUP BY r.id, kind, create_date, close_date, authority_name, termination_basis;</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
@ -81,14 +85,11 @@ join ugol on r.id = ugol.recruit_id;</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>N</specify_fields>
<table>prosecution</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>

7
v1_apache-hop dash mapping/jobs_once_a_day/punishment_once.hpl
View file

@ -41,10 +41,10 @@
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>with ugol as
(select jsonb_array_elements(ri.info->'svedSudim'->'sudim'->'osuzhdenie') u,
(select jsonb_array_elements(ri.info->'svedFL'->'svedSudim'->'sudim'->'osuzhdenie') u,
ri.recruit_id
from public.recruits_info ri
where jsonb_typeof(ri.info->'svedSudim'->'sudim'->'osuzhdenie') = 'array')
where jsonb_typeof(ri.info->'svedFL'->'svedSudim'->'sudim'->'osuzhdenie') = 'array')
select
r.id recruit_id,
u->>'naimSuda' authority_name,
@ -86,14 +86,11 @@ join ugol on r.id = ugol.recruit_id;</sql>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>N</specify_fields>
<table>punishment</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>