micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

7

Browse source
This commit is contained in:
Ruslan 2025-03-21 14:38:18 +03:00
parent c6de11c9f0
commit 01e78b831c
5 changed files with 362 additions and 202 deletions
Download patch file Download diff file Expand all files Collapse all files

74
v1_apache-hop dash mapping/mil_com/job_recruitments_milcom.hwf
View file

@ -114,43 +114,13 @@
<yloc>736</yloc>
<attributes_hac/>
</action>
<action>
<name>main_dashboard.recruitment_campaign(m_c).hpl</name>
<description/>
<type>PIPELINE</type>
<attributes/>
<add_date>N</add_date>
<add_time>N</add_time>
<clear_files>N</clear_files>
<clear_rows>N</clear_rows>
<create_parent_folder>N</create_parent_folder>
<exec_per_row>Y</exec_per_row>
<filename>${PROJECT_HOME}/mil_com/main_dashboard.recruitment_campaign(m_c).hpl</filename>
<loglevel>Basic</loglevel>
<parameters>
<parameter>
<name>ARR_MC</name>
<stream_name>ARRAY_MIL_COM</stream_name>
</parameter>
<pass_all_parameters>Y</pass_all_parameters>
</parameters>
<params_from_previous>Y</params_from_previous>
<run_configuration>local</run_configuration>
<set_append_logfile>N</set_append_logfile>
<set_logfile>N</set_logfile>
<wait_until_finished>Y</wait_until_finished>
<parallel>N</parallel>
<xloc>1392</xloc>
<yloc>640</yloc>
<attributes_hac/>
</action>
<action>
<name>Success 2</name>
<description/>
<type>SUCCESS</type>
<attributes/>
<parallel>N</parallel>
<xloc>1456</xloc>
<xloc>1536</xloc>
<yloc>800</yloc>
<attributes_hac/>
</action>
@ -986,6 +956,27 @@ WHERE recording_date = current_date
<yloc>800</yloc>
<attributes_hac/>
</action>
<action>
<name>SQL main_dashboard.recruitment_campaign clearing null</name>
<description/>
<type>SQL</type>
<attributes/>
<connection>ervu-dashboard</connection>
<sendOneStatement>N</sendOneStatement>
<sql>DELETE FROM main_dashboard.recruitment_campaign
WHERE recording_date = current_date
AND new_recruits IS NULL
AND subpoenas_sent IS NULL
AND appeared_on_subpoenas IS NULL
AND postponement_have_right IS NULL
AND postponement_granted IS NULL;</sql>
<sqlfromfile>N</sqlfromfile>
<useVariableSubstitution>N</useVariableSubstitution>
<parallel>N</parallel>
<xloc>1296</xloc>
<yloc>800</yloc>
<attributes_hac/>
</action>
</actions>
<hops>
<hop>
@ -1310,13 +1301,6 @@ WHERE recording_date = current_date
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>main_dashboard.recruitment_campaign(m_c).hwf</from>
<to>Success 2</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>Start</from>
<to>SQL main_dashboard.recruitment_campaign</to>
@ -1331,6 +1315,20 @@ WHERE recording_date = current_date
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>main_dashboard.recruitment_campaign(m_c).hwf</from>
<to>SQL main_dashboard.recruitment_campaign clearing null</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>SQL main_dashboard.recruitment_campaign clearing null</from>
<to>Success 2</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
</hops>
<notepads>
<notepad>

34
v1_apache-hop dash mapping/mil_com/main_dashboard.recruitment_campaign(m_c).hwf
View file

@ -411,8 +411,8 @@ WHERE recording_date = current_date
<type>SUCCESS</type>
<attributes/>
<parallel>N</parallel>
<xloc>1936</xloc>
<yloc>256</yloc>
<xloc>2144</xloc>
<yloc>240</yloc>
<attributes_hac/>
</action>
</actions>
@ -420,7 +420,7 @@ WHERE recording_date = current_date
<hop>
<from>m_d.rec_cam(m_c)_to_person_registry_uniq.hpl</from>
<to>m_d.rec_cam(m_c)_to_subpoena_registry_uniq.hpl</to>
<enabled>N</enabled>
<enabled>Y</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
@ -434,14 +434,14 @@ WHERE recording_date = current_date
<hop>
<from>m_d.rec_cam(m_c)_to_decision_service_uniq.hpl</from>
<to>Success uniq</to>
<enabled>N</enabled>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>recruitment_season_combination_uniq.hpl</from>
<to>m_d.rec_cam(m_c)_to_person_registry_uniq.hpl</to>
<enabled>N</enabled>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>Y</unconditional>
</hop>
@ -469,7 +469,7 @@ WHERE recording_date = current_date
<hop>
<from>m_d.rec_cam(m_c)_to_person_registry.hpl</from>
<to>m_d.rec_cam(m_c)_to_subpoena_registry.hpl</to>
<enabled>N</enabled>
<enabled>Y</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
@ -483,7 +483,7 @@ WHERE recording_date = current_date
<hop>
<from>recruitment_season_combination.hpl</from>
<to>m_d.rec_cam(m_c)_to_person_registry.hpl</to>
<enabled>N</enabled>
<enabled>Y</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
@ -497,7 +497,7 @@ WHERE recording_date = current_date
<hop>
<from>m_d.rec_cam(m_c)_to_decision_service.hpl</from>
<to>Success all</to>
<enabled>N</enabled>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
@ -567,31 +567,17 @@ WHERE recording_date = current_date
<hop>
<from>recruitment_season_combination.hpl</from>
<to>m_d.rec_cam(m_c)_to_subpoena_registry.hpl</to>
<enabled>Y</enabled>
<enabled>N</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>recruitment_season_combination_uniq.hpl</from>
<to>m_d.rec_cam(m_c)_to_subpoena_registry_uniq.hpl</to>
<enabled>Y</enabled>
<enabled>N</enabled>
<evaluation>N</evaluation>
<unconditional>Y</unconditional>
</hop>
<hop>
<from>m_d.rec_cam(m_c)_to_decision_service.hpl</from>
<to>Success 4</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
<hop>
<from>m_d.rec_cam(m_c)_to_decision_service_uniq.hpl</from>
<to>Success 4</to>
<enabled>Y</enabled>
<evaluation>Y</evaluation>
<unconditional>N</unconditional>
</hop>
</hops>
<notepads>
<notepad>

166
v1_apache-hop dash mapping/mil_com/main_dashboard.recruitment_campaign(m_c)/m_d.rec_cam(m_c)_to_person_registry.hpl
View file

@ -23,67 +23,10 @@
<order>
<hop>
<from>Table input (person_registry) регион/осень</from>
<to>Insert / update (main_dashboard.recruitment_campaign)</to>
<to>Update</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Insert / update (main_dashboard.recruitment_campaign)</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>spring_autumn</field>
<name>spring_autumn</name>
<name2/>
</key>
<schema>main_dashboard</schema>
<table>recruitment_campaign</table>
<value>
<name>recruitment_id</name>
<rename>recruitment_id</rename>
<update>N</update>
</value>
<value>
<name>spring_autumn</name>
<rename>spring_autumn</rename>
<update>N</update>
</value>
<value>
<name>recording_date</name>
<rename>recording_date</rename>
<update>N</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>Table input (person_registry) регион/осень</name>
<type>TableInput</type>
@ -103,17 +46,61 @@ ids AS (
-- Преобразуем строковый массив в таблицу UUID
SELECT unnest(string_to_array(trim(both '{}' from '${ARR_MC}'), ','))::uuid AS recruitment_id
),
season_separations AS (
SELECT 'Весна' AS spring_autumn
UNION ALL
SELECT 'Осень' AS spring_autumn
last_status as (select recruit_id,
max(date_time) filter (where status not in ('1', '1.3', '12', '12.1', '12.2', '12.3', '12.4', '12.5', '13')) max_date_time
from recruits_history
where status not in ('1', '1.3', '12', '12.1', '12.2', '12.3', '12.4', '12.5', '13')
group by recruit_id),
registered as (
select
distinct rh.recruit_id,
rh.id as recruit_history_id
from recruits_history rh
join last_status ls on rh.recruit_id=ls. recruit_id and rh.date_time=ls.max_date_time
where rh.status not in ('1', '1.3', '12', '12.1', '12.2', '12.3', '12.4', '12.5', '13') -- критерии от 03.06.2025
--and rh.source is not null
),
vse AS (
SELECT
r.id AS r_id,
ri.recruit_id AS recruit_id,
r.gender,
coalesce (r.current_recruitment_id, rh.current_recruitment_id) AS recruitment_id,
CASE
WHEN (r.birth_date BETWEEN DATE (extract(year from now())-30||'-04-02') and DATE (extract(year from now())-18||'-07-13'))
THEN 'Весна'
WHEN (r.birth_date BETWEEN DATE (extract(year from now())-30||'-10-02') and DATE (extract(year from now())-18||'-12-30'))
THEN 'Осень'
END AS spring_autumn,
r.birth_date,
CASE
WHEN ddd.hidden IS false THEN ddd.hidden
ELSE true
END AS hidden, -- когда фолс отсрочка есть, когда тру или нулл - нет
ri.info
FROM public.recruits AS r
JOIN public.recruits_info AS ri ON ri.recruit_id = r.id
JOIN public.recruits_history rh ON rh.recruit_id=r.id
LEFT JOIN public.decision_deferment_dto ddd ON ddd.recruit_id = r.id
WHERE r.gender = 'MALE' AND
r.id in (SELECT recruit_id FROM registered) and
rh.id in (SELECT recruit_history_id FROM registered) and
coalesce(r.current_recruitment_id,rh.current_recruitment_id) ::uuid IN (SELECT recruitment_id FROM ids)
AND
((r.birth_date BETWEEN DATE (extract(year from now())-30||'-04-02') and DATE (extract(year from now())-18||'-07-13'))
OR
(r.birth_date BETWEEN DATE (extract(year from now())-30||'-10-02') and DATE (extract(year from now())-18||'-12-30')))
)
SELECT
SELECT
ri.recruitment_id,
ri.spring_autumn,
current_date AS recording_date,
i.recruitment_id AS recruitment_id,
ss.spring_autumn
FROM ids i
CROSS JOIN season_separations ss</sql>
COUNT(DISTINCT ri.recruit_id) FILTER (
WHERE
ri.hidden IS true -- нет отсрочки и нет освобождения от службы
) AS recruit
FROM vse AS ri
GROUP BY ri.recruitment_id, ri.spring_autumn</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -121,6 +108,51 @@ CROSS JOIN season_separations ss</sql>
<yloc>320</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
</key>
<key>
<condition>=</condition>
<field>spring_autumn</field>
<name>spring_autumn</name>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
</key>
<schema>main_dashboard</schema>
<table>recruitment_campaign</table>
<value>
<name>new_recruits</name>
<rename>recruit</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>1056</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>

177
v1_apache-hop dash mapping/mil_com/main_dashboard.recruitment_campaign(m_c)/m_d.rec_cam(m_c)_to_person_registry_uniq.hpl
View file

@ -21,71 +21,12 @@
<notepads>
</notepads>
<order>
<hop>
<from>Table input (person_registry) регион/осень</from>
<to>Insert / update (main_dashboard.recruitment_campaign)</to>
<enabled>N</enabled>
</hop>
<hop>
<from>Table input (person_registry) регион/осень</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Insert / update (main_dashboard.recruitment_campaign)</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<lookup>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
</key>
<key>
<condition>=</condition>
<field>spring_autumn</field>
<name>spring_autumn</name>
</key>
<schema>main_dashboard</schema>
<table>recruitment_campaign</table>
<value>
<name>recruitment_id</name>
<rename>recruitment_id</rename>
<update>N</update>
</value>
<value>
<name>spring_autumn</name>
<rename>spring_autumn</rename>
<update>N</update>
</value>
<value>
<name>recording_date</name>
<rename>recording_date</rename>
<update>N</update>
</value>
</lookup>
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
<name>Table input (person_registry) регион/осень</name>
<type>TableInput</type>
@ -100,22 +41,88 @@
<connection>postgres.person_registry</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>WITH
<sql>WITH
ids AS (
-- Преобразуем строковый массив в таблицу UUID
SELECT unnest(string_to_array(trim(both '{}' from '${ARR_MC}'), ','))::uuid AS recruitment_id
),
season_separations AS (
SELECT 'Весна' AS spring_autumn
-- Весенние ID
SELECT unnest(string_to_array(nullif(trim(both '{}' from '${ARR_MC_SP}'), ''), ','))::uuid AS recruitment_id,
'Весна' AS spring_autumn
UNION ALL
SELECT 'Осень' AS spring_autumn
-- Осенние ID
SELECT unnest(string_to_array(nullif(trim(both '{}' from '${ARR_MC_AU}'), ''), ','))::uuid AS recruitment_id,
'Осень' AS spring_autumn
),
last_status as (select recruit_id,
max(date_time) filter (where status not in ('1', '1.3', '12', '12.1', '12.2', '12.3', '12.4', '12.5', '13')) max_date_time
from recruits_history
where status not in ('1', '1.3', '12', '12.1', '12.2', '12.3', '12.4', '12.5', '13')
group by recruit_id),
registered as (
select
distinct rh.recruit_id,
rh.id as recruit_history_id
from recruits_history rh
join last_status ls on rh.recruit_id=ls. recruit_id and rh.date_time=ls.max_date_time
where rh.status not in ('1', '1.3', '12', '12.1', '12.2', '12.3', '12.4', '12.5', '13') -- критерии от 03.06.2025
--and rh.source is not null
),
vse AS (
SELECT
r.id AS r_id,
ri.recruit_id AS recruit_id,
r.gender,
coalesce (r.current_recruitment_id, rh.current_recruitment_id) AS recruitment_id,
'Весна' AS spring_autumn,
r.birth_date,
CASE
WHEN ddd.hidden IS false THEN ddd.hidden
ELSE true
END AS hidden, -- когда фолс отсрочка есть, когда тру или нулл - нет
ri.info
FROM public.recruits AS r
JOIN public.recruits_info AS ri ON ri.recruit_id = r.id
JOIN public.recruits_history rh ON rh.recruit_id=r.id
LEFT JOIN public.decision_deferment_dto ddd ON ddd.recruit_id = r.id
WHERE r.gender = 'MALE' AND
r.id in (SELECT recruit_id FROM registered) and
rh.id in (SELECT recruit_history_id FROM registered) and
coalesce(r.current_recruitment_id,rh.current_recruitment_id) ::uuid IN (SELECT recruitment_id FROM ids WHERE spring_autumn = 'Весна')
AND (r.birth_date BETWEEN DATE (extract(year from now())-30||'-04-02') and DATE (extract(year from now())-18||'-07-13'))
UNION ALL
SELECT
r.id AS r_id,
ri.recruit_id AS recruit_id,
r.gender,
coalesce (r.current_recruitment_id, rh.current_recruitment_id) AS recruitment_id,
'Осень' AS spring_autumn,
r.birth_date,
CASE
WHEN ddd.hidden IS false THEN ddd.hidden
ELSE true
END AS hidden, -- когда фолс отсрочка есть, когда тру или нулл - нет
ri.info
FROM public.recruits AS r
JOIN public.recruits_info AS ri ON ri.recruit_id = r.id
JOIN public.recruits_history rh ON rh.recruit_id=r.id
LEFT JOIN public.decision_deferment_dto ddd ON ddd.recruit_id = r.id
WHERE r.gender = 'MALE' AND
r.id in (SELECT recruit_id FROM registered) and
rh.id in (SELECT recruit_history_id FROM registered) and
coalesce(r.current_recruitment_id,rh.current_recruitment_id) ::uuid IN (SELECT recruitment_id FROM ids WHERE spring_autumn = 'Осень')
AND (r.birth_date BETWEEN DATE (extract(year from now())-30||'-10-02') and DATE (extract(year from now())-18||'-12-30'))
)
SELECT
SELECT
ri.recruitment_id,
ri.spring_autumn,
current_date AS recording_date,
i.recruitment_id AS recruitment_id,
ss.spring_autumn
FROM ids i
CROSS JOIN season_separations ss</sql>
COUNT(DISTINCT ri.recruit_id) FILTER (
WHERE
ri.hidden IS true -- нет отсрочки и нет освобождения от службы
) AS recruit
FROM vse AS ri
GROUP BY ri.recruitment_id, ri.spring_autumn</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -135,10 +142,34 @@ CROSS JOIN season_separations ss</sql>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<ignore_flag_field/>
<lookup>
<schema/>
<table>lookup table</table>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>spring_autumn</field>
<name>spring_autumn</name>
<name2/>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
<name2/>
</key>
<schema>main_dashboard</schema>
<table>recruitment_campaign</table>
<value>
<name>new_recruits</name>
<rename>recruit</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>N</use_batch>

113
v1_apache-hop dash mapping/mil_com/main_dashboard.recruitment_campaign(m_c)/postponement_percents.hpl Normal file
View file

@ -0,0 +1,113 @@
<?xml version="1.0" encoding="UTF-8"?>
<pipeline>
<info>
<name>postponement_percents</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<parameters>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
<created_user>-</created_user>
<created_date>2025/03/21 14:21:43.089</created_date>
<modified_user>-</modified_user>
<modified_date>2025/03/21 14:21:43.089</modified_date>
</info>
<notepads>
</notepads>
<order>
<hop>
<from>Table input</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit>0</limit>
<sql>SELECT
current_date AS recording_date,
spring_autumn,
recruitment_id,
COALESCE(SUM(postponement_have_right) * 100 / NULLIF(SUM(new_recruits), 0), 0) AS postponement_have_right_percent,
COALESCE(SUM(postponement_granted) * 100 / NULLIF(SUM(new_recruits), 0), 0) AS postponement_granted_percent
FROM main_dashboard.recruitment_campaign
WHERE recording_date = current_date
AND schema = 'empty'
GROUP BY recruitment_id, spring_autumn;
</sql>
<variables_active>N</variables_active>
<attributes/>
<GUI>
<xloc>480</xloc>
<yloc>256</yloc>
</GUI>
</transform>
<transform>
<name>Update</name>
<type>Update</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>100</commit>
<connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored>
<lookup>
<key>
<condition>=</condition>
<field>spring_autumn</field>
<name>spring_autumn</name>
</key>
<key>
<condition>=</condition>
<field>recruitment_id</field>
<name>recruitment_id</name>
</key>
<key>
<condition>=</condition>
<field>recording_date</field>
<name>recording_date</name>
</key>
<schema>main_dashboard</schema>
<table>recruitment_campaign</table>
<value>
<name>postponement_have_right_percent</name>
<rename>postponement_have_right_percent</rename>
</value>
<value>
<name>postponement_granted_percent</name>
<rename>postponement_granted_percent</rename>
</value>
</lookup>
<skip_lookup>N</skip_lookup>
<use_batch>Y</use_batch>
<attributes/>
<GUI>
<xloc>800</xloc>
<yloc>256</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>
</pipeline>