micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

SUPPORT-8925 fix for job_recruitments_milcom

Browse source
This commit is contained in:
y.kuznetsova 2025-02-21 13:57:47 +03:00
parent 5865f89b95
commit 65a40b8b7a
7 changed files with 68 additions and 204 deletions
Download patch file Download diff file Expand all files Collapse all files

83
v1_apache-hop dash mapping/mil_com/total_registered.age(m_c).hpl
View file

@ -292,26 +292,6 @@ ids AS (
-- Преобразуем строковый массив в таблицу UUID
SELECT unnest(string_to_array(trim(both '{}' from '${ARR_MC}'), ','))::uuid AS recruitment_id
),
gender_separations AS (
SELECT 'MALE' AS gender
UNION ALL
SELECT 'FEMALE' AS gender
),
registered_separation AS (
SELECT 'true' AS registered
UNION ALL
SELECT 'false' AS registered
),
data_combinations AS (
-- Создаём все комбинации recruitment_id, gender и registered
SELECT
i.recruitment_id AS recruitment_id,
gs.gender,
rs.registered
FROM ids i
CROSS JOIN gender_separations gs
CROSS JOIN registered_separation rs
),
recruit_data AS (
SELECT
r.target_recruitment_id,
@ -328,16 +308,12 @@ recruit_data AS (
),
total_count AS (
SELECT
dc.recruitment_id,
dc.gender,
dc.registered,
rd.target_recruitment_id as recruitment_id,
rd.gender,
rd.registered,
COUNT(rd.recruit_id) AS total
FROM data_combinations dc
LEFT JOIN recruit_data rd
ON rd.target_recruitment_id = dc.recruitment_id
AND rd.gender = dc.gender
AND rd.registered = dc.registered
GROUP BY dc.recruitment_id, dc.gender, dc.registered
FROM recruit_data rd
GROUP BY rd.target_recruitment_id, rd.gender, rd.registered
),
total_count_all AS (
SELECT
@ -353,9 +329,9 @@ total_count_all AS (
),
age_counts AS (
SELECT
dc.recruitment_id,
dc.gender,
dc.registered,
rd.target_recruitment_id as recruitment_id,
rd.gender,
rd.registered,
COUNT(rd.recruit_id) FILTER (WHERE EXTRACT(YEAR FROM AGE(rd.birth_date)) = 17) AS "17_year_count",
COUNT(rd.recruit_id) FILTER (WHERE EXTRACT(YEAR FROM AGE(rd.birth_date)) BETWEEN 18 AND 26) AS "18-26_year_count",
COUNT(rd.recruit_id) FILTER (WHERE EXTRACT(YEAR FROM AGE(rd.birth_date)) BETWEEN 27 AND 30) AS "27-30_year_count",
@ -364,12 +340,8 @@ age_counts AS (
COUNT(rd.recruit_id) FILTER (WHERE EXTRACT(YEAR FROM AGE(rd.birth_date)) BETWEEN 41 AND 45) AS "41-45_year_count",
COUNT(rd.recruit_id) FILTER (WHERE EXTRACT(YEAR FROM AGE(rd.birth_date)) BETWEEN 46 AND 50) AS "46-50_year_count",
COUNT(rd.recruit_id) FILTER (WHERE EXTRACT(YEAR FROM AGE(rd.birth_date)) >= 51) AS "51+_year_count"
FROM data_combinations dc
LEFT JOIN recruit_data rd
ON rd.target_recruitment_id = dc.recruitment_id
AND rd.gender = dc.gender
AND rd.registered = dc.registered
GROUP BY dc.recruitment_id, dc.gender, dc.registered
FROM recruit_data rd
GROUP BY rd.target_recruitment_id, rd.gender, rd.registered
),
age_counts_all AS (
SELECT
@ -409,22 +381,17 @@ SELECT
COALESCE(ac."51+_year_count", 0) AS "51+_year_count",
COALESCE(ROUND((COALESCE(ac."51+_year_count", 0) * 100.0) / NULLIF(tc.total, 0), 2), 0) AS "51+_year_percent",
COALESCE(tc.total, 0) AS total_count,
dc.recruitment_id,
tc.recruitment_id,
(CASE
WHEN dc.gender = 'MALE' THEN 'M'
WHEN dc.gender = 'FEMALE' THEN 'W'
ELSE null
WHEN tc.gender = 'MALE' THEN 'M'
ELSE 'W'
END)::text AS gender,
dc.registered::text
FROM data_combinations dc
LEFT JOIN total_count tc
ON tc.recruitment_id = dc.recruitment_id
AND tc.gender = dc.gender
AND tc.registered = dc.registered
tc.registered::text
FROM total_count tc
LEFT JOIN age_counts ac
ON ac.recruitment_id = dc.recruitment_id
AND ac.gender = dc.gender
AND ac.registered = dc.registered
ON ac.recruitment_id = tc.recruitment_id
AND ac.gender = tc.gender
AND ac.registered = tc.registered
UNION ALL
SELECT
current_date AS recording_date,
@ -445,17 +412,13 @@ SELECT
COALESCE(aca."51+_year_count", 0) AS "51+_year_count",
COALESCE(ROUND((COALESCE(aca."51+_year_count", 0) * 100.0) / NULLIF(tca.total, 0), 2), 0) AS "51+_year_percent",
COALESCE(tca.total, 0) AS total_count,
dc.recruitment_id,
tca.recruitment_id,
'ALL' AS gender,
dc.registered::text
FROM data_combinations dc
LEFT JOIN total_count_all tca
ON tca.recruitment_id = dc.recruitment_id
AND tca.registered = dc.registered
tca.registered::text
FROM total_count_all tca
LEFT JOIN age_counts_all aca
ON aca.recruitment_id = dc.recruitment_id
AND aca.registered = dc.registered
</sql>
ON aca.recruitment_id = tca.recruitment_id
AND aca.registered = tca.registered</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>

46
v1_apache-hop dash mapping/mil_com/total_registered.busyness(m_c).hpl
View file

@ -249,30 +249,7 @@
<limit>0</limit>
<sql>WITH
ids AS (
-- Преобразуем строковый массив в таблицу UUID
SELECT unnest(string_to_array(trim(both '{}' from
'${ARR_MC}'
), ','))::uuid AS recruitment_id
),
gender_separations AS (
SELECT 'MALE' AS gender
UNION ALL
SELECT 'FEMALE' AS gender
),
registered_separation AS (
SELECT 'true' AS registered
UNION ALL
SELECT 'false' AS registered
),
data_combinations AS (
-- Создаём все комбинации recruitment_id, gender и registered
SELECT
i.recruitment_id AS recruitment_id,
gs.gender,
rs.registered
FROM ids i
CROSS JOIN gender_separations gs
CROSS JOIN registered_separation rs
SELECT unnest(string_to_array(trim(both '{}' from '${ARR_MC}'), ','))::uuid AS recruitment_id
),
status_info AS (
SELECT
@ -328,20 +305,19 @@ status_info AS (
r.vu_current_info->>'recruitmentCommercialInfo' AS sidit
FROM public.recruits_info ri
JOIN public.recruits r ON ri.recruit_id = r.id
WHERE (r.vu_current_info -> 'isMilitaryRegistered' = 'true' OR r.vu_current_info -> 'isMilitaryRegistered' = 'false')
AND (r.gender = 'MALE' OR r.gender = 'FEMALE')
WHERE r.vu_current_info -> 'isMilitaryRegistered' IN ('true', 'false')
AND r.gender IN ('MALE', 'FEMALE')
AND r.current_recruitment_id IS NOT NULL
AND r.target_recruitment_id::uuid IN (SELECT recruitment_id FROM ids)
),
summary AS (
SELECT
dc.recruitment_id,
si.recruitment_id,
CASE
WHEN dc.gender = 'MALE' THEN 'M'
WHEN dc.gender = 'FEMALE' THEN 'W'
ELSE NULL
WHEN si.gender = 'MALE' THEN 'M'
ELSE 'W'
END AS gender,
dc.registered,
si.registered,
-- Количество людей, которые учатся (в вузе или колледже)
COUNT(CASE WHEN is_studying_in_vuz OR is_studying_in_college THEN 1 END) AS study,
@ -356,12 +332,8 @@ summary AS (
COUNT(CASE WHEN sidit = 'true' THEN 1 END) AS convicts
FROM data_combinations dc
LEFT JOIN status_info si
ON si.recruitment_id = dc.recruitment_id
AND si.gender = dc.gender
AND si.registered::text = dc.registered
GROUP BY dc.recruitment_id, dc.gender, dc.registered
FROM status_info si
GROUP BY si.recruitment_id, si.gender, si.registered
),
summary_all AS (
SELECT

34
v1_apache-hop dash mapping/mil_com/total_registered.child_minor(m_c).hpl
View file

@ -251,18 +251,6 @@
ids AS (
SELECT unnest(string_to_array(trim(both '{}' from '${ARR_MC}'), ','))::uuid AS recruitment_id
),
gender_separations AS (
SELECT 'MALE' AS gender
UNION ALL
SELECT 'FEMALE' AS gender
),
data_combinations AS (
SELECT
i.recruitment_id AS recruitment_id,
g.gender
FROM ids i
CROSS JOIN gender_separations g
),
extracted_children AS (
SELECT
ri.recruit_id,
@ -273,7 +261,7 @@ extracted_children AS (
WHERE
jsonb_typeof(ri.info->'svedDeti'->'rebenok') = 'array'
AND r.vu_current_info->>'isMilitaryRegistered' = 'true'
AND (r.gender = 'MALE' OR r.gender = 'FEMALE')
AND r.gender IN ('MALE', 'FEMALE')
AND r.current_recruitment_id IS NOT NULL
AND r.target_recruitment_id::uuid IN (SELECT recruitment_id FROM ids)
),
@ -296,11 +284,10 @@ gender_and_recruitment_data AS (
SELECT
current_date AS recording_date,
CASE
WHEN dc.gender = 'MALE' THEN 'M'
WHEN dc.gender = 'FEMALE' THEN 'W'
ELSE NULL
WHEN r.gender = 'MALE' THEN 'M'
ELSE 'W'
END gender,
dc.recruitment_id,
r.target_recruitment_id::uuid AS recruitment_id,
COUNT(r.id) AS total_count,
COUNT(r.id) FILTER (WHERE COALESCE(children_under_18, 0) = 0) AS no_children,
COUNT(r.id) FILTER (WHERE COALESCE(children_under_18, 0) = 1) AS one_child,
@ -313,17 +300,14 @@ gender_and_recruitment_data AS (
ROUND(COUNT(r.id) FILTER (WHERE COALESCE(children_under_18, 0) = 3) * 100.0 / NULLIF(COUNT(r.id), 0), 2) AS three_children_percent,
ROUND(COUNT(r.id) FILTER (WHERE COALESCE(children_under_18, 0) >= 4) * 100.0 / NULLIF(COUNT(r.id), 0), 2) AS four_or_more_children_percent
FROM
data_combinations dc
LEFT JOIN public.recruits r
ON r.gender = dc.gender
AND r.target_recruitment_id::uuid = dc.recruitment_id
AND r.vu_current_info->>'isMilitaryRegistered' = 'true'
AND r.current_recruitment_id IS NOT NULL
public.recruits r
LEFT JOIN children_count cc
ON r.id = cc.recruit_id
WHERE r.vu_current_info->>'isMilitaryRegistered' = 'true'
AND r.current_recruitment_id IS NOT NULL
GROUP BY
dc.gender,
dc.recruitment_id
r.gender,
recruitment_id
)
SELECT
recording_date,

39
v1_apache-hop dash mapping/mil_com/total_registered.driver_license(m_c).hpl
View file

@ -261,19 +261,6 @@
ids AS (
SELECT unnest(string_to_array(trim(both '{}' from '${ARR_MC}'), ','))::uuid AS recruitment_id
),
gender_separations AS (
SELECT 'MALE' AS gender
UNION ALL
SELECT 'FEMALE' AS gender
),
data_combinations AS (
-- Создаём все комбинации target_recruitment_id и gender
SELECT
i.recruitment_id AS recruitment_id,
gs.gender
FROM ids i
CROSS JOIN gender_separations gs
),
categorized AS (
SELECT
r.id,
@ -298,11 +285,10 @@ aggregated AS (
SELECT
current_date AS recording_date,
CASE
WHEN dc.gender = 'MALE' THEN 'M'
WHEN dc.gender = 'FEMALE' THEN 'W'
ELSE NULL
WHEN r.gender = 'MALE' THEN 'M'
ELSE 'W'
END AS gender,
dc.recruitment_id,
r.recruitment_id,
-- Считаем количество уникальных рекрутов с каждой категорией
COUNT(DISTINCT r.id) FILTER (WHERE has_A > 0) AS a,
COUNT(DISTINCT r.id) FILTER (WHERE has_B > 0) AS b,
@ -311,17 +297,14 @@ aggregated AS (
COUNT(DISTINCT r.id) FILTER (WHERE has_E > 0) AS e,
COUNT(DISTINCT r.id) FILTER (WHERE has_A = 0 AND has_B = 0 AND has_C = 0 AND has_D = 0 AND has_E = 0) AS nope,
COUNT(DISTINCT r.id) AS total
FROM data_combinations dc
LEFT JOIN categorized r ON
r.recruitment_id = dc.recruitment_id AND
r.gender = dc.gender
GROUP BY dc.gender, dc.recruitment_id
FROM categorized r
GROUP BY r.gender, r.recruitment_id
),
aggregated_all AS (
SELECT
current_date AS recording_date,
'ALL' AS gender,
dc.recruitment_id,
r.recruitment_id,
COUNT(DISTINCT r.id) FILTER (WHERE has_A > 0) AS a,
COUNT(DISTINCT r.id) FILTER (WHERE has_B > 0) AS b,
COUNT(DISTINCT r.id) FILTER (WHERE has_C > 0) AS c,
@ -329,11 +312,8 @@ aggregated_all AS (
COUNT(DISTINCT r.id) FILTER (WHERE has_E > 0) AS e,
COUNT(DISTINCT r.id) FILTER (WHERE has_A = 0 AND has_B = 0 AND has_C = 0 AND has_D = 0 AND has_E = 0) AS nope,
COUNT(DISTINCT r.id) AS total
FROM data_combinations dc
LEFT JOIN categorized r ON
r.recruitment_id = dc.recruitment_id AND
r.gender IN ('MALE', 'FEMALE') -- Комбинированная агрегация по обоим гендерам
GROUP BY dc.recruitment_id
FROM categorized r
GROUP BY r.recruitment_id
)
-- Объединение результатов с вычислением процентов
@ -355,8 +335,7 @@ SELECT
COALESCE(ROUND((d * 100.0) / NULLIF(total, 0), 2), 0) AS d_percent,
COALESCE(ROUND((e * 100.0) / NULLIF(total, 0), 2), 0) AS e_percent,
COALESCE(ROUND((nope * 100.0) / NULLIF(total, 0), 2), 0) AS nope_percent
FROM aggregated_all
ORDER BY recruitment_id, gender;</sql>
FROM aggregated_all</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>

34
v1_apache-hop dash mapping/mil_com/total_registered.education_level(m_c).hpl
View file

@ -241,18 +241,6 @@
ids AS (
SELECT unnest(string_to_array(trim(both '{}' from '${ARR_MC}'), ','))::uuid AS recruitment_id
),
gender_separations AS (
SELECT 'MALE' AS gender
UNION ALL
SELECT 'FEMALE' AS gender
),
data_combinations AS (
SELECT
i.recruitment_id AS recruitment_id,
gs.gender
FROM ids i
CROSS JOIN gender_separations gs
),
education_status AS (
SELECT
ri.id,
@ -268,7 +256,7 @@ education_status AS (
) OR EXISTS (
SELECT 1
FROM jsonb_array_elements(ri.info->'svedInObr'->'inObr') AS education
WHERE jsonb_typeof(ri.info->'svedInObr'-&gt;'inObr') = 'array'
WHERE jsonb_typeof(ri.info->'svedInObr'->'inObr') = 'array'
AND education->>'kodUrObr' IN ('1', '2', '3', '4', '5')
)
) AS has_higher,
@ -326,24 +314,21 @@ education_status AS (
FROM public.recruits_info ri
JOIN public.recruits r ON ri.recruit_id = r.id
WHERE r.vu_current_info->>'isMilitaryRegistered' = 'true'
AND (r.gender = 'MALE' OR r.gender = 'FEMALE')
AND r.current_recruitment_id IS NOT NULL
AND r.gender IN ('MALE','FEMALE')
AND r.current_recruitment_id IS NOT NULL
AND r.target_recruitment_id::uuid IN (SELECT recruitment_id FROM ids)
),
final_counts AS (
SELECT
dc.gender,
dc.recruitment_id,
es.gender,
es.recruitment_id,
COUNT(DISTINCT es.id) AS total,
COUNT(DISTINCT CASE WHEN es.has_higher THEN es.id END) AS higher,
COUNT(DISTINCT CASE WHEN NOT es.has_higher AND es.has_average_prof THEN es.id END) AS average_prof,
COUNT(DISTINCT CASE WHEN NOT es.has_higher AND NOT es.has_average_prof AND es.has_only_general THEN es.id END) AS only_general,
COUNT(DISTINCT CASE WHEN es.has_no_data THEN es.id END) AS count_nodata
FROM data_combinations dc
LEFT JOIN education_status es
ON dc.recruitment_id = es.recruitment_id
AND dc.gender = es.gender
GROUP BY dc.recruitment_id, dc.gender
FROM education_status es
GROUP BY es.recruitment_id, es.gender
),
final_counts_all AS (
SELECT
@ -359,10 +344,9 @@ final_counts_all AS (
)
SELECT
current_date AS recording_date,
CASE
CASE
WHEN fc.gender = 'MALE' THEN 'M'
WHEN fc.gender = 'FEMALE' THEN 'W'
ELSE NULL
ELSE 'W'
END AS gender,
fc.recruitment_id,
fc.higher,

30
v1_apache-hop dash mapping/mil_com/total_registered.marital_status(m_c).hpl
View file

@ -229,19 +229,7 @@
<limit>0</limit>
<sql>WITH
ids AS (
SELECT unnest(string_to_array(trim(both '{}' from '${ARR_MC}'), ','))::uuid AS recruitment_id
),
gender_separations AS (
SELECT 'MALE' AS gender
UNION ALL
SELECT 'FEMALE' AS gender
),
data_combinations AS (
SELECT
i.recruitment_id AS recruitment_id,
gs.gender
FROM ids i
CROSS JOIN gender_separations gs
SELECT unnest(string_to_array(trim(both '{}' from '${ARR_MC}'), ','))::uuid AS recruitment_id
),
recruit_data AS (
SELECT
@ -258,17 +246,14 @@ recruit_data AS (
married_counts AS (
-- Считаем значения для каждого гендера
SELECT
dc.recruitment_id,
dc.gender,
rd.recruitment_id,
rd.gender,
COUNT(*) FILTER (WHERE rd.sem_pol IN ('Вдова/Вдовец', 'В разводе', 'Вдовец', 'Холост')) AS not_married,
COUNT(*) FILTER (WHERE rd.sem_pol = 'В браке') AS married,
COUNT(*) FILTER (WHERE rd.sem_pol IS NULL OR rd.sem_pol = 'Отсутствуют сведения о семейном положении') AS other,
COUNT(*) AS total
FROM data_combinations dc
LEFT JOIN recruit_data rd
ON rd.recruitment_id = dc.recruitment_id
AND rd.gender = dc.gender
GROUP BY dc.recruitment_id, dc.gender
FROM recruit_data rd
GROUP BY rd.recruitment_id, rd.gender
),
all_counts AS (
-- Считаем значения для ALL, суммируя данные по всем гендерам
@ -293,8 +278,7 @@ SELECT
COALESCE(ROUND(mc.other * 100.0 / NULLIF(mc.total, 0), 2), 0) AS other_percent,
CASE
WHEN mc.gender = 'MALE' THEN 'M'
WHEN mc.gender = 'FEMALE' THEN 'W'
ELSE NULL
ELSE 'W'
END AS gender,
current_date AS recording_date,
mc.recruitment_id
@ -311,7 +295,7 @@ SELECT
ac.gender,
current_date AS recording_date,
ac.recruitment_id
FROM all_counts ac;</sql>
FROM all_counts ac</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>

6
v1_apache-hop dash mapping/mil_com/total_registered.reg_mil_cat(m_c).hpl
View file

@ -271,8 +271,7 @@ base_data AS (
ELSE null
END)::numeric AS is_military_registered
FROM public.recruits r
WHERE (r.vu_current_info ->> 'isMilitaryRegistered' = 'true'
OR r.vu_current_info ->> 'isMilitaryRegistered' = 'false')
WHERE r.vu_current_info ->> 'isMilitaryRegistered' IN ('true', 'false')
AND r.current_recruitment_id IS NOT NULL
AND r.target_recruitment_id::uuid IN (SELECT recruitment_id FROM ids)
),
@ -390,8 +389,7 @@ LEFT JOIN t4 ON dc.target_recruitment_id = t4.target_recruitment_id
LEFT JOIN t5 ON dc.target_recruitment_id = t5.target_recruitment_id
AND dc.is_military_registered = t5.is_military_registered
LEFT JOIN t6 ON dc.target_recruitment_id = t6.target_recruitment_id
AND dc.is_military_registered = t6.is_military_registered
ORDER BY dc.target_recruitment_id, dc.is_military_registered;</sql>
AND dc.is_military_registered = t6.is_military_registered;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>