micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
ervu-dashboard-etl/apache-hop mapping/country/total_registered.driver_license.hpl

552 lines
17 KiB
Text
Raw Normal View History

+apache-hop mapping
2024-10-16 10:38:29 +03:00
<pipeline>
<info>
<name>total_registered.driver_license</name>
<name_sync_with_filename>Y</name_sync_with_filename>
<description/>
<extended_description/>
<pipeline_version/>
<pipeline_type>Normal</pipeline_type>
<pipeline_status>0</pipeline_status>
<directory>/</directory>
<parameters>
</parameters>
<capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
<transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
<created_user>-</created_user>
<created_date>2024/08/02 11:56:22.507</created_date>
<modified_user>-</modified_user>
<modified_date>2024/08/02 11:56:22.507</modified_date>
<key_for_session_key>H4sIAAAAAAAAAAMAAAAAAAAAAAA=</key_for_session_key>
<is_key_private>N</is_key_private>
</info>
<notepads>
</notepads>
<order>
<hop>
<from>Table input (person_registry) РФ/женщины</from>
<to>Insert / update (total_registered.driver_license) 2 2 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input (person_registry) РФ/мужчины</from>
<to>Insert / update (total_registered.driver_license) 2 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input (person_registry) РФ/все</from>
<to>Insert / update (total_registered.driver_license) 2</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Insert / update (total_registered.driver_license) 2</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<commit>100</commit>
<update_bypassed>Y</update_bypassed>
<lookup>
<schema>total_registered</schema>
<table>driver_license</table>
<key>
<name>recruitment_id</name>
<field>recruitment_id</field>
<condition>=</condition>
<name2/>
</key>
<key>
<name>gender</name>
<field>"all_M_W"</field>
<condition>=</condition>
<name2/>
</key>
<value>
<name>"A"</name>
<rename>a</rename>
<update>Y</update>
</value>
<value>
<name>"B"</name>
<rename>b</rename>
<update>Y</update>
</value>
<value>
<name>"C"</name>
<rename>c</rename>
<update>Y</update>
</value>
<value>
<name>"D"</name>
<rename>d</rename>
<update>Y</update>
</value>
<value>
<name>"E"</name>
<rename>e</rename>
<update>Y</update>
</value>
<value>
<name>nope</name>
<rename>nope</rename>
<update>Y</update>
</value>
<value>
<name>"A_repcent"</name>
<rename>a_percent</rename>
<update>Y</update>
</value>
<value>
<name>"B_repcent"</name>
<rename>b_percent</rename>
<update>Y</update>
</value>
<value>
<name>"C_repcent"</name>
<rename>c_percent</rename>
<update>Y</update>
</value>
<value>
<name>"D_repcent"</name>
<rename>d_percent</rename>
<update>Y</update>
</value>
<value>
<name>"E_repcent"</name>
<rename>e_percent</rename>
<update>Y</update>
</value>
<value>
<name>"all_M_W"</name>
<rename>gender</rename>
<update>N</update>
</value>
<value>
<name>recruitment_id</name>
<rename>recruitment_id</rename>
<update>N</update>
</value>
</lookup>
<attributes/>
<cluster_schema/>
<GUI>
<xloc>1216</xloc>
<yloc>176</yloc>
<draw>Y</draw>
</GUI>
</transform>
<transform>
<name>Insert / update (total_registered.driver_license) 2 2</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<commit>100</commit>
<update_bypassed>Y</update_bypassed>
<lookup>
<schema>total_registered</schema>
<table>driver_license</table>
<key>
<name>recruitment_id</name>
<field>recruitment_id</field>
<condition>=</condition>
<name2/>
</key>
<key>
<name>gender</name>
<field>"all_M_W"</field>
<condition>=</condition>
<name2/>
</key>
<value>
<name>"A"</name>
<rename>a</rename>
<update>Y</update>
</value>
<value>
<name>"B"</name>
<rename>b</rename>
<update>Y</update>
</value>
<value>
<name>"C"</name>
<rename>c</rename>
<update>Y</update>
</value>
<value>
<name>"D"</name>
<rename>d</rename>
<update>Y</update>
</value>
<value>
<name>"E"</name>
<rename>e</rename>
<update>Y</update>
</value>
<value>
<name>nope</name>
<rename>nope</rename>
<update>Y</update>
</value>
<value>
<name>"A_repcent"</name>
<rename>a_percent</rename>
<update>Y</update>
</value>
<value>
<name>"B_repcent"</name>
<rename>b_percent</rename>
<update>Y</update>
</value>
<value>
<name>"C_repcent"</name>
<rename>c_percent</rename>
<update>Y</update>
</value>
<value>
<name>"D_repcent"</name>
<rename>d_percent</rename>
<update>Y</update>
</value>
<value>
<name>"E_repcent"</name>
<rename>e_percent</rename>
<update>Y</update>
</value>
<value>
<name>"all_M_W"</name>
<rename>gender</rename>
<update>N</update>
</value>
<value>
<name>recruitment_id</name>
<rename>recruitment_id</rename>
<update>N</update>
</value>
</lookup>
<attributes/>
<cluster_schema/>
<GUI>
<xloc>1216</xloc>
<yloc>272</yloc>
<draw>Y</draw>
</GUI>
</transform>
<transform>
<name>Insert / update (total_registered.driver_license) 2 2 2</name>
<type>InsertUpdate</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<commit>100</commit>
<update_bypassed>Y</update_bypassed>
<lookup>
<schema>total_registered</schema>
<table>driver_license</table>
<key>
<name>recruitment_id</name>
<field>recruitment_id</field>
<condition>=</condition>
<name2/>
</key>
<key>
<name>gender</name>
<field>"all_M_W"</field>
<condition>=</condition>
<name2/>
</key>
<value>
<name>"A"</name>
<rename>a</rename>
<update>Y</update>
</value>
<value>
<name>"B"</name>
<rename>b</rename>
<update>Y</update>
</value>
<value>
<name>"C"</name>
<rename>c</rename>
<update>Y</update>
</value>
<value>
<name>"D"</name>
<rename>d</rename>
<update>Y</update>
</value>
<value>
<name>"E"</name>
<rename>e</rename>
<update>Y</update>
</value>
<value>
<name>nope</name>
<rename>nope</rename>
<update>Y</update>
</value>
<value>
<name>"A_repcent"</name>
<rename>a_percent</rename>
<update>Y</update>
</value>
<value>
<name>"B_repcent"</name>
<rename>b_percent</rename>
<update>Y</update>
</value>
<value>
<name>"C_repcent"</name>
<rename>c_percent</rename>
<update>Y</update>
</value>
<value>
<name>"D_repcent"</name>
<rename>d_percent</rename>
<update>Y</update>
</value>
<value>
<name>"E_repcent"</name>
<rename>e_percent</rename>
<update>Y</update>
</value>
<value>
<name>"all_M_W"</name>
<rename>gender</rename>
<update>N</update>
</value>
<value>
<name>recruitment_id</name>
<rename>recruitment_id</rename>
<update>N</update>
</value>
</lookup>
<attributes/>
<cluster_schema/>
<GUI>
<xloc>1216</xloc>
<yloc>368</yloc>
<draw>Y</draw>
</GUI>
</transform>
<transform>
<name>Table input (person_registry) РФ/все</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>postgres.person_registry</connection>
<sql>WITH categorized AS (
SELECT
r.id,
r.gender,
-- Проверяем наличие хотя бы одной категории, используем DISTINCT для уникальных рекрутов
MAX(CASE WHEN cat->>'kategoriya' like '%A%' THEN 1 ELSE 0 END) AS has_A,
MAX(CASE WHEN cat->>'kategoriya' like '%B%' THEN 1 ELSE 0 END) AS has_B,
MAX(CASE WHEN cat->>'kategoriya' like '%C%' THEN 1 ELSE 0 END) AS has_C,
MAX(CASE WHEN cat->>'kategoriya' like '%D%' THEN 1 ELSE 0 END) AS has_D,
MAX(CASE WHEN cat->>'kategoriya' like '%E%' THEN 1 ELSE 0 END) AS has_E
FROM public.recruits_info ri
JOIN public.recruits r ON ri.recruit_id = r.id
LEFT JOIN jsonb_array_elements(ri.info->'svedVoditUdost'->'voditUdost'->'svedKat') AS cat ON true
WHERE r.vu_current_info->>'isMilitaryRegistered' = 'true'
AND r.current_recruitment_id IS NOT NULL
AND r.target_recruitment_id IS NOT NULL
GROUP BY r.id, r.gender
),
aggregated AS (
SELECT
'ALL' AS gender,
'00' AS recruitment_id,
-- Считаем количество уникальных рекрутов с каждой категорией
COUNT(DISTINCT r.id) FILTER (WHERE has_A > 0) AS a,
COUNT(DISTINCT r.id) FILTER (WHERE has_B > 0) AS b,
COUNT(DISTINCT r.id) FILTER (WHERE has_C > 0) AS c,
COUNT(DISTINCT r.id) FILTER (WHERE has_D > 0) AS d,
COUNT(DISTINCT r.id) FILTER (WHERE has_E > 0) AS e,
COUNT(DISTINCT r.id) FILTER (WHERE has_A = 0 AND has_B = 0 AND has_C = 0 AND has_D = 0 AND has_E = 0) AS nope,
COUNT(DISTINCT r.id) AS total
FROM categorized r
)
SELECT *,
ROUND((a * 100.0) / NULLIF(total, 0), 2) AS a_percent,
ROUND((b * 100.0) / NULLIF(total, 0), 2) AS b_percent,
ROUND((c * 100.0) / NULLIF(total, 0), 2) AS c_percent,
ROUND((d * 100.0) / NULLIF(total, 0), 2) AS d_percent,
ROUND((e * 100.0) / NULLIF(total, 0), 2) AS e_percent,
ROUND((nope * 100.0) / NULLIF(total, 0), 2) AS nope_percent
FROM aggregated;</sql>
<limit>0</limit>
<lookup/>
<execute_each_row>N</execute_each_row>
<variables_active>N</variables_active>
<lazy_conversion_active>N</lazy_conversion_active>
<attributes/>
<cluster_schema/>
<GUI>
<xloc>592</xloc>
<yloc>176</yloc>
<draw>Y</draw>
</GUI>
</transform>
<transform>
<name>Table input (person_registry) РФ/женщины</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>postgres.person_registry</connection>
<sql>WITH categorized AS (
SELECT
r.id,
r.gender,
-- Проверяем наличие хотя бы одной категории, используем DISTINCT для уникальных рекрутов
MAX(CASE WHEN cat->>'kategoriya' like '%A%' THEN 1 ELSE 0 END) AS has_A,
MAX(CASE WHEN cat->>'kategoriya' like '%B%' THEN 1 ELSE 0 END) AS has_B,
MAX(CASE WHEN cat->>'kategoriya' like '%C%' THEN 1 ELSE 0 END) AS has_C,
MAX(CASE WHEN cat->>'kategoriya' like '%D%' THEN 1 ELSE 0 END) AS has_D,
MAX(CASE WHEN cat->>'kategoriya' like '%E%' THEN 1 ELSE 0 END) AS has_E
FROM public.recruits_info ri
JOIN public.recruits r ON ri.recruit_id = r.id
LEFT JOIN jsonb_array_elements(ri.info->'svedVoditUdost'->'voditUdost'->'svedKat') AS cat ON true
WHERE r.vu_current_info->>'isMilitaryRegistered' = 'true'
AND r.current_recruitment_id IS NOT NULL
AND r.target_recruitment_id IS NOT NULL
AND r.gender = 'FEMALE'
GROUP BY r.id, r.gender
),
aggregated AS (
SELECT
'W' AS gender,
'00' as recruitment_id,
-- Считаем количество уникальных рекрутов с каждой категорией
COUNT(DISTINCT r.id) FILTER (WHERE has_A > 0) AS a,
COUNT(DISTINCT r.id) FILTER (WHERE has_B > 0) AS b,
COUNT(DISTINCT r.id) FILTER (WHERE has_C > 0) AS c,
COUNT(DISTINCT r.id) FILTER (WHERE has_D > 0) AS d,
COUNT(DISTINCT r.id) FILTER (WHERE has_E > 0) AS e,
COUNT(DISTINCT r.id) FILTER (WHERE has_A = 0 AND has_B = 0 AND has_C = 0 AND has_D = 0 AND has_E = 0) AS nope,
COUNT(DISTINCT r.id) AS total
FROM categorized r
)
SELECT *,
ROUND((a * 100.0) / NULLIF(total, 0), 2) AS a_percent,
ROUND((b * 100.0) / NULLIF(total, 0), 2) AS b_percent,
ROUND((c * 100.0) / NULLIF(total, 0), 2) AS c_percent,
ROUND((d * 100.0) / NULLIF(total, 0), 2) AS d_percent,
ROUND((e * 100.0) / NULLIF(total, 0), 2) AS e_percent,
ROUND((nope * 100.0) / NULLIF(total, 0), 2) AS nope_percent
FROM aggregated;</sql>
<limit>0</limit>
<lookup/>
<execute_each_row>N</execute_each_row>
<variables_active>N</variables_active>
<lazy_conversion_active>N</lazy_conversion_active>
<attributes/>
<cluster_schema/>
<GUI>
<xloc>592</xloc>
<yloc>368</yloc>
<draw>Y</draw>
</GUI>
</transform>
<transform>
<name>Table input (person_registry) РФ/мужчины</name>
<type>TableInput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>postgres.person_registry</connection>
<sql>WITH categorized AS (
SELECT
r.id,
r.gender,
-- Проверяем наличие хотя бы одной категории, используем DISTINCT для уникальных рекрутов
MAX(CASE WHEN cat->>'kategoriya' like '%A%' THEN 1 ELSE 0 END) AS has_A,
MAX(CASE WHEN cat->>'kategoriya' like '%B%' THEN 1 ELSE 0 END) AS has_B,
MAX(CASE WHEN cat->>'kategoriya' like '%C%' THEN 1 ELSE 0 END) AS has_C,
MAX(CASE WHEN cat->>'kategoriya' like '%D%' THEN 1 ELSE 0 END) AS has_D,
MAX(CASE WHEN cat->>'kategoriya' like '%E%' THEN 1 ELSE 0 END) AS has_E
FROM public.recruits_info ri
JOIN public.recruits r ON ri.recruit_id = r.id
LEFT JOIN jsonb_array_elements(ri.info->'svedVoditUdost'->'voditUdost'->'svedKat') AS cat ON true
WHERE r.vu_current_info->>'isMilitaryRegistered' = 'true'
AND r.current_recruitment_id IS NOT NULL
AND r.target_recruitment_id IS NOT NULL
AND r.gender = 'MALE'
GROUP BY r.id, r.gender
),
aggregated AS (
SELECT
'M' AS gender,
'00' as recruitment_id,
-- Считаем количество уникальных рекрутов с каждой категорией
COUNT(DISTINCT r.id) FILTER (WHERE has_A > 0) AS a,
COUNT(DISTINCT r.id) FILTER (WHERE has_B > 0) AS b,
COUNT(DISTINCT r.id) FILTER (WHERE has_C > 0) AS c,
COUNT(DISTINCT r.id) FILTER (WHERE has_D > 0) AS d,
COUNT(DISTINCT r.id) FILTER (WHERE has_E > 0) AS e,
COUNT(DISTINCT r.id) FILTER (WHERE has_A = 0 AND has_B = 0 AND has_C = 0 AND has_D = 0 AND has_E = 0) AS nope,
COUNT(DISTINCT r.id) AS total
FROM categorized r
)
SELECT *,
ROUND((a * 100.0) / NULLIF(total, 0), 2) AS a_percent,
ROUND((b * 100.0) / NULLIF(total, 0), 2) AS b_percent,
ROUND((c * 100.0) / NULLIF(total, 0), 2) AS c_percent,
ROUND((d * 100.0) / NULLIF(total, 0), 2) AS d_percent,
ROUND((e * 100.0) / NULLIF(total, 0), 2) AS e_percent,
ROUND((nope * 100.0) / NULLIF(total, 0), 2) AS nope_percent
FROM aggregated;</sql>
<limit>0</limit>
<lookup/>
<execute_each_row>N</execute_each_row>
<variables_active>N</variables_active>
<lazy_conversion_active>N</lazy_conversion_active>
<attributes/>
<cluster_schema/>
<GUI>
<xloc>592</xloc>
<yloc>272</yloc>
<draw>Y</draw>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>
</pipeline>
Reference in a new issue Copy permalink