<?xml version="1.0" encoding="UTF-8"?>
<pipeline>
  <info>
    <name>nlvl_main_profile</name>
    <name_sync_with_filename>Y</name_sync_with_filename>
    <description/>
    <extended_description/>
    <pipeline_version/>
    <pipeline_type>Normal</pipeline_type>
    <parameters>
    </parameters>
    <capture_transform_performance>N</capture_transform_performance>
    <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
    <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
    <created_user>-</created_user>
    <created_date>2025/06/17 10:49:22.576</created_date>
    <modified_user>-</modified_user>
    <modified_date>2025/06/17 10:49:22.576</modified_date>
  </info>
  <notepads>
  </notepads>
  <order>
    <hop>
      <from>Table input nlvl main profile</from>
      <to>Insert / update nlvl main profile</to>
      <enabled>Y</enabled>
    </hop>
  </order>
  <transform>
    <name>Table input nlvl main profile</name>
    <type>TableInput</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <connection>ervu-dashboard</connection>
    <execute_each_row>N</execute_each_row>
    <limit>0</limit>
    <sql>--- next_level_org

WITH latest_hierarchy AS (
    -- Берем только записи с максимальной версией для каждого idm_id
    SELECT DISTINCT ON (idm_id) *
    FROM ervu_dashboard.recruitment
    ORDER BY idm_id
),
latest_date AS (
    -- Определяем последнюю дату загрузки
    SELECT MAX(recording_date) AS max_date FROM appeals.main_profile
),
filtered_records AS (
    -- Фильтруем данные, оставляя только записи с последней датой загрузки
    SELECT id_main_profile, gender, age, child_min_18, education, employment, recording_date, recruitment_id, "schema" FROM appeals.main_profile
    WHERE recording_date = (SELECT max_date FROM latest_date)
),
hierarchy_cte AS (
    -- Строим связи между уровнями (Department → Organization → Region → Ministry)
    SELECT 
        h.idm_id AS department_id,
        h.parent_id AS organization_id,
        h2.parent_id AS region_id,
        h3.parent_id AS ministry_id
    FROM latest_hierarchy h
    LEFT JOIN latest_hierarchy h2 ON h.parent_id = h2.idm_id -- Organization
    LEFT JOIN latest_hierarchy h3 ON h2.parent_id = h3.idm_id -- Region
    WHERE h.schema = 'Department' -- Начинаем с Department
),
aggregated_counts AS (
    -- Агрегируем для Organization
    SELECT 
        h.organization_id AS level_id, 
        ROUND(AVG(age)::numeric, 0) avg_age,
        MODE() WITHIN GROUP (ORDER BY gender) AS most_common_gender,
        MODE() WITHIN GROUP (ORDER BY employment) AS most_common_employment,
        ROUND(AVG(child_min_18)::numeric, 0) AS avg_child_min_18,
        MODE() WITHIN GROUP (ORDER BY education) AS most_common_education,
        MAX(r.recording_date) as recording_date,
        'Organization' AS level
    FROM filtered_records r
    JOIN hierarchy_cte h ON r.recruitment_id = h.department_id
    WHERE h.organization_id IS NOT NULL
    GROUP BY h.organization_id

    UNION ALL

    -- Агрегируем для Region
    SELECT 
        h.region_id AS level_id, 
        ROUND(AVG(age)::numeric, 0) avg_age,
        MODE() WITHIN GROUP (ORDER BY gender) AS most_common_gender,
        MODE() WITHIN GROUP (ORDER BY employment) AS most_common_employment,
        ROUND(AVG(child_min_18)::numeric, 0) AS avg_child_min_18,
        MODE() WITHIN GROUP (ORDER BY education) AS most_common_education,
        MAX(r.recording_date) as recording_date,
        'Region' AS level
    FROM filtered_records r
    JOIN hierarchy_cte h ON r.recruitment_id = h.department_id
    WHERE h.region_id IS NOT NULL
    GROUP BY h.region_id

    UNION ALL

    -- Агрегируем для Ministry
    SELECT 
        h.ministry_id AS level_id, 
        ROUND(AVG(age)::numeric, 0) avg_age,
        MODE() WITHIN GROUP (ORDER BY gender) AS most_common_gender,
        MODE() WITHIN GROUP (ORDER BY employment) AS most_common_employment,
        ROUND(AVG(child_min_18)::numeric, 0) AS avg_child_min_18,
        MODE() WITHIN GROUP (ORDER BY education) AS most_common_education,
        MAX(r.recording_date) as recording_date,
        'Ministry' AS level
    FROM filtered_records r
    JOIN hierarchy_cte h ON r.recruitment_id = h.department_id
    WHERE h.ministry_id IS NOT NULL
    GROUP BY h.ministry_id
)
SELECT * FROM aggregated_counts;
</sql>
    <variables_active>N</variables_active>
    <attributes/>
    <GUI>
      <xloc>608</xloc>
      <yloc>352</yloc>
    </GUI>
  </transform>
  <transform>
    <name>Insert / update nlvl main profile</name>
    <type>InsertUpdate</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <commit>100</commit>
    <connection>ervu-dashboard</connection>
    <lookup>
      <key>
        <condition>=</condition>
        <field>recruitment_id</field>
        <name>level_id</name>
        <name2/>
      </key>
      <key>
        <condition>=</condition>
        <field>recording_date</field>
        <name>recording_date</name>
        <name2/>
      </key>
      <schema>appeals</schema>
      <table>main_profile</table>
      <value>
        <name>age</name>
        <rename>avg_age</rename>
        <update>Y</update>
      </value>
      <value>
        <name>child_min_18</name>
        <rename>avg_child_min_18</rename>
        <update>Y</update>
      </value>
      <value>
        <name>education</name>
        <rename>most_common_education</rename>
        <update>Y</update>
      </value>
      <value>
        <name>employment</name>
        <rename>most_common_employment</rename>
        <update>Y</update>
      </value>
      <value>
        <name>gender</name>
        <rename>most_common_gender</rename>
        <update>Y</update>
      </value>
      <value>
        <name>recording_date</name>
        <rename>recording_date</rename>
        <update>Y</update>
      </value>
      <value>
        <name>schema</name>
        <rename>level</rename>
        <update>Y</update>
      </value>
      <value>
        <name>recruitment_id</name>
        <rename>level_id</rename>
        <update>Y</update>
      </value>
    </lookup>
    <update_bypassed>N</update_bypassed>
    <attributes/>
    <GUI>
      <xloc>784</xloc>
      <yloc>368</yloc>
    </GUI>
  </transform>
  <transform_error_handling>
  </transform_error_handling>
  <attributes/>
</pipeline>
