<?xml version="1.0" encoding="UTF-8"?>
<pipeline>
  <info>
    <name>citizen_spouse_flow</name>
    <name_sync_with_filename>Y</name_sync_with_filename>
    <description/>
    <extended_description/>
    <pipeline_version/>
    <pipeline_type>Normal</pipeline_type>
    <parameters>
    </parameters>
    <capture_transform_performance>N</capture_transform_performance>
    <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
    <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
    <created_user>-</created_user>
    <created_date>2025/08/05 14:31:22.799</created_date>
    <modified_user>-</modified_user>
    <modified_date>2025/08/05 14:31:22.799</modified_date>
  </info>
  <notepads>
  </notepads>
  <order>
    <hop>
      <from>Create job execution record</from>
      <to>spouse_input</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>citizen_spouse_output</from>
      <to>spouse_error_job_status</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>spouse_detect_empty_stream</from>
      <to>spouse_success_job_status</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>spouse_error_job_status</from>
      <to>Abort</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>marriage_registry_output </from>
      <to>spouse_error_job_status</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>spouse_input</from>
      <to>citizen_spouse_output</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>spouse_input</from>
      <to>spouse_sort_rows</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>spouse_sort_rows</from>
      <to>marriage_registry_output </to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>spouse_input</from>
      <to>spouse_detect_empty_stream</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>spouse_sort_rows</from>
      <to>spouse_update_flags</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>spouse_update_flags</from>
      <to>spouse_error_job_status</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>citizen_spouse_output</from>
      <to>Identify last row in a stream</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>Identify last row in a stream</from>
      <to>Filter rows</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>Filter rows</from>
      <to>spouse_success_job_status</to>
      <enabled>Y</enabled>
    </hop>
  </order>
  <transform>
    <name>Abort</name>
    <type>Abort</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <abort_option>ABORT_WITH_ERROR</abort_option>
    <always_log_rows>Y</always_log_rows>
    <row_threshold>0</row_threshold>
    <attributes/>
    <GUI>
      <xloc>1040</xloc>
      <yloc>192</yloc>
    </GUI>
  </transform>
  <transform>
    <name>Create job execution record</name>
    <type>ExecSql</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <arguments>
</arguments>
    <connection>ervu-dashboard</connection>
    <execute_each_row>N</execute_each_row>
    <quoteString>N</quoteString>
    <replace_variables>Y</replace_variables>
    <set_params>N</set_params>
    <single_statement>N</single_statement>
    <sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');

</sql>
    <attributes/>
    <GUI>
      <xloc>64</xloc>
      <yloc>576</yloc>
    </GUI>
  </transform>
  <transform>
    <name>Filter rows</name>
    <type>FilterRows</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <compare>
      <condition>
        <conditions>
</conditions>
        <function>=</function>
        <leftvalue>last_row</leftvalue>
        <negated>N</negated>
        <operator>-</operator>
        <value>
          <isnull>N</isnull>
          <length>-1</length>
          <name>constant</name>
          <precision>-1</precision>
          <text>Y</text>
          <type>Boolean</type>
        </value>
      </condition>
    </compare>
    <send_true_to>spouse_success_job_status</send_true_to>
    <attributes/>
    <GUI>
      <xloc>592</xloc>
      <yloc>912</yloc>
    </GUI>
  </transform>
  <transform>
    <name>citizen_spouse_output</name>
    <type>TableOutput</type>
    <description/>
    <distribute>N</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <commit>10000</commit>
    <connection>ervu-dashboard</connection>
    <fields>
      <field>
        <column_name>recruit_id</column_name>
        <stream_name>recruit_id</stream_name>
      </field>
      <field>
        <column_name>spouse_external_id</column_name>
        <stream_name>spouse_external_id</stream_name>
      </field>
      <field>
        <column_name>spouse_id_ern</column_name>
        <stream_name>spouse_id_ern</stream_name>
      </field>
      <field>
        <column_name>kinship_type</column_name>
        <stream_name>kinship_type</stream_name>
      </field>
      <field>
        <column_name>last_name</column_name>
        <stream_name>last_name</stream_name>
      </field>
      <field>
        <column_name>first_name</column_name>
        <stream_name>first_name</stream_name>
      </field>
      <field>
        <column_name>middle_name</column_name>
        <stream_name>middle_name</stream_name>
      </field>
      <field>
        <column_name>full_name</column_name>
        <stream_name>full_name</stream_name>
      </field>
      <field>
        <column_name>birth_date</column_name>
        <stream_name>birth_date</stream_name>
      </field>
      <field>
        <column_name>death_date</column_name>
        <stream_name>death_date</stream_name>
      </field>
      <field>
        <column_name>death_az_number</column_name>
        <stream_name>death_az_number</stream_name>
      </field>
      <field>
        <column_name>information_excluded</column_name>
        <stream_name>information_excluded</stream_name>
      </field>
    </fields>
    <ignore_errors>N</ignore_errors>
    <only_when_have_rows>N</only_when_have_rows>
    <partitioning_daily>N</partitioning_daily>
    <partitioning_enabled>N</partitioning_enabled>
    <partitioning_monthly>Y</partitioning_monthly>
    <return_keys>N</return_keys>
    <schema>ervu_dashboard</schema>
    <specify_fields>Y</specify_fields>
    <table>citizen_spouse</table>
    <tablename_in_field>N</tablename_in_field>
    <tablename_in_table>Y</tablename_in_table>
    <truncate>N</truncate>
    <use_batch>Y</use_batch>
    <attributes/>
    <GUI>
      <xloc>800</xloc>
      <yloc>576</yloc>
    </GUI>
  </transform>
  <transform>
    <name>marriage_registry_output </name>
    <type>TableOutput</type>
    <description/>
    <distribute>N</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <commit>10000</commit>
    <connection>ervu-dashboard</connection>
    <fields>
      <field>
        <column_name>recruit_id</column_name>
        <stream_name>recruit_id</stream_name>
      </field>
      <field>
        <column_name>marriage_az_number</column_name>
        <stream_name>marriage_az_number</stream_name>
      </field>
      <field>
        <column_name>marriage_date</column_name>
        <stream_name>marriage_date</stream_name>
      </field>
      <field>
        <column_name>divorce_az_number</column_name>
        <stream_name>divorce_az_number</stream_name>
      </field>
      <field>
        <column_name>divorce_date</column_name>
        <stream_name>divorce_date</stream_name>
      </field>
    </fields>
    <ignore_errors>N</ignore_errors>
    <only_when_have_rows>N</only_when_have_rows>
    <partitioning_daily>N</partitioning_daily>
    <partitioning_enabled>N</partitioning_enabled>
    <partitioning_monthly>Y</partitioning_monthly>
    <return_keys>N</return_keys>
    <schema>ervu_dashboard</schema>
    <specify_fields>Y</specify_fields>
    <table>marriage_registry</table>
    <tablename_in_field>N</tablename_in_field>
    <tablename_in_table>Y</tablename_in_table>
    <truncate>N</truncate>
    <use_batch>Y</use_batch>
    <attributes/>
    <GUI>
      <xloc>576</xloc>
      <yloc>192</yloc>
    </GUI>
  </transform>
  <transform>
    <name>spouse_detect_empty_stream</name>
    <type>DetectEmptyStream</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <attributes/>
    <GUI>
      <xloc>352</xloc>
      <yloc>720</yloc>
    </GUI>
  </transform>
  <transform>
    <name>spouse_error_job_status</name>
    <type>ExecSql</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <arguments>
      <argument>
        <name>error_description</name>
      </argument>
    </arguments>
    <connection>ervu-dashboard</connection>
    <execute_each_row>Y</execute_each_row>
    <quoteString>N</quoteString>
    <replace_variables>Y</replace_variables>
    <set_params>Y</set_params>
    <single_statement>N</single_statement>
    <sql>UPDATE etl.job_execution
SET status = 'ERROR',
    error_description = ?,
    execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}' 
and recruitment_id = '${IDM_ID}';

</sql>
    <attributes/>
    <GUI>
      <xloc>800</xloc>
      <yloc>192</yloc>
    </GUI>
  </transform>
  <transform>
    <name>spouse_input</name>
    <type>TableInput</type>
    <description/>
    <distribute>N</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <connection>ervu-dashboard</connection>
    <execute_each_row>N</execute_each_row>
    <limit/>
    <sql>SELECT
    ri.recruit_id AS recruit_id,
    (supr->>'id')::uuid AS spouse_external_id,
    NULLIF(supr->>'idERN', '') AS spouse_id_ern,
    NULLIF(supr->>'rodstvSvyazSuprug', '')::int AS kinship_type,
    supr->'svedFLBS'->'fio'->>'familiya' AS last_name,
    supr->'svedFLBS'->'fio'->>'imya' AS first_name,
    supr->'svedFLBS'->'fio'->>'otchestvo' AS middle_name,
    CONCAT_WS(' ',
              supr->'svedFLBS'->'fio'->>'familiya',
              supr->'svedFLBS'->'fio'->>'imya',
              supr->'svedFLBS'->'fio'->>'otchestvo'
    ) AS full_name,
    MAKE_DATE(
            NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'god', '')::int,
            NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'mesyacz', '')::int,
            NULLIF(supr->'svedFLBS'->'dataRozhdDok'->>'den', '')::int
    ) AS birth_date,
    (supr->'svedSmert'->'extend'->>'dataSmert')::date AS death_date,
    supr->'svedSmert'->>'nomerZapis' AS death_az_number,
    ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'nomerZapis' AS marriage_az_number,
    (ri.info->'svedFL'->'svedSemPolozh'->'svedAZBrak'->>'dataBrakKalend')::date AS marriage_date,
    ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'nomerZapis' AS divorce_az_number,
    (ri.info->'svedFL'->'svedSemPolozh'->'svedAZRazvod'->>'dataRastBrakKalend')::date AS divorce_date,
    NULLIF(ri.info->'svedFL'->'svedSemPolozh'->>'semPolozh','')::int AS marital_status,
    CASE
        WHEN ri.info->'svedFL'->'svedSemPolozh'->>'svedSuprIskl' = '1' THEN true
        ELSE false
        END AS information_excluded
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
CROSS JOIN LATERAL (
    SELECT supri
    FROM jsonb_array_elements(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') AS supri
    WHERE jsonb_typeof(ri.info->'svedFL'->'svedSemPolozh'->'suprugi') = 'array'
    ) AS supr(supr)
WHERE
    '${IDM_ID}' != ''
    AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'</sql>
    <variables_active>Y</variables_active>
    <attributes/>
    <GUI>
      <xloc>352</xloc>
      <yloc>576</yloc>
    </GUI>
  </transform>
  <transform>
    <name>spouse_sort_rows</name>
    <type>SortRows</type>
    <description/>
    <distribute>N</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <directory>${java.io.tmpdir}</directory>
    <prefix>out</prefix>
    <sort_size>100000</sort_size>
    <free_memory/>
    <compress>N</compress>
    <compress_variable/>
    <unique_rows>Y</unique_rows>
    <fields>
      <field>
        <name>recruit_id</name>
        <ascending>Y</ascending>
        <case_sensitive>N</case_sensitive>
        <collator_enabled>N</collator_enabled>
        <collator_strength>0</collator_strength>
        <presorted>N</presorted>
      </field>
    </fields>
    <attributes/>
    <GUI>
      <xloc>352</xloc>
      <yloc>192</yloc>
    </GUI>
  </transform>
  <transform>
    <name>spouse_success_job_status</name>
    <type>ExecSql</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <arguments>
</arguments>
    <connection>ervu-dashboard</connection>
    <execute_each_row>Y</execute_each_row>
    <quoteString>N</quoteString>
    <replace_variables>Y</replace_variables>
    <set_params>N</set_params>
    <single_statement>N</single_statement>
    <sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
    execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'
and status = 'PROCESSING';

</sql>
    <attributes/>
    <GUI>
      <xloc>592</xloc>
      <yloc>720</yloc>
    </GUI>
  </transform>
  <transform>
    <name>spouse_update_flags</name>
    <type>Update</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <commit>10000</commit>
    <connection>ervu-dashboard</connection>
    <error_ignored>Y</error_ignored>
    <lookup>
      <key>
        <condition>=</condition>
        <field>recruit_id</field>
        <name>recruit_id</name>
      </key>
      <key>
        <condition>&lt;></condition>
        <field>marital_status</field>
        <name>marital_status</name>
      </key>
      <schema>ervu_dashboard</schema>
      <table>citizen</table>
      <value>
        <name>marital_status</name>
        <rename>marital_status</rename>
      </value>
    </lookup>
    <skip_lookup>N</skip_lookup>
    <use_batch>Y</use_batch>
    <attributes/>
    <GUI>
      <xloc>576</xloc>
      <yloc>32</yloc>
    </GUI>
  </transform>
  <transform>
    <name>Identify last row in a stream</name>
    <type>DetectLastRow</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <resultfieldname>last_row</resultfieldname>
    <attributes/>
    <GUI>
      <xloc>800</xloc>
      <yloc>912</yloc>
    </GUI>
  </transform>
  <transform_error_handling>
    <error>
      <source_transform>citizen_spouse_output</source_transform>
      <target_transform>spouse_error_job_status</target_transform>
      <is_enabled>Y</is_enabled>
      <nr_valuename/>
      <descriptions_valuename>error_description</descriptions_valuename>
      <fields_valuename/>
      <codes_valuename/>
      <max_errors/>
      <max_pct_errors/>
      <min_pct_rows/>
    </error>
    <error>
      <source_transform>marriage_registry_output </source_transform>
      <target_transform>spouse_error_job_status</target_transform>
      <is_enabled>Y</is_enabled>
      <nr_valuename/>
      <descriptions_valuename>error_description</descriptions_valuename>
      <fields_valuename/>
      <codes_valuename/>
      <max_errors/>
      <max_pct_errors/>
      <min_pct_rows/>
    </error>
    <error>
      <source_transform>spouse_update_flags</source_transform>
      <target_transform>spouse_error_job_status</target_transform>
      <is_enabled>Y</is_enabled>
      <nr_valuename/>
      <descriptions_valuename>error_description</descriptions_valuename>
      <fields_valuename/>
      <codes_valuename/>
      <max_errors/>
      <max_pct_errors/>
      <min_pct_rows/>
    </error>
  </transform_error_handling>
  <attributes/>
</pipeline>
