<?xml version="1.0" encoding="UTF-8"?>
<pipeline>
  <info>
    <name>health_state_flow_delta</name>
    <name_sync_with_filename>Y</name_sync_with_filename>
    <description/>
    <extended_description/>
    <pipeline_version/>
    <pipeline_type>Normal</pipeline_type>
    <pipeline_status>0</pipeline_status>
    <parameters>
      <parameter>
        <name>IDM_ID</name>
        <default_value>42c79094-4767-4898-9c45-e2453d0665c7</default_value>
        <description/>
      </parameter>
      <parameter>
        <name>MAX_SOURCE_UPDATE_DATE</name>
        <default_value>2024-05-10</default_value>
        <description/>
      </parameter>
    </parameters>
    <capture_transform_performance>N</capture_transform_performance>
    <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
    <transform_performance_capturing_size_limit>100</transform_performance_capturing_size_limit>
    <created_user>-</created_user>
    <created_date>2025/09/08 22:10:33.736</created_date>
    <modified_user>-</modified_user>
    <modified_date>2025/09/08 22:10:33.736</modified_date>
  </info>
  <notepads>
  </notepads>
  <order>
    <hop>
      <from>Change job status on error</from>
      <to>Abort</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>Create job execution record</from>
      <to>health_state_input</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>Filter rows</from>
      <to>Change job status on success</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>Identify last row in a stream</from>
      <to>Detect empty stream</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>Identify last row in a stream</from>
      <to>health_state_upsert</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>health_state_upsert</from>
      <to>Change job status on error</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>Detect empty stream</from>
      <to>Change job status on success</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>Filter rows 2</from>
      <to>Update</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>Filter rows 2</from>
      <to>Identify last row in a stream</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>health_state_input</from>
      <to>Filter rows 2</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>health_state_upsert</from>
      <to>Update 2</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>Update 2</from>
      <to>Change job status on error</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>Change job status on error 2</from>
      <to>Abort 2</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>Update</from>
      <to>Change job status on error 2</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>Update 2</from>
      <to>Block until transforms finish</to>
      <enabled>Y</enabled>
    </hop>
    <hop>
      <from>Block until transforms finish</from>
      <to>Filter rows</to>
      <enabled>Y</enabled>
    </hop>
  </order>
  <transform>
    <name>Abort</name>
    <type>Abort</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <abort_option>ABORT</abort_option>
    <always_log_rows>Y</always_log_rows>
    <row_threshold>0</row_threshold>
    <attributes/>
    <GUI>
      <xloc>2256</xloc>
      <yloc>704</yloc>
    </GUI>
  </transform>
  <transform>
    <name>Abort 2</name>
    <type>Abort</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <abort_option>ABORT</abort_option>
    <always_log_rows>Y</always_log_rows>
    <row_threshold>0</row_threshold>
    <attributes/>
    <GUI>
      <xloc>1328</xloc>
      <yloc>848</yloc>
    </GUI>
  </transform>
  <transform>
    <name>Block until transforms finish</name>
    <type>BlockUntilTransformsFinish</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <transforms>
      <transform>
        <name>Update</name>
      </transform>
    </transforms>
    <attributes/>
    <GUI>
      <xloc>2064</xloc>
      <yloc>512</yloc>
    </GUI>
  </transform>
  <transform>
    <name>Change job status on error</name>
    <type>ExecSql</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <arguments>
      <argument>
        <name>error_description</name>
      </argument>
    </arguments>
    <connection>ervu-dashboard</connection>
    <execute_each_row>Y</execute_each_row>
    <quoteString>N</quoteString>
    <replace_variables>Y</replace_variables>
    <set_params>Y</set_params>
    <single_statement>N</single_statement>
    <sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
    error_description = ?,
    execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}' 
and recruitment_id = '${IDM_ID}';

</sql>
    <attributes/>
    <GUI>
      <xloc>1856</xloc>
      <yloc>704</yloc>
    </GUI>
  </transform>
  <transform>
    <name>Change job status on error 2</name>
    <type>ExecSql</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <arguments>
      <argument>
        <name>error_description</name>
      </argument>
    </arguments>
    <connection>ervu-dashboard</connection>
    <execute_each_row>Y</execute_each_row>
    <quoteString>N</quoteString>
    <replace_variables>Y</replace_variables>
    <set_params>Y</set_params>
    <single_statement>N</single_statement>
    <sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
    error_description = ?,
    execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}' 
and recruitment_id = '${IDM_ID}';

</sql>
    <attributes/>
    <GUI>
      <xloc>1120</xloc>
      <yloc>848</yloc>
    </GUI>
  </transform>
  <transform>
    <name>Change job status on success</name>
    <type>ExecSql</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <arguments>
</arguments>
    <connection>ervu-dashboard</connection>
    <execute_each_row>Y</execute_each_row>
    <quoteString>N</quoteString>
    <replace_variables>Y</replace_variables>
    <set_params>N</set_params>
    <single_statement>N</single_statement>
    <sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS',
    execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';

</sql>
    <attributes/>
    <GUI>
      <xloc>2256</xloc>
      <yloc>336</yloc>
    </GUI>
  </transform>
  <transform>
    <name>Create job execution record</name>
    <type>ExecSql</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <arguments>
</arguments>
    <connection>ervu-dashboard</connection>
    <execute_each_row>N</execute_each_row>
    <quoteString>N</quoteString>
    <replace_variables>Y</replace_variables>
    <set_params>N</set_params>
    <single_statement>N</single_statement>
    <sql>UPDATE etl.job_execution  
SET
    status = 'DELTA_PROCESSING',
    execution_datetime = current_timestamp,
    error_description = NULL
where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql>
    <attributes/>
    <GUI>
      <xloc>656</xloc>
      <yloc>512</yloc>
    </GUI>
  </transform>
  <transform>
    <name>Detect empty stream</name>
    <type>DetectEmptyStream</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <attributes/>
    <GUI>
      <xloc>1376</xloc>
      <yloc>336</yloc>
    </GUI>
  </transform>
  <transform>
    <name>Filter rows</name>
    <type>FilterRows</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <compare>
      <condition>
        <conditions>
</conditions>
        <function>=</function>
        <leftvalue>last_row</leftvalue>
        <negated>N</negated>
        <operator>-</operator>
        <value>
          <isnull>N</isnull>
          <length>-1</length>
          <name>constant</name>
          <precision>-1</precision>
          <text>Y</text>
          <type>Boolean</type>
        </value>
      </condition>
    </compare>
    <send_true_to>Change job status on success</send_true_to>
    <attributes/>
    <GUI>
      <xloc>2256</xloc>
      <yloc>512</yloc>
    </GUI>
  </transform>
  <transform>
    <name>Filter rows 2</name>
    <type>FilterRows</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <compare>
      <condition>
        <conditions>
</conditions>
        <function>=</function>
        <leftvalue>has_disease</leftvalue>
        <negated>N</negated>
        <operator>-</operator>
        <value>
          <isnull>N</isnull>
          <length>-1</length>
          <name>constant</name>
          <precision>-1</precision>
          <text>Y</text>
          <type>Boolean</type>
        </value>
      </condition>
    </compare>
    <send_false_to>Update</send_false_to>
    <send_true_to>Identify last row in a stream</send_true_to>
    <attributes/>
    <GUI>
      <xloc>1120</xloc>
      <yloc>512</yloc>
    </GUI>
  </transform>
  <transform>
    <name>Identify last row in a stream</name>
    <type>DetectLastRow</type>
    <description/>
    <distribute>N</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <resultfieldname>last_row</resultfieldname>
    <attributes/>
    <GUI>
      <xloc>1376</xloc>
      <yloc>512</yloc>
    </GUI>
  </transform>
  <transform>
    <name>Update</name>
    <type>Update</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <commit>10000</commit>
    <connection>ervu-dashboard</connection>
    <error_ignored>N</error_ignored>
    <lookup>
      <key>
        <condition>=</condition>
        <field>recruit_id</field>
        <name>recruit_id</name>
      </key>
      <schema>ervu_dashboard</schema>
      <table>citizen</table>
      <value>
        <name>has_health_state_info</name>
        <rename>has_health_state_info</rename>
      </value>
    </lookup>
    <skip_lookup>N</skip_lookup>
    <use_batch>Y</use_batch>
    <attributes/>
    <GUI>
      <xloc>1120</xloc>
      <yloc>704</yloc>
    </GUI>
  </transform>
  <transform>
    <name>Update 2</name>
    <type>Update</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <commit>10000</commit>
    <connection>ervu-dashboard</connection>
    <error_ignored>N</error_ignored>
    <lookup>
      <key>
        <condition>=</condition>
        <field>recruit_id</field>
        <name>recruit_id</name>
      </key>
      <schema>ervu_dashboard</schema>
      <table>citizen</table>
      <value>
        <name>has_disease</name>
        <rename>has_disease</rename>
      </value>
    </lookup>
    <skip_lookup>N</skip_lookup>
    <use_batch>Y</use_batch>
    <attributes/>
    <GUI>
      <xloc>1856</xloc>
      <yloc>512</yloc>
    </GUI>
  </transform>
  <transform>
    <name>health_state_input</name>
    <type>TableInput</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <connection>ervu-dashboard</connection>
    <execute_each_row>N</execute_each_row>
    <limit/>
    <sql>WITH filteredData AS (SELECT ri.recruit_id,
                             ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved'  as source_update_date,
                             ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' -> 'sostZdorov' as zdorov_arr,
                             coalesce(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'priznakNalichSved' =
                                      '1', false)                                                    as has_health_state_info
                      FROM ervu_dashboard.recruits_info ri
                      WHERE ri.info -> 'svedFL' -> 'svedSostZdorov' ->> 'svedZdorov' &lt;&gt; 'null'
                        AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
                        and case
                                when '${MAX_SOURCE_UPDATE_DATE}' &lt;&gt; ''
                                    then to_date(ri.info -> 'svedFL' -> 'svedSostZdorov' -> 'svedZdorov' ->> 'dataSved',
                                                 'YYYY-MM-DD') >
                                         to_date('${MAX_SOURCE_UPDATE_DATE}', 'YYYY-MM-DD')
                                else true end
                      )
SELECT fd.recruit_id,
       has_health_state_info,
       zdorov_elem ->> 'id'                               AS source_id,
       source_update_date,
       zdorov_elem ->> 'kodZabol'                         AS disease_code,
       to_date(zdorov_elem ->> 'dataDiagn', 'YYYY-MM-DD') AS diagnosis_date,
       coalesce(zdorov_elem ->> 'kodZabol' &lt;&gt; 'null', false) as has_disease
FROM filteredData fd
         CROSS JOIN LATERAL jsonb_array_elements(COALESCE(
                NULLIF(zdorov_arr, 'null'::jsonb),
                '[]'::jsonb)) AS zdorov_elem;</sql>
    <variables_active>Y</variables_active>
    <attributes/>
    <GUI>
      <xloc>880</xloc>
      <yloc>512</yloc>
    </GUI>
  </transform>
  <transform>
    <name>health_state_upsert</name>
    <type>InsertUpdate</type>
    <description/>
    <distribute>Y</distribute>
    <custom_distribution/>
    <copies>1</copies>
    <partitioning>
      <method>none</method>
      <schema_name/>
    </partitioning>
    <commit>10000</commit>
    <connection>ervu-dashboard</connection>
    <lookup>
      <key>
        <condition>=</condition>
        <field>source_id</field>
        <name>source_id</name>
      </key>
      <schema>ervu_dashboard</schema>
      <table>health_state</table>
      <value>
        <name>recruit_id</name>
        <rename>recruit_id</rename>
        <update>Y</update>
      </value>
      <value>
        <name>source_id</name>
        <rename>source_id</rename>
        <update>Y</update>
      </value>
      <value>
        <name>source_update_date</name>
        <rename>source_update_date</rename>
        <update>Y</update>
      </value>
      <value>
        <name>disease_code</name>
        <rename>disease_code</rename>
        <update>Y</update>
      </value>
      <value>
        <name>diagnosis_date</name>
        <rename>diagnosis_date</rename>
        <update>Y</update>
      </value>
    </lookup>
    <update_bypassed>N</update_bypassed>
    <attributes/>
    <GUI>
      <xloc>1664</xloc>
      <yloc>512</yloc>
    </GUI>
  </transform>
  <transform_error_handling>
    <error>
      <source_transform>Update</source_transform>
      <target_transform>Change job status on error 2</target_transform>
      <is_enabled>Y</is_enabled>
      <nr_valuename/>
      <descriptions_valuename>error_description</descriptions_valuename>
      <fields_valuename/>
      <codes_valuename/>
      <max_errors/>
      <max_pct_errors/>
      <min_pct_rows/>
    </error>
    <error>
      <source_transform>Update 2</source_transform>
      <target_transform>Change job status on error</target_transform>
      <is_enabled>Y</is_enabled>
      <nr_valuename/>
      <descriptions_valuename/>
      <fields_valuename/>
      <codes_valuename/>
      <max_errors/>
      <max_pct_errors/>
      <min_pct_rows/>
    </error>
    <error>
      <source_transform>health_state_upsert</source_transform>
      <target_transform>Change job status on error</target_transform>
      <is_enabled>Y</is_enabled>
      <nr_valuename/>
      <descriptions_valuename>error_description</descriptions_valuename>
      <fields_valuename/>
      <codes_valuename/>
      <max_errors/>
      <max_pct_errors/>
      <min_pct_rows/>
    </error>
  </transform_error_handling>
  <attributes/>
</pipeline>
